From c931a78081034e543760fc93fac9227f1068e15b Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:01:54 +0200 Subject: [PATCH 01/10] update sauce connect proxy to 4.6.2 --- tests/frontend/travis/sauce_tunnel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/frontend/travis/sauce_tunnel.sh b/tests/frontend/travis/sauce_tunnel.sh index b8e1bc59e..0c4dd5245 100755 --- a/tests/frontend/travis/sauce_tunnel.sh +++ b/tests/frontend/travis/sauce_tunnel.sh @@ -9,7 +9,7 @@ if [ -z "${SAUCE_USERNAME}" ]; then echo "SAUCE_USERNAME is unset - exiting"; exit 1; fi if [ -z "${SAUCE_ACCESS_KEY}" ]; then echo "SAUCE_ACCESS_KEY is unset - exiting"; exit 1; fi -curl https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz > /tmp/sauce.tar.gz +curl https://saucelabs.com/downloads/sc-4.6.2-linux.tar.gz > /tmp/sauce.tar.gz tar zxf /tmp/sauce.tar.gz --directory /tmp mv /tmp/sc-*-linux /tmp/sauce_connect From d407a890b3b42d11f60fc4adcfe990e02876dbdd Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:07:50 +0200 Subject: [PATCH 02/10] include tunnelIdentifier in webdriver capabilities --- tests/frontend/travis/remote_runner.js | 1 + tests/frontend/travis/sauce_tunnel.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index 1e04b2a0d..aba5435e8 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -17,6 +17,7 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { testSettings.name = name; testSettings["public"] = true; testSettings["build"] = process.env.GIT_HASH; + testSettings["tunnelIdentifier"] = process.env.TRAVIS_JOB_NUMBER; // we wait 10 seconds here with the hope it was enough time for the minified files to be built etc. setTimeout(function(){ diff --git a/tests/frontend/travis/sauce_tunnel.sh b/tests/frontend/travis/sauce_tunnel.sh index 0c4dd5245..4ab6e81a4 100755 --- a/tests/frontend/travis/sauce_tunnel.sh +++ b/tests/frontend/travis/sauce_tunnel.sh @@ -14,7 +14,7 @@ tar zxf /tmp/sauce.tar.gz --directory /tmp mv /tmp/sc-*-linux /tmp/sauce_connect # start the sauce connector in background and make sure it doesn't output the secret key -(/tmp/sauce_connect/bin/sc --user "${SAUCE_USERNAME}" --key "${SAUCE_ACCESS_KEY}" --pidfile /tmp/sauce.pid --readyfile /tmp/tunnel > /dev/null )& +(/tmp/sauce_connect/bin/sc --user "${SAUCE_USERNAME}" --key "${SAUCE_ACCESS_KEY}" -i "${TRAVIS_JOB_NUMBER}" --pidfile /tmp/sauce.pid --readyfile /tmp/tunnel > /dev/null )& # wait for the tunnel to build up while [ ! -e "/tmp/tunnel" ] From 8832b074e46762fbe2f0f7141c6f900f22339b9b Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:09:00 +0200 Subject: [PATCH 03/10] add platform in console output --- tests/frontend/travis/remote_runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index aba5435e8..a4960db4c 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -39,7 +39,7 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { var testResult = knownConsoleText.replace(/\[red\]/g,'\x1B[31m').replace(/\[yellow\]/g,'\x1B[33m') .replace(/\[green\]/g,'\x1B[32m').replace(/\[clear\]/g, '\x1B[39m'); testResult = testResult.split("\\n").map(function(line){ - return "[" + testSettings.browserName + (testSettings.version === "" ? '' : (" " + testSettings.version)) + "] " + line; + return "[" + testSettings.browserName + " " + testSettings.platform + (testSettings.version === "" ? '' : (" " + testSettings.version)) + "] " + line; }).join("\n"); console.log(testResult); From ac54bf49097dddc02f33a038d3aaaffa535f3139 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:12:36 +0200 Subject: [PATCH 04/10] include extendedDebugging in webdriver capabilities to get browser console logs --- tests/frontend/travis/remote_runner.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index a4960db4c..6925d9267 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -17,6 +17,7 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { testSettings.name = name; testSettings["public"] = true; testSettings["build"] = process.env.GIT_HASH; + testSettings["extendedDebugging"] = true; // console.json can be downloaded via saucelabs, don't know how to print them into output of the tests testSettings["tunnelIdentifier"] = process.env.TRAVIS_JOB_NUMBER; // we wait 10 seconds here with the hope it was enough time for the minified files to be built etc. From 5e1b82388774973d513cc0bccf247546ce73e709 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:14:50 +0200 Subject: [PATCH 05/10] informative: add comment for timeouts during tests --- tests/frontend/travis/remote_runner.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index 6925d9267..4475456dc 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -49,7 +49,10 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { callback(); } - //timeout for the case the test hangs + /** + * timeout for the case the test hangs + * @todo this should be configured in testSettings, see https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-Timeouts + */ var timeout = setTimeout(function(){ stopSauce(false); }, 1200000 * 10); From f0934076fab7b96ef98a5002af5373347a888c46 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:18:30 +0200 Subject: [PATCH 06/10] When the killTimeout in runner.js stops the tests, it's an failure. --- tests/frontend/travis/remote_runner.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index 4475456dc..c0161634d 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -66,8 +66,13 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { knownConsoleText = consoleText; if(knownConsoleText.indexOf("FINISHED") > 0){ - var success = knownConsoleText.indexOf("FAILED") === -1; - stopSauce(success); + let match = knownConsoleText.match(/FINISHED - ([0-9]+) tests passed, ([0-9]+) tests failed/); + if (match[2] && match[2] == 0){ + stopSauce(true); + } + else { + stopSauce(false); + } } }); }, 5000); From bc9566146f3ae498d0f166e21fa6468b28a3a7b3 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:24:24 +0200 Subject: [PATCH 07/10] do not wait a hardcoded amount of 10 seconds for files to be minified. this setup time is not included in the total time of the first test. --- tests/frontend/travis/remote_runner.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index c0161634d..a77e98c2b 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -20,8 +20,6 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { testSettings["extendedDebugging"] = true; // console.json can be downloaded via saucelabs, don't know how to print them into output of the tests testSettings["tunnelIdentifier"] = process.env.TRAVIS_JOB_NUMBER; - // we wait 10 seconds here with the hope it was enough time for the minified files to be built etc. - setTimeout(function(){ browser.init(testSettings).get("http://localhost:9001/tests/frontend/", function(){ var url = "https://saucelabs.com/jobs/" + browser.sessionID; console.log("Remote sauce test '" + name + "' started! " + url); @@ -78,8 +76,6 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { }, 5000); }); - }, 10000); - }, 1); //run 1 test in parrallel // 1) Firefox on Linux From 5d605d5cdf0f2b1d7d5eeef9e09f88aea38fcd8f Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:29:48 +0200 Subject: [PATCH 08/10] run 4 browsers at a time during frontend testing --- tests/frontend/travis/remote_runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js index a77e98c2b..115963d00 100644 --- a/tests/frontend/travis/remote_runner.js +++ b/tests/frontend/travis/remote_runner.js @@ -76,7 +76,7 @@ var sauceTestWorker = async.queue(function (testSettings, callback) { }, 5000); }); -}, 1); //run 1 test in parrallel +}, 4); //run 4 test in parrallel // 1) Firefox on Linux sauceTestWorker.push({ From a974d45066bd136a4661e30ac8d49c87d7eef9ce Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:38:12 +0200 Subject: [PATCH 09/10] try to include test.speed in output --- tests/frontend/runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/frontend/runner.js b/tests/frontend/runner.js index f233e7a9a..5a8da6e9f 100644 --- a/tests/frontend/runner.js +++ b/tests/frontend/runner.js @@ -60,7 +60,7 @@ $(function(){ stats.tests = stats.tests || 0; stats.tests++; if ('passed' == test.state) { - append("->","[green]PASSED[clear] :", test.title); + append("->","[green]PASSED[clear] :", test.title," ",test.speed,"ms"); } else if (test.pending) { append("->","[yellow]PENDING[clear]:", test.title); } else { From af3ab060e150c705237f7596d537e947482c5894 Mon Sep 17 00:00:00 2001 From: webzwo0i Date: Fri, 10 Jul 2020 21:50:43 +0200 Subject: [PATCH 10/10] time is in test.duration, not test.speed --- tests/frontend/runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/frontend/runner.js b/tests/frontend/runner.js index 5a8da6e9f..e9eedc646 100644 --- a/tests/frontend/runner.js +++ b/tests/frontend/runner.js @@ -60,7 +60,7 @@ $(function(){ stats.tests = stats.tests || 0; stats.tests++; if ('passed' == test.state) { - append("->","[green]PASSED[clear] :", test.title," ",test.speed,"ms"); + append("->","[green]PASSED[clear] :", test.title," ",test.duration,"ms"); } else if (test.pending) { append("->","[yellow]PENDING[clear]:", test.title); } else {