From b67c9cc13660ce0522fc33850c1a6361213978f8 Mon Sep 17 00:00:00 2001
From: webzwo0i <webzwo0i@c3d2.de>
Date: Fri, 30 Oct 2020 11:40:11 +0100
Subject: [PATCH] tests: Speed up and fix travis (#4453)

Various test runner fixes.
---
 .travis.yml                            | 26 ++++++++++----------------
 tests/frontend/specs/helper.js         |  4 ++--
 tests/frontend/travis/remote_runner.js |  8 ++++++--
 3 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 883dec49c..99264b9cc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,20 +8,6 @@ services:
 
 cache: false
 
-before_install:
-  - sudo add-apt-repository -y ppa:libreoffice/ppa
-  - sudo apt-get update
-  - sudo apt-get -y install libreoffice
-  - sudo apt-get -y install libreoffice-pdfimport
-
-install:
-  - "bin/installDeps.sh"
-  - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
-  # Installing some plugins
-
-script:
-  - "tests/frontend/travis/runner.sh"
-
 env:
   global:
     - secure: "WMGxFkOeTTlhWB+ChMucRtIqVmMbwzYdNHuHQjKCcj8HBEPdZLfCuK/kf4rG\nVLcLQiIsyllqzNhBGVHG1nyqWr0/LTm8JRqSCDDVIhpyzp9KpCJQQJG2Uwjk\n6/HIJJh/wbxsEdLNV2crYU/EiVO3A4Bq0YTHUlbhUqG3mSCr5Ec="
@@ -53,9 +39,13 @@ jobs:
         - "bin/installDeps.sh"
         - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
       script:
-        - "tests/frontend/travis/runner.sh"
+        - "travis_wait 15 tests/frontend/travis/runner.sh"
     - name: "Run the Backend tests without Plugins"
       install:
+        - "sudo add-apt-repository -y ppa:libreoffice/ppa"
+        - "sudo apt-get update"
+        - "sudo apt-get -y install libreoffice"
+        - "sudo apt-get -y install libreoffice-pdfimport"
         - "bin/installDeps.sh"
         - "cd src && npm install && cd -"
       script:
@@ -87,7 +77,7 @@ jobs:
         - *install_plugins
         - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
       script:
-        - "tests/frontend/travis/runner.sh"
+        - "travis_wait 15 tests/frontend/travis/runner.sh"
     - name: "Lint test package-lock.json"
       install:
         - "npm install lockfile-lint"
@@ -95,6 +85,10 @@ jobs:
         - npx lockfile-lint --path src/package-lock.json --validate-https --allowed-hosts npm
     - name: "Run the Backend tests with Plugins"
       install:
+        - "sudo add-apt-repository -y ppa:libreoffice/ppa"
+        - "sudo apt-get update"
+        - "sudo apt-get -y install libreoffice"
+        - "sudo apt-get -y install libreoffice-pdfimport"
         - "bin/installDeps.sh"
         - *install_plugins
         - "cd src && npm install && cd -"
diff --git a/tests/frontend/specs/helper.js b/tests/frontend/specs/helper.js
index 94a0236fe..e6b06412d 100644
--- a/tests/frontend/specs/helper.js
+++ b/tests/frontend/specs/helper.js
@@ -151,7 +151,7 @@ describe("the test helper", function(){
       }, 2000, 100).fail(function(){
         // One at the beginning, and 19-20 more depending on whether it's the timeout or the final
         // poll that wins at 2000ms.
-        expect(checks).to.be.greaterThan(17);
+        expect(checks).to.be.greaterThan(15);
         expect(checks).to.be.lessThan(24);
         done();
       });
@@ -243,7 +243,7 @@ describe("the test helper", function(){
        // `checks` is expected to be 20 or 21: one at the beginning, plus 19 or 20 more depending on
        // whether it's the timeout or the final poll that wins at 2000ms. Margin is added to reduce
        // flakiness on slow test machines.
-      expect(checks).to.be.greaterThan(17);
+      expect(checks).to.be.greaterThan(15);
       expect(checks).to.be.lessThan(24);
     });
   });
diff --git a/tests/frontend/travis/remote_runner.js b/tests/frontend/travis/remote_runner.js
index 569b121f5..3fb98ccca 100644
--- a/tests/frontend/travis/remote_runner.js
+++ b/tests/frontend/travis/remote_runner.js
@@ -10,6 +10,10 @@ var config = {
 }
 
 var allTestsPassed = true;
+// overwrite the default exit code
+// in case not all worker can be run (due to saucelabs limits), `queue.drain` below will not be called
+// and the script would silently exit with error code 0
+process.exitCode = 1;
 
 var sauceTestWorker = async.queue(function (testSettings, callback) {
   var browser = wd.promiseChainRemote(config.host, config.port, config.username, config.accessKey);
@@ -53,13 +57,13 @@ var sauceTestWorker = async.queue(function (testSettings, callback) {
       }
 
       /**
-       * timeout if a test hangs or the job exceeds 9.5 minutes
+       * timeout if a test hangs or the job exceeds 14.5 minutes
        * It's necessary because if travis kills the saucelabs session due to inactivity, we don't get any output
        * @todo this should be configured in testSettings, see https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-Timeouts
        */
       var timeout = setTimeout(function(){
         stopSauce(false,true);
-      }, 570000); // travis timeout is 10 minutes, set this to a slightly lower value
+      }, 870000); // travis timeout is 15 minutes, set this to a slightly lower value
 
       var knownConsoleText = "";
       var getStatusInterval = setInterval(function(){