remove newlines only in the context of ordered lists - TODO check unordered

run all other workflows on push and on PRs from external repos

This should prevent running workflows twice in case of a PR from a
branch of the main repo

.github/workflows/backend-tests.yml

@@ -0,0 +1,60 @@
+name: "Backend tests"
+
+# any branch is useful for testing before a PR is submitted
+on: [push, pull_request]
+
+jobs:
+  withoutplugins:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: without plugins
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Install libreoffice
+      run: |
+        sudo add-apt-repository -y ppa:libreoffice/ppa
+        sudo apt update
+        sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
+
+    - name: Install all dependencies and symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    # configures some settings and runs npm run test
+    - name: Run the backend tests
+      run: tests/frontend/travis/runnerBackend.sh
+
+  withplugins:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: with Plugins
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Install libreoffice
+      run: |
+        sudo add-apt-repository -y ppa:libreoffice/ppa
+        sudo apt update
+        sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
+
+    - name: Install all dependencies and symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    - name: Install etherpad plugins
+      run: npm install ep_align ep_author_hover ep_cursortrace ep_font_size ep_hash_auth ep_headings2 ep_markdown ep_readonly_guest ep_spellcheck ep_subscript_and_superscript ep_table_of_contents
+
+    # configures some settings and runs npm run test
+    - name: Run the backend tests
+      run: tests/frontend/travis/runnerBackend.sh

.github/workflows/dockerfile.yml

@@ -0,0 +1,26 @@
+name: "Dockerfile"
+
+# any branch is useful for testing before a PR is submitted
+on: [push, pull_request]
+
+jobs:
+  dockerfile:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: build image and run connectivity test
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: docker build
+      run: |
+        docker build -t etherpad:test .
+        docker run -d -p 9001:9001 etherpad:test
+        ./bin/installDeps.sh
+        sleep 3 # delay for startup?
+        cd src && npm run test-container

.github/workflows/frontend-tests.yml

@@ -0,0 +1,83 @@
+name: "Frontend tests"
+
+on: [push]
+
+jobs:
+  withoutplugins:
+    name: without plugins
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Run sauce-connect-action
+      shell: bash
+      env:
+        SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+        SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
+        TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }}
+      run: tests/frontend/travis/sauce_tunnel.sh
+
+    - name: Install all dependencies and symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    - name: export GIT_HASH to env
+      id: environment
+      run: echo "::set-output name=sha_short::$(git rev-parse --short ${{ github.sha }})"
+
+    - name: Write custom settings.json with loglevel WARN
+      run: "sed 's/\"loglevel\": \"INFO\",/\"loglevel\": \"WARN\",/' < settings.json.template > settings.json"
+
+    - name: Run the frontend tests
+      shell: bash
+      env:
+        SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+        SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
+        TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }}
+        GIT_HASH: ${{ steps.environment.outputs.sha_short }}
+      run: |
+        tests/frontend/travis/runner.sh
+
+  withplugins:
+    name: with plugins
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Run sauce-connect-action
+      shell: bash
+      env:
+        SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+        SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
+        TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }}
+      run: tests/frontend/travis/sauce_tunnel.sh
+
+    - name: Install all dependencies and symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    - name: Install etherpad plugins
+      run: npm install ep_align ep_author_hover ep_cursortrace ep_font_size ep_hash_auth ep_headings2 ep_markdown ep_readonly_guest ep_spellcheck ep_subscript_and_superscript ep_table_of_contents
+
+    - name: export GIT_HASH to env
+      id: environment
+      run: echo "::set-output name=sha_short::$(git rev-parse --short ${{ github.sha }})"
+
+    - name: Write custom settings.json with loglevel WARN
+      run: "sed 's/\"loglevel\": \"INFO\",/\"loglevel\": \"WARN\",/' < settings.json.template > settings.json"
+
+      # XXX we should probably run all tests, because plugins could effect their results
+    - name: Remove standard frontend test files, so only plugin tests are run
+      run: rm tests/frontend/specs/*
+
+    - name: Run the frontend tests
+      shell: bash
+      env:
+        SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+        SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
+        TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }}
+        GIT_HASH: ${{ steps.environment.outputs.sha_short }}
+      run: |
+        tests/frontend/travis/runner.sh

.github/workflows/lint-package-lock.yml

@@ -0,0 +1,24 @@
+name: "Lint"
+
+# any branch is useful for testing before a PR is submitted
+on: [push, pull_request]
+
+jobs:
+  lint-package-lock:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: package-lock.json
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Install lockfile-lint
+      run: npm install lockfile-lint
+
+    - name: Run lockfile-lint on package-lock.json
+      run: npx lockfile-lint --path src/package-lock.json --validate-https --allowed-hosts npm

.github/workflows/load-test.yml

@@ -0,0 +1,53 @@
+name: "Loadtest"
+
+# any branch is useful for testing before a PR is submitted
+on: [push, pull_request]
+
+jobs:
+  withoutplugins:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: without plugins
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Install all dependencies and symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    - name: Install etherpad-load-test
+      run: sudo npm install -g etherpad-load-test
+
+    - name: Run load test
+      run: tests/frontend/travis/runnerLoadTest.sh
+
+  withplugins:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: with Plugins
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Install all dependencies and symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    - name: Install etherpad-load-test
+      run: sudo npm install -g etherpad-load-test
+
+    - name: Install etherpad plugins
+      run: npm install ep_align ep_author_hover ep_cursortrace ep_font_size ep_hash_auth ep_headings2 ep_markdown ep_readonly_guest ep_spellcheck ep_subscript_and_superscript ep_table_of_contents
+
+    # configures some settings and runs npm run test
+    - name: Run load test
+      run: tests/frontend/travis/runnerLoadTest.sh

.github/workflows/rate-limit.yml

@@ -0,0 +1,39 @@
+name: "rate limit"
+
+# any branch is useful for testing before a PR is submitted
+on: [push, pull_request]
+
+jobs:
+  ratelimit:
+    # run on pushes to any branch
+    # run on PRs from external forks
+    if: |
+         (github.event_name != 'pull_request')
+         || (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id)
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: docker network
+      run: docker network create --subnet=172.23.42.0/16 ep_net
+
+    - name: build docker image
+      run: |
+        docker build -f Dockerfile -t epl-debian-slim .
+        docker build -f tests/ratelimit/Dockerfile.nginx -t nginx-latest .
+        docker build -f tests/ratelimit/Dockerfile.anotherip -t anotherip .
+    - name: run docker images
+      run: |
+        docker run --name etherpad-docker -p 9000:9001 --rm --network ep_net --ip 172.23.42.2 -e 'TRUST_PROXY=true' epl-debian-slim &
+        docker run -p 8081:80 --rm --network ep_net --ip 172.23.42.1 -d nginx-latest
+        docker run --rm --network ep_net --ip 172.23.42.3 --name anotherip -dt anotherip
+
+    - name: install dependencies and create symlink for ep_etherpad-lite
+      run: bin/installDeps.sh
+
+    - name: run rate limit test
+      run: |
+        cd tests/ratelimit
+        ./testlimits.sh

src/node/utils/ImportHtml.js

@@ -33,7 +33,12 @@ exports.setPadHTML = async (pad, html) => {
   rehype()
     .use(format, opts)
     .process(html, function(err, output){
-      html = String(output).replace(/(\r\n|\n|\r)/gm,"");
+      html = String(output)
+        .replace(/<ol>\n/gm,"<ol>")
+        .replace(/<\/ol>\n/gm,"</ol>")
+        .replace(/<li>\n/gm,"<li>")
+        .replace(/<\/li>\n/gm,"</li>")
+        .replace(/>\n<\/li>/gm,"></li>")
   })
 
   var $ = cheerio.load(html);

tests/frontend/travis/remote_runner.js

@@ -13,7 +13,12 @@ var allTestsPassed = true;
 // overwrite the default exit code
 // in case not all worker can be run (due to saucelabs limits), `queue.drain` below will not be called
 // and the script would silently exit with error code 0
-process.exitCode = 1;
+process.exitCode = 2;
+process.on('exit', (code) => {
+  if (code === 2){
+    console.log("\x1B[31mFAILED\x1B[39m Not all saucelabs runner have been started.");
+  }
+})
 
 var sauceTestWorker = async.queue(function (testSettings, callback) {
   var browser = wd.promiseChainRemote(config.host, config.port, config.username, config.accessKey);