From 2b112ac85170e5b8c36abe818e69e288b1c82fa9 Mon Sep 17 00:00:00 2001 From: John McLear Date: Sun, 7 Feb 2021 11:32:57 +0000 Subject: [PATCH] tests: Admin Frontend Test Coverage(#4717) Covers all frontend admin operations, runs separated in CI. --- .github/workflows/frontend-admin-tests.yml | 54 +++++++++ .github/workflows/frontend-tests.yml | 3 + .travis.yml | 7 ++ settings.json.template | 5 +- src/node/hooks/express/tests.js | 8 +- src/node/utils/Settings.js | 6 + src/tests/frontend/helper.js | 17 +++ src/tests/frontend/specs/adminplugins.js | 113 ++++++++++++++++++ src/tests/frontend/specs/adminroot.js | 29 +++++ src/tests/frontend/specs/adminsettings.js | 73 +++++++++++ .../frontend/specs/admintroubleshooting.js | 47 ++++++++ src/tests/frontend/travis/adminrunner.sh | 44 +++++++ src/tests/frontend/travis/remote_runner.js | 107 +++++++++-------- 13 files changed, 463 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/frontend-admin-tests.yml create mode 100755 src/tests/frontend/specs/adminplugins.js create mode 100644 src/tests/frontend/specs/adminroot.js create mode 100644 src/tests/frontend/specs/adminsettings.js create mode 100755 src/tests/frontend/specs/admintroubleshooting.js create mode 100755 src/tests/frontend/travis/adminrunner.sh diff --git a/.github/workflows/frontend-admin-tests.yml b/.github/workflows/frontend-admin-tests.yml new file mode 100644 index 000000000..6f95a6c7f --- /dev/null +++ b/.github/workflows/frontend-admin-tests.yml @@ -0,0 +1,54 @@ +name: "Frontend admin tests" + +on: [push] + +jobs: + withplugins: + name: with plugins + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Run sauce-connect-action + shell: bash + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }} + run: src/tests/frontend/travis/sauce_tunnel.sh + + - name: Install all dependencies and symlink for ep_etherpad-lite + run: src/bin/installDeps.sh + + # We intentionally install a much old ep_align version to test update minor versions + - name: Install etherpad plugins + run: npm install ep_align@0.2.27 + + # Nuke plugin tests + - name: Install etherpad plugins + run: rm -Rf node_modules/ep_align/static/tests/* + + - name: export GIT_HASH to env + id: environment + run: echo "::set-output name=sha_short::$(git rev-parse --short ${{ github.sha }})" + + - name: Write custom settings.json with loglevel WARN + run: "sed 's/\"loglevel\": \"INFO\",/\"loglevel\": \"WARN\",/' < settings.json.template > settings.json" + + - name: Write custom settings.json that enables the Admin UI tests + run: "sed -i 's/\"enableAdminUITests\": false/\"enableAdminUITests\": true,\\n\"users\":{\"admin\":{\"password\":\"changeme\",\"is_admin\":true}}/' settings.json" + + - name: Remove standard frontend test files, so only admin tests are run + run: mv src/tests/frontend/specs/* /tmp && mv /tmp/admin*.js src/tests/frontend/specs + + - name: Run the frontend admin tests + shell: bash + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + TRAVIS_JOB_NUMBER: ${{ github.run_id }}-${{ github.run_number }}-${{ github.job }} + GIT_HASH: ${{ steps.environment.outputs.sha_short }} + run: | + src/tests/frontend/travis/adminrunner.sh diff --git a/.github/workflows/frontend-tests.yml b/.github/workflows/frontend-tests.yml index f8a0e76b4..70cb197a3 100644 --- a/.github/workflows/frontend-tests.yml +++ b/.github/workflows/frontend-tests.yml @@ -88,6 +88,9 @@ jobs: - name: Write custom settings.json with loglevel WARN run: "sed 's/\"loglevel\": \"INFO\",/\"loglevel\": \"WARN\",/' < settings.json.template > settings.json" + - name: Write custom settings.json that enables the Admin UI tests + run: "sed -i 's/\"enableAdminUITests\": false/\"enableAdminUITests\": true,\\n\"users\":{\"admin\":{\"password\":\"changeme\",\"is_admin\":true}}/' settings.json" + # XXX we should probably run all tests, because plugins could effect their results - name: Remove standard frontend test files, so only plugin tests are run run: rm src/tests/frontend/specs/* diff --git a/.travis.yml b/.travis.yml index f6cf4dba8..8517e2a89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,11 @@ _set_loglevel_warn: &set_loglevel_warn | settings.json.template >settings.json.template.new && mv settings.json.template.new settings.json.template +_enable_admin_tests: &enable_admin_tests | +sed -e 's/"enableAdminUITests": false/"enableAdminUITests": true,\n"users":{"admin":{"password":"changeme","is_admin":true}}/' \ + settings.json.template >settings.json.template.new && + mv settings.json.template.new settings.json.template + _install_libreoffice: &install_libreoffice >- sudo add-apt-repository -y ppa:libreoffice/ppa && sudo apt-get update && @@ -46,6 +51,7 @@ jobs: name: "Test the Frontend without Plugins" install: - *set_loglevel_warn + - *enable_admin_tests - "src/tests/frontend/travis/sauce_tunnel.sh" - "src/bin/installDeps.sh" - "export GIT_HASH=$(git rev-parse --verify --short HEAD)" @@ -80,6 +86,7 @@ jobs: name: "Test the Frontend Plugins only" install: - *set_loglevel_warn + - *enable_admin_tests - "src/tests/frontend/travis/sauce_tunnel.sh" - "src/bin/installDeps.sh" - "rm src/tests/frontend/specs/*" diff --git a/settings.json.template b/settings.json.template index 3c9c50837..b15b024ab 100644 --- a/settings.json.template +++ b/settings.json.template @@ -600,5 +600,8 @@ }, // logconfig /* Override any strings found in locale directories */ - "customLocaleStrings": {} + "customLocaleStrings": {}, + + /* Disable Admin UI tests */ + "enableAdminUITests": false } diff --git a/src/node/hooks/express/tests.js b/src/node/hooks/express/tests.js index de868ba3b..825c495ca 100644 --- a/src/node/hooks/express/tests.js +++ b/src/node/hooks/express/tests.js @@ -4,6 +4,7 @@ const path = require('path'); const npm = require('npm'); const fs = require('fs'); const util = require('util'); +const settings = require('../../utils/Settings'); exports.expressCreateServer = (hookName, args, cb) => { args.app.get('/tests/frontend/specs_list.js', async (req, res) => { @@ -18,6 +19,11 @@ exports.expressCreateServer = (hookName, args, cb) => { // Keep only *.js files files = files.filter((f) => f.endsWith('.js')); + // remove admin tests if the setting to enable them isn't in settings.json + if (!settings.enableAdminUITests) { + files = files.filter((file) => file.indexOf('admin') !== 0); + } + console.debug('Sent browser the following test specs:', files); res.setHeader('content-type', 'application/javascript'); res.end(`var specs_list = ${JSON.stringify(files)};\n`); @@ -49,7 +55,7 @@ exports.expressCreateServer = (hookName, args, cb) => { fs.readFile(specFilePath, (err, content) => { if (err) { return res.send(500); } - content = `describe(${JSON.stringify(specFileName)}, function(){ ${content} });`; + content = `describe(${JSON.stringify(specFileName)}, function(){${content}});`; if (!specFilePath.endsWith('index.html')) { res.setHeader('content-type', 'application/javascript'); diff --git a/src/node/utils/Settings.js b/src/node/utils/Settings.js index 65455ed18..9db195f64 100644 --- a/src/node/utils/Settings.js +++ b/src/node/utils/Settings.js @@ -380,6 +380,12 @@ exports.commitRateLimiting = { */ exports.importMaxFileSize = 50 * 1024 * 1024; +/* + * Disable Admin UI tests + */ +exports.enableAdminUITests = false; + + // checks if abiword is avaiable exports.abiwordAvailable = () => { if (exports.abiword != null) { diff --git a/src/tests/frontend/helper.js b/src/tests/frontend/helper.js index c38175fe1..0f8a32f1a 100644 --- a/src/tests/frontend/helper.js +++ b/src/tests/frontend/helper.js @@ -1,4 +1,5 @@ 'use strict'; + const helper = {}; // eslint-disable-line no-redeclare (function () { @@ -180,6 +181,22 @@ const helper = {}; // eslint-disable-line no-redeclare return padName; }; + helper.newAdmin = async function (page) { + // define the iframe + $iframe = $(``); + + // clean up inner iframe references + helper.admin$ = null; + + // remove old iframe + $('#iframe-container iframe').remove(); + // set new iframe + $('#iframe-container').append($iframe); + $iframe.one('load', () => { + helper.admin$ = getFrameJQuery($('#iframe-container iframe')); + }); + }; + helper.waitFor = function (conditionFunc, timeoutTime = 1900, intervalTime = 10) { const deferred = new $.Deferred(); diff --git a/src/tests/frontend/specs/adminplugins.js b/src/tests/frontend/specs/adminplugins.js new file mode 100755 index 000000000..ba00d3e48 --- /dev/null +++ b/src/tests/frontend/specs/adminplugins.js @@ -0,0 +1,113 @@ +'use strict'; + +describe('Plugins page', function () { + function timeout(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + before(async function () { + let success = false; + $.ajax({ + url: `${location.protocol}//admin:changeme@${location.hostname}:${location.port}/admin`, + type: 'GET', + success: () => success = true, + }); + await helper.waitForPromise(() => success === true); + }); + + // create a new pad before each test run + beforeEach(async function () { + helper.newAdmin('plugins'); + await helper.waitForPromise( + () => helper.admin$ && helper.admin$('.menu').find('li').length >= 3); + }); + + it('Lists some plugins', async function () { + await helper.waitForPromise(() => helper.admin$('.results').children().length > 50); + }); + + it('Searches for plugin', async function () { + helper.admin$('#search-query').val('ep_font_color'); + await helper.waitForPromise(() => helper.admin$('.results').children().length < 300, 5000); + await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 5000); + }); + + it('Attempt to Update a plugin', async function () { + this.timeout(120000); + + if (helper.admin$('.ep_align').length === 0) this.skip(); + + await helper.waitForPromise( + () => helper.admin$('.ep_align .version').text().split('.').length >= 2); + + const minorVersionBefore = + parseInt(helper.admin$('.ep_align .version').text().split('.')[1]); + + if (!minorVersionBefore) { + throw new Error('Unable to get minor number of plugin, is the plugin installed?'); + } + + if (minorVersionBefore !== 2) this.skip(); + + helper.waitForPromise( + () => helper.admin$('.ep_align .do-update').length === 1); + + await timeout(500); // HACK! Please submit better fix.. + const $doUpdateButton = helper.admin$('.ep_align .do-update'); + $doUpdateButton.click(); + + // ensure its showing as Updating + await helper.waitForPromise( + () => helper.admin$('.ep_align .message').text() === 'Updating'); + + // Ensure it's a higher minor version IE 0.3.x as 0.2.x was installed + // Coverage for https://github.com/ether/etherpad-lite/issues/4536 + await helper.waitForPromise(() => parseInt(helper.admin$( + '.ep_align .version' + ) + .text() + .split('.')[1]) > minorVersionBefore, 60000, 1000); + // allow 50 seconds, check every 1 second. + }); + it('Attempt to Install a plugin', async function () { + this.timeout(240000); + + helper.admin$('#search-query').val('ep_activepads'); + await helper.waitForPromise(() => helper.admin$('.results').children().length < 300, 6000); + await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 6000); + + // skip if we already have ep_activepads installed.. + if (helper.admin$('.ep_activepads .do-install').is(':visible') === false) this.skip(); + + helper.admin$('.ep_activepads .do-install').click(); + // ensure install has attempted to be started + await helper.waitForPromise( + () => helper.admin$('.ep_activepads .do-install').length !== 0, 120000); + // ensure its not showing installing any more + await helper.waitForPromise( + () => helper.admin$('.ep_activepads .message').text() === '', 180000); + // ensure uninstall button is visible + await helper.waitForPromise( + () => helper.admin$('.ep_activepads .do-uninstall').length !== 0, 120000); + }); + + it('Attempt to Uninstall a plugin', async function () { + this.timeout(360000); + await helper.waitForPromise( + () => helper.admin$('.ep_activepads .do-uninstall').length !== 0, 120000); + + helper.admin$('.ep_activepads .do-uninstall').click(); + + // ensure its showing uninstalling + await helper.waitForPromise( + () => helper.admin$('.ep_activepads .message') + .text() === 'Uninstalling', 120000); + // ensure its gone + await helper.waitForPromise( + () => helper.admin$('.ep_activepads').length === 0, 240000); + + helper.admin$('#search-query').val('ep_font'); + await helper.waitForPromise(() => helper.admin$('.results').children().length < 300, 240000); + await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 1000); + }); +}); diff --git a/src/tests/frontend/specs/adminroot.js b/src/tests/frontend/specs/adminroot.js new file mode 100644 index 000000000..7416c21b4 --- /dev/null +++ b/src/tests/frontend/specs/adminroot.js @@ -0,0 +1,29 @@ +'use strict'; + +describe('Admin page', function () { + before(async function () { + let success = false; + $.ajax({ + url: `${location.protocol}//admin:changeme@${location.hostname}:${location.port}/admin/`, + type: 'GET', + success: () => success = true, + }); + await helper.waitForPromise(() => success === true); + }); + + beforeEach(async function () { + helper.newAdmin(''); + await helper.waitForPromise( + () => helper.admin$ && helper.admin$('.menu').find('li').length >= 3); + }); + + it('Shows Plugin Manager Link', async function () { + helper.admin$('a[data-l10n-id="admin_plugins"]').is(':visible'); + }); + it('Shows Troubleshooting Info Link', async function () { + helper.admin$('a[data-l10n-id="admin_plugins_info"]').is(':visible'); + }); + it('Shows Settings Link', async function () { + helper.admin$('a[data-l10n-id="admin_settings"]').is(':visible'); + }); +}); diff --git a/src/tests/frontend/specs/adminsettings.js b/src/tests/frontend/specs/adminsettings.js new file mode 100644 index 000000000..0d7d0accb --- /dev/null +++ b/src/tests/frontend/specs/adminsettings.js @@ -0,0 +1,73 @@ +'use strict'; + +describe('Admin > Settings', function () { + this.timeout(480000); + + before(async function () { + let success = false; + $.ajax({ + url: `${location.protocol}//admin:changeme@${location.hostname}:${location.port}/admin/`, + type: 'GET', + success: () => success = true, + }); + await helper.waitForPromise(() => success === true); + }); + + beforeEach(async function () { + helper.newAdmin('settings'); + // needed, because the load event is fired to early + await helper.waitForPromise(() => helper.admin$ && helper.admin$('.settings').val().length > 0); + }); + + it('Are Settings visible, populated, does save work', async function () { + // save old value + const settings = helper.admin$('.settings').val(); + const settingsLength = settings.length; + + // set new value + helper.admin$('.settings').val((_, text) => `/* test */\n${text}`); + await helper.waitForPromise( + () => settingsLength + 11 === helper.admin$('.settings').val().length); + + // saves + helper.admin$('#saveSettings').click(); + await helper.waitForPromise(() => helper.admin$('#response').is(':visible')); + + // new value for settings.json should now be saved + // reset it to the old value + helper.newAdmin('settings'); + await helper.waitForPromise(() => helper.admin$ && helper.admin$('.settings').val().length > 0); + + // replace the test value with a line break + helper.admin$('.settings').val((_, text) => text.replace('/* test */\n', '')); + await helper.waitForPromise(() => settingsLength === helper.admin$('.settings').val().length); + + helper.admin$('#saveSettings').click(); // saves + await helper.waitForPromise(() => helper.admin$('#response').is(':visible')); + + // settings should have the old value + helper.newAdmin('settings'); + await helper.waitForPromise( + () => helper.admin$ && helper.admin$('.settings').val().length > 0, 36000); + expect(settings).to.be(helper.admin$('.settings').val()); + }); + + function timeout(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + it('restart works', async function () { + // restarts + helper.admin$('#restartEtherpad').click(); + + // Hacky... Other suggestions welcome.. + await timeout(200000); + let success = false; + $.ajax({ + url: `${location.protocol}//admin:changeme@${location.hostname}:${location.port}/admin`, + type: 'GET', + success: () => success = true, + }); + await helper.waitForPromise(() => success === true); + }); +}); diff --git a/src/tests/frontend/specs/admintroubleshooting.js b/src/tests/frontend/specs/admintroubleshooting.js new file mode 100755 index 000000000..6e428d3b1 --- /dev/null +++ b/src/tests/frontend/specs/admintroubleshooting.js @@ -0,0 +1,47 @@ +'use strict'; + +describe('Admin Troupbleshooting page', function () { + before(async function () { + let success = false; + $.ajax({ + url: `${location.protocol}//admin:changeme@${location.hostname}:${location.port}/admin`, + type: 'GET', + success: () => success = true, + }); + await helper.waitForPromise(() => success === true); + }); + + // create a new pad before each test run + beforeEach(async function () { + helper.newAdmin('plugins/info'); + await helper.waitForPromise( + () => helper.admin$ && helper.admin$('.menu').find('li').length >= 3); + }); + + it('Shows Troubleshooting page Manager', async function () { + helper.admin$('a[data-l10n-id="admin_plugins_info"]')[0].click(); + }); + + it('Shows a version number', async function () { + const content = helper.admin$('span[data-l10n-id="admin_plugins_info.version_number"]') + .parent().text(); + const version = content.split(': ')[1].split('.'); + if (version.length !== 3) { + throw new Error('Not displaying a semver version number'); + } + }); + + it('Lists installed parts', async function () { + const parts = helper.admin$('pre')[1]; + if (parts.textContent.indexOf('ep_etherpad-lite/adminsettings') === -1) { + throw new Error('No admin setting part being displayed...'); + } + }); + + it('Lists installed hooks', async function () { + const parts = helper.admin$('dt'); + if (parts.length <= 20) { + throw new Error('Not enough hooks being displayed...'); + } + }); +}); diff --git a/src/tests/frontend/travis/adminrunner.sh b/src/tests/frontend/travis/adminrunner.sh new file mode 100755 index 000000000..da20d2801 --- /dev/null +++ b/src/tests/frontend/travis/adminrunner.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +pecho() { printf %s\\n "$*"; } +log() { pecho "$@"; } +error() { log "ERROR: $@" >&2; } +fatal() { error "$@"; exit 1; } +try() { "$@" || fatal "'$@' failed"; } + +[ -n "${SAUCE_USERNAME}" ] || fatal "SAUCE_USERNAME is unset - exiting" +[ -n "${SAUCE_ACCESS_KEY}" ] || fatal "SAUCE_ACCESS_KEY is unset - exiting" + +# Move to the Etherpad base directory. +MY_DIR=$(try cd "${0%/*}" && try pwd -P) || exit 1 +try cd "${MY_DIR}/../../../.." + +log "Assuming src/bin/installDeps.sh has already been run" +node node_modules/ep_etherpad-lite/node/server.js --experimental-worker "${@}" & +ep_pid=$! + +log "Waiting for Etherpad to accept connections (http://localhost:9001)..." +connected=false +can_connect() { + curl -sSfo /dev/null http://localhost:9001/ || return 1 + connected=true +} +now() { date +%s; } +start=$(now) +while [ $(($(now) - $start)) -le 15 ] && ! can_connect; do + sleep 1 +done +[ "$connected" = true ] \ + || fatal "Timed out waiting for Etherpad to accept connections" +log "Successfully connected to Etherpad on http://localhost:9001" + +# start the remote runner +try cd "${MY_DIR}" +log "Starting the remote runner..." +node remote_runner.js admin +exit_code=$? + +kill "$(cat /tmp/sauce.pid)" +kill "$ep_pid" && wait "$ep_pid" +log "Done." +exit "$exit_code" diff --git a/src/tests/frontend/travis/remote_runner.js b/src/tests/frontend/travis/remote_runner.js index c5f3f1466..5a75dbe66 100644 --- a/src/tests/frontend/travis/remote_runner.js +++ b/src/tests/frontend/travis/remote_runner.js @@ -10,6 +10,8 @@ const config = { accessKey: process.env.SAUCE_ACCESS_KEY, }; +const isAdminRunner = process.argv[2] === 'admin'; + let allTestsPassed = true; // overwrite the default exit code // in case not all worker can be run (due to saucelabs limits), @@ -126,57 +128,66 @@ const sauceTestWorker = async.queue((testSettings, callback) => { }); }, 6); // run 6 tests in parrallel -// 1) Firefox on Linux -sauceTestWorker.push({ - platform: 'Windows 7', - browserName: 'firefox', - version: '52.0', -}); +if (!isAdminRunner) { + // 1) Firefox on Linux + sauceTestWorker.push({ + platform: 'Windows 7', + browserName: 'firefox', + version: '52.0', + }); -// 2) Chrome on Linux -sauceTestWorker.push({ - platform: 'Windows 7', - browserName: 'chrome', - version: '55.0', - args: ['--use-fake-device-for-media-stream'], -}); + // 2) Chrome on Linux + sauceTestWorker.push({ + platform: 'Windows 7', + browserName: 'chrome', + version: '55.0', + args: ['--use-fake-device-for-media-stream'], + }); -/* -// 3) Safari on OSX 10.15 -sauceTestWorker.push({ - 'platform' : 'OS X 10.15' - , 'browserName' : 'safari' - , 'version' : '13.1' -}); -*/ + /* + // 3) Safari on OSX 10.15 + sauceTestWorker.push({ + 'platform' : 'OS X 10.15' + , 'browserName' : 'safari' + , 'version' : '13.1' + }); + */ -// 4) Safari on OSX 10.14 -sauceTestWorker.push({ - platform: 'OS X 10.15', - browserName: 'safari', - version: '13.1', -}); -// IE 10 doesn't appear to be working anyway -/* -// 4) IE 10 on Win 8 -sauceTestWorker.push({ - 'platform' : 'Windows 8' - , 'browserName' : 'iexplore' - , 'version' : '10.0' -}); -*/ -// 5) Edge on Win 10 -sauceTestWorker.push({ - platform: 'Windows 10', - browserName: 'microsoftedge', - version: '83.0', -}); -// 6) Firefox on Win 7 -sauceTestWorker.push({ - platform: 'Windows 7', - browserName: 'firefox', - version: '78.0', -}); + // 4) Safari on OSX 10.14 + sauceTestWorker.push({ + platform: 'OS X 10.15', + browserName: 'safari', + version: '13.1', + }); + // IE 10 doesn't appear to be working anyway + /* + // 4) IE 10 on Win 8 + sauceTestWorker.push({ + 'platform' : 'Windows 8' + , 'browserName' : 'iexplore' + , 'version' : '10.0' + }); + */ + // 5) Edge on Win 10 + sauceTestWorker.push({ + platform: 'Windows 10', + browserName: 'microsoftedge', + version: '83.0', + }); + // 6) Firefox on Win 7 + sauceTestWorker.push({ + platform: 'Windows 7', + browserName: 'firefox', + version: '78.0', + }); +} else { + // 4) Safari on OSX 10.14 + sauceTestWorker.push({ + platform: 'OS X 10.15', + browserName: 'safari', + version: '13.1', + }); +} sauceTestWorker.drain(() => { process.exit(allTestsPassed ? 0 : 1);