From 1d491c0059e6865752c8a82a178e0c3e5802c43b Mon Sep 17 00:00:00 2001 From: Richard Hansen Date: Fri, 20 Nov 2020 19:19:39 -0500 Subject: [PATCH] tests: Clean up Travis runner scripts * Avoid bashisms. * Simplify `sed` of `settings.json`. * Wrap long lines. * Define and use the conventional log functions. * Quote variable expansions. --- tests/frontend/travis/runner.sh | 67 +++++++++++------------ tests/frontend/travis/runnerBackend.sh | 71 ++++++++++++------------- tests/frontend/travis/runnerLoadTest.sh | 69 ++++++++++++------------ tests/frontend/travis/sauce_tunnel.sh | 46 ++++++++++------ 4 files changed, 130 insertions(+), 123 deletions(-) diff --git a/tests/frontend/travis/runner.sh b/tests/frontend/travis/runner.sh index 73d04e296..da28ec1ef 100755 --- a/tests/frontend/travis/runner.sh +++ b/tests/frontend/travis/runner.sh @@ -1,50 +1,45 @@ -#!/bin/bash -if [ -z "${SAUCE_USERNAME}" ]; then echo "SAUCE_USERNAME is unset - exiting"; exit 1; fi -if [ -z "${SAUCE_ACCESS_KEY}" ]; then echo "SAUCE_ACCESS_KEY is unset - exiting"; exit 1; fi +#!/bin/sh -# do not continue if there is an error -set -eu +pecho() { printf %s\\n "$*"; } +log() { pecho "$@"; } +error() { log "ERROR: $@" >&2; } +fatal() { error "$@"; exit 1; } +try() { "$@" || fatal "'$@' failed"; } -# source: https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself#246128 -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +[ -n "${SAUCE_USERNAME}" ] || fatal "SAUCE_USERNAME is unset - exiting" +[ -n "${SAUCE_ACCESS_KEY}" ] || fatal "SAUCE_ACCESS_KEY is unset - exiting" + +MY_DIR=$(try cd "${0%/*}" && try pwd) || exit 1 # reliably move to the etherpad base folder before running it -cd "${MY_DIR}/../../../" +try cd "${MY_DIR}/../../../" -# start Etherpad, assuming all dependencies are already installed. -# -# This is possible because the "install" section of .travis.yml already contains -# a call to bin/installDeps.sh -echo "Running Etherpad directly, assuming bin/installDeps.sh has already been run" +log "Assuming bin/installDeps.sh has already been run" node node_modules/ep_etherpad-lite/node/server.js --experimental-worker "${@}" & ep_pid=$! -echo "Now I will try for 15 seconds to connect to Etherpad on http://localhost:9001" - -# wait for at most 15 seconds until Etherpad starts accepting connections -# -# modified from: -# https://unix.stackexchange.com/questions/5277/how-do-i-tell-a-script-to-wait-for-a-process-to-start-accepting-requests-on-a-po#349138 -# -(timeout 15 bash -c 'until echo > /dev/tcp/localhost/9001; do sleep 0.5; done') || \ - (echo "Could not connect to Etherpad on http://localhost:9001" ; exit 1) - -echo "Successfully connected to Etherpad on http://localhost:9001" - -# On the Travis VM, remote_runner.js is found at -# /home/travis/build/ether/[secure]/tests/frontend/travis/remote_runner.js -# which is the same directory that contains this script. -# Let's move back there. -# -# Probably remote_runner.js is injected by Saucelabs. -cd "${MY_DIR}" +log "Waiting for Etherpad to accept connections (http://localhost:9001)..." +connected=false +can_connect() { + curl -sSfo /dev/null http://localhost:9001/ || return 1 + connected=true +} +now() { date +%s; } +start=$(now) +while [ $(($(now) - $start)) -le 15 ] && ! can_connect; do + sleep 1 +done +[ "$connected" = true ] \ + || fatal "Timed out waiting for Etherpad to accept connections" +log "Successfully connected to Etherpad on http://localhost:9001" # start the remote runner -echo "Now starting the remote runner" +try cd "${MY_DIR}" +log "Starting the remote runner..." node remote_runner.js exit_code=$? -kill $(cat /tmp/sauce.pid) +kill "$(cat /tmp/sauce.pid)" kill "$ep_pid" && wait "$ep_pid" - -exit $exit_code +log "Done." +exit "$exit_code" diff --git a/tests/frontend/travis/runnerBackend.sh b/tests/frontend/travis/runnerBackend.sh index 806fe8cf1..de7e49528 100755 --- a/tests/frontend/travis/runnerBackend.sh +++ b/tests/frontend/travis/runnerBackend.sh @@ -1,48 +1,47 @@ -#!/bin/bash +#!/bin/sh -# do not continue if there is an error -set -u +pecho() { printf %s\\n "$*"; } +log() { pecho "$@"; } +error() { log "ERROR: $@" >&2; } +fatal() { error "$@"; exit 1; } +try() { "$@" || fatal "'$@' failed"; } -# source: https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself#246128 -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +MY_DIR=$(try cd "${0%/*}" && try pwd) || fatal "failed to find script directory" # reliably move to the etherpad base folder before running it -cd "${MY_DIR}/../../../" +try cd "${MY_DIR}/../../../" -# Set soffice to /usr/bin/soffice -sed 's#\"soffice\": null,#\"soffice\":\"/usr/bin/soffice\",#g' settings.json.template > settings.json.soffice +try sed -e ' +s!"soffice":[^,]*!"soffice": "/usr/bin/soffice"! +# Reduce rate limit aggressiveness +s!"max":[^,]*!"max": 100! +s!"points":[^,]*!"points": 1000! +' settings.json.template >settings.json -# Set "max": 10 to 100 to not agressively rate limit -sed 's/\"max\": 10/\"max\": 100/g' settings.json.soffice > settings.json.rateLimit - -# Set "points": 10 to 1000 to not agressively rate limit commits -sed 's/\"points\": 10/\"points\": 1000/g' settings.json.rateLimit > settings.json - -# start Etherpad, assuming all dependencies are already installed. -# -# This is possible because the "install" section of .travis.yml already contains -# a call to bin/installDeps.sh -echo "Running Etherpad directly, assuming bin/installDeps.sh has already been run" +log "Assuming bin/installDeps.sh has already been run" node node_modules/ep_etherpad-lite/node/server.js "${@}" & ep_pid=$! -echo "Now I will try for 15 seconds to connect to Etherpad on http://localhost:9001" +log "Waiting for Etherpad to accept connections (http://localhost:9001)..." +connected=false +can_connect() { + curl -sSfo /dev/null http://localhost:9001/ || return 1 + connected=true +} +now() { date +%s; } +start=$(now) +while [ $(($(now) - $start)) -le 15 ] && ! can_connect; do + sleep 1 +done +[ "$connected" = true ] \ + || fatal "Timed out waiting for Etherpad to accept connections" +log "Successfully connected to Etherpad on http://localhost:9001" -# wait for at most 15 seconds until Etherpad starts accepting connections -# -# modified from: -# https://unix.stackexchange.com/questions/5277/how-do-i-tell-a-script-to-wait-for-a-process-to-start-accepting-requests-on-a-po#349138 -# -(timeout 15 bash -c 'until echo > /dev/tcp/localhost/9001; do sleep 0.5; done') || \ - (echo "Could not connect to Etherpad on http://localhost:9001" ; exit 1) +log "Running the backend tests..." +try cd src +npm test +exit_code=$? -echo "Successfully connected to Etherpad on http://localhost:9001" - -# run the backend tests -echo "Now run the backend tests" -cd src - -failed=0 -npm run test || failed=1 kill "$ep_pid" && wait "$ep_pid" -exit $failed +log "Done." +exit "$exit_code" diff --git a/tests/frontend/travis/runnerLoadTest.sh b/tests/frontend/travis/runnerLoadTest.sh index 8fe854321..50ffcbb49 100755 --- a/tests/frontend/travis/runnerLoadTest.sh +++ b/tests/frontend/travis/runnerLoadTest.sh @@ -1,52 +1,51 @@ -#!/bin/bash +#!/bin/sh -# do not continue if there is an error -set -eu +pecho() { printf %s\\n "$*"; } +log() { pecho "$@"; } +error() { log "ERROR: $@" >&2; } +fatal() { error "$@"; exit 1; } +try() { "$@" || fatal "'$@' failed"; } -# source: https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself#246128 -MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +MY_DIR=$(try cd "${0%/*}" && try pwd) || exit 1 # reliably move to the etherpad base folder before running it -cd "${MY_DIR}/../../../" +try cd "${MY_DIR}/../../../" -# Set "points": 10 to 1000 to not agressively rate limit commits -sed 's/\"points\": 10/\"points\": 1000/g' settings.json.template > settings.json.points -# And enable loadTest -sed 's/\"loadTest\": false,/\"loadTest\": true,/g' settings.json.points > settings.json +try sed -e ' +s!"loadTest":[^,]*!"loadTest": true! +# Reduce rate limit aggressiveness +s!"points":[^,]*!"points": 1000! +' settings.json.template >settings.json -# start Etherpad, assuming all dependencies are already installed. -# -# This is possible because the "install" section of .travis.yml already contains -# a call to bin/installDeps.sh -echo "Running Etherpad directly, assuming bin/installDeps.sh has already been run" - -node node_modules/ep_etherpad-lite/node/server.js "${@}" & +log "Assuming bin/installDeps.sh has already been run" +node node_modules/ep_etherpad-lite/node/server.js "${@}" >/dev/null & ep_pid=$! -echo "Now I will try for 15 seconds to connect to Etherpad on http://localhost:9001" +log "Waiting for Etherpad to accept connections (http://localhost:9001)..." +connected=false +can_connect() { + curl -sSfo /dev/null http://localhost:9001/ || return 1 + connected=true +} +now() { date +%s; } +start=$(now) +while [ $(($(now) - $start)) -le 15 ] && ! can_connect; do + sleep 1 +done +[ "$connected" = true ] \ + || fatal "Timed out waiting for Etherpad to accept connections" +log "Successfully connected to Etherpad on http://localhost:9001" -# wait for at most 15 seconds until Etherpad starts accepting connections -# -# modified from: -# https://unix.stackexchange.com/questions/5277/how-do-i-tell-a-script-to-wait-for-a-process-to-start-accepting-requests-on-a-po#349138 -# -(timeout 15 bash -c 'until echo > /dev/tcp/localhost/9001; do sleep 0.5; done') || \ - (echo "Could not connect to Etherpad on http://localhost:9001" ; exit 1) - -echo "Successfully connected to Etherpad on http://localhost:9001" - -# Build the minified files? -curl http://localhost:9001/p/minifyme -f -s > /dev/null +# Build the minified files +try curl http://localhost:9001/p/minifyme -f -s >/dev/null # just in case, let's wait for another 10 seconds before going on sleep 10 -# run the backend tests -echo "Now run the load tests for 25 seconds and if it stalls before 100 then error" +log "Running the load tests..." etherpad-loadtest -d 25 exit_code=$? kill "$ep_pid" && wait "$ep_pid" -sleep 5 - -exit $exit_code +log "Done." +exit "$exit_code" diff --git a/tests/frontend/travis/sauce_tunnel.sh b/tests/frontend/travis/sauce_tunnel.sh index 4ab6e81a4..45827d31e 100755 --- a/tests/frontend/travis/sauce_tunnel.sh +++ b/tests/frontend/travis/sauce_tunnel.sh @@ -1,23 +1,37 @@ -#!/bin/bash +#!/bin/sh + +pecho() { printf %s\\n "$*"; } +log() { pecho "$@"; } +error() { log "ERROR: $@" >&2; } +fatal() { error "$@"; exit 1; } +try() { "$@" || fatal "'$@' failed"; } + +[ -n "${SAUCE_USERNAME}" ] || fatal "SAUCE_USERNAME is unset - exiting" +[ -n "${SAUCE_ACCESS_KEY}" ] || fatal "SAUCE_ACCESS_KEY is unset - exiting" + # download and unzip the sauce connector # # ACHTUNG: as of 2019-12-21, downloading sc-latest-linux.tar.gz does not work. -# It is necessary to explicitly download a specific version, for -# example https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz -# Supported versions are currently listed at: -# https://wiki.saucelabs.com/display/DOCS/Downloading+Sauce+Connect+Proxy -if [ -z "${SAUCE_USERNAME}" ]; then echo "SAUCE_USERNAME is unset - exiting"; exit 1; fi -if [ -z "${SAUCE_ACCESS_KEY}" ]; then echo "SAUCE_ACCESS_KEY is unset - exiting"; exit 1; fi +# It is necessary to explicitly download a specific version, for example +# https://saucelabs.com/downloads/sc-4.5.4-linux.tar.gz Supported versions are +# currently listed at: +# https://wiki.saucelabs.com/display/DOCS/Downloading+Sauce+Connect+Proxy +try curl -o /tmp/sauce.tar.gz \ + https://saucelabs.com/downloads/sc-4.6.2-linux.tar.gz +try tar zxf /tmp/sauce.tar.gz --directory /tmp +try mv /tmp/sc-*-linux /tmp/sauce_connect -curl https://saucelabs.com/downloads/sc-4.6.2-linux.tar.gz > /tmp/sauce.tar.gz -tar zxf /tmp/sauce.tar.gz --directory /tmp -mv /tmp/sc-*-linux /tmp/sauce_connect - -# start the sauce connector in background and make sure it doesn't output the secret key -(/tmp/sauce_connect/bin/sc --user "${SAUCE_USERNAME}" --key "${SAUCE_ACCESS_KEY}" -i "${TRAVIS_JOB_NUMBER}" --pidfile /tmp/sauce.pid --readyfile /tmp/tunnel > /dev/null )& +# start the sauce connector in background and make sure it doesn't output the +# secret key +try rm -f /tmp/tunnel +/tmp/sauce_connect/bin/sc \ + --user "${SAUCE_USERNAME}" \ + --key "${SAUCE_ACCESS_KEY}" \ + -i "${TRAVIS_JOB_NUMBER}" \ + --pidfile /tmp/sauce.pid \ + --readyfile /tmp/tunnel >/dev/null & # wait for the tunnel to build up -while [ ! -e "/tmp/tunnel" ] - do - sleep 1 +while ! [ -e "/tmp/tunnel" ]; do + sleep 1 done