From 40014d82301c522639d43a5ac01e0c2c8b201b39 Mon Sep 17 00:00:00 2001 From: John McLear Date: Sun, 19 Jul 2020 22:44:24 +0100 Subject: [PATCH] Rate limit Socket IO communication - WIP (#4036) Includes settings Includes i18n Includes a nice notification Disconnects on rate limit Includes feeding into metrics/stats Include console warn to server console. --- .travis.yml | 2 -- settings.json.template | 17 +++++++++++++++++ src/locales/en.json | 3 +++ src/node/handler/PadMessageHandler.js | 19 +++++++++++++++++++ src/node/utils/Settings.js | 16 ++++++++++++++++ src/package-lock.json | 5 +++++ src/package.json | 1 + src/static/js/pad_connectionstatus.js | 3 +-- src/templates/pad.html | 4 ++++ tests/frontend/travis/runnerBackend.sh | 5 ++++- tests/frontend/travis/runnerLoadTest.sh | 8 +++++--- 11 files changed, 75 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e003fc45..fd750adf3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,8 +57,6 @@ jobs: - "bin/installDeps.sh" - "cd src && npm install && cd -" - "npm install -g etherpad-load-test" - # I set loadTest to true - - "sed 's/\"loadTest\": false,/\"loadTest\": true,/g' settings.json.template > settings.json" script: - "tests/frontend/travis/runnerLoadTest.sh" diff --git a/settings.json.template b/settings.json.template index 1e11557fb..c6b7f394f 100644 --- a/settings.json.template +++ b/settings.json.template @@ -480,6 +480,23 @@ */ "allowAnyoneToImport": false, + /* + * From Etherpad 1.9.0 onwards, when Etherpad is in production mode commits from individual users are rate limited + * + * The default is to allow at most 10 changes per IP in a 1 second window. + * After that the change is rejected. + * + * See https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#websocket-single-connection-prevent-flooding for more options + */ + "commitRateLimiting": { + // duration of the rate limit window (seconds) + "duration": 1, + + // maximum number of chanes per IP to allow during the rate limit window + "points": 10 + }, + + /* * Toolbar buttons configuration. * diff --git a/src/locales/en.json b/src/locales/en.json index c67bad104..fff48f2b2 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -87,6 +87,9 @@ "pad.modals.deleted": "Deleted.", "pad.modals.deleted.explanation": "This pad has been removed.", + "pad.modals.rateLimited": "Rate Limited.", + "pad.modals.rateLimited.explanation": "You sent too many messages to this pad so it disconnected you.", + "pad.modals.disconnected": "You have been disconnected.", "pad.modals.disconnected.explanation": "The connection to the server was lost", "pad.modals.disconnected.cause": "The server may be unavailable. Please notify the service administrator if this continues to happen.", diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js index c68569e61..b6012d364 100644 --- a/src/node/handler/PadMessageHandler.js +++ b/src/node/handler/PadMessageHandler.js @@ -37,6 +37,12 @@ var channels = require("channels"); var stats = require('../stats'); var remoteAddress = require("../utils/RemoteAddress").remoteAddress; const nodeify = require("nodeify"); +const { RateLimiterMemory } = require('rate-limiter-flexible'); + +const rateLimiter = new RateLimiterMemory({ + points: settings.commitRateLimiting.points, + duration: settings.commitRateLimiting.duration +}); /** * A associative array that saves informations about a session @@ -164,6 +170,19 @@ exports.handleDisconnect = async function(client) */ exports.handleMessage = async function(client, message) { + var env = process.env.NODE_ENV || 'development'; + + if (env === 'production') { + try { + await rateLimiter.consume(client.handshake.address); // consume 1 point per event from IP + }catch(e){ + console.warn("Rate limited: ", client.handshake.address, " to reduce the amount of rate limiting that happens edit the rateLimit values in settings.json"); + stats.meter('rateLimited').mark(); + client.json.send({disconnect:"rateLimited"}); + return; + } + } + if (message == null) { return; } diff --git a/src/node/utils/Settings.js b/src/node/utils/Settings.js index e80c32e4b..6a03668c6 100644 --- a/src/node/utils/Settings.js +++ b/src/node/utils/Settings.js @@ -343,6 +343,22 @@ exports.importExportRateLimiting = { "max": 10 }; +/* + * From Etherpad 1.9.0 onwards, commits from individual users are rate limited + * + * The default is to allow at most 10 changes per IP in a 1 second window. + * After that the change is rejected. + * + * See https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#websocket-single-connection-prevent-flooding for more options + */ +exports.commitRateLimiting = { + // duration of the rate limit window (seconds) + "duration": 1, + + // maximum number of chanes per IP to allow during the rate limit window + "points": 10 +}; + /* * From Etherpad 1.8.3 onwards, the maximum allowed size for a single imported * file is always bounded. diff --git a/src/package-lock.json b/src/package-lock.json index 98bda7167..fef6d4b49 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -7472,6 +7472,11 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, + "rate-limiter-flexible": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-2.1.4.tgz", + "integrity": "sha512-wtbWcqZbCqyAO1k63moagJlCZuPCEqbJJ6il1y2JVoiUyxlE36+cM7ETta9K6tTom9O5pNK+CxwHMgyyyJ31Gg==" + }, "raw-body": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", diff --git a/src/package.json b/src/package.json index d5327a10b..4c06b1781 100644 --- a/src/package.json +++ b/src/package.json @@ -54,6 +54,7 @@ "nodeify": "1.0.1", "npm": "6.14.5", "openapi-backend": "2.4.1", + "rate-limiter-flexible": "^2.1.4", "rehype": "^10.0.0", "rehype-format": "^3.0.1", "request": "2.88.2", diff --git a/src/static/js/pad_connectionstatus.js b/src/static/js/pad_connectionstatus.js index 4e5f41be0..ad9fe0855 100644 --- a/src/static/js/pad_connectionstatus.js +++ b/src/static/js/pad_connectionstatus.js @@ -63,9 +63,8 @@ var padconnectionstatus = (function() what: 'disconnected', why: msg }; - var k = String(msg); // known reason why - if (!(k == 'userdup' || k == 'deleted' || k == 'looping' || k == 'slowcommit' || k == 'initsocketfail' || k == 'unauth' || k == 'badChangeset' || k == 'corruptPad')) + if (!(k == 'userdup' || k == 'deleted' || k == 'looping' || k == 'slowcommit' || k == 'initsocketfail' || k == 'unauth' || k == 'rateLimited' || k == 'badChangeset' || k == 'corruptPad')) { k = 'disconnected'; } diff --git a/src/templates/pad.html b/src/templates/pad.html index 25496b4ec..a23892d07 100644 --- a/src/templates/pad.html +++ b/src/templates/pad.html @@ -279,6 +279,10 @@

+
+

+

+
<% e.begin_block("disconnected"); %>

diff --git a/tests/frontend/travis/runnerBackend.sh b/tests/frontend/travis/runnerBackend.sh index 2bbe17cb3..680e96f9e 100755 --- a/tests/frontend/travis/runnerBackend.sh +++ b/tests/frontend/travis/runnerBackend.sh @@ -16,7 +16,10 @@ sed 's#\"soffice\": null,#\"soffice\":\"/usr/bin/soffice\",#g' settings.json.tem sed 's/\"allowAnyoneToImport\": false,/\"allowAnyoneToImport\": true,/g' settings.json.soffice > settings.json.allowImport # Set "max": 10 to 100 to not agressively rate limit -sed 's/\"max\": 10/\"max\": 100/g' settings.json.allowImport > settings.json +sed 's/\"max\": 10/\"max\": 100/g' settings.json.allowImport > settings.json.rateLimit + +# Set "points": 10 to 1000 to not agressively rate limit commits +sed 's/\"points\": 10/\"points\": 1000/g' settings.json.rateLimit > settings.json # start Etherpad, assuming all dependencies are already installed. # diff --git a/tests/frontend/travis/runnerLoadTest.sh b/tests/frontend/travis/runnerLoadTest.sh index c91655335..5ac447758 100755 --- a/tests/frontend/travis/runnerLoadTest.sh +++ b/tests/frontend/travis/runnerLoadTest.sh @@ -9,6 +9,11 @@ MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" # reliably move to the etherpad base folder before running it cd "${MY_DIR}/../../../" +# Set "points": 10 to 1000 to not agressively rate limit commits +sed 's/\"points\": 10/\"points\": 1000/g' settings.json.template > settings.json.points +# And enable loadTest +sed 's/\"loadTest\": false,/\"loadTest\": true,/g' settings.json.points > settings.json + # start Etherpad, assuming all dependencies are already installed. # # This is possible because the "install" section of .travis.yml already contains @@ -29,9 +34,6 @@ echo "Now I will try for 15 seconds to connect to Etherpad on http://localhost:9 echo "Successfully connected to Etherpad on http://localhost:9001" -# a copy of settings.json is necessary for the backend tests to work -cp settings.json.template settings.json - # Build the minified files? curl http://localhost:9001/p/minifyme -f -s > /dev/null