npm bump: attempting to simulate security does not exist bug

resilient.

.github/workflows/backend-tests.yml

@@ -38,7 +38,7 @@ jobs:
           sudo apt update
           sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
       -
-        name: Install all dependencies and symlink for ep_etherpad-lite
+        name: Install the ep_etherpad-lite package from ./src
         run: src/bin/installDeps.sh
       -
         name: Run the backend tests
@@ -74,12 +74,13 @@ jobs:
           sudo add-apt-repository -y ppa:libreoffice/ppa
           sudo apt update
           sudo apt install -y --no-install-recommends libreoffice libreoffice-pdfimport
+      -
+        name: Install the ep_etherpad-lite package from ./src
+        run: src/bin/installDeps.sh
       -
         name: Install Etherpad plugins
-        # The --legacy-peer-deps flag is required to work around a bug in npm v7:
-        # https://github.com/npm/cli/issues/2199
         run: >
-          npm install --no-save --legacy-peer-deps
+          npm install
           ep_align
           ep_author_hover
           ep_cursortrace
@@ -93,18 +94,6 @@ jobs:
           ep_spellcheck
           ep_subscript_and_superscript
           ep_table_of_contents
-      # Etherpad core dependencies must be installed after installing the
-      # plugin's dependencies, otherwise npm will try to hoist common
-      # dependencies by removing them from src/node_modules and installing them
-      # in the top-level node_modules. As of v6.14.10, npm's hoist logic appears
-      # to be buggy, because it sometimes removes dependencies from
-      # src/node_modules but fails to add them to the top-level node_modules.
-      # Even if npm correctly hoists the dependencies, the hoisting seems to
-      # confuse tools such as `npm outdated`, `npm update`, and some ESLint
-      # rules.
-      -
-        name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
       -
         name: Run the backend tests
         run: cd src && npm test
@@ -130,8 +119,11 @@ jobs:
             src/package-lock.json
             src/bin/doc/package-lock.json
       -
-        name: Install all dependencies and symlink for ep_etherpad-lite
+        name: Install the ep_etherpad-lite package from ./src
         run: src/bin/installOnWindows.bat
+      -
+        name: Run npm i once to make bin files available - why is this needed at all?
+        run: npm i
       -
         name:  Fix up the settings.json
         run: |
@@ -139,7 +131,7 @@ jobs:
           powershell -Command "(gc settings.json.holder) -replace '\"points\": 10', '\"points\": 1000' | Out-File -encoding ASCII settings.json"
       -
         name: Run the backend tests
-        run: cd src && npm test
+        run: cd src && npm run test-on-windows
 
   withpluginsWindows:
     # run on pushes to any branch
@@ -162,12 +154,16 @@ jobs:
           cache-dependency-path: |
             src/package-lock.json
             src/bin/doc/package-lock.json
+      -
+        name: Install the ep_etherpad-lite package from ./src
+        run: src/bin/installOnWindows.bat
+      -
+        name: Run npm i once to make bin files available - why is this needed at all?
+        run: npm i
       -
         name: Install Etherpad plugins
-        # The --legacy-peer-deps flag is required to work around a bug in npm
-        # v7: https://github.com/npm/cli/issues/2199
         run: >
-          npm install --no-save --legacy-peer-deps
+          npm install
           ep_align
           ep_author_hover
           ep_cursortrace
@@ -181,18 +177,6 @@ jobs:
           ep_spellcheck
           ep_subscript_and_superscript
           ep_table_of_contents
-      # Etherpad core dependencies must be installed after installing the
-      # plugin's dependencies, otherwise npm will try to hoist common
-      # dependencies by removing them from src/node_modules and installing them
-      # in the top-level node_modules. As of v6.14.10, npm's hoist logic appears
-      # to be buggy, because it sometimes removes dependencies from
-      # src/node_modules but fails to add them to the top-level node_modules.
-      # Even if npm correctly hoists the dependencies, the hoisting seems to
-      # confuse tools such as `npm outdated`, `npm update`, and some ESLint
-      # rules.
-      -
-        name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installOnWindows.bat
       -
         name:  Fix up the settings.json
         run: |
@@ -200,4 +184,4 @@ jobs:
           powershell -Command "(gc settings.json.holder) -replace '\"points\": 10', '\"points\": 1000' | Out-File -encoding ASCII settings.json"
       -
         name: Run the backend tests
-        run: cd src && npm test
+        run: cd src && npm run test-on-windows

.github/workflows/frontend-admin-tests.yml

@@ -46,23 +46,11 @@ jobs:
             src/package-lock.json
             src/bin/doc/package-lock.json
       -
-        name: Install etherpad plugins
+        name: Install the ep_etherpad-lite package from ./src
-        # We intentionally install an old ep_align version to test upgrades to
-        # the minor version number. The --legacy-peer-deps flag is required to
-        # work around a bug in npm v7: https://github.com/npm/cli/issues/2199
-        run: npm install --no-save --legacy-peer-deps ep_align@0.2.27
-      # Etherpad core dependencies must be installed after installing the
-      # plugin's dependencies, otherwise npm will try to hoist common
-      # dependencies by removing them from src/node_modules and installing them
-      # in the top-level node_modules. As of v6.14.10, npm's hoist logic appears
-      # to be buggy, because it sometimes removes dependencies from
-      # src/node_modules but fails to add them to the top-level node_modules.
-      # Even if npm correctly hoists the dependencies, the hoisting seems to
-      # confuse tools such as `npm outdated`, `npm update`, and some ESLint
-      # rules.
-      -
-        name: Install all dependencies and symlink for ep_etherpad-lite
         run: src/bin/installDeps.sh
+      -
+        name: Install etherpad plugins
+        run: npm install ep_align@0.2.27
       -
         name: Install etherpad plugins
         run: rm -Rf node_modules/ep_align/static/tests/*

.github/workflows/frontend-tests.yml

@@ -40,7 +40,7 @@ jobs:
             src/package-lock.json
             src/bin/doc/package-lock.json
       -
-        name: Install all dependencies and symlink for ep_etherpad-lite
+        name: Install the ep_etherpad-lite package from ./src
         run: src/bin/installDeps.sh
       -
         name: export GIT_HASH to env
@@ -103,12 +103,13 @@ jobs:
           cache-dependency-path: |
             src/package-lock.json
             src/bin/doc/package-lock.json
+      -
+        name: Install the ep_etherpad-lite package from ./src
+        run: src/bin/installDeps.sh
       -
         name: Install Etherpad plugins
-        # The --legacy-peer-deps flag is required to work around a bug in npm v7:
-        # https://github.com/npm/cli/issues/2199
         run: >
-          npm install --no-save --legacy-peer-deps
+          npm install
           ep_align
           ep_author_hover
           ep_cursortrace
@@ -123,18 +124,6 @@ jobs:
           ep_spellcheck
           ep_subscript_and_superscript
           ep_table_of_contents
-      # Etherpad core dependencies must be installed after installing the
-      # plugin's dependencies, otherwise npm will try to hoist common
-      # dependencies by removing them from src/node_modules and installing them
-      # in the top-level node_modules. As of v6.20.10, npm's hoist logic appears
-      # to be buggy, because it sometimes removes dependencies from
-      # src/node_modules but fails to add them to the top-level node_modules.
-      # Even if npm correctly hoists the dependencies, the hoisting seems to
-      # confuse tools such as `npm outdated`, `npm update`, and some ESLint
-      # rules.
-      -
-        name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
       -
         name: export GIT_HASH to env
         id: environment

.github/workflows/lint-package-lock.yml

@@ -38,4 +38,6 @@ jobs:
           --allowed-hosts npm
           --allowed-schemes https:
           --allowed-schemes github:
-          --allowed-urls github:mapbox/node-sqlite3#593c9d498be2510d286349134537e3bf89401c4a
+          --allowed-schemes git+ssh:
+          --allowed-schemes file:
+          --allowed-urls git+ssh://git@github.com/mapbox/node-sqlite3.git#593c9d498be2510d286349134537e3bf89401c4a

.github/workflows/load-test.yml

@@ -28,7 +28,7 @@ jobs:
             src/package-lock.json
             src/bin/doc/package-lock.json
       -
-        name: Install all dependencies and symlink for ep_etherpad-lite
+        name: Install the ep_etherpad-lite package from ./src
         run: src/bin/installDeps.sh
       -
         name: Install etherpad-load-test
@@ -57,15 +57,16 @@ jobs:
           cache-dependency-path: |
             src/package-lock.json
             src/bin/doc/package-lock.json
+      -
+        name: Install the ep_etherpad-lite package from ./src
+        run: src/bin/installDeps.sh
       -
         name: Install etherpad-load-test
         run: sudo npm install -g etherpad-load-test
       -
         name: Install etherpad plugins
-        # The --legacy-peer-deps flag is required to work around a bug in npm v7:
-        # https://github.com/npm/cli/issues/2199
         run: >
-          npm install --no-save --legacy-peer-deps
+          npm install
           ep_align
           ep_author_hover
           ep_cursortrace
@@ -78,18 +79,6 @@ jobs:
           ep_spellcheck
           ep_subscript_and_superscript
           ep_table_of_contents
-      # Etherpad core dependencies must be installed after installing the
-      # plugin's dependencies, otherwise npm will try to hoist common
-      # dependencies by removing them from src/node_modules and installing them
-      # in the top-level node_modules. As of v6.14.10, npm's hoist logic appears
-      # to be buggy, because it sometimes removes dependencies from
-      # src/node_modules but fails to add them to the top-level node_modules.
-      # Even if npm correctly hoists the dependencies, the hoisting seems to
-      # confuse tools such as `npm outdated`, `npm update`, and some ESLint
-      # rules.
-      -
-        name: Install all dependencies and symlink for ep_etherpad-lite
-        run: src/bin/installDeps.sh
       -
         name: Run load test
         run: src/tests/frontend/travis/runnerLoadTest.sh 25 50
@@ -115,7 +104,7 @@ jobs:
             src/package-lock.json
             src/bin/doc/package-lock.json
       -
-        name: Install all dependencies and symlink for ep_etherpad-lite
+        name: Install the ep_etherpad-lite package from ./src
         run: src/bin/installDeps.sh
       -
         name: Install etherpad-load-test

.github/workflows/upgrade-from-latest-release.yml

@@ -33,12 +33,12 @@ jobs:
           cache-dependency-path: |
             src/package-lock.json
             src/bin/doc/package-lock.json
+      # FIXME after 1.9.2
+      # After 1.9.2 is released, we need to remove '--legacy-peer-deps --no-save'
       -
         name: Install Etherpad plugins
-        # The --legacy-peer-deps flag is required to work around a bug in npm
-        # v7: https://github.com/npm/cli/issues/2199
         run: >
-          npm install --no-save --legacy-peer-deps
+          npm install --legacy-peer-deps --no-save
           ep_align
           ep_author_hover
           ep_cursortrace
@@ -52,15 +52,6 @@ jobs:
           ep_spellcheck
           ep_subscript_and_superscript
           ep_table_of_contents
-      # Etherpad core dependencies must be installed after installing the
-      # plugin's dependencies, otherwise npm will try to hoist common
-      # dependencies by removing them from src/node_modules and installing them
-      # in the top-level node_modules. As of v6.14.10, npm's hoist logic appears
-      # to be buggy, because it sometimes removes dependencies from
-      # src/node_modules but fails to add them to the top-level node_modules.
-      # Even if npm correctly hoists the dependencies, the hoisting seems to
-      # confuse tools such as `npm outdated`, `npm update`, and some ESLint
-      # rules.
       -
         name: Install all dependencies and symlink for ep_etherpad-lite
         run: src/bin/installDeps.sh
@@ -83,17 +74,37 @@ jobs:
         # commit that merges the PR's source branch to its destination branch.
         run: git checkout "${GITHUB_SHA}"
       -
-        name: Install all dependencies and symlink for ep_etherpad-lite
+        name: Temporary remove ./node_modules; Remove this after 1.9.2 release
+        run: rm -rf ./node_modules
+      -
+        name: Install the ep_etherpad-lite package from ./src
         run: src/bin/installDeps.sh
+      -
+        name: Install Etherpad plugins again; Remove this after 1.9.2 release
+        run: >
+          npm install
+          ep_align
+          ep_author_hover
+          ep_cursortrace
+          ep_font_size
+          ep_hash_auth
+          ep_headings2
+          ep_image_upload
+          ep_markdown
+          ep_readonly_guest
+          ep_set_title_on_pad
+          ep_spellcheck
+          ep_subscript_and_superscript
+          ep_table_of_contents
       -
         name: Run the backend tests
         run: cd src && npm test
       -
         name: Install Cypress
-        run: cd src && npm install cypress
+        run: npm install cypress
       -
         name: Run Etherpad & Test Frontend
         run: |
           node src/node/server.js &
           curl --connect-timeout 10 --max-time 20 --retry 5 --retry-delay 10 --retry-max-time 60 --retry-connrefused http://127.0.0.1:9001/p/test
-          ./src/node_modules/cypress/bin/cypress run --config-file src/tests/frontend/cypress/cypress.config.js
+          ./node_modules/cypress/bin/cypress run --config-file src/tests/frontend/cypress/cypress.config.js

.github/workflows/windows.yml

@@ -34,13 +34,16 @@ jobs:
             src/package-lock.json
             src/bin/doc/package-lock.json
       -
-        name: Install all dependencies and symlink for ep_etherpad-lite
+        name: Install the ep_etherpad-lite package from ./src
         shell: msys2 {0}
         run: src/bin/installDeps.sh
+      -
+        name: Run npm i once to make bin files available - why is this needed at all?
+        run: npm i
       -
         name: Run the backend tests
         shell: msys2 {0}
-        run: cd src && npm test
+        run: cd src && npm run test-on-windows
       -
         name: Build the .zip
         shell: msys2 {0}
@@ -115,11 +118,11 @@ jobs:
             etherpad/src/bin/doc/package-lock.json
       -
         name: Install Cypress
-        run: cd etherpad && cd src && npm install cypress
+        run: cd etherpad && npm install cypress
       -
         name: Run Etherpad
         run: |
           cd etherpad
           node node_modules\ep_etherpad-lite\node\server.js &
           curl --connect-timeout 10 --max-time 20 --retry 5 --retry-delay 10 --retry-max-time 60 --retry-connrefused http://127.0.0.1:9001/p/test
-          src\node_modules\cypress\bin\cypress run --config-file src\tests\frontendcypress\cypress.config.js
+          node_modules\cypress\bin\cypress run --config-file src\tests\frontendcypress\cypress.config.js

.travis.yml

@@ -28,10 +28,8 @@ _install_libreoffice: &install_libreoffice >-
   sudo apt-get update &&
   sudo apt-get -y install libreoffice libreoffice-pdfimport
 
-# The --legacy-peer-deps flag is required to work around a bug in npm v7:
-# https://github.com/npm/cli/issues/2199
 _install_plugins: &install_plugins >-
-  npm install --no-save --legacy-peer-deps
+  npm install
   ep_align
   ep_author_hover
   ep_cursortrace
@@ -64,12 +62,11 @@ jobs:
         - *install_libreoffice
         - *set_loglevel_warn
         - "src/bin/installDeps.sh"
-        - "cd src && npm install && cd -"
       script:
         - "cd src && npm test"
     - name: "Test the Dockerfile"
       install:
-        - "cd src && npm install && cd -"
+        - "src/bin/installDeps.sh"
       script:
         - "docker build -t etherpad:test ."
         - "docker run -d -p 9001:9001 etherpad:test && sleep 3"
@@ -78,7 +75,6 @@ jobs:
       install:
         - *set_loglevel_warn
         - "src/bin/installDeps.sh"
-        - "cd src && npm install && cd -"
         - "npm install -g etherpad-load-test"
       script:
         - "src/tests/frontend/travis/runnerLoadTest.sh"
@@ -112,7 +108,7 @@ jobs:
         - "cd src && npm test"
     - name: "Test the Dockerfile"
       install:
-        - "cd src && npm install && cd -"
+        - "src/bin/installDeps.sh"
       script:
         - "docker build -t etherpad:test ."
         - "docker run -d -p 9001:9001 etherpad:test && sleep 3"

CHANGELOG.md

@@ -1,3 +1,35 @@
+# Next release
+
+#### Note for admins
+Etherpad does no longer store it's dependencies in ./src/node_modules by default. Also, Etherpad now
+stores installed plugins in a package.json file in the root directory and no longer requires quirks
+like `--legacy-peer-deps` or `--no-save` when invoking npm during plugin installation.
+
+When you're updating, it's best to use the `./src/bin/installDeps.sh` script. It will `npm link` the
+src directory, using the package.json file in ./src. This will create the well-known symlink ep_etherpad-lite
+in ./node_modules, that we've been using for years. However, this will also add a dependency in ./package.json.
+
+`./src/bin/installDeps.sh` will fail, if you have no ./package.json or ./package-lock.json and your
+./node_modules directory is not empty, as this is an indicator of installed plugins. You need to remove
+./node_modules and install all your plugins in the next step.
+
+`./src/bin/installDeps.sh` will remove any existing directories in `./src/node_modules`.
+
+After running `./src/bin/installDeps.sh`, install your plugins with `npm i ep_plugin1 ep_plugin2...` or via
+`/admin/plugins`.
+
+
+#### Note for plugin authors
+You can no longer depend on core's dependencies via `require('ep_etherpad-lite/node_modules/$dep')`.
+Please run `src/bin/checkPlugins.sh` or manually change to `require('$dep')`. We don't recommend
+that you rely on Etherpad to include specific dependencies in the future. So it's best if you add
+the dependency in your package.json.
+For convenience we have added symlinks in ./src/node_modules for the following dependencies:
+async, cheerio, express, formidable, log4js and supertest.
+Please note that those symlinks will be removed in a future version, so we strongly recommend that
+you adapt your require statements.
+
+
 # 1.9.1
 
 ### Notable enhancements and fixes

Dockerfile

@@ -87,17 +87,12 @@ WORKDIR "${EP_DIR}"
 
 COPY --chown=etherpad:etherpad ./ ./
 
-# Plugins must be installed before installing Etherpad's dependencies, otherwise
+RUN npm config set prefix "${EP_DIR}/.npm-packages"
-# npm will try to hoist common dependencies by removing them from
+
-# src/node_modules and installing them in the top-level node_modules. As of
+RUN ./src/bin/installDeps.sh
-# v6.14.10, npm's hoist logic appears to be buggy, because it sometimes removes
+
-# dependencies from src/node_modules but fails to add them to the top-level
-# node_modules. Even if npm correctly hoists the dependencies, the hoisting
-# seems to confuse tools such as `npm outdated`, `npm update`, and some ESLint
-# rules.
 RUN { [ -z "${ETHERPAD_PLUGINS}" ] || \
-      npm install --no-save --legacy-peer-deps ${ETHERPAD_PLUGINS}; } && \
+    npm install ${ETHERPAD_PLUGINS}; } && \
-    src/bin/installDeps.sh && \
     rm -rf ~/.npm
 
 # Copy the configuration file.
@@ -106,11 +101,9 @@ COPY --chown=etherpad:etherpad ./settings.json.docker "${EP_DIR}"/settings.json
 # Fix group permissions
 RUN chmod -R g=u .
 
-USER root
-RUN cd src && npm link
 USER etherpad
 
-HEALTHCHECK --interval=20s --timeout=3s CMD ["etherpad-healthcheck"]
+HEALTHCHECK --interval=20s --timeout=3s CMD ["./src/bin/etherpad-healthcheck"]
 
 EXPOSE 9001
-CMD ["etherpad"]
+CMD ["./src/node/server.js"]

README.md

@@ -140,9 +140,7 @@ Alternatively, you can install plugins from the command line:
 
 ```sh
 cd /path/to/etherpad-lite
-# The `--no-save` and `--legacy-peer-deps` arguments are necessary to work
+npm install ep_${plugin_name}
-# around npm quirks.
-npm install --no-save --legacy-peer-deps ep_${plugin_name}
 ```
 
 Also see [the plugin wiki
@@ -154,7 +152,7 @@ Run the following command in your Etherpad folder to get all of the features
 visible in the above demo gif:
 
 ```sh
-npm install --no-save --legacy-peer-deps \
+npm install \
   ep_align \
   ep_comments_page \
   ep_embedded_hyperlinks2 \

doc/plugins.adoc

@@ -7,8 +7,7 @@ execute its own functionality based on these events.
 Publicly available plugins can be found in the npm registry (see
 <https://npmjs.org>). Etherpad's naming convention for plugins is to prefix your
 plugins with `ep_`. So, e.g. it's `ep_flubberworms`. Thus you can install
-plugins from npm, using `npm install --no-save --legacy-peer-deps
+plugins from npm, using `npm install ep_flubberworm` in Etherpad's root directory.
-ep_flubberworm` in Etherpad's root directory.
 
 You can also browse to `http://yourEtherpadInstan.ce/admin/plugins`, which will
 list all installed plugins and those available on npm. It even provides

node_modules/ep_etherpad-lite

@@ -1 +0,0 @@
-../src

src/bin/buildForWindows.sh

@@ -46,6 +46,9 @@ try ./src/bin/installDeps.sh
 log "copy the windows settings template..."
 try cp settings.json.template settings.json
 
+log "Because this is a production build, we delete supertest"
+try rm src/node_modules/supertest
+
 log "resolve symbolic links..."
 try cp -rL node_modules node_modules_resolved
 try rm -rf node_modules

src/bin/functions.sh

@@ -1,10 +1,10 @@
 # minimum required node version
-REQUIRED_NODE_MAJOR=12
+REQUIRED_NODE_MAJOR=16
-REQUIRED_NODE_MINOR=13
+REQUIRED_NODE_MINOR=0
 
 # minimum required npm version
-REQUIRED_NPM_MAJOR=5
+REQUIRED_NPM_MAJOR=8
-REQUIRED_NPM_MINOR=5
+REQUIRED_NPM_MINOR=19
 
 pecho() { printf %s\\n "$*"; }
 log() { pecho "$@"; }

src/bin/installDeps.sh

@@ -37,23 +37,63 @@ if [ ! -f "$settings" ]; then
   cp settings.json.template "$settings" || exit 1
 fi
 
+log "Removing src/node_modules."
+rm -rf ./src/node_modules || true
+
+# try to determine if plugins were installed using --no-save
+ROOT_PLUGINS_EXIST=1
+for file in node_modules/*
+do
+  if [ ! -e "$file" ]; then break; fi
+  if [ -L "$file" ] && [ "$file" = "node_modules/ep_etherpad-lite" ]; then break; fi
+
+  if expr "$file" : "node_modules/ep_*" > /dev/null; then
+    ROOT_PLUGINS_EXIST=0
+  fi
+done
+
+PACKAGE_EXISTS=1
+PACKAGELOCK_EXISTS=1
+if test -f ./package.json; then PACKAGE_EXISTS=0;fi
+if test -f ./package-lock.json; then PACKAGELOCK_EXISTS=0;fi
+
+if [ "$PACKAGE_EXISTS" = "1" ] || [ "$PACKAGELOCK_EXISTS" = "1" ]; then
+  if [ "$ROOT_PLUGINS_EXIST" = "0" ]; then
+    log "You have plugins in ./node_modules but don't have a package.json or package-lock.json file."
+    log "Please manually remove your ./node_modules directory, run this script again and install any plugins with npm i ep_plugin1 ep_plugin2 afterwards"
+    exit 1
+  fi
+fi
+
+log "Linking src as new package ep_etherpad-lite."
+exit_code=0
+(cd ./src && npm link --bin-links=false) || exit_code=$?
+
+if [ "$exit_code" != 0 ]; then
+  log "npm link failed. If there was a permission error, please set a prefix for npm."
+  log "The prefix can be set e.g. with npm config set prefix $HOME/.npm-packages"
+  log "This will create a symlink in $HOME/.npm-packages/lib/node_modules that points to this directory."
+  exit 1
+fi
 
 log "Installing dependencies..."
-(mkdir -p node_modules &&
+if [ "$NODE_ENV" = "production" ]; then
-cd node_modules &&
-{ [ -d ep_etherpad-lite ] || ln -sf ../src ep_etherpad-lite; } &&
-cd ep_etherpad-lite)
-
-cd src
-
-if [ -z "${ETHERPAD_PRODUCTION}" ]; then
-  log "Installing dev dependencies"
- npm ci --no-optional --omit=optional --include=dev --lockfile-version 1 || exit 1
-else
   log "Installing production dependencies"
-  npm ci --no-optional --omit=optional --omit=dev --lockfile-version 1 --production || exit 1
+  npm link ep_etherpad-lite --omit=optional --omit=dev --save --package-lock=true --bin-links=false || exit 1
+else
+  log "Installing dev dependencies"
+  npm link ep_etherpad-lite --omit=optional --save --package-lock=true --bin-links=false || exit 1
 fi
 
+log "Adding symlinks for plugin backwards compatibility"
+mkdir src/node_modules -p
+ln -s ../../node_modules/async src/node_modules/async
+ln -s ../../node_modules/express src/node_modules/express
+ln -s ../../node_modules/formidable src/node_modules/formidable
+ln -s ../../node_modules/log4js src/node_modules/log4js
+ln -s ../../node_modules/supertest src/node_modules/supertest
+
+
 # Remove all minified data to force node creating it new
 log "Clearing minified cache..."
 rm -f var/minified*

src/bin/installOnWindows.bat

@@ -9,19 +9,34 @@ cmd /C node -e "" || ( echo "Please install node.js ( https://nodejs.org )" && e
 echo _
 echo Ensure that all dependencies are up to date...  If this is the first time you have run Etherpad please be patient.
 
-mkdir node_modules
+echo Deleting old node_modules and src/node_modules
-cd /D node_modules
+del /q .\node_modules
-mklink /D "ep_etherpad-lite" "..\src"
+del /q .\src\node_modules
+echo Deleting old package.json and package-lock.json
+del /q .\package.json
+del /q .\package-lock.json
 
-cd /D "ep_etherpad-lite"
+cd /D src
-cmd /C npm ci || exit /B 1
+cmd /C npm link --bin-links=false || exit /B 1
 
-cd /D "%~dp0\..\.."
+cd ..
+
+cmd /C npm link ep_etherpad-lite --omit=optional --omit=dev --save --package-lock=true --bin-links=false || exit /B 1
 
 echo _
 echo Clearing cache...
 del /S var\minified*
 
+echo Adding symlinks for plugin backwards compatibility
+mkdir src\node_modules
+cd /D src\node_modules
+mklink /D "async" "..\..\node_modules\async"
+mklink /D "express" "..\..\node_modules\express"
+mklink /D "formidable" "..\..\node_modules\formidable"
+mklink /D "log4js" "..\..\node_modules\log4js"
+mklink /D "supertest" "..\..\node_modules\supertest"
+cd ..\..
+
 echo _
 echo Setting up settings.json...
 IF NOT EXIST settings.json (

src/bin/plugins/checkPlugin.js

@@ -356,18 +356,17 @@ const logger = log4js.getLogger('checkPlugin');
     logger.warn('Test files not found, please create tests.  https://github.com/ether/etherpad-lite/wiki/Creating-a-plugin#writing-and-running-front-end-tests-for-your-plugin');
   }
 
-  // Install dependencies so we can run ESLint. This should also create or update package-lock.json
-  // if autoFix is enabled.
-  const npmInstall = `npm install${autoFix ? '' : ' --no-package-lock'}`;
-  execSync(npmInstall, {stdio: 'inherit'});
-  // Create the ep_etherpad-lite symlink if necessary. This must be done after running `npm install`
-  // because that command nukes the symlink.
   try {
     const d = await fsp.realpath(path.join(pluginPath, 'node_modules/ep_etherpad-lite'));
     assert.equal(d, epSrcDir);
   } catch (err) {
-    execSync(`${npmInstall} --no-save ep_etherpad-lite@file:${epSrcDir}`, {stdio: 'inherit'});
+    execSync('./src/bin/installDeps.sh', {stdio: 'inherit'});
   }
+
+  // Install dependencies so we can run ESLint. This should also create or update package-lock.json
+  // if autoFix is enabled.
+  const npmInstall = `npm install${autoFix ? '' : ' --no-package-lock'}`;
+  execSync(npmInstall, {stdio: 'inherit'});
   // linting begins
   try {
     logger.info('Linting...');

src/bin/plugins/lib/README.md

@@ -11,7 +11,7 @@ TODO: Describe the plugin.
 From the Etherpad working directory, run:
 
 ```shell
-npm install --no-save --legacy-peer-deps [plugin_name]
+npm install [plugin_name]
 ```
 
 Or, install from Etherpad's `/admin/plugins` page.

src/bin/updatePlugins.sh

@@ -8,4 +8,4 @@ OUTDATED=$(npm outdated --depth=0 | awk '{print $1}' | grep '^ep_') || {
 }
 set -- ${OUTDATED}
 echo "Updating plugins: $*"
-exec npm install --no-save "$@"
+exec npm install "$@"

src/node/hooks/express/adminplugins.js

@@ -62,7 +62,7 @@ exports.socketio = (hookName, args, cb) => {
           const currentVersion = pluginDefs.plugins[plugin].package.version;
 
           return semver.gt(latestVersion, currentVersion);
-        });
+        }).map((plugin) => ({name: plugin, version: results[plugin].version}));
 
         socket.emit('results:updatable', {updatable});
       } catch (err) {
@@ -98,8 +98,8 @@ exports.socketio = (hookName, args, cb) => {
       }
     });
 
-    socket.on('install', (pluginName) => {
+    socket.on('install', (pluginName, version) => {
-      installer.install(pluginName, (err) => {
+      installer.install(pluginName, version, (err) => {
         if (err) console.warn(err.stack || err.toString());
 
         socket.emit('finished:install', {

src/node/server.js

@@ -111,7 +111,7 @@ exports.start = async () => {
       // eslint-disable-next-line promise/no-promise-in-callback
       exports.exit(err)
           .catch((err) => {
-            logger.error('Error in process exit', err);
+            logger.error('Error in process exit', JSON.stringify(err));
             // eslint-disable-next-line n/no-process-exit
             process.exit(1);
           });

src/node/utils/Minify.js

@@ -163,7 +163,7 @@ const minify = async (req, res) => {
       // Go straight into node_modules
       // Avoid `require.resolve()`, since 'mustache' and 'mustache/index.js'
       // would end up resolving to logically distinct resources.
-      filename = path.join('../node_modules/', library, libraryPath);
+      filename = path.join('../../node_modules/', library, libraryPath);
     }
   }
   const [, testf] = /^plugins\/ep_etherpad-lite\/(tests\/frontend\/.*)/.exec(filename) || [];

src/node_modules/async

@@ -0,0 +1 @@
+../../node_modules/async

src/node_modules/express

@@ -0,0 +1 @@
+../../node_modules/express

src/node_modules/formidable

@@ -0,0 +1 @@
+../../node_modules/formidable

src/node_modules/log4js

@@ -0,0 +1 @@
+../../node_modules/log4js

src/node_modules/supertest

@@ -0,0 +1 @@
+../../node_modules/supertest

src/package.json

@@ -53,7 +53,6 @@
     "log4js": "0.6.38",
     "measured-core": "^2.0.0",
     "mime-types": "^2.1.35",
-    "npm": "^6.14.18",
     "openapi-backend": "^5.9.2",
     "proxy-addr": "^2.0.7",
     "rate-limiter-flexible": "^2.4.1",
@@ -94,16 +93,17 @@
   },
   "engines": {
     "node": ">=16.20.1",
-    "npm": ">=6.14.0"
+    "npm": ">=9.8.0"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/ether/etherpad-lite.git"
   },
   "scripts": {
-    "lint": "eslint .",
+    "lint": "../node_modules/eslint/bin/eslint.js .",
-    "test": "mocha --timeout 120000 --recursive tests/backend/specs ../node_modules/ep_*/static/tests/backend/specs",
+    "test": "../node_modules/mocha/bin/_mocha --timeout 120000 --recursive tests/backend/specs ../node_modules/ep_*/static/tests/backend/specs",
-    "test-container": "mocha --timeout 5000 tests/container/specs/api",
+    "test-on-windows": "mocha --timeout 120000 --recursive tests/backend/specs ../node_modules/ep_*/static/tests/backend/specs",
+    "test-container": "../node_modules/mocha/bin/_mocha --timeout 5000 tests/container/specs/api",
     "dev": "bash ./bin/run.sh"
   },
   "version": "1.9.1",

src/static/js/admin/plugins.js

@@ -123,13 +123,17 @@ $(document).ready(() => {
     $('.do-install, .do-update').unbind('click').click(function (e) {
       const $row = $(e.target).closest('tr');
       const plugin = $row.data('plugin');
+      const update = $row.find('input.do-update');
+      // TODO dont store it here in DOM
+      // version after update or after installing a new package
+      const version = update.length !== 0 ? update.attr('version') : $row.find('.version').text();
       if ($(this).hasClass('do-install')) {
         $row.remove().appendTo('#installed-plugins');
         installed.progress.show(plugin, 'Installing');
       } else {
         installed.progress.show(plugin, 'Updating');
       }
-      socket.emit('install', plugin);
+      socket.emit('install', plugin, version);
       installed.messages.hide('nothing-installed');
     });
 
@@ -217,11 +221,13 @@ $(document).ready(() => {
   });
 
   socket.on('results:updatable', (data) => {
-    data.updatable.forEach((pluginName) => {
+    data.updatable.forEach((plugin) => {
-      const actions = $(`#installed-plugins > tr.${pluginName} .actions`);
+      const {name, version} = plugin;
+      const actions = $(`#installed-plugins > tr.${name} .actions`);
       actions.find('.do-update').remove();
+      // TODO dont store version here in DOM
       actions.append(
-          $('<input>').addClass('do-update').attr('type', 'button').attr('value', 'Update'));
+          $('<input>').addClass('do-update').attr('type', 'button').attr('version', version).attr('value', 'Update'));
     });
     updateHandlers();
   });

src/static/js/pluginfw/installer.js

@@ -31,10 +31,7 @@ exports.uninstall = async (pluginName, cb = null) => {
   cb = wrapTaskCb(cb);
   logger.info(`Uninstalling plugin ${pluginName}...`);
   try {
-    // The --no-save flag prevents npm from creating package.json or package-lock.json.
+    await runCmd(['npm', 'uninstall', pluginName]);
-    // The --legacy-peer-deps flag is required to work around a bug in npm v7:
-    // https://github.com/npm/cli/issues/2199
-    await runCmd(['npm', 'uninstall', '--no-save', '--legacy-peer-deps', pluginName]);
   } catch (err) {
     logger.error(`Failed to uninstall plugin ${pluginName}`);
     cb(err || new Error(err));
@@ -46,20 +43,17 @@ exports.uninstall = async (pluginName, cb = null) => {
   cb(null);
 };
 
-exports.install = async (pluginName, cb = null) => {
+exports.install = async (pluginName, version, cb = null) => {
   cb = wrapTaskCb(cb);
-  logger.info(`Installing plugin ${pluginName}...`);
+  logger.info(`Installing plugin ${pluginName}@${version}...`);
   try {
-    // The --no-save flag prevents npm from creating package.json or package-lock.json.
+    await runCmd(['npm', 'install', `${pluginName}@${version}`]);
-    // The --legacy-peer-deps flag is required to work around a bug in npm v7:
-    // https://github.com/npm/cli/issues/2199
-    await runCmd(['npm', 'install', '--no-save', '--legacy-peer-deps', pluginName]);
   } catch (err) {
-    logger.error(`Failed to install plugin ${pluginName}`);
+    logger.error(`Failed to install plugin ${pluginName}@${version}`);
     cb(err || new Error(err));
     throw err;
   }
-  logger.info(`Successfully installed plugin ${pluginName}`);
+  logger.info(`Successfully installed plugin ${pluginName}@${version}`);
   await hooks.aCallAll('pluginInstall', {pluginName});
   await plugins.update();
   cb(null);

src/tests/frontend/specs/adminupdateplugins.js

@@ -18,96 +18,127 @@ describe('Plugins page', function () {
   // create a new pad before each test run
   beforeEach(async function () {
     helper.newAdmin('plugins');
+    // menu is plugins, settings, help - so at least three entries atm
     await helper.waitForPromise(
         () => helper.admin$ && helper.admin$('.menu').find('li').length >= 3, 30000);
   });
 
-  it('Lists some plugins', async function () {
+  it('Lists some plugins assuming more than 50 available plugins', async function () {
     await helper.waitForPromise(() => helper.admin$('.results').children().length > 50, 20000);
   });
 
-  it('Searches for plugin', async function () {
+  it('Searches for plugin ep_font_color', async function () {
     helper.admin$('#search-query').val('ep_font_color');
     await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 10000);
-    await helper.waitForPromise(() => helper.admin$('.results').children().length < 300, 10000);
+    // multiple packages may be found
+    await helper.waitForPromise(() => helper.admin$('.results').children().length < 20, 10000);
+    await helper.waitForPromise(() => helper.admin$('.results .ep_font_color').length === 1, 10000);
   });
 
-  it('Attempt to Update a plugin', async function () {
+  it('Second search for ep_font_size does not return old result', async function () {
+    helper.admin$('#search-query').val('ep_font_size');
+    await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 10000);
+    // multiple packages may be found
+    await helper.waitForPromise(() => helper.admin$('.results').children().length < 20, 10000);
+    await helper.waitForPromise(() => helper.admin$('.results .ep_font_size').length === 1, 10000);
+    await helper.waitForPromise(() => helper.admin$('.results .ep_font_color').length === 0, 10000);
+  });
+
+  it('Searches for plugins ep_font_ (partial match)', async function () {
+    helper.admin$('#search-query').val('ep_font');
+    await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 10000);
+    // multiple packages may be found
+    await helper.waitForPromise(() => helper.admin$('.results').children().length < 50, 10000);
+    await helper.waitForPromise(() => helper.admin$('.results .ep_font_size').length === 1, 10000);
+    await helper.waitForPromise(() => helper.admin$('.results .ep_font_color').length === 1, 10000);
+  });
+
+  it('Attempt to Update a plugin (minor version update)', async function () {
     this.timeout(280000);
 
+    // available plugin list should load
     await helper.waitForPromise(() => helper.admin$('.results').children().length > 50, 20000);
 
-    if (helper.admin$('.ep_align').length === 0) this.skip();
+    // ep_align should be installed (via step in workflow)
+    await helper.waitForPromise(() => helper.admin$('#installed-plugins .ep_align').length >= 1, 10000);
 
+    let latestVersion;
     await helper.waitForPromise(
-        () => helper.admin$('.ep_align .version').text().split('.').length >= 2);
+        () => {
+          latestVersion = helper.admin$('#installed-plugins .ep_align .version').text();
+          return latestVersion === '0.2.27';
+        }
+    );
 
     const minorVersionBefore =
-        parseInt(helper.admin$('.ep_align .version').text().split('.')[1]);
+        parseInt(latestVersion.split('.')[1]);
 
-    if (!minorVersionBefore) {
+    await helper.waitForPromise(
-      throw new Error('Unable to get minor number of plugin, is the plugin installed?');
+        () => helper.admin$('#installed-plugins .ep_align .do-update').length === 1);
-    }
 
-    if (minorVersionBefore !== 2) this.skip();
+    helper.admin$('#installed-plugins .ep_align .do-update').click();
-
-    helper.waitForPromise(
-        () => helper.admin$('.ep_align .do-update').length === 1);
-
-    await timeout(500); // HACK!  Please submit better fix..
-    const $doUpdateButton = helper.admin$('.ep_align .do-update');
-    $doUpdateButton.click();
 
     // ensure its showing as Updating
+    // this assumes that updating will take some time, so there is a message showing up
+    // in the mean time
     await helper.waitForPromise(
-        () => helper.admin$('.ep_align .message').text() === 'Updating');
+        () => {
+		const message = helper.admin$('#installed-plugins .ep_align .message').text();
+		return message === 'Updating';
+	}, 120000)
 
     // Ensure it's a higher minor version IE 0.3.x as 0.2.x was installed
     // Coverage for https://github.com/ether/etherpad-lite/issues/4536
-    await helper.waitForPromise(() => parseInt(helper.admin$('.ep_align .version')
+    await helper.waitForPromise(() => parseInt(helper.admin$('#installed-plugins .ep_align .version')
         .text()
         .split('.')[1]) > minorVersionBefore, 60000, 1000);
-    // allow 50 seconds, check every 1 second.
+
+    // ensure it's the latest version
+    await helper.waitForPromise(
+        () => helper.admin$('.ep_align .do-update').length === 0);
   });
+
   it('Attempt to Install a plugin', async function () {
     this.timeout(280000);
 
     helper.admin$('#search-query').val('ep_headings2');
-    await helper.waitForPromise(() => helper.admin$('.results').children().length < 300, 6000);
+    await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 10000);
-    await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 6000);
+    await helper.waitForPromise(() => helper.admin$('.results').children().length < 50, 10000);
-
-    // skip if we already have ep_headings2 installed..
-    if (helper.admin$('.ep_headings2 .do-install').is(':visible') === false) this.skip();
 
     helper.admin$('.ep_headings2 .do-install').click();
     // ensure install has attempted to be started
     await helper.waitForPromise(
-        () => helper.admin$('.ep_headings2 .do-install').length !== 0, 120000);
+        () => helper.admin$('.ep_headings2 .do-install').length > 0, 120000);
     // ensure its not showing installing any more
     await helper.waitForPromise(
         () => helper.admin$('.ep_headings2 .message').text() === '', 180000);
     // ensure uninstall button is visible
     await helper.waitForPromise(
-        () => helper.admin$('.ep_headings2 .do-uninstall').length !== 0, 120000);
+        () => helper.admin$('#installed-plugins .ep_headings2 .do-uninstall').length > 0, 120000);
   });
 
   it('Attempt to Uninstall a plugin', async function () {
-    this.timeout(360000);
+    this.timeout(280000);
-    await helper.waitForPromise(
-        () => helper.admin$('.ep_headings2 .do-uninstall').length !== 0, 120000);
 
-    helper.admin$('.ep_headings2 .do-uninstall').click();
+    await helper.waitForPromise(
+        () => helper.admin$('#installed-plugins .ep_headings2 .do-uninstall').length > 0, 120000);
+
+    helper.admin$('#installed-plugins .ep_headings2 .do-uninstall').click();
 
     // ensure its showing uninstalling
+    // this assumes that uninstalling will take some time, so there is a message showing up
+    // in the mean time
     await helper.waitForPromise(
-        () => helper.admin$('.ep_headings2 .message')
+        () => helper.admin$('#installed-plugins .ep_headings2 .message')
             .text() === 'Uninstalling', 120000);
     // ensure its gone
-    await helper.waitForPromise(
+    await helper.waitForPromise(() => helper.admin$('#installed-plugins .ep_headings2').length === 0, 200000);
-        () => helper.admin$('.ep_headings2').length === 0, 240000);
 
+    // ensure search still works
     helper.admin$('#search-query').val('ep_font');
-    await helper.waitForPromise(() => helper.admin$('.results').children().length < 300, 240000);
+    await helper.waitForPromise(() => helper.admin$('.results').children().length < 50, 240000);
     await helper.waitForPromise(() => helper.admin$('.results').children().length > 0, 1000);
+    await helper.waitForPromise(() => helper.admin$('.results .ep_font_size').length === 1, 10000);
+    await helper.waitForPromise(() => helper.admin$('.results .ep_font_color').length === 1, 10000);
   });
 });

src/tests/frontend/specs/language.js

@@ -11,7 +11,7 @@ describe('Language select and change', function () {
 
   // Destroy language cookies
   it('makes text german', async function () {
-    this.timeout(1000);
+    this.timeout(3000);
     const chrome$ = helper.padChrome$;
 
     // click on the settings button to make settings visible
@@ -40,7 +40,7 @@ describe('Language select and change', function () {
   });
 
   it('makes text English', async function () {
-    this.timeout(1000);
+    this.timeout(3000);
     const chrome$ = helper.padChrome$;
 
     // click on the settings button to make settings visible