diff --git a/bin/checkAllPads.js b/bin/checkAllPads.js index f90e57aef..6e1f14842 100644 --- a/bin/checkAllPads.js +++ b/bin/checkAllPads.js @@ -1,28 +1,31 @@ +'use strict'; /* * This is a debug tool. It checks all revisions for data corruption */ -if (process.argv.length != 2) { - console.error('Use: node bin/checkAllPads.js'); - process.exit(1); -} +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + +if (process.argv.length !== 2) throw new Error('Use: node bin/checkAllPads.js'); // load and initialize NPM -const npm = require('../src/node_modules/npm'); +const npm = require('ep_etherpad-lite/node_modules/npm'); npm.load({}, async () => { try { // initialize the database - const settings = require('../src/node/utils/Settings'); - const db = require('../src/node/db/DB'); + require('ep_etherpad-lite/node/utils/Settings'); + const db = require('ep_etherpad-lite/node/db/DB'); await db.init(); // load modules - const Changeset = require('../src/static/js/Changeset'); - const padManager = require('../src/node/db/PadManager'); + const Changeset = require('ep_etherpad-lite/static/js/Changeset'); + const padManager = require('ep_etherpad-lite/node/db/PadManager'); + + let revTestedCount = 0; // get all pads const res = await padManager.listAllPads(); - for (const padId of res.padIDs) { const pad = await padManager.getPad(padId); @@ -31,7 +34,6 @@ npm.load({}, async () => { console.error(`[${pad.id}] Missing attribute pool`); continue; } - // create an array with key kevisions // key revisions always save the full pad atext const head = pad.getHeadRevisionNumber(); @@ -71,21 +73,23 @@ npm.load({}, async () => { const apool = pad.pool; let atext = revisions[keyRev].meta.atext; - for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) { try { const cs = revisions[rev].changeset; atext = Changeset.applyToAText(cs, atext, apool); + revTestedCount++; } catch (e) { - console.error(`[${pad.id}] Bad changeset at revision ${i} - ${e.message}`); + console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`); } } } - console.log('finished'); - process.exit(0); } + if (revTestedCount === 0) { + throw new Error('No revisions tested'); + } + console.log(`Finished: Tested ${revTestedCount} revisions`); } catch (err) { console.trace(err); - process.exit(1); + throw err; } }); diff --git a/bin/checkPad.js b/bin/checkPad.js index 323840e72..de1c51402 100644 --- a/bin/checkPad.js +++ b/bin/checkPad.js @@ -1,33 +1,33 @@ +'use strict'; /* * This is a debug tool. It checks all revisions for data corruption */ -if (process.argv.length != 3) { - console.error('Use: node bin/checkPad.js $PADID'); - process.exit(1); -} +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + +if (process.argv.length !== 3) throw new Error('Use: node bin/checkPad.js $PADID'); // get the padID const padId = process.argv[2]; +let checkRevisionCount = 0; // load and initialize NPM; -const npm = require('../src/node_modules/npm'); +const npm = require('ep_etherpad-lite/node_modules/npm'); npm.load({}, async () => { try { // initialize database - const settings = require('../src/node/utils/Settings'); - const db = require('../src/node/db/DB'); + require('ep_etherpad-lite/node/utils/Settings'); + const db = require('ep_etherpad-lite/node/db/DB'); await db.init(); // load modules const Changeset = require('ep_etherpad-lite/static/js/Changeset'); - const padManager = require('../src/node/db/PadManager'); + const padManager = require('ep_etherpad-lite/node/db/PadManager'); const exists = await padManager.doesPadExists(padId); - if (!exists) { - console.error('Pad does not exist'); - process.exit(1); - } + if (!exists) throw new Error('Pad does not exist'); // get the pad const pad = await padManager.getPad(padId); @@ -41,7 +41,8 @@ npm.load({}, async () => { } // run through all key revisions - for (const keyRev of keyRevisions) { + for (let keyRev of keyRevisions) { + keyRev = parseInt(keyRev); // create an array of revisions we need till the next keyRevision or the End const revisionsNeeded = []; for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) { @@ -58,13 +59,12 @@ npm.load({}, async () => { } // check if the pad has a pool - if (pad.pool === undefined) { - console.error('Attribute pool is missing'); - process.exit(1); - } + if (pad.pool === undefined) throw new Error('Attribute pool is missing'); // check if there is an atext in the keyRevisions - if (revisions[keyRev] === undefined || revisions[keyRev].meta === undefined || revisions[keyRev].meta.atext === undefined) { + if (revisions[keyRev] === undefined || + revisions[keyRev].meta === undefined || + revisions[keyRev].meta.atext === undefined) { console.error(`No atext in key revision ${keyRev}`); continue; } @@ -73,8 +73,8 @@ npm.load({}, async () => { let atext = revisions[keyRev].meta.atext; for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) { + checkRevisionCount++; try { - // console.log("check revision " + rev); const cs = revisions[rev].changeset; atext = Changeset.applyToAText(cs, atext, apool); } catch (e) { @@ -82,11 +82,10 @@ npm.load({}, async () => { continue; } } - console.log('finished'); - process.exit(0); + console.log(`Finished: Checked ${checkRevisionCount} revisions`); } - } catch (e) { - console.trace(e); - process.exit(1); + } catch (err) { + console.trace(err); + throw err; } }); diff --git a/bin/checkPadDeltas.js b/bin/checkPadDeltas.js index 1e45f7148..ecbb20846 100644 --- a/bin/checkPadDeltas.js +++ b/bin/checkPadDeltas.js @@ -1,111 +1,107 @@ +'use strict'; /* * This is a debug tool. It checks all revisions for data corruption */ -if (process.argv.length != 3) { - console.error('Use: node bin/checkPadDeltas.js $PADID'); - process.exit(1); -} +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + +if (process.argv.length !== 3) throw new Error('Use: node bin/checkPadDeltas.js $PADID'); // get the padID const padId = process.argv[2]; // load and initialize NPM; -const expect = require('expect.js'); -const diff = require('diff'); -var async = require('async'); - -const npm = require('../src/node_modules/npm'); -var async = require('ep_etherpad-lite/node_modules/async'); -const Changeset = require('ep_etherpad-lite/static/js/Changeset'); +const expect = require('../tests/frontend/lib/expect'); +const diff = require('ep_etherpad-lite/node_modules/diff'); +const npm = require('ep_etherpad-lite/node_modules/npm'); npm.load({}, async () => { - try { - // initialize database - const settings = require('../src/node/utils/Settings'); - const db = require('../src/node/db/DB'); - await db.init(); + // initialize database + require('ep_etherpad-lite/node/utils/Settings'); + const db = require('ep_etherpad-lite/node/db/DB'); + await db.init(); - // load modules - const Changeset = require('ep_etherpad-lite/static/js/Changeset'); - const padManager = require('../src/node/db/PadManager'); + // load modules + const Changeset = require('ep_etherpad-lite/static/js/Changeset'); + const padManager = require('ep_etherpad-lite/node/db/PadManager'); - const exists = await padManager.doesPadExists(padId); - if (!exists) { - console.error('Pad does not exist'); - process.exit(1); + const exists = await padManager.doesPadExists(padId); + if (!exists) throw new Error('Pad does not exist'); + + // get the pad + const pad = await padManager.getPad(padId); + + // create an array with key revisions + // key revisions always save the full pad atext + const head = pad.getHeadRevisionNumber(); + const keyRevisions = []; + for (let i = 0; i < head; i += 100) { + keyRevisions.push(i); + } + + // create an array with all revisions + const revisions = []; + for (let i = 0; i <= head; i++) { + revisions.push(i); + } + + let atext = Changeset.makeAText('\n'); + + // run trough all revisions + for (const revNum of revisions) { + // console.log('Fetching', revNum) + const revision = await db.get(`pad:${padId}:revs:${revNum}`); + // check if there is a atext in the keyRevisions + if (~keyRevisions.indexOf(revNum) && + (revision === undefined || + revision.meta === undefined || + revision.meta.atext === undefined)) { + console.error(`No atext in key revision ${revNum}`); + continue; } - // get the pad - const pad = await padManager.getPad(padId); - - // create an array with key revisions - // key revisions always save the full pad atext - const head = pad.getHeadRevisionNumber(); - const keyRevisions = []; - for (var i = 0; i < head; i += 100) { - keyRevisions.push(i); + // try glue everything together + try { + // console.log("check revision ", revNum); + const cs = revision.changeset; + atext = Changeset.applyToAText(cs, atext, pad.pool); + } catch (e) { + console.error(`Bad changeset at revision ${revNum} - ${e.message}`); + continue; } - // create an array with all revisions - const revisions = []; - for (var i = 0; i <= head; i++) { - revisions.push(i); - } - - let atext = Changeset.makeAText('\n'); - - // run trough all revisions - async.forEachSeries(revisions, (revNum, callback) => { - // console.log('Fetching', revNum) - db.db.get(`pad:${padId}:revs:${revNum}`, (err, revision) => { - if (err) return callback(err); - - // check if there is a atext in the keyRevisions - if (~keyRevisions.indexOf(revNum) && (revision === undefined || revision.meta === undefined || revision.meta.atext === undefined)) { - console.error(`No atext in key revision ${revNum}`); - callback(); - return; - } - - try { - // console.log("check revision ", revNum); - const cs = revision.changeset; - atext = Changeset.applyToAText(cs, atext, pad.pool); - } catch (e) { - console.error(`Bad changeset at revision ${revNum} - ${e.message}`); - callback(); - return; - } - - if (~keyRevisions.indexOf(revNum)) { - try { - expect(revision.meta.atext.text).to.eql(atext.text); - expect(revision.meta.atext.attribs).to.eql(atext.attribs); - } catch (e) { - console.error(`Atext in key revision ${revNum} doesn't match computed one.`); - console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => { if (!op.added && !op.removed) op.value = op.value.length; return op; })); - // console.error(e) - // console.log('KeyRev. :', revision.meta.atext) - // console.log('Computed:', atext) - callback(); - return; - } - } - - setImmediate(callback); - }); - }, (er) => { - if (pad.atext.text == atext.text) { console.log('ok'); } else { - console.error('Pad AText doesn\'t match computed one! (Computed ', atext.text.length, ', db', pad.atext.text.length, ')'); - console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => { if (!op.added && !op.removed) op.value = op.value.length; return op; })); + // check things are working properly + if (~keyRevisions.indexOf(revNum)) { + try { + expect(revision.meta.atext.text).to.eql(atext.text); + expect(revision.meta.atext.attribs).to.eql(atext.attribs); + } catch (e) { + console.error(`Atext in key revision ${revNum} doesn't match computed one.`); + console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => { + if (!op.added && !op.removed) op.value = op.value.length; + return op; + })); + // console.error(e) + // console.log('KeyRev. :', revision.meta.atext) + // console.log('Computed:', atext) + continue; } - callback(er); - }); + } + } - process.exit(0); - } catch (e) { - console.trace(e); - process.exit(1); + // check final text is right... + if (pad.atext.text === atext.text) { + console.log('ok'); + } else { + console.error('Pad AText doesn\'t match computed one! (Computed ', + atext.text.length, ', db', pad.atext.text.length, ')'); + console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => { + if (!op.added && !op.removed) { + op.value = op.value.length; + return op; + } + })); } }); diff --git a/bin/createUserSession.js b/bin/createUserSession.js index 324941ec8..292fde8ba 100644 --- a/bin/createUserSession.js +++ b/bin/createUserSession.js @@ -1,15 +1,19 @@ +'use strict'; + /* * A tool for generating a test user session which can be used for debugging configs * that require sessions. */ -const m = (f) => `${__dirname}/../${f}`; + +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); const fs = require('fs'); const path = require('path'); const querystring = require('querystring'); -const request = require(m('src/node_modules/request')); -const settings = require(m('src/node/utils/Settings')); -const supertest = require(m('src/node_modules/supertest')); +const settings = require('ep_etherpad-lite/node/utils/Settings'); +const supertest = require('ep_etherpad-lite/node_modules/supertest'); (async () => { const api = supertest(`http://${settings.ip}:${settings.port}`); diff --git a/bin/deleteAllGroupSessions.js b/bin/deleteAllGroupSessions.js index ee0058ffa..fd8ba5341 100644 --- a/bin/deleteAllGroupSessions.js +++ b/bin/deleteAllGroupSessions.js @@ -1,51 +1,47 @@ +'use strict'; + /* * A tool for deleting ALL GROUP sessions Etherpad user sessions from the CLI, * because sometimes a brick is required to fix a face. */ -const request = require('../src/node_modules/request'); -const settings = require(`${__dirname}/../tests/container/loadSettings`).loadSettings(); -const supertest = require(`${__dirname}/../src/node_modules/supertest`); -const api = supertest(`http://${settings.ip}:${settings.port}`); +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + +const supertest = require('ep_etherpad-lite/node_modules/supertest'); const path = require('path'); const fs = require('fs'); +// Set a delete counter which will increment on each delete attempt +// TODO: Check delete is successful before incrementing +let deleteCount = 0; + // get the API Key const filePath = path.join(__dirname, '../APIKEY.txt'); -const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'}); +console.log('Deleting all group sessions, please be patient.'); -// Set apiVersion to base value, we change this later. -let apiVersion = 1; -let guids; +(async () => { + const settings = require('../tests/container/loadSettings').loadSettings(); + const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'}); + const api = supertest(`http://${settings.ip}:${settings.port}`); -// Update the apiVersion -api.get('/api/') - .expect((res) => { - apiVersion = res.body.currentVersion; - if (!res.body.currentVersion) throw new Error('No version set in API'); - return; - }) - .then(() => { - const guri = `/api/${apiVersion}/listAllGroups?apikey=${apikey}`; - api.get(guri) - .then((res) => { - guids = res.body.data.groupIDs; - guids.forEach((groupID) => { - const luri = `/api/${apiVersion}/listSessionsOfGroup?apikey=${apikey}&groupID=${groupID}`; - api.get(luri) - .then((res) => { - if (res.body.data) { - Object.keys(res.body.data).forEach((sessionID) => { - if (sessionID) { - console.log('Deleting', sessionID); - const duri = `/api/${apiVersion}/deleteSession?apikey=${apikey}&sessionID=${sessionID}`; - api.post(duri); // deletes - } - }); - } else { - // no session in this group. - } - }); - }); - }); - }); + const apiVersionResponse = await api.get('/api/'); + const apiVersion = apiVersionResponse.body.currentVersion; // 1.12.5 + + const groupsResponse = await api.get(`/api/${apiVersion}/listAllGroups?apikey=${apikey}`); + const groups = groupsResponse.body.data.groupIDs; // ['whateverGroupID'] + + for (const groupID of groups) { + const sessionURI = `/api/${apiVersion}/listSessionsOfGroup?apikey=${apikey}&groupID=${groupID}`; + const sessionsResponse = await api.get(sessionURI); + const sessions = sessionsResponse.body.data; + + for (const sessionID of Object.keys(sessions)) { + const deleteURI = `/api/${apiVersion}/deleteSession?apikey=${apikey}&sessionID=${sessionID}`; + await api.post(deleteURI); // delete + deleteCount++; + } + } + console.log(`Deleted ${deleteCount} sessions`); +})(); diff --git a/bin/deletePad.js b/bin/deletePad.js index e145d63a0..ea9aea7e0 100644 --- a/bin/deletePad.js +++ b/bin/deletePad.js @@ -1,18 +1,21 @@ +'use strict'; + /* * A tool for deleting pads from the CLI, because sometimes a brick is required * to fix a window. */ -const request = require('../src/node_modules/request'); -const settings = require(`${__dirname}/../tests/container/loadSettings`).loadSettings(); -const supertest = require(`${__dirname}/../src/node_modules/supertest`); +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + +const settings = require('../tests/container/loadSettings').loadSettings(); +const supertest = require('ep_etherpad-lite/node_modules/supertest'); const api = supertest(`http://${settings.ip}:${settings.port}`); const path = require('path'); const fs = require('fs'); -if (process.argv.length != 3) { - console.error('Use: node deletePad.js $PADID'); - process.exit(1); -} + +if (process.argv.length !== 3) throw new Error('Use: node deletePad.js $PADID'); // get the padID const padId = process.argv[2]; @@ -21,28 +24,14 @@ const padId = process.argv[2]; const filePath = path.join(__dirname, '../APIKEY.txt'); const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'}); -// Set apiVersion to base value, we change this later. -let apiVersion = 1; +(async () => { + let apiVersion = await api.get('/api/'); + apiVersion = apiVersion.body.currentVersion; + if (!apiVersion) throw new Error('No version set in API'); -// Update the apiVersion -api.get('/api/') - .expect((res) => { - apiVersion = res.body.currentVersion; - if (!res.body.currentVersion) throw new Error('No version set in API'); - return; - }) - .end((err, res) => { - // Now we know the latest API version, let's delete pad - const uri = `/api/${apiVersion}/deletePad?apikey=${apikey}&padID=${padId}`; - api.post(uri) - .expect((res) => { - if (res.body.code === 1) { - console.error('Error deleting pad', res.body); - } else { - console.log('Deleted pad', res.body); - } - return; - }) - .end(() => {}); - }); -// end + // Now we know the latest API version, let's delete pad + const uri = `/api/${apiVersion}/deletePad?apikey=${apikey}&padID=${padId}`; + const deleteAttempt = await api.post(uri); + if (deleteAttempt.body.code === 1) throw new Error(`Error deleting pad ${deleteAttempt.body}`); + console.log('Deleted pad', deleteAttempt.body); +})(); diff --git a/bin/extractPadData.js b/bin/extractPadData.js index a811076ef..3b182a571 100644 --- a/bin/extractPadData.js +++ b/bin/extractPadData.js @@ -1,34 +1,34 @@ +'use strict'; + /* * This is a debug tool. It helps to extract all datas of a pad and move it from * a productive environment and to a develop environment to reproduce bugs * there. It outputs a dirtydb file */ -if (process.argv.length != 3) { - console.error('Use: node extractPadData.js $PADID'); - process.exit(1); -} +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + +if (process.argv.length !== 3) throw new Error('Use: node extractPadData.js $PADID'); // get the padID const padId = process.argv[2]; -const npm = require('../src/node_modules/npm'); +const npm = require('ep_etherpad-lite/node_modules/npm'); -npm.load({}, async (er) => { - if (er) { - console.error(`Could not load NPM: ${er}`); - process.exit(1); - } +npm.load({}, async (err) => { + if (err) throw err; try { // initialize database - const settings = require('../src/node/utils/Settings'); - const db = require('../src/node/db/DB'); + require('ep_etherpad-lite/node/utils/Settings'); + const db = require('ep_etherpad-lite/node/db/DB'); await db.init(); // load extra modules - const dirtyDB = require('../src/node_modules/dirty'); - const padManager = require('../src/node/db/PadManager'); + const dirtyDB = require('ep_etherpad-lite/node_modules/dirty'); + const padManager = require('ep_etherpad-lite/node/db/PadManager'); const util = require('util'); // initialize output database @@ -67,9 +67,8 @@ npm.load({}, async (er) => { } console.log('finished'); - process.exit(0); - } catch (er) { - console.error(er); - process.exit(1); + } catch (err) { + console.error(err); + throw err; } }); diff --git a/bin/importSqlFile.js b/bin/importSqlFile.js index 870c02cce..35fe4f323 100644 --- a/bin/importSqlFile.js +++ b/bin/importSqlFile.js @@ -1,77 +1,18 @@ +'use strict'; + +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + const startTime = Date.now(); -require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => { - const fs = require('fs'); - - const ueberDB = require('ep_etherpad-lite/node_modules/ueberdb2'); - const settings = require('ep_etherpad-lite/node/utils/Settings'); - const log4js = require('ep_etherpad-lite/node_modules/log4js'); - - const dbWrapperSettings = { - cache: 0, - writeInterval: 100, - json: false, // data is already json encoded - }; - const db = new ueberDB.database(settings.dbType, settings.dbSettings, dbWrapperSettings, log4js.getLogger('ueberDB')); - - const sqlFile = process.argv[2]; - - // stop if the settings file is not set - if (!sqlFile) { - console.error('Use: node importSqlFile.js $SQLFILE'); - process.exit(1); - } - - log('initializing db'); - db.init((err) => { - // there was an error while initializing the database, output it and stop - if (err) { - console.error('ERROR: Problem while initializing the database'); - console.error(err.stack ? err.stack : err); - process.exit(1); - } else { - log('done'); - - log('open output file...'); - const lines = fs.readFileSync(sqlFile, 'utf8').split('\n'); - - const count = lines.length; - let keyNo = 0; - - process.stdout.write(`Start importing ${count} keys...\n`); - lines.forEach((l) => { - if (l.substr(0, 27) == 'REPLACE INTO store VALUES (') { - const pos = l.indexOf("', '"); - const key = l.substr(28, pos - 28); - let value = l.substr(pos + 3); - value = value.substr(0, value.length - 2); - console.log(`key: ${key} val: ${value}`); - console.log(`unval: ${unescape(value)}`); - db.set(key, unescape(value), null); - keyNo++; - if (keyNo % 1000 == 0) { - process.stdout.write(` ${keyNo}/${count}\n`); - } - } - }); - process.stdout.write('\n'); - process.stdout.write('done. waiting for db to finish transaction. depended on dbms this may take some time...\n'); - - db.close(() => { - log(`finished, imported ${keyNo} keys.`); - process.exit(0); - }); - } - }); -}); - -function log(str) { +const log = (str) => { console.log(`${(Date.now() - startTime) / 1000}\t${str}`); -} +}; -unescape = function (val) { +const unescape = (val) => { // value is a string - if (val.substr(0, 1) == "'") { + if (val.substr(0, 1) === "'") { val = val.substr(0, val.length - 1).substr(1); return val.replace(/\\[0nrbtZ\\'"]/g, (s) => { @@ -88,16 +29,81 @@ unescape = function (val) { } // value is a boolean or NULL - if (val == 'NULL') { + if (val === 'NULL') { return null; } - if (val == 'true') { + if (val === 'true') { return true; } - if (val == 'false') { + if (val === 'false') { return false; } // value is a number return val; }; + + +require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => { + const fs = require('fs'); + + const ueberDB = require('ep_etherpad-lite/node_modules/ueberdb2'); + const settings = require('ep_etherpad-lite/node/utils/Settings'); + const log4js = require('ep_etherpad-lite/node_modules/log4js'); + + const dbWrapperSettings = { + cache: 0, + writeInterval: 100, + json: false, // data is already json encoded + }; + const db = new ueberDB.database( // eslint-disable-line new-cap + settings.dbType, + settings.dbSettings, + dbWrapperSettings, + log4js.getLogger('ueberDB')); + + const sqlFile = process.argv[2]; + + // stop if the settings file is not set + if (!sqlFile) throw new Error('Use: node importSqlFile.js $SQLFILE'); + + log('initializing db'); + db.init((err) => { + // there was an error while initializing the database, output it and stop + if (err) { + throw err; + } else { + log('done'); + + log('open output file...'); + const lines = fs.readFileSync(sqlFile, 'utf8').split('\n'); + + const count = lines.length; + let keyNo = 0; + + process.stdout.write(`Start importing ${count} keys...\n`); + lines.forEach((l) => { + if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') { + const pos = l.indexOf("', '"); + const key = l.substr(28, pos - 28); + let value = l.substr(pos + 3); + value = value.substr(0, value.length - 2); + console.log(`key: ${key} val: ${value}`); + console.log(`unval: ${unescape(value)}`); + db.set(key, unescape(value), null); + keyNo++; + if (keyNo % 1000 === 0) { + process.stdout.write(` ${keyNo}/${count}\n`); + } + } + }); + process.stdout.write('\n'); + process.stdout.write('done. waiting for db to finish transaction. ' + + 'depended on dbms this may take some time..\n'); + + db.close(() => { + log(`finished, imported ${keyNo} keys.`); + }); + } + }); +}); diff --git a/bin/migrateDirtyDBtoRealDB.js b/bin/migrateDirtyDBtoRealDB.js index 63425cab7..48760b8ba 100644 --- a/bin/migrateDirtyDBtoRealDB.js +++ b/bin/migrateDirtyDBtoRealDB.js @@ -1,4 +1,12 @@ -require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => { +'use strict'; + +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + +const util = require('util'); + +require('ep_etherpad-lite/node_modules/npm').load({}, async (er, npm) => { process.chdir(`${npm.root}/..`); // This script requires that you have modified your settings.json file @@ -10,39 +18,42 @@ require('ep_etherpad-lite/node_modules/npm').load({}, (er, npm) => { const settings = require('ep_etherpad-lite/node/utils/Settings'); - let dirty = require('../src/node_modules/dirty'); - const ueberDB = require('../src/node_modules/ueberdb2'); - const log4js = require('../src/node_modules/log4js'); + const dirtyDb = require('ep_etherpad-lite/node_modules/dirty'); + const ueberDB = require('ep_etherpad-lite/node_modules/ueberdb2'); + const log4js = require('ep_etherpad-lite/node_modules/log4js'); const dbWrapperSettings = { cache: '0', // The cache slows things down when you're mostly writing. writeInterval: 0, // Write directly to the database, don't buffer }; - const db = new ueberDB.database(settings.dbType, settings.dbSettings, dbWrapperSettings, log4js.getLogger('ueberDB')); - let i = 0; - let length = 0; + const db = new ueberDB.database( // eslint-disable-line new-cap + settings.dbType, + settings.dbSettings, + dbWrapperSettings, + log4js.getLogger('ueberDB')); + await db.init(); - db.init(() => { - console.log('Waiting for dirtyDB to parse its file.'); - dirty = dirty('var/dirty.db').on('load', () => { - dirty.forEach(() => { - length++; - }); - console.log(`Found ${length} records, processing now.`); + console.log('Waiting for dirtyDB to parse its file.'); + const dirty = dirtyDb('var/dirty.db'); + const length = await new Promise((resolve) => { dirty.once('load', resolve); }); - dirty.forEach(async (key, value) => { - const error = await db.set(key, value); - console.log(`Wrote record ${i}`); - i++; - - if (i === length) { - console.log('finished, just clearing up for a bit...'); - setTimeout(() => { - process.exit(0); - }, 5000); - } - }); - console.log('Please wait for all records to flush to database, then kill this process.'); - }); - console.log('done?'); + console.log(`Found ${length} records, processing now.`); + const p = []; + let numWritten = 0; + dirty.forEach((key, value) => { + let bcb, wcb; + p.push(new Promise((resolve, reject) => { + bcb = (err) => { if (err != null) return reject(err); }; + wcb = (err) => { + if (err != null) return reject(err); + if (++numWritten % 100 === 0) console.log(`Wrote record ${numWritten} of ${length}`); + resolve(); + }; + })); + db.set(key, value, bcb, wcb); }); + await Promise.all(p); + console.log(`Wrote all ${numWritten} records`); + + await util.promisify(db.close.bind(db))(); + console.log('Finished.'); }); diff --git a/bin/plugins/checkPlugin.js b/bin/plugins/checkPlugin.js index 570c3bda0..3b8c0b31a 100755 --- a/bin/plugins/checkPlugin.js +++ b/bin/plugins/checkPlugin.js @@ -9,16 +9,17 @@ * node bin/plugins/checkPlugin.js ep_whatever autocommit */ +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + const fs = require('fs'); const childProcess = require('child_process'); // get plugin name & path from user input const pluginName = process.argv[2]; -if (!pluginName) { - console.error('no plugin name specified'); - process.exit(1); -} +if (!pluginName) throw new Error('no plugin name specified'); const pluginPath = `node_modules/${pluginName}`; @@ -107,10 +108,7 @@ fs.readdir(pluginPath, (err, rootFiles) => { files.push(rootFiles[i].toLowerCase()); } - if (files.indexOf('.git') === -1) { - console.error('No .git folder, aborting'); - process.exit(1); - } + if (files.indexOf('.git') === -1) throw new Error('No .git folder, aborting'); prepareRepo(); try { @@ -302,8 +300,10 @@ fs.readdir(pluginPath, (err, rootFiles) => { if (files.indexOf('contributing') === -1 && files.indexOf('contributing.md') === -1) { console.warn('CONTRIBUTING.md file not found, please create'); if (autoFix) { - console.log('Autofixing missing CONTRIBUTING.md file, please edit the CONTRIBUTING.md file further to include plugin specific details.'); - let contributing = fs.readFileSync('bin/plugins/lib/CONTRIBUTING.md', {encoding: 'utf8', flag: 'r'}); + console.log('Autofixing missing CONTRIBUTING.md file, please edit the CONTRIBUTING.md ' + + 'file further to include plugin specific details.'); + let contributing = + fs.readFileSync('bin/plugins/lib/CONTRIBUTING.md', {encoding: 'utf8', flag: 'r'}); contributing = contributing.replace(/\[plugin_name\]/g, pluginName); fs.writeFileSync(`${pluginPath}/CONTRIBUTING.md`, contributing); } @@ -311,7 +311,8 @@ fs.readdir(pluginPath, (err, rootFiles) => { if (files.indexOf('readme') !== -1 && files.indexOf('readme.md') !== -1) { - const readme = fs.readFileSync(`${pluginPath}/${readMeFileName}`, {encoding: 'utf8', flag: 'r'}); + const readme = + fs.readFileSync(`${pluginPath}/${readMeFileName}`, {encoding: 'utf8', flag: 'r'}); if (readme.toLowerCase().indexOf('license') === -1) { console.warn('No license section in README'); if (autoFix) { @@ -335,7 +336,9 @@ fs.readdir(pluginPath, (err, rootFiles) => { travisConfig = travisConfig.replace(/\[plugin_name\]/g, pluginName); if (files.indexOf('.travis.yml') === -1) { - console.warn('.travis.yml file not found, please create. .travis.yml is used for automatically CI testing Etherpad. It is useful to know if your plugin breaks another feature for example.'); + console.warn('.travis.yml file not found, please create. ' + + '.travis.yml is used for automatically CI testing Etherpad. ' + + 'It is useful to know if your plugin breaks another feature for example.'); // TODO: Make it check version of the .travis file to see if it needs an update. if (autoFix) { console.log('Autofixing missing .travis.yml file'); @@ -345,9 +348,11 @@ fs.readdir(pluginPath, (err, rootFiles) => { } if (autoFix) { // checks the file versioning of .travis and updates it to the latest. - const existingConfig = fs.readFileSync(`${pluginPath}/.travis.yml`, {encoding: 'utf8', flag: 'r'}); + const existingConfig = + fs.readFileSync(`${pluginPath}/.travis.yml`, {encoding: 'utf8', flag: 'r'}); const existingConfigLocation = existingConfig.indexOf('##ETHERPAD_TRAVIS_V='); - const existingValue = parseInt(existingConfig.substr(existingConfigLocation + 20, existingConfig.length)); + const existingValue = + parseInt(existingConfig.substr(existingConfigLocation + 20, existingConfig.length)); const newConfigLocation = travisConfig.indexOf('##ETHERPAD_TRAVIS_V='); const newValue = parseInt(travisConfig.substr(newConfigLocation + 20, travisConfig.length)); @@ -362,7 +367,8 @@ fs.readdir(pluginPath, (err, rootFiles) => { } if (files.indexOf('.gitignore') === -1) { - console.warn(".gitignore file not found, please create. .gitignore files are useful to ensure files aren't incorrectly commited to a repository."); + console.warn('.gitignore file not found, please create. .gitignore files are useful to ' + + "ensure files aren't incorrectly commited to a repository."); if (autoFix) { console.log('Autofixing missing .gitignore file'); const gitignore = fs.readFileSync('bin/plugins/lib/gitignore', {encoding: 'utf8', flag: 'r'}); @@ -382,12 +388,15 @@ fs.readdir(pluginPath, (err, rootFiles) => { // if we include templates but don't have translations... if (files.indexOf('templates') !== -1 && files.indexOf('locales') === -1) { - console.warn('Translations not found, please create. Translation files help with Etherpad accessibility.'); + console.warn('Translations not found, please create. ' + + 'Translation files help with Etherpad accessibility.'); } if (files.indexOf('.ep_initialized') !== -1) { - console.warn('.ep_initialized found, please remove. .ep_initialized should never be commited to git and should only exist once the plugin has been executed one time.'); + console.warn( + '.ep_initialized found, please remove. .ep_initialized should never be commited to git ' + + 'and should only exist once the plugin has been executed one time.'); if (autoFix) { console.log('Autofixing incorrectly existing .ep_initialized file'); fs.unlinkSync(`${pluginPath}/.ep_initialized`); @@ -395,7 +404,8 @@ fs.readdir(pluginPath, (err, rootFiles) => { } if (files.indexOf('npm-debug.log') !== -1) { - console.warn('npm-debug.log found, please remove. npm-debug.log should never be commited to your repository.'); + console.warn('npm-debug.log found, please remove. npm-debug.log should never be commited to ' + + 'your repository.'); if (autoFix) { console.log('Autofixing incorrectly existing npm-debug.log file'); fs.unlinkSync(`${pluginPath}/npm-debug.log`); diff --git a/bin/rebuildPad.js b/bin/rebuildPad.js index 12ff21847..0034fe336 100644 --- a/bin/rebuildPad.js +++ b/bin/rebuildPad.js @@ -1,43 +1,39 @@ +'use strict'; + /* This is a repair tool. It rebuilds an old pad at a new pad location up to a known "good" revision. */ -if (process.argv.length != 4 && process.argv.length != 5) { - console.error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]'); - process.exit(1); +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + +if (process.argv.length !== 4 && process.argv.length !== 5) { + throw new Error('Use: node bin/repairPad.js $PADID $REV [$NEWPADID]'); } -const npm = require('../src/node_modules/npm'); -const async = require('../src/node_modules/async'); -const ueberDB = require('../src/node_modules/ueberdb2'); +const async = require('ep_etherpad-lite/node_modules/async'); +const npm = require('ep_etherpad-lite/node_modules/npm'); +const util = require('util'); const padId = process.argv[2]; const newRevHead = process.argv[3]; const newPadId = process.argv[4] || `${padId}-rebuilt`; -let db, oldPad, newPad, settings; -let AuthorManager, ChangeSet, Pad, PadManager; +let db, oldPad, newPad; +let Pad, PadManager; async.series([ - function (callback) { - npm.load({}, (err) => { - if (err) { - console.error(`Could not load NPM: ${err}`); - process.exit(1); - } else { - callback(); - } - }); - }, - function (callback) { + (callback) => npm.load({}, callback), + (callback) => { // Get a handle into the database - db = require('../src/node/db/DB'); + db = require('ep_etherpad-lite/node/db/DB'); db.init(callback); }, - function (callback) { - PadManager = require('../src/node/db/PadManager'); - Pad = require('../src/node/db/Pad').Pad; + (callback) => { + Pad = require('ep_etherpad-lite/node/db/Pad').Pad; + PadManager = require('ep_etherpad-lite/node/db/PadManager'); // Get references to the original pad and to a newly created pad // HACK: This is a standalone script, so we want to write everything // out to the database immediately. The only problem with this is @@ -46,14 +42,10 @@ async.series([ // Validate the newPadId if specified and that a pad with that ID does // not already exist to avoid overwriting it. if (!PadManager.isValidPadId(newPadId)) { - console.error('Cannot create a pad with that id as it is invalid'); - process.exit(1); + throw new Error('Cannot create a pad with that id as it is invalid'); } PadManager.doesPadExists(newPadId, (err, exists) => { - if (exists) { - console.error('Cannot create a pad with that id as it already exists'); - process.exit(1); - } + if (exists) throw new Error('Cannot create a pad with that id as it already exists'); }); PadManager.getPad(padId, (err, pad) => { oldPad = pad; @@ -61,10 +53,10 @@ async.series([ callback(); }); }, - function (callback) { + (callback) => { // Clone all Chat revisions const chatHead = oldPad.chatHead; - for (var i = 0, curHeadNum = 0; i <= chatHead; i++) { + for (let i = 0, curHeadNum = 0; i <= chatHead; i++) { db.db.get(`pad:${padId}:chat:${i}`, (err, chat) => { db.db.set(`pad:${newPadId}:chat:${curHeadNum++}`, chat); console.log(`Created: Chat Revision: pad:${newPadId}:chat:${curHeadNum}`); @@ -72,10 +64,10 @@ async.series([ } callback(); }, - function (callback) { + (callback) => { // Rebuild Pad from revisions up to and including the new revision head - AuthorManager = require('../src/node/db/AuthorManager'); - Changeset = require('ep_etherpad-lite/static/js/Changeset'); + const AuthorManager = require('ep_etherpad-lite/node/db/AuthorManager'); + const Changeset = require('ep_etherpad-lite/static/js/Changeset'); // Author attributes are derived from changesets, but there can also be // non-author attributes with specific mappings that changesets depend on // and, AFAICT, cannot be recreated any other way @@ -83,7 +75,7 @@ async.series([ for (let curRevNum = 0; curRevNum <= newRevHead; curRevNum++) { db.db.get(`pad:${padId}:revs:${curRevNum}`, (err, rev) => { if (rev.meta) { - throw 'The specified revision number could not be found.'; + throw new Error('The specified revision number could not be found.'); } const newRevNum = ++newPad.head; const newRevId = `pad:${newPad.id}:revs:${newRevNum}`; @@ -91,18 +83,17 @@ async.series([ AuthorManager.addPad(rev.meta.author, newPad.id); newPad.atext = Changeset.applyToAText(rev.changeset, newPad.atext, newPad.pool); console.log(`Created: Revision: pad:${newPad.id}:revs:${newRevNum}`); - if (newRevNum == newRevHead) { + if (newRevNum === newRevHead) { callback(); } }); } }, - function (callback) { + (callback) => { // Add saved revisions up to the new revision head console.log(newPad.head); const newSavedRevisions = []; - for (const i in oldPad.savedRevisions) { - savedRev = oldPad.savedRevisions[i]; + for (const savedRev of oldPad.savedRevisions) { if (savedRev.revNum <= newRevHead) { newSavedRevisions.push(savedRev); console.log(`Added: Saved Revision: ${savedRev.revNum}`); @@ -111,16 +102,14 @@ async.series([ newPad.savedRevisions = newSavedRevisions; callback(); }, - function (callback) { + (callback) => { // Save the source pad db.db.set(`pad:${newPadId}`, newPad, (err) => { console.log(`Created: Source Pad: pad:${newPadId}`); - newPad.saveToDatabase().then(() => callback(), callback); + util.callbackify(newPad.saveToDatabase.bind(newPad))(callback); }); }, ], (err) => { - if (err) { throw err; } else { - console.info('finished'); - process.exit(0); - } + if (err) throw err; + console.info('finished'); }); diff --git a/bin/release.js b/bin/release.js index 36b1194e6..f06679e48 100644 --- a/bin/release.js +++ b/bin/release.js @@ -1,8 +1,12 @@ 'use strict'; +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + const fs = require('fs'); -const child_process = require('child_process'); -const semver = require('../src/node_modules/semver'); +const childProcess = require('child_process'); +const semver = require('ep_etherpad-lite/node_modules/semver'); /* @@ -15,7 +19,7 @@ const usage = 'node bin/release.js [patch/minor/major] -- example: "node bin/rel const release = process.argv[2]; -if(!release) { +if (!release) { console.log(usage); throw new Error('No release type included'); } @@ -26,14 +30,14 @@ packageJson = JSON.parse(packageJson); const currentVersion = packageJson.version; const newVersion = semver.inc(currentVersion, release); -if(!newVersion) { +if (!newVersion) { console.log(usage); throw new Error('Unable to generate new version from input'); } const changelogIncludesVersion = changelog.indexOf(newVersion) !== -1; -if(!changelogIncludesVersion) { +if (!changelogIncludesVersion) { throw new Error('No changelog record for ', newVersion, ' - please create changelog record'); } @@ -44,24 +48,27 @@ packageJson.version = newVersion; fs.writeFileSync('src/package.json', JSON.stringify(packageJson, null, 2)); // run npm version `release` where release is patch, minor or major -child_process.execSync('npm install --package-lock-only', {cwd: `src/`}); +childProcess.execSync('npm install --package-lock-only', {cwd: 'src/'}); // run npm install --package-lock-only <-- required??? -child_process.execSync(`git checkout -b release/${newVersion}`); -child_process.execSync(`git add src/package.json`); -child_process.execSync(`git add src/package-lock.json`); -child_process.execSync(`git commit -m 'bump version'`); -child_process.execSync(`git push origin release/${newVersion}`); +childProcess.execSync(`git checkout -b release/${newVersion}`); +childProcess.execSync('git add src/package.json'); +childProcess.execSync('git add src/package-lock.json'); +childProcess.execSync('git commit -m "bump version"'); +childProcess.execSync(`git push origin release/${newVersion}`); -child_process.execSync(`make docs`); -child_process.execSync(`git clone git@github.com:ether/ether.github.com.git`); -child_process.execSync(`cp -R out/doc/ ether.github.com/doc/v${newVersion}`); +childProcess.execSync('make docs'); +childProcess.execSync('git clone git@github.com:ether/ether.github.com.git'); +childProcess.execSync(`cp -R out/doc/ ether.github.com/doc/v${newVersion}`); console.log('Once merged into master please run the following commands'); console.log(`git tag -a ${newVersion} -m ${newVersion} && git push origin master`); console.log(`cd ether.github.com && git add . && git commit -m '${newVersion} docs'`); -console.log(`Build the windows zip`) -console.log(`Visit https://github.com/ether/etherpad-lite/releases/new and create a new release with 'master' as the target and the version is ${newVersion}. Include the windows zip as an assett`) -console.log('Once the new docs are uploaded then modify the download link on etherpad.org and then pull master onto develop'); +console.log('Build the windows zip'); +console.log('Visit https://github.com/ether/etherpad-lite/releases/new and create a new release ' + + `with 'master' as the target and the version is ${newVersion}. Include the windows ` + + 'zip as an asset'); +console.log(`Once the new docs are uploaded then modify the download + link on etherpad.org and then pull master onto develop`); console.log('Finally go public with an announcement via our comms channels :)'); diff --git a/bin/repairPad.js b/bin/repairPad.js index 8408e4b72..e083f30b9 100644 --- a/bin/repairPad.js +++ b/bin/repairPad.js @@ -1,77 +1,59 @@ +'use strict'; + /* * This is a repair tool. It extracts all datas of a pad, removes and inserts them again. */ +// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an +// unhandled rejection into an uncaught exception, which does cause Node.js to exit. +process.on('unhandledRejection', (err) => { throw err; }); + console.warn('WARNING: This script must not be used while etherpad is running!'); -if (process.argv.length != 3) { - console.error('Use: node bin/repairPad.js $PADID'); - process.exit(1); -} +if (process.argv.length !== 3) throw new Error('Use: node bin/repairPad.js $PADID'); // get the padID const padId = process.argv[2]; -const npm = require('../src/node_modules/npm'); -npm.load({}, async (er) => { - if (er) { - console.error(`Could not load NPM: ${er}`); - process.exit(1); +let valueCount = 0; + +const npm = require('ep_etherpad-lite/node_modules/npm'); +npm.load({}, async (err) => { + if (err) throw err; + + // intialize database + require('ep_etherpad-lite/node/utils/Settings'); + const db = require('ep_etherpad-lite/node/db/DB'); + await db.init(); + + // get the pad + const padManager = require('ep_etherpad-lite/node/db/PadManager'); + const pad = await padManager.getPad(padId); + + // accumulate the required keys + const neededDBValues = [`pad:${padId}`]; + + // add all authors + neededDBValues.push(...pad.getAllAuthors().map((author) => `globalAuthor:${author}`)); + + // add all revisions + for (let rev = 0; rev <= pad.head; ++rev) { + neededDBValues.push(`pad:${padId}:revs:${rev}`); } - try { - // intialize database - const settings = require('../src/node/utils/Settings'); - const db = require('../src/node/db/DB'); - await db.init(); - - // get the pad - const padManager = require('../src/node/db/PadManager'); - const pad = await padManager.getPad(padId); - - // accumulate the required keys - const neededDBValues = [`pad:${padId}`]; - - // add all authors - neededDBValues.push(...pad.getAllAuthors().map((author) => 'globalAuthor:')); - - // add all revisions - for (let rev = 0; rev <= pad.head; ++rev) { - neededDBValues.push(`pad:${padId}:revs:${rev}`); - } - - // add all chat values - for (let chat = 0; chat <= pad.chatHead; ++chat) { - neededDBValues.push(`pad:${padId}:chat:${chat}`); - } - - // - // NB: this script doesn't actually does what's documented - // since the `value` fields in the following `.forEach` - // block are just the array index numbers - // - // the script therefore craps out now before it can do - // any damage. - // - // See gitlab issue #3545 - // - console.info('aborting [gitlab #3545]'); - process.exit(1); - - // now fetch and reinsert every key - neededDBValues.forEach((key, value) => { - console.log(`Key: ${key}, value: ${value}`); - db.remove(key); - db.set(key, value); - }); - - console.info('finished'); - process.exit(0); - } catch (er) { - if (er.name === 'apierror') { - console.error(er); - } else { - console.trace(er); - } + // add all chat values + for (let chat = 0; chat <= pad.chatHead; ++chat) { + neededDBValues.push(`pad:${padId}:chat:${chat}`); } + // now fetch and reinsert every key + for (const key of neededDBValues) { + const value = await db.get(key); + // if it isn't a globalAuthor value which we want to ignore.. + // console.log(`Key: ${key}, value: ${JSON.stringify(value)}`); + await db.remove(key); + await db.set(key, value); + valueCount++; + } + + console.info(`Finished: Replaced ${valueCount} values in the database`); });