diff --git a/src/bin/checkAllPads.js b/src/bin/checkAllPads.js index d15e5ec5b..d2a5f837c 100644 --- a/src/bin/checkAllPads.js +++ b/src/bin/checkAllPads.js @@ -10,79 +10,18 @@ process.on('unhandledRejection', (err) => { throw err; }); if (process.argv.length !== 2) throw new Error('Use: node src/bin/checkAllPads.js'); (async () => { - // initialize the database - require('../node/utils/Settings'); const db = require('../node/db/DB'); await db.init(); - - // load modules - const Changeset = require('../static/js/Changeset'); const padManager = require('../node/db/PadManager'); - - let revTestedCount = 0; - - // get all pads - const res = await padManager.listAllPads(); - for (const padId of res.padIDs) { + await Promise.all((await padManager.listAllPads()).padIDs.map(async (padId) => { const pad = await padManager.getPad(padId); - - // check if the pad has a pool - if (pad.pool == null) { - console.error(`[${pad.id}] Missing attribute pool`); - continue; + try { + await pad.check(); + } catch (err) { + console.error(`Error in pad ${padId}: ${err.stack || err}`); + return; } - // create an array with key kevisions - // key revisions always save the full pad atext - const head = pad.getHeadRevisionNumber(); - const keyRevisions = []; - for (let rev = 0; rev < head; rev += 100) { - keyRevisions.push(rev); - } - - // run through all key revisions - for (const keyRev of keyRevisions) { - // create an array of revisions we need till the next keyRevision or the End - const revisionsNeeded = []; - for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) { - revisionsNeeded.push(rev); - } - - // this array will hold all revision changesets - const revisions = []; - - // run through all needed revisions and get them from the database - for (const revNum of revisionsNeeded) { - const revision = await db.get(`pad:${pad.id}:revs:${revNum}`); - revisions[revNum] = revision; - } - - // check if the revision exists - if (revisions[keyRev] == null) { - console.error(`[${pad.id}] Missing revision ${keyRev}`); - continue; - } - - // check if there is a atext in the keyRevisions - let {meta: {atext} = {}} = revisions[keyRev]; - if (atext == null) { - console.error(`[${pad.id}] Missing atext in revision ${keyRev}`); - continue; - } - - const apool = pad.pool; - for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) { - try { - const cs = revisions[rev].changeset; - atext = Changeset.applyToAText(cs, atext, apool); - revTestedCount++; - } catch (e) { - console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`); - } - } - } - } - if (revTestedCount === 0) { - throw new Error('No revisions tested'); - } - console.log(`Finished: Tested ${revTestedCount} revisions`); + console.log(`Pad ${padId}: OK`); + })); + console.log('Finished.'); })(); diff --git a/src/bin/checkPad.js b/src/bin/checkPad.js index 5b17fa31a..6aa4b3034 100644 --- a/src/bin/checkPad.js +++ b/src/bin/checkPad.js @@ -8,75 +8,13 @@ process.on('unhandledRejection', (err) => { throw err; }); if (process.argv.length !== 3) throw new Error('Use: node src/bin/checkPad.js $PADID'); - -// get the padID const padId = process.argv[2]; -let checkRevisionCount = 0; - (async () => { - // initialize database - require('../node/utils/Settings'); const db = require('../node/db/DB'); await db.init(); - - // load modules - const Changeset = require('../static/js/Changeset'); const padManager = require('../node/db/PadManager'); - - const exists = await padManager.doesPadExists(padId); - if (!exists) throw new Error('Pad does not exist'); - - // get the pad + if (!await padManager.doesPadExists(padId)) throw new Error('Pad does not exist'); const pad = await padManager.getPad(padId); - - // create an array with key revisions - // key revisions always save the full pad atext - const head = pad.getHeadRevisionNumber(); - const keyRevisions = []; - for (let rev = 0; rev < head; rev += 100) { - keyRevisions.push(rev); - } - - // run through all key revisions - for (let keyRev of keyRevisions) { - keyRev = parseInt(keyRev); - // create an array of revisions we need till the next keyRevision or the End - const revisionsNeeded = []; - for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) { - revisionsNeeded.push(rev); - } - - // this array will hold all revision changesets - const revisions = []; - - // run through all needed revisions and get them from the database - for (const revNum of revisionsNeeded) { - const revision = await db.get(`pad:${padId}:revs:${revNum}`); - revisions[revNum] = revision; - } - - // check if the pad has a pool - if (pad.pool == null) throw new Error('Attribute pool is missing'); - - // check if there is an atext in the keyRevisions - let {meta: {atext} = {}} = revisions[keyRev] || {}; - if (atext == null) { - console.error(`No atext in key revision ${keyRev}`); - continue; - } - - const apool = pad.pool; - - for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) { - checkRevisionCount++; - try { - const cs = revisions[rev].changeset; - atext = Changeset.applyToAText(cs, atext, apool); - } catch (e) { - console.error(`Bad changeset at revision ${rev} - ${e.message}`); - continue; - } - } - console.log(`Finished: Checked ${checkRevisionCount} revisions`); - } + await pad.check(); + console.log('Finished.'); })(); diff --git a/src/bin/checkPadDeltas.js b/src/bin/checkPadDeltas.js deleted file mode 100644 index 852c68332..000000000 --- a/src/bin/checkPadDeltas.js +++ /dev/null @@ -1,103 +0,0 @@ -'use strict'; -/* - * This is a debug tool. It checks all revisions for data corruption - */ - -// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an -// unhandled rejection into an uncaught exception, which does cause Node.js to exit. -process.on('unhandledRejection', (err) => { throw err; }); - -if (process.argv.length !== 3) throw new Error('Use: node src/bin/checkPadDeltas.js $PADID'); - -// get the padID -const padId = process.argv[2]; - -const expect = require('../tests/frontend/lib/expect'); -const diff = require('diff'); - -(async () => { - // initialize database - require('../node/utils/Settings'); - const db = require('../node/db/DB'); - await db.init(); - - // load modules - const Changeset = require('../static/js/Changeset'); - const padManager = require('../node/db/PadManager'); - - const exists = await padManager.doesPadExists(padId); - if (!exists) throw new Error('Pad does not exist'); - - // get the pad - const pad = await padManager.getPad(padId); - - // create an array with key revisions - // key revisions always save the full pad atext - const head = pad.getHeadRevisionNumber(); - const keyRevisions = []; - for (let i = 0; i < head; i += 100) { - keyRevisions.push(i); - } - - // create an array with all revisions - const revisions = []; - for (let i = 0; i <= head; i++) { - revisions.push(i); - } - - let atext = Changeset.makeAText('\n'); - - // run through all revisions - for (const revNum of revisions) { - // console.log('Fetching', revNum) - const revision = await db.get(`pad:${padId}:revs:${revNum}`); - // check if there is a atext in the keyRevisions - const {meta: {atext: revAtext} = {}} = revision || {}; - if (~keyRevisions.indexOf(revNum) && revAtext == null) { - console.error(`No atext in key revision ${revNum}`); - continue; - } - - // try glue everything together - try { - // console.log("check revision ", revNum); - const cs = revision.changeset; - atext = Changeset.applyToAText(cs, atext, pad.pool); - } catch (e) { - console.error(`Bad changeset at revision ${revNum} - ${e.message}`); - continue; - } - - // check things are working properly - if (~keyRevisions.indexOf(revNum)) { - try { - expect(revision.meta.atext.text).to.eql(atext.text); - expect(revision.meta.atext.attribs).to.eql(atext.attribs); - } catch (e) { - console.error(`Atext in key revision ${revNum} doesn't match computed one.`); - console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => { - if (!op.added && !op.removed) op.value = op.value.length; - return op; - })); - // console.error(e) - // console.log('KeyRev. :', revision.meta.atext) - // console.log('Computed:', atext) - continue; - } - } - } - - // check final text is right... - if (pad.atext.text === atext.text) { - console.log('ok'); - } else { - console.error('Pad AText doesn\'t match computed one! (Computed ', - atext.text.length, ', db', pad.atext.text.length, ')'); - console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => { - if (!op.added && !op.removed) { - op.value = op.value.length; - return op; - } - })); - } -})(); diff --git a/src/node/db/Pad.js b/src/node/db/Pad.js index 677e9c014..08e980958 100644 --- a/src/node/db/Pad.js +++ b/src/node/db/Pad.js @@ -7,6 +7,7 @@ const Changeset = require('../../static/js/Changeset'); const ChatMessage = require('../../static/js/ChatMessage'); const AttributePool = require('../../static/js/AttributePool'); +const assert = require('assert').strict; const db = require('./DB'); const settings = require('../utils/Settings'); const authorManager = require('./AuthorManager'); @@ -602,3 +603,127 @@ Pad.prototype.addSavedRevision = async function (revNum, savedById, label) { Pad.prototype.getSavedRevisions = function () { return this.savedRevisions; }; + +/** + * Asserts that all pad data is consistent. Throws if inconsistent. + */ +Pad.prototype.check = async function () { + assert(this.id != null); + assert.equal(typeof this.id, 'string'); + + const head = this.getHeadRevisionNumber(); + assert(Number.isInteger(head)); + assert(head >= -1); + + const savedRevisionsList = this.getSavedRevisionsList(); + assert(Array.isArray(savedRevisionsList)); + assert.equal(this.getSavedRevisionsNumber(), savedRevisionsList.length); + let prevSavedRev = null; + for (const rev of savedRevisionsList) { + assert(Number.isInteger(rev)); + assert(rev >= 0); + assert(rev <= head); + assert(prevSavedRev == null || rev > prevSavedRev); + prevSavedRev = rev; + } + const savedRevisions = this.getSavedRevisions(); + assert(Array.isArray(savedRevisions)); + assert.equal(savedRevisions.length, savedRevisionsList.length); + const savedRevisionsIds = new Set(); + for (const savedRev of savedRevisions) { + assert(savedRev != null); + assert.equal(typeof savedRev, 'object'); + assert(savedRevisionsList.includes(savedRev.revNum)); + assert(savedRev.id != null); + assert.equal(typeof savedRev.id, 'string'); + assert(!savedRevisionsIds.has(savedRev.id)); + savedRevisionsIds.add(savedRev.id); + } + + const pool = this.apool(); + assert(pool instanceof AttributePool); + await pool.check(); + + const decodeAttribString = function* (str) { + const re = /\*([0-9a-z]+)|./gy; + let match; + while ((match = re.exec(str)) != null) { + const [m, n] = match; + if (n == null) throw new Error(`invalid character in attribute string: ${m}`); + yield Number.parseInt(n, 36); + } + }; + + const authors = new Set(); + pool.eachAttrib((k, v) => { + if (k === 'author' && v) authors.add(v); + }); + let atext = Changeset.makeAText('\n'); + let r; + try { + for (r = 0; r <= head; ++r) { + const [changeset, author, timestamp] = await Promise.all([ + this.getRevisionChangeset(r), + this.getRevisionAuthor(r), + this.getRevisionDate(r), + ]); + assert(author != null); + assert.equal(typeof author, 'string'); + if (author) authors.add(author); + assert(timestamp != null); + assert.equal(typeof timestamp, 'number'); + assert(timestamp > 0); + assert(changeset != null); + assert.equal(typeof changeset, 'string'); + Changeset.checkRep(changeset); + const unpacked = Changeset.unpack(changeset); + let text = atext.text; + const iter = Changeset.opIterator(unpacked.ops); + while (iter.hasNext()) { + const op = iter.next(); + if (['=', '-'].includes(op.opcode)) { + assert(text.length >= op.chars); + const consumed = text.slice(0, op.chars); + const nlines = (consumed.match(/\n/g) || []).length; + assert.equal(op.lines, nlines); + if (op.lines > 0) assert(consumed.endsWith('\n')); + text = text.slice(op.chars); + } + let prevK = null; + for (const n of decodeAttribString(op.attribs)) { + const attrib = pool.getAttrib(n); + assert(attrib != null); + const [k] = attrib; + assert(prevK == null || prevK < k); + prevK = k; + } + } + atext = Changeset.applyToAText(changeset, atext, pool); + assert.deepEqual(await this.getInternalRevisionAText(r), atext); + } + } catch (err) { + const pfx = `(pad ${this.id} revision ${r}) `; + if (err.stack) err.stack = pfx + err.stack; + err.message = pfx + err.message; + throw err; + } + assert.equal(this.text(), atext.text); + assert.deepEqual(this.atext, atext); + assert.deepEqual(this.getAllAuthors().sort(), [...authors].sort()); + + assert(Number.isInteger(this.chatHead)); + assert(this.chatHead >= -1); + let c; + try { + for (c = 0; c <= this.chatHead; ++c) { + const msg = await this.getChatMessage(c); + assert(msg != null); + assert(msg instanceof ChatMessage); + } + } catch (err) { + const pfx = `(pad ${this.id} chat message ${c}) `; + if (err.stack) err.stack = pfx + err.stack; + err.message = pfx + err.message; + throw err; + } +}; diff --git a/src/static/js/AttributePool.js b/src/static/js/AttributePool.js index d8e587654..3479e03d1 100644 --- a/src/static/js/AttributePool.js +++ b/src/static/js/AttributePool.js @@ -188,6 +188,35 @@ class AttributePool { } return this; } + + /** + * Asserts that the data in the pool is consistent. Throws if inconsistent. + */ + check() { + if (!Number.isInteger(this.nextNum)) throw new Error('nextNum property is not an integer'); + if (this.nextNum < 0) throw new Error('nextNum property is negative'); + for (const prop of ['numToAttrib', 'attribToNum']) { + const obj = this[prop]; + if (obj == null) throw new Error(`${prop} property is null`); + if (typeof obj !== 'object') throw new TypeError(`${prop} property is not an object`); + const keys = Object.keys(obj); + if (keys.length !== this.nextNum) { + throw new Error(`${prop} size mismatch (want ${this.nextNum}, got ${keys.length})`); + } + } + for (let i = 0; i < this.nextNum; ++i) { + const attr = this.numToAttrib[`${i}`]; + if (!Array.isArray(attr)) throw new TypeError(`attrib ${i} is not an array`); + if (attr.length !== 2) throw new Error(`attrib ${i} is not an array of length 2`); + const [k, v] = attr; + if (k == null) throw new TypeError(`attrib ${i} key is null`); + if (typeof k !== 'string') throw new TypeError(`attrib ${i} key is not a string`); + if (v == null) throw new TypeError(`attrib ${i} value is null`); + if (typeof v !== 'string') throw new TypeError(`attrib ${i} value is not a string`); + const attrStr = String(attr); + if (this.attribToNum[attrStr] !== i) throw new Error(`attribToNum for ${attrStr} !== ${i}`); + } + } } module.exports = AttributePool;