Pad: Add strict validation checks

src/bin/checkAllPads.js

@@ -10,79 +10,18 @@ process.on('unhandledRejection', (err) => { throw err; });
 if (process.argv.length !== 2) throw new Error('Use: node src/bin/checkAllPads.js');
 
 (async () => {
-  // initialize the database
-  require('../node/utils/Settings');
   const db = require('../node/db/DB');
   await db.init();
-
-  // load modules
-  const Changeset = require('../static/js/Changeset');
   const padManager = require('../node/db/PadManager');
-
+  await Promise.all((await padManager.listAllPads()).padIDs.map(async (padId) => {
-  let revTestedCount = 0;
-
-  // get all pads
-  const res = await padManager.listAllPads();
-  for (const padId of res.padIDs) {
     const pad = await padManager.getPad(padId);
-
+    try {
-    // check if the pad has a pool
+      await pad.check();
-    if (pad.pool == null) {
+    } catch (err) {
-      console.error(`[${pad.id}] Missing attribute pool`);
+      console.error(`Error in pad ${padId}: ${err.stack || err}`);
-      continue;
+      return;
     }
-    // create an array with key kevisions
+    console.log(`Pad ${padId}: OK`);
-    // key revisions always save the full pad atext
+  }));
-    const head = pad.getHeadRevisionNumber();
+  console.log('Finished.');
-    const keyRevisions = [];
-    for (let rev = 0; rev < head; rev += 100) {
-      keyRevisions.push(rev);
-    }
-
-    // run through all key revisions
-    for (const keyRev of keyRevisions) {
-      // create an array of revisions we need till the next keyRevision or the End
-      const revisionsNeeded = [];
-      for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
-        revisionsNeeded.push(rev);
-      }
-
-      // this array will hold all revision changesets
-      const revisions = [];
-
-      // run through all needed revisions and get them from the database
-      for (const revNum of revisionsNeeded) {
-        const revision = await db.get(`pad:${pad.id}:revs:${revNum}`);
-        revisions[revNum] = revision;
-      }
-
-      // check if the revision exists
-      if (revisions[keyRev] == null) {
-        console.error(`[${pad.id}] Missing revision ${keyRev}`);
-        continue;
-      }
-
-      // check if there is a atext in the keyRevisions
-      let {meta: {atext} = {}} = revisions[keyRev];
-      if (atext == null) {
-        console.error(`[${pad.id}] Missing atext in revision ${keyRev}`);
-        continue;
-      }
-
-      const apool = pad.pool;
-      for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
-        try {
-          const cs = revisions[rev].changeset;
-          atext = Changeset.applyToAText(cs, atext, apool);
-          revTestedCount++;
-        } catch (e) {
-          console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`);
-        }
-      }
-    }
-  }
-  if (revTestedCount === 0) {
-    throw new Error('No revisions tested');
-  }
-  console.log(`Finished: Tested ${revTestedCount} revisions`);
 })();

src/bin/checkPad.js

@@ -8,75 +8,13 @@
 process.on('unhandledRejection', (err) => { throw err; });
 
 if (process.argv.length !== 3) throw new Error('Use: node src/bin/checkPad.js $PADID');
-
-// get the padID
 const padId = process.argv[2];
-let checkRevisionCount = 0;
-
 (async () => {
-  // initialize database
-  require('../node/utils/Settings');
   const db = require('../node/db/DB');
   await db.init();
-
-  // load modules
-  const Changeset = require('../static/js/Changeset');
   const padManager = require('../node/db/PadManager');
-
+  if (!await padManager.doesPadExists(padId)) throw new Error('Pad does not exist');
-  const exists = await padManager.doesPadExists(padId);
-  if (!exists) throw new Error('Pad does not exist');
-
-  // get the pad
   const pad = await padManager.getPad(padId);
-
+  await pad.check();
-  // create an array with key revisions
+  console.log('Finished.');
-  // key revisions always save the full pad atext
-  const head = pad.getHeadRevisionNumber();
-  const keyRevisions = [];
-  for (let rev = 0; rev < head; rev += 100) {
-    keyRevisions.push(rev);
-  }
-
-  // run through all key revisions
-  for (let keyRev of keyRevisions) {
-    keyRev = parseInt(keyRev);
-    // create an array of revisions we need till the next keyRevision or the End
-    const revisionsNeeded = [];
-    for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
-      revisionsNeeded.push(rev);
-    }
-
-    // this array will hold all revision changesets
-    const revisions = [];
-
-    // run through all needed revisions and get them from the database
-    for (const revNum of revisionsNeeded) {
-      const revision = await db.get(`pad:${padId}:revs:${revNum}`);
-      revisions[revNum] = revision;
-    }
-
-    // check if the pad has a pool
-    if (pad.pool == null) throw new Error('Attribute pool is missing');
-
-    // check if there is an atext in the keyRevisions
-    let {meta: {atext} = {}} = revisions[keyRev] || {};
-    if (atext == null) {
-      console.error(`No atext in key revision ${keyRev}`);
-      continue;
-    }
-
-    const apool = pad.pool;
-
-    for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
-      checkRevisionCount++;
-      try {
-        const cs = revisions[rev].changeset;
-        atext = Changeset.applyToAText(cs, atext, apool);
-      } catch (e) {
-        console.error(`Bad changeset at revision ${rev} - ${e.message}`);
-        continue;
-      }
-    }
-    console.log(`Finished: Checked ${checkRevisionCount} revisions`);
-  }
 })();

src/bin/checkPadDeltas.js

@@ -1,103 +0,0 @@
-'use strict';
-/*
- * This is a debug tool. It checks all revisions for data corruption
- */
-
-// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
-// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
-process.on('unhandledRejection', (err) => { throw err; });
-
-if (process.argv.length !== 3) throw new Error('Use: node src/bin/checkPadDeltas.js $PADID');
-
-// get the padID
-const padId = process.argv[2];
-
-const expect = require('../tests/frontend/lib/expect');
-const diff = require('diff');
-
-(async () => {
-  // initialize database
-  require('../node/utils/Settings');
-  const db = require('../node/db/DB');
-  await db.init();
-
-  // load modules
-  const Changeset = require('../static/js/Changeset');
-  const padManager = require('../node/db/PadManager');
-
-  const exists = await padManager.doesPadExists(padId);
-  if (!exists) throw new Error('Pad does not exist');
-
-  // get the pad
-  const pad = await padManager.getPad(padId);
-
-  // create an array with key revisions
-  // key revisions always save the full pad atext
-  const head = pad.getHeadRevisionNumber();
-  const keyRevisions = [];
-  for (let i = 0; i < head; i += 100) {
-    keyRevisions.push(i);
-  }
-
-  // create an array with all revisions
-  const revisions = [];
-  for (let i = 0; i <= head; i++) {
-    revisions.push(i);
-  }
-
-  let atext = Changeset.makeAText('\n');
-
-  // run through all revisions
-  for (const revNum of revisions) {
-    // console.log('Fetching', revNum)
-    const revision = await db.get(`pad:${padId}:revs:${revNum}`);
-    // check if there is a atext in the keyRevisions
-    const {meta: {atext: revAtext} = {}} = revision || {};
-    if (~keyRevisions.indexOf(revNum) && revAtext == null) {
-      console.error(`No atext in key revision ${revNum}`);
-      continue;
-    }
-
-    // try glue everything together
-    try {
-      // console.log("check revision ", revNum);
-      const cs = revision.changeset;
-      atext = Changeset.applyToAText(cs, atext, pad.pool);
-    } catch (e) {
-      console.error(`Bad changeset at revision ${revNum} - ${e.message}`);
-      continue;
-    }
-
-    // check things are working properly
-    if (~keyRevisions.indexOf(revNum)) {
-      try {
-        expect(revision.meta.atext.text).to.eql(atext.text);
-        expect(revision.meta.atext.attribs).to.eql(atext.attribs);
-      } catch (e) {
-        console.error(`Atext in key revision ${revNum} doesn't match computed one.`);
-        console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => {
-          if (!op.added && !op.removed) op.value = op.value.length;
-          return op;
-        }));
-        // console.error(e)
-        // console.log('KeyRev. :', revision.meta.atext)
-        // console.log('Computed:', atext)
-        continue;
-      }
-    }
-  }
-
-  // check final text is right...
-  if (pad.atext.text === atext.text) {
-    console.log('ok');
-  } else {
-    console.error('Pad AText doesn\'t match computed one! (Computed ',
-        atext.text.length, ', db', pad.atext.text.length, ')');
-    console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => {
-      if (!op.added && !op.removed) {
-        op.value = op.value.length;
-        return op;
-      }
-    }));
-  }
-})();

src/node/db/Pad.js

@@ -7,6 +7,7 @@
 const Changeset = require('../../static/js/Changeset');
 const ChatMessage = require('../../static/js/ChatMessage');
 const AttributePool = require('../../static/js/AttributePool');
+const assert = require('assert').strict;
 const db = require('./DB');
 const settings = require('../utils/Settings');
 const authorManager = require('./AuthorManager');
@@ -602,3 +603,127 @@ Pad.prototype.addSavedRevision = async function (revNum, savedById, label) {
 Pad.prototype.getSavedRevisions = function () {
   return this.savedRevisions;
 };
+
+/**
+ * Asserts that all pad data is consistent. Throws if inconsistent.
+ */
+Pad.prototype.check = async function () {
+  assert(this.id != null);
+  assert.equal(typeof this.id, 'string');
+
+  const head = this.getHeadRevisionNumber();
+  assert(Number.isInteger(head));
+  assert(head >= -1);
+
+  const savedRevisionsList = this.getSavedRevisionsList();
+  assert(Array.isArray(savedRevisionsList));
+  assert.equal(this.getSavedRevisionsNumber(), savedRevisionsList.length);
+  let prevSavedRev = null;
+  for (const rev of savedRevisionsList) {
+    assert(Number.isInteger(rev));
+    assert(rev >= 0);
+    assert(rev <= head);
+    assert(prevSavedRev == null || rev > prevSavedRev);
+    prevSavedRev = rev;
+  }
+  const savedRevisions = this.getSavedRevisions();
+  assert(Array.isArray(savedRevisions));
+  assert.equal(savedRevisions.length, savedRevisionsList.length);
+  const savedRevisionsIds = new Set();
+  for (const savedRev of savedRevisions) {
+    assert(savedRev != null);
+    assert.equal(typeof savedRev, 'object');
+    assert(savedRevisionsList.includes(savedRev.revNum));
+    assert(savedRev.id != null);
+    assert.equal(typeof savedRev.id, 'string');
+    assert(!savedRevisionsIds.has(savedRev.id));
+    savedRevisionsIds.add(savedRev.id);
+  }
+
+  const pool = this.apool();
+  assert(pool instanceof AttributePool);
+  await pool.check();
+
+  const decodeAttribString = function* (str) {
+    const re = /\*([0-9a-z]+)|./gy;
+    let match;
+    while ((match = re.exec(str)) != null) {
+      const [m, n] = match;
+      if (n == null) throw new Error(`invalid character in attribute string: ${m}`);
+      yield Number.parseInt(n, 36);
+    }
+  };
+
+  const authors = new Set();
+  pool.eachAttrib((k, v) => {
+    if (k === 'author' && v) authors.add(v);
+  });
+  let atext = Changeset.makeAText('\n');
+  let r;
+  try {
+    for (r = 0; r <= head; ++r) {
+      const [changeset, author, timestamp] = await Promise.all([
+        this.getRevisionChangeset(r),
+        this.getRevisionAuthor(r),
+        this.getRevisionDate(r),
+      ]);
+      assert(author != null);
+      assert.equal(typeof author, 'string');
+      if (author) authors.add(author);
+      assert(timestamp != null);
+      assert.equal(typeof timestamp, 'number');
+      assert(timestamp > 0);
+      assert(changeset != null);
+      assert.equal(typeof changeset, 'string');
+      Changeset.checkRep(changeset);
+      const unpacked = Changeset.unpack(changeset);
+      let text = atext.text;
+      const iter = Changeset.opIterator(unpacked.ops);
+      while (iter.hasNext()) {
+        const op = iter.next();
+        if (['=', '-'].includes(op.opcode)) {
+          assert(text.length >= op.chars);
+          const consumed = text.slice(0, op.chars);
+          const nlines = (consumed.match(/\n/g) || []).length;
+          assert.equal(op.lines, nlines);
+          if (op.lines > 0) assert(consumed.endsWith('\n'));
+          text = text.slice(op.chars);
+        }
+        let prevK = null;
+        for (const n of decodeAttribString(op.attribs)) {
+          const attrib = pool.getAttrib(n);
+          assert(attrib != null);
+          const [k] = attrib;
+          assert(prevK == null || prevK < k);
+          prevK = k;
+        }
+      }
+      atext = Changeset.applyToAText(changeset, atext, pool);
+      assert.deepEqual(await this.getInternalRevisionAText(r), atext);
+    }
+  } catch (err) {
+    const pfx = `(pad ${this.id} revision ${r}) `;
+    if (err.stack) err.stack = pfx + err.stack;
+    err.message = pfx + err.message;
+    throw err;
+  }
+  assert.equal(this.text(), atext.text);
+  assert.deepEqual(this.atext, atext);
+  assert.deepEqual(this.getAllAuthors().sort(), [...authors].sort());
+
+  assert(Number.isInteger(this.chatHead));
+  assert(this.chatHead >= -1);
+  let c;
+  try {
+    for (c = 0; c <= this.chatHead; ++c) {
+      const msg = await this.getChatMessage(c);
+      assert(msg != null);
+      assert(msg instanceof ChatMessage);
+    }
+  } catch (err) {
+    const pfx = `(pad ${this.id} chat message ${c}) `;
+    if (err.stack) err.stack = pfx + err.stack;
+    err.message = pfx + err.message;
+    throw err;
+  }
+};

src/static/js/AttributePool.js

@@ -188,6 +188,35 @@ class AttributePool {
     }
     return this;
   }
+
+  /**
+   * Asserts that the data in the pool is consistent. Throws if inconsistent.
+   */
+  check() {
+    if (!Number.isInteger(this.nextNum)) throw new Error('nextNum property is not an integer');
+    if (this.nextNum < 0) throw new Error('nextNum property is negative');
+    for (const prop of ['numToAttrib', 'attribToNum']) {
+      const obj = this[prop];
+      if (obj == null) throw new Error(`${prop} property is null`);
+      if (typeof obj !== 'object') throw new TypeError(`${prop} property is not an object`);
+      const keys = Object.keys(obj);
+      if (keys.length !== this.nextNum) {
+        throw new Error(`${prop} size mismatch (want ${this.nextNum}, got ${keys.length})`);
+      }
+    }
+    for (let i = 0; i < this.nextNum; ++i) {
+      const attr = this.numToAttrib[`${i}`];
+      if (!Array.isArray(attr)) throw new TypeError(`attrib ${i} is not an array`);
+      if (attr.length !== 2) throw new Error(`attrib ${i} is not an array of length 2`);
+      const [k, v] = attr;
+      if (k == null) throw new TypeError(`attrib ${i} key is null`);
+      if (typeof k !== 'string') throw new TypeError(`attrib ${i} key is not a string`);
+      if (v == null) throw new TypeError(`attrib ${i} value is null`);
+      if (typeof v !== 'string') throw new TypeError(`attrib ${i} value is not a string`);
+      const attrStr = String(attr);
+      if (this.attribToNum[attrStr] !== i) throw new Error(`attribToNum for ${attrStr} !== ${i}`);
+    }
+  }
 }
 
 module.exports = AttributePool;