Pad: Add strict validation checks
Richard Hansen
parent
2608a81654
commit
7c870f8a58
|
|
@ -10,79 +10,18 @@ process.on('unhandledRejection', (err) => { throw err; });
|
|||
if (process.argv.length !== 2) throw new Error('Use: node src/bin/checkAllPads.js');
|
||||
|
||||
(async () => {
|
||||
// initialize the database
|
||||
require('../node/utils/Settings');
|
||||
const db = require('../node/db/DB');
|
||||
await db.init();
|
||||
|
||||
// load modules
|
||||
const Changeset = require('../static/js/Changeset');
|
||||
const padManager = require('../node/db/PadManager');
|
||||
|
||||
let revTestedCount = 0;
|
||||
|
||||
// get all pads
|
||||
const res = await padManager.listAllPads();
|
||||
for (const padId of res.padIDs) {
|
||||
await Promise.all((await padManager.listAllPads()).padIDs.map(async (padId) => {
|
||||
const pad = await padManager.getPad(padId);
|
||||
|
||||
// check if the pad has a pool
|
||||
if (pad.pool == null) {
|
||||
console.error(`[${pad.id}] Missing attribute pool`);
|
||||
continue;
|
||||
}
|
||||
// create an array with key kevisions
|
||||
// key revisions always save the full pad atext
|
||||
const head = pad.getHeadRevisionNumber();
|
||||
const keyRevisions = [];
|
||||
for (let rev = 0; rev < head; rev += 100) {
|
||||
keyRevisions.push(rev);
|
||||
}
|
||||
|
||||
// run through all key revisions
|
||||
for (const keyRev of keyRevisions) {
|
||||
// create an array of revisions we need till the next keyRevision or the End
|
||||
const revisionsNeeded = [];
|
||||
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
|
||||
revisionsNeeded.push(rev);
|
||||
}
|
||||
|
||||
// this array will hold all revision changesets
|
||||
const revisions = [];
|
||||
|
||||
// run through all needed revisions and get them from the database
|
||||
for (const revNum of revisionsNeeded) {
|
||||
const revision = await db.get(`pad:${pad.id}:revs:${revNum}`);
|
||||
revisions[revNum] = revision;
|
||||
}
|
||||
|
||||
// check if the revision exists
|
||||
if (revisions[keyRev] == null) {
|
||||
console.error(`[${pad.id}] Missing revision ${keyRev}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// check if there is a atext in the keyRevisions
|
||||
let {meta: {atext} = {}} = revisions[keyRev];
|
||||
if (atext == null) {
|
||||
console.error(`[${pad.id}] Missing atext in revision ${keyRev}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const apool = pad.pool;
|
||||
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
|
||||
try {
|
||||
const cs = revisions[rev].changeset;
|
||||
atext = Changeset.applyToAText(cs, atext, apool);
|
||||
revTestedCount++;
|
||||
} catch (e) {
|
||||
console.error(`[${pad.id}] Bad changeset at revision ${rev} - ${e.message}`);
|
||||
await pad.check();
|
||||
} catch (err) {
|
||||
console.error(`Error in pad ${padId}: ${err.stack || err}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (revTestedCount === 0) {
|
||||
throw new Error('No revisions tested');
|
||||
}
|
||||
console.log(`Finished: Tested ${revTestedCount} revisions`);
|
||||
console.log(`Pad ${padId}: OK`);
|
||||
}));
|
||||
console.log('Finished.');
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -8,75 +8,13 @@
|
|||
process.on('unhandledRejection', (err) => { throw err; });
|
||||
|
||||
if (process.argv.length !== 3) throw new Error('Use: node src/bin/checkPad.js $PADID');
|
||||
|
||||
// get the padID
|
||||
const padId = process.argv[2];
|
||||
let checkRevisionCount = 0;
|
||||
|
||||
(async () => {
|
||||
// initialize database
|
||||
require('../node/utils/Settings');
|
||||
const db = require('../node/db/DB');
|
||||
await db.init();
|
||||
|
||||
// load modules
|
||||
const Changeset = require('../static/js/Changeset');
|
||||
const padManager = require('../node/db/PadManager');
|
||||
|
||||
const exists = await padManager.doesPadExists(padId);
|
||||
if (!exists) throw new Error('Pad does not exist');
|
||||
|
||||
// get the pad
|
||||
if (!await padManager.doesPadExists(padId)) throw new Error('Pad does not exist');
|
||||
const pad = await padManager.getPad(padId);
|
||||
|
||||
// create an array with key revisions
|
||||
// key revisions always save the full pad atext
|
||||
const head = pad.getHeadRevisionNumber();
|
||||
const keyRevisions = [];
|
||||
for (let rev = 0; rev < head; rev += 100) {
|
||||
keyRevisions.push(rev);
|
||||
}
|
||||
|
||||
// run through all key revisions
|
||||
for (let keyRev of keyRevisions) {
|
||||
keyRev = parseInt(keyRev);
|
||||
// create an array of revisions we need till the next keyRevision or the End
|
||||
const revisionsNeeded = [];
|
||||
for (let rev = keyRev; rev <= keyRev + 100 && rev <= head; rev++) {
|
||||
revisionsNeeded.push(rev);
|
||||
}
|
||||
|
||||
// this array will hold all revision changesets
|
||||
const revisions = [];
|
||||
|
||||
// run through all needed revisions and get them from the database
|
||||
for (const revNum of revisionsNeeded) {
|
||||
const revision = await db.get(`pad:${padId}:revs:${revNum}`);
|
||||
revisions[revNum] = revision;
|
||||
}
|
||||
|
||||
// check if the pad has a pool
|
||||
if (pad.pool == null) throw new Error('Attribute pool is missing');
|
||||
|
||||
// check if there is an atext in the keyRevisions
|
||||
let {meta: {atext} = {}} = revisions[keyRev] || {};
|
||||
if (atext == null) {
|
||||
console.error(`No atext in key revision ${keyRev}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const apool = pad.pool;
|
||||
|
||||
for (let rev = keyRev + 1; rev <= keyRev + 100 && rev <= head; rev++) {
|
||||
checkRevisionCount++;
|
||||
try {
|
||||
const cs = revisions[rev].changeset;
|
||||
atext = Changeset.applyToAText(cs, atext, apool);
|
||||
} catch (e) {
|
||||
console.error(`Bad changeset at revision ${rev} - ${e.message}`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
console.log(`Finished: Checked ${checkRevisionCount} revisions`);
|
||||
}
|
||||
await pad.check();
|
||||
console.log('Finished.');
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -1,103 +0,0 @@
|
|||
'use strict';
|
||||
/*
|
||||
* This is a debug tool. It checks all revisions for data corruption
|
||||
*/
|
||||
|
||||
// As of v14, Node.js does not exit when there is an unhandled Promise rejection. Convert an
|
||||
// unhandled rejection into an uncaught exception, which does cause Node.js to exit.
|
||||
process.on('unhandledRejection', (err) => { throw err; });
|
||||
|
||||
if (process.argv.length !== 3) throw new Error('Use: node src/bin/checkPadDeltas.js $PADID');
|
||||
|
||||
// get the padID
|
||||
const padId = process.argv[2];
|
||||
|
||||
const expect = require('../tests/frontend/lib/expect');
|
||||
const diff = require('diff');
|
||||
|
||||
(async () => {
|
||||
// initialize database
|
||||
require('../node/utils/Settings');
|
||||
const db = require('../node/db/DB');
|
||||
await db.init();
|
||||
|
||||
// load modules
|
||||
const Changeset = require('../static/js/Changeset');
|
||||
const padManager = require('../node/db/PadManager');
|
||||
|
||||
const exists = await padManager.doesPadExists(padId);
|
||||
if (!exists) throw new Error('Pad does not exist');
|
||||
|
||||
// get the pad
|
||||
const pad = await padManager.getPad(padId);
|
||||
|
||||
// create an array with key revisions
|
||||
// key revisions always save the full pad atext
|
||||
const head = pad.getHeadRevisionNumber();
|
||||
const keyRevisions = [];
|
||||
for (let i = 0; i < head; i += 100) {
|
||||
keyRevisions.push(i);
|
||||
}
|
||||
|
||||
// create an array with all revisions
|
||||
const revisions = [];
|
||||
for (let i = 0; i <= head; i++) {
|
||||
revisions.push(i);
|
||||
}
|
||||
|
||||
let atext = Changeset.makeAText('\n');
|
||||
|
||||
// run through all revisions
|
||||
for (const revNum of revisions) {
|
||||
// console.log('Fetching', revNum)
|
||||
const revision = await db.get(`pad:${padId}:revs:${revNum}`);
|
||||
// check if there is a atext in the keyRevisions
|
||||
const {meta: {atext: revAtext} = {}} = revision || {};
|
||||
if (~keyRevisions.indexOf(revNum) && revAtext == null) {
|
||||
console.error(`No atext in key revision ${revNum}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// try glue everything together
|
||||
try {
|
||||
// console.log("check revision ", revNum);
|
||||
const cs = revision.changeset;
|
||||
atext = Changeset.applyToAText(cs, atext, pad.pool);
|
||||
} catch (e) {
|
||||
console.error(`Bad changeset at revision ${revNum} - ${e.message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// check things are working properly
|
||||
if (~keyRevisions.indexOf(revNum)) {
|
||||
try {
|
||||
expect(revision.meta.atext.text).to.eql(atext.text);
|
||||
expect(revision.meta.atext.attribs).to.eql(atext.attribs);
|
||||
} catch (e) {
|
||||
console.error(`Atext in key revision ${revNum} doesn't match computed one.`);
|
||||
console.log(diff.diffChars(atext.text, revision.meta.atext.text).map((op) => {
|
||||
if (!op.added && !op.removed) op.value = op.value.length;
|
||||
return op;
|
||||
}));
|
||||
// console.error(e)
|
||||
// console.log('KeyRev. :', revision.meta.atext)
|
||||
// console.log('Computed:', atext)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check final text is right...
|
||||
if (pad.atext.text === atext.text) {
|
||||
console.log('ok');
|
||||
} else {
|
||||
console.error('Pad AText doesn\'t match computed one! (Computed ',
|
||||
atext.text.length, ', db', pad.atext.text.length, ')');
|
||||
console.log(diff.diffChars(atext.text, pad.atext.text).map((op) => {
|
||||
if (!op.added && !op.removed) {
|
||||
op.value = op.value.length;
|
||||
return op;
|
||||
}
|
||||
}));
|
||||
}
|
||||
})();
|
||||
|
|
@ -7,6 +7,7 @@
|
|||
const Changeset = require('../../static/js/Changeset');
|
||||
const ChatMessage = require('../../static/js/ChatMessage');
|
||||
const AttributePool = require('../../static/js/AttributePool');
|
||||
const assert = require('assert').strict;
|
||||
const db = require('./DB');
|
||||
const settings = require('../utils/Settings');
|
||||
const authorManager = require('./AuthorManager');
|
||||
|
|
@ -602,3 +603,127 @@ Pad.prototype.addSavedRevision = async function (revNum, savedById, label) {
|
|||
Pad.prototype.getSavedRevisions = function () {
|
||||
return this.savedRevisions;
|
||||
};
|
||||
|
||||
/**
|
||||
* Asserts that all pad data is consistent. Throws if inconsistent.
|
||||
*/
|
||||
Pad.prototype.check = async function () {
|
||||
assert(this.id != null);
|
||||
assert.equal(typeof this.id, 'string');
|
||||
|
||||
const head = this.getHeadRevisionNumber();
|
||||
assert(Number.isInteger(head));
|
||||
assert(head >= -1);
|
||||
|
||||
const savedRevisionsList = this.getSavedRevisionsList();
|
||||
assert(Array.isArray(savedRevisionsList));
|
||||
assert.equal(this.getSavedRevisionsNumber(), savedRevisionsList.length);
|
||||
let prevSavedRev = null;
|
||||
for (const rev of savedRevisionsList) {
|
||||
assert(Number.isInteger(rev));
|
||||
assert(rev >= 0);
|
||||
assert(rev <= head);
|
||||
assert(prevSavedRev == null || rev > prevSavedRev);
|
||||
prevSavedRev = rev;
|
||||
}
|
||||
const savedRevisions = this.getSavedRevisions();
|
||||
assert(Array.isArray(savedRevisions));
|
||||
assert.equal(savedRevisions.length, savedRevisionsList.length);
|
||||
const savedRevisionsIds = new Set();
|
||||
for (const savedRev of savedRevisions) {
|
||||
assert(savedRev != null);
|
||||
assert.equal(typeof savedRev, 'object');
|
||||
assert(savedRevisionsList.includes(savedRev.revNum));
|
||||
assert(savedRev.id != null);
|
||||
assert.equal(typeof savedRev.id, 'string');
|
||||
assert(!savedRevisionsIds.has(savedRev.id));
|
||||
savedRevisionsIds.add(savedRev.id);
|
||||
}
|
||||
|
||||
const pool = this.apool();
|
||||
assert(pool instanceof AttributePool);
|
||||
await pool.check();
|
||||
|
||||
const decodeAttribString = function* (str) {
|
||||
const re = /\*([0-9a-z]+)|./gy;
|
||||
let match;
|
||||
while ((match = re.exec(str)) != null) {
|
||||
const [m, n] = match;
|
||||
if (n == null) throw new Error(`invalid character in attribute string: ${m}`);
|
||||
yield Number.parseInt(n, 36);
|
||||
}
|
||||
};
|
||||
|
||||
const authors = new Set();
|
||||
pool.eachAttrib((k, v) => {
|
||||
if (k === 'author' && v) authors.add(v);
|
||||
});
|
||||
let atext = Changeset.makeAText('\n');
|
||||
let r;
|
||||
try {
|
||||
for (r = 0; r <= head; ++r) {
|
||||
const [changeset, author, timestamp] = await Promise.all([
|
||||
this.getRevisionChangeset(r),
|
||||
this.getRevisionAuthor(r),
|
||||
this.getRevisionDate(r),
|
||||
]);
|
||||
assert(author != null);
|
||||
assert.equal(typeof author, 'string');
|
||||
if (author) authors.add(author);
|
||||
assert(timestamp != null);
|
||||
assert.equal(typeof timestamp, 'number');
|
||||
assert(timestamp > 0);
|
||||
assert(changeset != null);
|
||||
assert.equal(typeof changeset, 'string');
|
||||
Changeset.checkRep(changeset);
|
||||
const unpacked = Changeset.unpack(changeset);
|
||||
let text = atext.text;
|
||||
const iter = Changeset.opIterator(unpacked.ops);
|
||||
while (iter.hasNext()) {
|
||||
const op = iter.next();
|
||||
if (['=', '-'].includes(op.opcode)) {
|
||||
assert(text.length >= op.chars);
|
||||
const consumed = text.slice(0, op.chars);
|
||||
const nlines = (consumed.match(/\n/g) || []).length;
|
||||
assert.equal(op.lines, nlines);
|
||||
if (op.lines > 0) assert(consumed.endsWith('\n'));
|
||||
text = text.slice(op.chars);
|
||||
}
|
||||
let prevK = null;
|
||||
for (const n of decodeAttribString(op.attribs)) {
|
||||
const attrib = pool.getAttrib(n);
|
||||
assert(attrib != null);
|
||||
const [k] = attrib;
|
||||
assert(prevK == null || prevK < k);
|
||||
prevK = k;
|
||||
}
|
||||
}
|
||||
atext = Changeset.applyToAText(changeset, atext, pool);
|
||||
assert.deepEqual(await this.getInternalRevisionAText(r), atext);
|
||||
}
|
||||
} catch (err) {
|
||||
const pfx = `(pad ${this.id} revision ${r}) `;
|
||||
if (err.stack) err.stack = pfx + err.stack;
|
||||
err.message = pfx + err.message;
|
||||
throw err;
|
||||
}
|
||||
assert.equal(this.text(), atext.text);
|
||||
assert.deepEqual(this.atext, atext);
|
||||
assert.deepEqual(this.getAllAuthors().sort(), [...authors].sort());
|
||||
|
||||
assert(Number.isInteger(this.chatHead));
|
||||
assert(this.chatHead >= -1);
|
||||
let c;
|
||||
try {
|
||||
for (c = 0; c <= this.chatHead; ++c) {
|
||||
const msg = await this.getChatMessage(c);
|
||||
assert(msg != null);
|
||||
assert(msg instanceof ChatMessage);
|
||||
}
|
||||
} catch (err) {
|
||||
const pfx = `(pad ${this.id} chat message ${c}) `;
|
||||
if (err.stack) err.stack = pfx + err.stack;
|
||||
err.message = pfx + err.message;
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -188,6 +188,35 @@ class AttributePool {
|
|||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that the data in the pool is consistent. Throws if inconsistent.
|
||||
*/
|
||||
check() {
|
||||
if (!Number.isInteger(this.nextNum)) throw new Error('nextNum property is not an integer');
|
||||
if (this.nextNum < 0) throw new Error('nextNum property is negative');
|
||||
for (const prop of ['numToAttrib', 'attribToNum']) {
|
||||
const obj = this[prop];
|
||||
if (obj == null) throw new Error(`${prop} property is null`);
|
||||
if (typeof obj !== 'object') throw new TypeError(`${prop} property is not an object`);
|
||||
const keys = Object.keys(obj);
|
||||
if (keys.length !== this.nextNum) {
|
||||
throw new Error(`${prop} size mismatch (want ${this.nextNum}, got ${keys.length})`);
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < this.nextNum; ++i) {
|
||||
const attr = this.numToAttrib[`${i}`];
|
||||
if (!Array.isArray(attr)) throw new TypeError(`attrib ${i} is not an array`);
|
||||
if (attr.length !== 2) throw new Error(`attrib ${i} is not an array of length 2`);
|
||||
const [k, v] = attr;
|
||||
if (k == null) throw new TypeError(`attrib ${i} key is null`);
|
||||
if (typeof k !== 'string') throw new TypeError(`attrib ${i} key is not a string`);
|
||||
if (v == null) throw new TypeError(`attrib ${i} value is null`);
|
||||
if (typeof v !== 'string') throw new TypeError(`attrib ${i} value is not a string`);
|
||||
const attrStr = String(attr);
|
||||
if (this.attribToNum[attrStr] !== i) throw new Error(`attribToNum for ${attrStr} !== ${i}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = AttributePool;
|
||||
|
|
|
|||
Loading…
Reference in New Issue