public-offering
/
pad.pub0.org
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bin/importSqlFile.js: Read the file one line at a time 

This avoids running out of memory if the file is large.
pull/5087/head
Richard Hansen 2021-06-17 19:41:44 -04:00
parent 485538bd79
commit 4b3e47bd23
1 changed files with 7 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/bin/importSqlFile.js
View File

@ -48,6 +48,7 @@ const unescape = (val) => {
(async () => { (async () => {
const fs = require('fs'); const fs = require('fs');
const log4js = require('log4js'); const log4js = require('log4js');
const readline = require('readline');
const settings = require('../node/utils/Settings'); const settings = require('../node/utils/Settings');
const ueberDB = require('ueberdb2'); const ueberDB = require('ueberdb2');
@ -71,14 +72,12 @@ const unescape = (val) => {
await util.promisify(db.init.bind(db))(); await util.promisify(db.init.bind(db))();
log('done'); log('done');
log('open output file...'); log(`Opening ${sqlFile}...`);
const lines = fs.readFileSync(sqlFile, 'utf8').split('\n'); const stream = fs.createReadStream(sqlFile, {encoding: 'utf8'});
const count = lines.length; log(`Reading ${sqlFile}...`);
let keyNo = 0; let keyNo = 0;
for await (const l of readline.createInterface({input: stream, crlfDelay: Infinity})) {
process.stdout.write(`Start importing ${count} keys...\n`);
lines.forEach((l) => {
if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') { if (l.substr(0, 27) === 'REPLACE INTO store VALUES (') {
const pos = l.indexOf("', '"); const pos = l.indexOf("', '");
const key = l.substr(28, pos - 28); const key = l.substr(28, pos - 28);
@ -88,11 +87,9 @@ const unescape = (val) => {
console.log(`unval: ${unescape(value)}`); console.log(`unval: ${unescape(value)}`);
db.set(key, unescape(value), null); db.set(key, unescape(value), null);
keyNo++; keyNo++;
if (keyNo % 1000 === 0) { if (keyNo % 1000 === 0) log(` ${keyNo}`);
process.stdout.write(` ${keyNo}/${count}\n`);
} }
} }
});
process.stdout.write('\n'); process.stdout.write('\n');
process.stdout.write('done. waiting for db to finish transaction. ' + process.stdout.write('done. waiting for db to finish transaction. ' +
'depended on dbms this may take some time..\n'); 'depended on dbms this may take some time..\n');