public-offering
/
pad.pub0.org
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2602 from devoidfury/fix/import-large-etherpad 

fix large etherpad import, fixes #2524
pull/2603/head
John McLear 2015-04-11 20:09:01 +01:00
parent 43687f6e8b a896d67e8c
commit 4a70837de9
2 changed files with 21 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/node/handler/ImportHandler.js
View File

@ -113,10 +113,8 @@ exports.doImport = function(req, res, padId)
if(ERR(err, callback)) return callback();
if(result.length > 0){ // This feels hacky and wrong..
importHandledByPlugin = true;
callback();
}else{
callback();
}
callback();
});
},
function(callback) {
@ -145,7 +143,7 @@ exports.doImport = function(req, res, padId)
},
//convert file to html
function(callback) {
if(!importHandledByPlugin || !directDatabaseAccess){
if(!importHandledByPlugin && !directDatabaseAccess){
var fileEnding = path.extname(srcFile).toLowerCase();
var fileIsHTML = (fileEnding === ".html" || fileEnding === ".htm");
var fileIsTXT = (fileEnding === ".txt");
@ -171,28 +169,24 @@ exports.doImport = function(req, res, padId)
},
function(callback) {
if (!abiword){
if(!directDatabaseAccess) {
// Read the file with no encoding for raw buffer access.
fs.readFile(destFile, function(err, buf) {
if (err) throw err;
var isAscii = true;
// Check if there are only ascii chars in the uploaded file
for (var i=0, len=buf.length; i<len; i++) {
if (buf[i] > 240) {
isAscii=false;
break;
}
if (!abiword && !directDatabaseAccess){
// Read the file with no encoding for raw buffer access.
fs.readFile(destFile, function(err, buf) {
if (err) throw err;
var isAscii = true;
// Check if there are only ascii chars in the uploaded file
for (var i=0, len=buf.length; i<len; i++) {
if (buf[i] > 240) {
isAscii=false;
break;
}
if (isAscii) {
callback();
} else {
callback("uploadFailed");
}
});
}else{
callback();
}
}
if (isAscii) {
callback();
} else {
callback("uploadFailed");
}
});
} else {
callback();
}

13
src/node/utils/ImportEtherpad.js
View File

@ -21,20 +21,11 @@ var db = require("../db/DB").db;
exports.setPadRaw = function(padId, records, callback){
records = JSON.parse(records);
// !! HACK !!
// If you have a really large pad it will cause a Maximum Range Stack crash
// This is a temporary patch for that so things are kept stable.
var recordCount = Object.keys(records).length;
if(recordCount >= 50000){
console.warn("Etherpad file is too large to import.. We need to fix this. See https://github.com/ether/etherpad-lite/issues/2524");
return callback("tooLarge", false);
}
async.eachSeries(Object.keys(records), function(key, cb){
var value = records[key]
if(!value){
cb(); // null values are bad.
return setImmediate(cb);
}
// Author data
@ -76,7 +67,7 @@ exports.setPadRaw = function(padId, records, callback){
// Write the value to the server
db.set(newKey, value);
cb();
setImmediate(cb);
}, function(){
callback(null, true);
});