From 931dec99d2d84e07f955a8d4b9c94b5332d2f226 Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Fri, 17 Jun 2016 22:34:46 +0200 Subject: [PATCH 1/5] save data in temporary file instead of memory since snoop crashes with OOM ~150 times a day for me right I am writing things to disk for now. --- app.js | 49 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/app.js b/app.js index 97451b72..b84dfa21 100644 --- a/app.js +++ b/app.js @@ -76,8 +76,14 @@ function generateFeatures(url, client, clientid) { var db = {}; var server; +var tempPath = 'temp' + (cluster.isWorker ? '-' + cluster.worker.process.pid: ''); function run(keys) { + try { + fs.mkdirSync(tempPath); + } catch(e) { + console.log('work dir already exists'); + } app.use('/static', express.static(__dirname + '/static')); if (keys === undefined) { @@ -109,6 +115,7 @@ function run(keys) { url: referer, userAgent: ua }; + var tempStream = fs.createWriteStream(tempPath + '/' + clientid); var baseStats = {}; @@ -142,22 +149,38 @@ function run(keys) { data[2] = statsMangler(data[2]); data[0] = 'getStats'; } - db[referer][clientid].peerConnections[data[1]].push({ - time: new Date(), - type: data[0], - value: data[2] - }); + data.time = new Date().getTime(); + tempStream.write(JSON.stringify(data) + '\n'); break; } }); client.on('close', function() { - console.log('closed'); - - var client = db[referer][clientid]; - dump(referer, client, clientid); - generateFeatures(referer, client, clientid); - delete db[referer][clientid]; + tempStream.on('finish', function() { + fs.readFile(tempStream.path, {encoding: 'utf-8'}, function(err, data) { + if (!err) { + data.split('\n').forEach(function(line) { + if (line.length) { + var data = JSON.parse(line); + db[referer][clientid].peerConnections[data[1]].push({ + time: new Date(data.time), + type: data[0], + value: data[2] + }); + } + }); + } + // we proceed even if there was an error. + var client = db[referer][clientid]; + delete db[referer][clientid]; + fs.unlink(tempStream.path, function(err, data) { + // we're good... + }); + dump(referer, client, clientid); + generateFeatures(referer, client, clientid); + }); + }); + tempStream.end(); }); }); } @@ -174,6 +197,10 @@ if (require.main === module && cluster.isMaster) { }); cluster.on('exit', function(worker, code, signal) { console.log('worker', worker.process.pid, 'died, restarting'); + // TODO: Possibly recover data. For now: throw it away. + fs.unlink('temp-' + worker.process.pid, function() { + console.log('removed temp-' + worker.process.pid); + }); cluster.fork(); }); } else { From 909a4b53dd39a2e150433c37f00ce428e39aa277 Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Sat, 18 Jun 2016 07:13:03 +0200 Subject: [PATCH 2/5] decompress only before process --- app.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/app.js b/app.js index b84dfa21..f5b4059c 100644 --- a/app.js +++ b/app.js @@ -137,18 +137,6 @@ function run(keys) { break; default: obfuscate(data); - if (!db[referer][clientid].peerConnections[data[1]]) { - db[referer][clientid].peerConnections[data[1]] = []; - baseStats[data[1]] = {}; - } - if (data[0] === 'getstats') { // delta-compressed - data[2] = statsDecompressor(baseStats[data[1]], data[2]); - baseStats[data[1]] = JSON.parse(JSON.stringify(data[2])); - } - if (data[0] === 'getStats' || data[0] === 'getstats') { - data[2] = statsMangler(data[2]); - data[0] = 'getStats'; - } data.time = new Date().getTime(); tempStream.write(JSON.stringify(data) + '\n'); break; @@ -162,8 +150,22 @@ function run(keys) { data.split('\n').forEach(function(line) { if (line.length) { var data = JSON.parse(line); + var time = new Date(data.time); + delete data.time; + if (!db[referer][clientid].peerConnections[data[1]]) { + db[referer][clientid].peerConnections[data[1]] = []; + baseStats[data[1]] = {}; + } + if (data[0] === 'getstats') { // delta-compressed + data[2] = statsDecompressor(baseStats[data[1]], data[2]); + baseStats[data[1]] = JSON.parse(JSON.stringify(data[2])); + } + if (data[0] === 'getStats' || data[0] === 'getstats') { + data[2] = statsMangler(data[2]); + data[0] = 'getStats'; + } db[referer][clientid].peerConnections[data[1]].push({ - time: new Date(data.time), + time: time, type: data[0], value: data[2] }); From 984e626631f859b5a057244a0b29880478ba9977 Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Mon, 20 Jun 2016 08:21:44 +0200 Subject: [PATCH 3/5] log err since directories dont get removed? --- app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.js b/app.js index f5b4059c..650f2a6f 100644 --- a/app.js +++ b/app.js @@ -200,8 +200,8 @@ if (require.main === module && cluster.isMaster) { cluster.on('exit', function(worker, code, signal) { console.log('worker', worker.process.pid, 'died, restarting'); // TODO: Possibly recover data. For now: throw it away. - fs.unlink('temp-' + worker.process.pid, function() { - console.log('removed temp-' + worker.process.pid); + fs.unlink('temp-' + worker.process.pid, function(err) { + console.log('removed temp-' + worker.process.pid, err); }); cluster.fork(); }); From d595c6e672bfb7c4c88041b9045047978c208f37 Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Mon, 20 Jun 2016 13:37:42 +0200 Subject: [PATCH 4/5] use rmdir, not unlink --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index 650f2a6f..c53d9ffa 100644 --- a/app.js +++ b/app.js @@ -200,7 +200,7 @@ if (require.main === module && cluster.isMaster) { cluster.on('exit', function(worker, code, signal) { console.log('worker', worker.process.pid, 'died, restarting'); // TODO: Possibly recover data. For now: throw it away. - fs.unlink('temp-' + worker.process.pid, function(err) { + fs.rmdir('temp-' + worker.process.pid, function(err) { console.log('removed temp-' + worker.process.pid, err); }); cluster.fork(); From 14023ba52ff925eb817c52e1c3e2ef704dcc8b9d Mon Sep 17 00:00:00 2001 From: Philipp Hancke Date: Mon, 20 Jun 2016 13:52:17 +0200 Subject: [PATCH 5/5] delete all files in directory first --- app.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app.js b/app.js index c53d9ffa..aae2c13d 100644 --- a/app.js +++ b/app.js @@ -199,11 +199,15 @@ if (require.main === module && cluster.isMaster) { }); cluster.on('exit', function(worker, code, signal) { console.log('worker', worker.process.pid, 'died, restarting'); + cluster.fork(); + + // clean up after worker. // TODO: Possibly recover data. For now: throw it away. - fs.rmdir('temp-' + worker.process.pid, function(err) { - console.log('removed temp-' + worker.process.pid, err); + var path = 'temp-' + worker.process.pid; + fs.readdirSync(path).forEach(function(fname) { + fs.unlinkSync(path + '/' + fname); }); - cluster.fork(); + fs.rmdirSync(path); }); } else { run();