From 062deb563cda2880c780b635f1978a3cdbc4a40b Mon Sep 17 00:00:00 2001 From: Amoki Date: Tue, 25 Nov 2014 18:41:32 +0100 Subject: [PATCH 1/7] add opbeat --- lib/index.js | 8 ++++++ lib/utils.js | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 79 insertions(+) create mode 100644 lib/utils.js diff --git a/lib/index.js b/lib/index.js index fd5245a..ca0a18a 100644 --- a/lib/index.js +++ b/lib/index.js @@ -5,6 +5,7 @@ var restify = require('restify'); var yaqs = require('yaqs'); var Childs = require('./helpers/Childs'); +var utils = require('./utils.js'); /** * Create a new hydration server. * This server will use `config.hydrater_function` as its main function, to turn a file into metadata. @@ -23,6 +24,13 @@ module.exports.createServer = function(config) { config.logger = config.logger || console.log; config.errLogger = config.errLogger || console.error; + utils.logError.config = config; + + if(config.opbeat && config.opbeat.secretToken) { + var opbeat = require('opbeat'); + utils.logError.opbeat = opbeat(config.opbeat); + } + var concurrency = config.concurrency || 1; var tasksPerProcess = config.tasksPerProcess || 100; diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..a341237 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,70 @@ +"use strict"; + + +module.exports.logError = function logError(err, req, extra) { + // No logging on test or if err is undefined + if(process.env.NODE_ENV === "test" || !err) { + return; + } + + if(!extra) { + extra = req; + req = null; + } + + delete err.domain; + delete err.domainThrown; + + if(err.__alreadyLogged) { + console.warn("Skipping an error already sent to Opbeat: ", err.toString()); + return; + } + + if(!extra) { + extra = {}; + } + + if(module.exports.logError.config) { + extra.hydrater = module.exports.logError.config.hydraterUrl; + } + + if(module.exports.logError.opbeat) { + var meta = { + extra: extra + }; + + if(req) { + meta.request = req; + + if(req.token) { + meta.user = { + is_authenticated: true, + id: req.token.anyfetchToken, + username: req.token.accountName, + email: req.token.accountName + }; + } + } + + module.exports.logError.opbeat.captureError(err, meta); + } + else { + var all = { + details: err.toString(), + err: err, + extra: extra + }; + + try { + all = JSON.stringify(all); + } + catch(e) { + // Converting circular structure to JSON. + // We can't do anything, let's log the raw object. + } + + console.warn("LOG-ERROR-DETAILS", all); + } + + err.__alreadyLogged = true; +}; diff --git a/package.json b/package.json index ad22af0..486c837 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ ], "dependencies": { "async": "^0.9.0", + "opbeat": "^1.0.5", "rarity": "^2.1.1", "redis": "^0.12.1", "restify": "^2.8.3", From 8b62ba0fcefc2e3eb5cf9a98a4ebcda9db18fe15 Mon Sep 17 00:00:00 2001 From: Amoki Date: Tue, 25 Nov 2014 18:59:10 +0100 Subject: [PATCH 2/7] add log errors --- lib/helpers/hydrater.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/helpers/hydrater.js b/lib/helpers/hydrater.js index 9e12329..b1d32e1 100644 --- a/lib/helpers/hydrater.js +++ b/lib/helpers/hydrater.js @@ -14,8 +14,8 @@ var rarity = require('rarity'); var util = require('util'); var lib = require('../index.js'); - var HydrationError = lib.HydrationError; +var logError = require('../utils').logError; @@ -52,6 +52,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { if(!cleaner.called) { cleaner.called = true; if(err) { + logError(err, {task: task, stdout: stdout, stderr: stderr}); child.reset(); } else { @@ -170,7 +171,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { cb(null, changes); }, function patchDocument(changes, cb) { - // Returning null means we won't complete the hdyration, and are waiting for something else. + // Returning null means we won't complete the hydration, and are waiting for something else. if(changes === null) { logger("Skipped task: " + ((task.file_path) ? task.file_path : task.document.id)); return cb(); @@ -189,6 +190,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { async.waterfall([ function logError(cb) { if(err) { + logError(err, {task: task}); errLogger("ERR hydrating " + ((task.file_path) ? task.file_path : task.document.id), err.toString()); } @@ -215,6 +217,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { } ], function(internalErr) { if(internalErr) { + logError(internalErr, {task: task}); errLogger("INTERNAL ERR", internalErr); } done(err || internalErr, changes); From da25369871ec2fc310637d39c4c56c0c1a0197a5 Mon Sep 17 00:00:00 2001 From: Amoki Date: Wed, 26 Nov 2014 10:07:40 +0100 Subject: [PATCH 3/7] fix logerrors --- lib/helpers/hydrater.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/helpers/hydrater.js b/lib/helpers/hydrater.js index b1d32e1..8f7cb96 100644 --- a/lib/helpers/hydrater.js +++ b/lib/helpers/hydrater.js @@ -188,7 +188,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { } ], function handleErrors(err, changes, res) { async.waterfall([ - function logError(cb) { + function logErrors(cb) { if(err) { logError(err, {task: task}); errLogger("ERR hydrating " + ((task.file_path) ? task.file_path : task.document.id), err.toString()); From 63ebcc4951ec34bc13932b36bacb30839f8582ab Mon Sep 17 00:00:00 2001 From: Amoki Date: Wed, 26 Nov 2014 10:30:49 +0100 Subject: [PATCH 4/7] improve opbeat --- lib/helpers/hydrater.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/helpers/hydrater.js b/lib/helpers/hydrater.js index 8f7cb96..400433e 100644 --- a/lib/helpers/hydrater.js +++ b/lib/helpers/hydrater.js @@ -52,7 +52,14 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { if(!cleaner.called) { cleaner.called = true; if(err) { - logError(err, {task: task, stdout: stdout, stderr: stderr}); + var extra = { + stdout: stdout, + stderr: stderr, + }; + extra.task = JSON.stringify(task); + extra.task = JSON.parse(extra.task); + + logError(err, extra); child.reset(); } else { @@ -190,7 +197,10 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { async.waterfall([ function logErrors(cb) { if(err) { - logError(err, {task: task}); + var extra = {}; + extra.task = JSON.stringify(task); + extra.task = JSON.parse(extra.task); + logError(err, extra); errLogger("ERR hydrating " + ((task.file_path) ? task.file_path : task.document.id), err.toString()); } @@ -217,7 +227,10 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { } ], function(internalErr) { if(internalErr) { - logError(internalErr, {task: task}); + var extra = {}; + extra.task = JSON.stringify(task); + extra.task = JSON.parse(extra.task); + logError(internalErr, extra); errLogger("INTERNAL ERR", internalErr); } done(err || internalErr, changes); From 57adac094e7330c301867bbe34251d93ffd4bd06 Mon Sep 17 00:00:00 2001 From: Amoki Date: Wed, 26 Nov 2014 10:34:35 +0100 Subject: [PATCH 5/7] ignore opbeat in istanbul --- lib/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils.js b/lib/utils.js index a341237..94094b6 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,6 +1,6 @@ "use strict"; - +/* istanbul ignore next */ module.exports.logError = function logError(err, req, extra) { // No logging on test or if err is undefined if(process.env.NODE_ENV === "test" || !err) { From 4ae91ca3902e9220fb7971394b13d63c5e94f25f Mon Sep 17 00:00:00 2001 From: Amoki Date: Wed, 26 Nov 2014 10:41:21 +0100 Subject: [PATCH 6/7] improve opbeat and some istabul ignores --- lib/helpers/hydrater.js | 18 ++++++------------ lib/index.js | 1 + 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/lib/helpers/hydrater.js b/lib/helpers/hydrater.js index 400433e..3b66348 100644 --- a/lib/helpers/hydrater.js +++ b/lib/helpers/hydrater.js @@ -52,12 +52,9 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { if(!cleaner.called) { cleaner.called = true; if(err) { - var extra = { - stdout: stdout, - stderr: stderr, - }; - extra.task = JSON.stringify(task); - extra.task = JSON.parse(extra.task); + var extra = JSON.parse(JSON.stringify(task)); + extra.stdout = stdout; + extra.stderr = stderr; logError(err, extra); child.reset(); @@ -197,9 +194,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { async.waterfall([ function logErrors(cb) { if(err) { - var extra = {}; - extra.task = JSON.stringify(task); - extra.task = JSON.parse(extra.task); + var extra = JSON.parse(JSON.stringify(task)); logError(err, extra); errLogger("ERR hydrating " + ((task.file_path) ? task.file_path : task.document.id), err.toString()); } @@ -226,10 +221,9 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { } } ], function(internalErr) { + /* istanbul ignore next */ if(internalErr) { - var extra = {}; - extra.task = JSON.stringify(task); - extra.task = JSON.parse(extra.task); + var extra = JSON.parse(JSON.stringify(task)); logError(internalErr, extra); errLogger("INTERNAL ERR", internalErr); } diff --git a/lib/index.js b/lib/index.js index ca0a18a..da26caf 100644 --- a/lib/index.js +++ b/lib/index.js @@ -26,6 +26,7 @@ module.exports.createServer = function(config) { utils.logError.config = config; + /* istanbul ignore next */ if(config.opbeat && config.opbeat.secretToken) { var opbeat = require('opbeat'); utils.logError.opbeat = opbeat(config.opbeat); From b46a7092c0cb78a99e03c4ca6727a4d4abf32a40 Mon Sep 17 00:00:00 2001 From: Amoki Date: Wed, 26 Nov 2014 10:46:02 +0100 Subject: [PATCH 7/7] add changes --- lib/helpers/hydrater.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/helpers/hydrater.js b/lib/helpers/hydrater.js index 3b66348..c2756f8 100644 --- a/lib/helpers/hydrater.js +++ b/lib/helpers/hydrater.js @@ -53,6 +53,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { cleaner.called = true; if(err) { var extra = JSON.parse(JSON.stringify(task)); + extra.changes = changes; extra.stdout = stdout; extra.stderr = stderr; @@ -195,6 +196,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { function logErrors(cb) { if(err) { var extra = JSON.parse(JSON.stringify(task)); + extra.changes = changes; logError(err, extra); errLogger("ERR hydrating " + ((task.file_path) ? task.file_path : task.document.id), err.toString()); } @@ -224,6 +226,7 @@ module.exports = function(hydraterFunction, childs, logger, errLogger) { /* istanbul ignore next */ if(internalErr) { var extra = JSON.parse(JSON.stringify(task)); + extra.changes = changes; logError(internalErr, extra); errLogger("INTERNAL ERR", internalErr); }