Permalink
Browse files

Much better behavior regarding the regular posting of the logs

  • Loading branch information...
hmalphettes committed May 29, 2013
1 parent 4929ea5 commit fedf2af47fba4eaf05f5f4044edb2f76599f5b69
Showing with 109 additions and 17 deletions.
  1. +1 −0 .gitignore
  2. +24 −2 README.md
  3. +83 −14 lib/elasticsearch-bulk.js
  4. +1 −1 package.json
View
@@ -11,3 +11,4 @@ html-report
build
cobertura-coverage.xml
metadata
+*.tgz
View
@@ -27,7 +27,7 @@ The default url of the ES server is http://localhost:9200
Usage: log4js configuration
---------------------------
-
+```javascript
var log4js = require('log4js');
log4js.configure({
"appenders": [
@@ -42,14 +42,36 @@ Usage: log4js configuration
},
{
"category": "tests",
- "type": "console",
+ "type": "console"
}
],
"levels": {
"tests": "DEBUG"
}
});
+ var log = log4js.getLogger('tests');
+
+ log.error('hello hello');
+
+ if (setTimeout(function() {}).unref === undefined) {
+ console.log('force flushing and goodbye for node <= 0.8');
+ require('log4js-elasticsearch').flushAll(true);
+ }
+```
+
+Note: related to clearing the interval
+--------------------------------------
+Sending the logs by batches on a regular basis is a lot more performant
+than one by one.
+
+However a node process will not exit until all its has no more refernced timers.
+Since node-0.9 it is possible to have a 'soft' timer that is not referened.
+Before that it is required to close the log4js elasticsearch appenders:
+```javascript
+require('log4js-elasticsearch').flushAll(true);
+```
+
Usage: custom
-------------
View
@@ -1,4 +1,3 @@
-
var esLogBuffers = [];
function EsOneAtaTime(layout, esclient, layoutES) {
@@ -29,9 +28,19 @@ function EsLogBuffer(layout, esclient, layoutES, timeout, buffersize) {
this.bulkCmds = [];
if (this.timeout > 0) {
var self = this;
- setTimeout(function() {
+ this.interv = setInterval(function() {
self.flush();
}, this.timeout);
+ if (typeof this.interv.unref === 'function') {
+ this.interv.unref();
+ } else {
+ console.warn('No support for Timer#unref:\n' +
+ ' the setInterval object will keep running the node\n' +
+ ' process until it stop for some other reason.\n' +
+ ' Call require(log4js-elasticsearch).flushAll(true)\n' +
+ ' Call EsLogBuffer#close to clear the setInterval.');
+ }
+ setupProcessListeners();
}
}
@@ -50,15 +59,27 @@ EsLogBuffer.prototype.log = function(loggingEvent) {
}
};
-EsLogBuffer.prototype.flush = function() {
+EsLogBuffer.prototype.close = function() {
+ if (this.interv) {
+ clearTimeout(this.interv);
+ }
+};
+
+EsLogBuffer.prototype.flush = function(optionalDone) {
if (this.bulkCmds.length === 0) {
+ if (optionalDone) {
+ optionalDone();
+ }
return;
}
+ if (!optionalDone) {
+ optionalDone = function() {
+ //emit an event to say it is over?
+ };
+ }
var current = this.bulkCmds;
this.bulkCmds = [];
- this.esclient.bulk(current, function() {
- //emit an event to say it is over?
- });
+ this.esclient.bulk(current, optionalDone);
};
/**
@@ -81,19 +102,67 @@ exports.makeEsLog = function(layout, esclient, layoutES, timeout, buffersize) {
return esBulk;
};
-function flushAll(doreset) {
+function flushAll(doreset, optionalDone) {
var flushthem = esLogBuffers;
if (doreset) {
esLogBuffers = [];
+ clearProcessListeners();
}
- flushthem.forEach(function (esLogBuffer) {
- esLogBuffer.flush();
+ if (typeof optionalDone === 'function') {
+ var i = -1;
+ var flushOneAtATime = function() {
+ i++;
+ var curr = flushthem[i];
+ if (curr) {
+ curr.flush(function(err) {
+ if (doreset) {
+ curr.close();
+ }
+ flushOneAtATime();
+ });
+ } else {
+ optionalDone();
+ }
+ };
+ flushOneAtATime();
+ } else {
+ flushthem.forEach(function (esLogBuffer) {
+ if (doreset) {
+ esLogBuffer.close();
+ }
+ esLogBuffer.flush();
+ });
+ }
+}
+
+var processListenersSetup = false;
+
+var onProcessSigint = function() {
+ clearProcessListeners();
+ flushAll(true, function() {
+ process.kill(process.pid, 'SIGINT');
});
+};
+var onProcessExit = function() {
+ clearProcessListeners();
+ flushAll(true, function() {
+ process.exit();
+ });
+};
+
+function setupProcessListeners() {
+ if (!processListenersSetup) {
+ processListenersSetup = true;
+ process.once('SIGINT', onProcessSigint);
+ process.once('exit', onProcessExit);
+ }
+}
+function clearProcessListeners() {
+ if (processListenersSetup) {
+ processListenersSetup = false;
+ process.removeListener('SIGINT', onProcessSigint);
+ process.removeListener('exit', onProcessExit);
+ }
}
exports.flushAll = flushAll;
-
-//on process exit send what's is missing.
-process.on('exit', function() {
- flushAll(true);
-});
View
@@ -22,9 +22,9 @@
},
"license": "MIT",
"dependencies": {
- "log4js": "*"
},
"devDependencies": {
+ "log4js": "*",
"chai": "*",
"mocha": "*",
"istanbul": "*",

0 comments on commit fedf2af

Please sign in to comment.