Permalink
Browse files

add a connect logger

  • Loading branch information...
1 parent c0a88ab commit ba3da752b63900467c5c839eebb9a24af94e73e8 @hmalphettes committed Jun 4, 2013
View
@@ -84,8 +84,7 @@ Usage: custom
var log4js = require('log4js');
var uuid = require('node-uuid');
log4js.configure({
- "appenders": [
- "appender": {
+ "appenders": [ {
"type": "log4js-elasticsearch",
"indexName": function(loggingEvent) {
return loggingEvent.categoryName;
@@ -113,6 +112,25 @@ Usage: custom
}
});
+Usage: connect logger
+---------------------
+```javascript
+ var log4js = require('log4js');
+ var logstashConnectFormatter = require('log4js-elasticsearch').logstashConnectFormatter;
+ log4js.configure({
+ "appenders": [
+ {
+ "type": "log4js-elasticsearch",
+ "esclient": mockElasticsearchClient,
+ "buffersize": 1,
+ "layout": { type: 'logstash' }
+ }
+ ]
+ });
+ var logger = log4js.getLogger('express');
+ var connectLogger = log4js.connectLogger(logger, { format: logstashConnectFormatter });
+ app.use(connectLogger); //where app is the express app.
+```
Appender configuration parameters
=================================
View
@@ -0,0 +1,31 @@
+/*
+ "@fields" => {
+ "client" => "127.0.0.1",
+ "duration_usec" => 240,
+ "status" => 404,
+ "request" => "/favicon.ico",
+ "method" => "GET",
+ "referrer" => "-"
+ },
+*/
+function logstashConnectFormatter(req, res, formattedOutput) {
+ var fields = {
+ url: req.originalUrl,
+ method: req.method,
+ status: res.__statusCode || res.statusCode,
+ 'response-time': res.responseTime,
+ referrer: req.headers.referer || req.headers.referrer || '',
+ 'http-version': req.httpVersionMajor + '.' + req.httpVersionMinor,
+ 'remote-addr': req.socket && (req.socket.remoteAddress || (req.socket.socket && req.socket.socket.remoteAddress)),
+ 'user-agent': req.headers['user-agent'] || '',
+ 'content-length': (res._headers && res._headers['content-length']) || (res.__headers && res.__headers['Content-Length']) || -1
+ };
+ var message = formattedOutput(':remote-addr - - ":method :url HTTP/:http-version" :status :content-length ":referrer" ":user-agent"', req, res);
+ var resul = {};
+ Object.defineProperty(resul, '@fields', { value: fields, enumerable: true });
+ resul.toString = function() {
+ return message;
+ };
+ return resul;
+}
+exports.logstashConnectFormatter = logstashConnectFormatter;
@@ -42,22 +42,28 @@ function simpleJsonLayout(loggingEvent) {
* @return The JSON
*/
function logstashLayout(loggingEvent) {
- var data = __formatData(loggingEvent);
- var message = data[0], errorMsg = data[1], stack = data[2];
+ var data, fields, message, errorMsg, stack;
+ if (loggingEvent.data[0]['@fields']) {
+ fields = loggingEvent.data[0]['@fields'];
+ message = loggingEvent.data.toString();
+ } else {
+ data = __formatData(loggingEvent);
+ message = data[0], errorMsg = data[1], stack = data[2];
+ fields = {};
+ }
var eslogger = loggingEvent.logger;
var base = {
'@timestamp': loggingEvent.startTime,
- '@message': message,
- '@fields': {
- level: loggingEvent.level.level,
- levelStr: loggingEvent.level.levelStr,
- category: loggingEvent.categoryName
- }
+ '@message': message
};
+ fields.level = loggingEvent.level.level;
+ fields.levelStr = loggingEvent.level.levelStr;
+ fields.category = loggingEvent.categoryName;
if (errorMsg) {
- base['@fields'].error = errorMsg;
- base['@fields'].stack = stack;
+ fields.error = errorMsg;
+ fields.stack = stack;
}
+ base['@fields'] = fields;
return base;
}
@@ -135,23 +141,51 @@ if (oriLayoutMaker.name !== 'layoutEs') {
};
}
+function __getOpt(path, options, defaultV) {
+ if (options === undefined) {
+ return defaultV;
+ }
+ var curr = options;
+ for(var i = 0; i < path.length; i++) {
+ var p = path[i];
+ if (p) {
+ curr = curr[p];
+ if (curr === undefined) {
+ return defaultV;
+ }
+ }
+ }
+ if (curr === '__delete__') {
+ return undefined;
+ } else {
+ return curr;
+ }
+}
+
layouts.esTemplateMakers = {};
// http://untergeek.com/2012/09/20/using-templates-to-improve-elasticsearch-caching-with-logstash/
-layouts.esTemplateMakers.logstash = function(templateName) {
+layouts.esTemplateMakers.logstash = function(templateName, templateConfig) {
+ var nbOfShards = __getOpt(['settings', 'number_of_shards'], templateConfig,
+ parseInt(process.env.ES_DEFAULT_SHARDS_NUMBER, 10) || 4);
+ var totalShardsPerNode = __getOpt(['settings', 'index.routing.allocation.total_shards_per_node'], templateConfig, 2);
+ var cacheFieldType = __getOpt(['settings', 'index.cache.field.type'], templateConfig, "soft");
+ var refreshInterval = __getOpt(['settings', 'index.refresh_interval'], templateConfig, "5s");
+ var defaultField = __getOpt(['settings', 'index.default_field'], templateConfig, "@message");
+ var enableAll = __getOpt(['mappings', '_default_', '_all', 'enabled'], templateConfig, false);
return {
"template" : templateName || "logstash-*",
"settings" : {
- "number_of_shards" : parseInt(process.env.ES_DEFAULT_SHARDS_NUMBER, 10) || 4,
- "index.cache.field.type" : "soft",
- "index.refresh_interval" : "5s",
+ "number_of_shards" : nbOfShards,
+ "index.cache.field.type" : cacheFieldType,
+ "index.refresh_interval" : refreshInterval,
"index.store.compress.stored" : true,
- "index.query.default_field" : "@message",
- "index.routing.allocation.total_shards_per_node" : 2
+ "index.query.default_field" : defaultField,
+ "index.routing.allocation.total_shards_per_node" : totalShardsPerNode
},
"mappings" : {
"_default_" : {
- "_all" : {"enabled" : false},
+ "_all" : {"enabled" : enableAll},
"properties" : {
"@fields" : { "type" : "object", "dynamic": true, "path": "full" },
"@message": { "type": "string", "index": "analyzed" },
@@ -167,7 +201,7 @@ layouts.esTemplateMakers.logstash = function(templateName) {
};
};
-layouts.esTemplateMakers.simpleJson = function(templateName) {
+layouts.esTemplateMakers.simpleJson = function(templateName, templateConfig) {
return {
"template" : templateName || "log4js*",
"settings" : {
@@ -1,9 +1,8 @@
var ElasticsearchClient = require('./elasticsearch-client');
var layouts = require('./log4js-elasticsearch-layouts');
+var logstashConnectFormatter = require('./connect-logger').logstashConnectFormatter;
var bulk = require('./elasticsearch-bulk');
-var defaultHostname = require('os').hostname();
-
function createAppender(layout, config, options, done) {
var layoutES = makeESHelper(layout, config);
var esclient = initESClient(config, options, layoutES.template, done);
@@ -109,11 +108,11 @@ function makeESHelper(layout, config) {
//'logstash-%{+YYYY.MM.dd}';
return prefix + vYearLong + '.' + vMonth + '.' + vDay;
};
- if (config.layout) {
+ if (config.layout && config.layout.template !== false) {
if (config.layout.type === 'logstash') {
- template = layouts.esTemplateMakers.logstash(templateName);
+ template = layouts.esTemplateMakers.logstash(templateName, config.layout.template);
} else if (config.layout.type === 'simpleJson') {
- template = layouts.esTemplateMakers.simpleJson(templateName);
+ template = layouts.esTemplateMakers.simpleJson(templateName, config.layout.template);
}
}
}
@@ -128,3 +127,4 @@ function makeESHelper(layout, config) {
exports.appender = createAppender;
exports.configure = configure;
exports.flushAll = bulk.flushAll;
+exports.logstashConnectFormatter = logstashConnectFormatter;
View
@@ -1,6 +1,6 @@
{
"name": "log4js-elasticsearch",
- "version": "0.0.1",
+ "version": "0.0.2",
"description": "log4js appender for node that targets elasticsearch.\nCompatible with logstash's elasticsearch_http output; Viewable with Kibana.",
"main": "lib/log4js-elasticsearch.js",
"scripts": {
View
@@ -0,0 +1,109 @@
+var sandbox = require('sandboxed-module');
+var libpath = process.env.COVERAGE ? '../lib-cov' : '../lib';
+var log4jsElasticSearch = require(libpath + '/log4js-elasticsearch');
+var logstashConnectFormatter = log4jsElasticSearch.logstashConnectFormatter;
+var expect = require('chai').expect;
+var levels = require('log4js').levels;
+
+function MockLogger() {
+
+ var that = this;
+ this.messages = [];
+
+ this.log = function(level, message, exception) {
+ that.messages.push({ level: level, message: message });
+ };
+
+ this.isLevelEnabled = function(level) {
+ return level.isGreaterThanOrEqualTo(that.level);
+ };
+
+ this.level = levels.TRACE;
+}
+
+function MockRequest(remoteAddr, method, originalUrl) {
+
+ this.socket = { remoteAddress: remoteAddr };
+ this.originalUrl = originalUrl;
+ this.method = method;
+ this.httpVersionMajor = '5';
+ this.httpVersionMinor = '0';
+ this.headers = {};
+
+}
+
+function MockResponse(statusCode) {
+
+ this.statusCode = statusCode;
+ this.end = function(chunk, encoding) {};
+
+}
+
+describe.skip('When using a connect logger', function() {
+ var log4js = require('log4js');
+ var ml, cl;
+ before(function() {
+ ml = new MockLogger();
+ cl = log4js.connectLogger(ml, { format: logstashConnectFormatter });
+ expect(cl).to.be.a['function'];
+ });
+ it('Must format a logging event for logstash from the execution of a connect request', function(done) {
+ var req = new MockRequest('my.remote.addr', 'GET', 'http://url');
+ var res = new MockResponse(200);
+ cl(req, res, function() {
+ res.end('chunk', 'encoding');
+ done();
+ });
+ });
+});
+describe('When using a connect logger on the es appender', function() {
+ var log4js = sandbox.require('log4js', {
+ requires: { 'log4js-elasticsearch': log4jsElasticSearch }
+ });
+ var ml, cl;
+ var mockElasticsearchClient = {
+ index: function(indexName, typeName, logObj, id, cb) {
+ // console.log('logObj', logObj);
+ expect(logObj['@message']).to.equal('my.remote.addr - - "GET http://url HTTP/5.0" 200 - "" ""');
+ var fields = logObj['@fields'];
+ expect(fields.url).to.equal('http://url');
+ expect(fields.method).to.equal('GET');
+ expect(fields['response-time']).to.be.a.number;
+ expect(fields['remote-addr']).to.equal('my.remote.addr');
+ expect(fields['content-length']).to.be.a.number;
+ expect(fields['http-version']).to.equal('5.0');
+ cb();
+ }, defineTemplate: function(templateName, template, cb) {
+ cb(null, 'ok');
+ }, getTemplate: function(templateName, cb) {
+ cb(null, '{}');
+ }
+ };
+ before(function() {
+ log4js.configure({
+ "appenders": [
+ {
+ "type": "log4js-elasticsearch",
+ "esclient": mockElasticsearchClient,
+ "buffersize": 1,
+ "layout": { type: 'logstash' }
+ }
+ ]
+ });
+ ml = log4js.getLogger('unittest');
+ cl = log4js.connectLogger(ml, { format: logstashConnectFormatter });
+ expect(cl).to.be.a['function'];
+ });
+ it('Must format a logging event for logstash from the execution of a connect request', function(done) {
+ var req = new MockRequest('my.remote.addr', 'GET', 'http://url');
+ var res = new MockResponse(200);
+ cl(req, res, function() {
+ res.end('chunk', 'encoding');
+ done();
+ });
+ });
+});
+
+
+
+
View
@@ -0,0 +1,31 @@
+var expect = require('chai').expect;
+var sandbox = require('sandboxed-module');
+var libpath = process.env.COVERAGE ? '../lib-cov' : '../lib';
+var lsLayouts = require(libpath + '/log4js-elasticsearch-layouts').esTemplateMakers.logstash;
+
+describe('When passing options to the es-template', function() {
+ it('Must return the template when no options are passed', function() {
+ var r = lsLayouts('test');
+ expect(r.template).to.equal('test');
+ });
+ it('Must keep the default value when no options are passed', function() {
+ var r = lsLayouts('test');
+ expect(r.settings['index.query.default_field']).to.equal('@message');
+ });
+ it('Must override the number of shards via the options', function() {
+ var r = lsLayouts('test', {settings: { number_of_shards: 1 }});
+ expect(r.settings.number_of_shards).to.equal(1);
+ });
+ it('Must override the total_shards_per_node via the options', function() {
+ var r = lsLayouts('test', {settings: { 'index.routing.allocation.total_shards_per_node': 10 }});
+ expect(r.settings['index.routing.allocation.total_shards_per_node']).to.equal(10);
+ });
+ it('Must delete the index.cache.field.type via the options', function() {
+ var r = lsLayouts('test', {settings: { 'index.cache.field.type': '__delete__' }});
+ expect(r.settings['index.cache.field.type']).to.not.exist;
+ });
+ it('Must enable the _all field via the options', function() {
+ var r = lsLayouts('test', {mappings: { _default_: { _all: {enabled: true} } }});
+ expect(r.mappings._default_._all.enabled).to.equal(true);
+ });
+});
View
@@ -139,6 +139,7 @@ describe('When configuring an elasticsearch logstash appender layout', function(
var currentMsg;
var defineTemplateWasCalled = false;
+ var expectedNumberOfShardsInTemplate = -1;
var mockElasticsearchClient = {
index: function(indexName, typeName, logObj, id, cb) {
expect(logObj['@message']).to.equal(currentMsg);
@@ -149,12 +150,17 @@ describe('When configuring an elasticsearch logstash appender layout', function(
cb();
}, defineTemplate: function(templateName, template, cb) {
defineTemplateWasCalled = true;
+ if (expectedNumberOfShardsInTemplate !== -1) {
+ expect(template.settings.number_of_shards).to.equal(expectedNumberOfShardsInTemplate);
+ expectedNumberOfShardsInTemplate = -1;
+ }
cb(null, 'something');
}, getTemplate: function(templateName, cb) {
cb(null, '{}');
}
};
it('Must have configured the appender with static params', function() {
+ expectedNumberOfShardsInTemplate = 1;
log4js.configure({
"appenders": [
{
@@ -165,7 +171,8 @@ describe('When configuring an elasticsearch logstash appender layout', function(
"layout": {
"type": "logstash",
"tags": [ "goodie" ],
- "sourceHost": "aspecialhost"
+ "sourceHost": "aspecialhost",
+ "template": {"settings": { "number_of_shards": 1 }}
}
}
]
@@ -179,6 +186,7 @@ describe('When configuring an elasticsearch logstash appender layout', function(
});
it('Must have configured the appender with dynamic params', function() {
+ expectedNumberOfShardsInTemplate = 4;
log4js.configure({
"appenders": [
{

0 comments on commit ba3da75

Please sign in to comment.