Permalink
Browse files

Don't get tripped up by un-finished erlang objects

Couchdb logs are basically the most obnoxious logs ever.
  • Loading branch information...
isaacs committed Aug 23, 2012
1 parent d6490e2 commit 0fd6191c78ce464235f81878a946b84029479906
Showing with 54 additions and 48 deletions.
  1. +32 −45 parser.js
  2. +2 −1 test/basic.js
  3. +2 −1 test/fixtures/log
  4. +18 −1 test/fixtures/log-parsed.js
View
@@ -2,7 +2,7 @@ var Stream = require('stream')
module.exports = LogParser
-var specials = new Buffer('{}[]<>"\'\n\\')
+var specials = new Buffer('{}[]<>"\'\n\\ ')
var openCurly = specials[0]
var closeCurly = specials[1]
var openSquare = specials[2]
@@ -13,6 +13,7 @@ var doubleQuote = specials[6]
var singleQuote = specials[7]
var lf = specials[8]
var escape = specials[9]
+var space = specials[10]
function LogParser () {
Stream.call(this)
@@ -27,6 +28,7 @@ function LogParser () {
this._quoted = false
this._escaped = false
+ this._sawLF = false
}
LogParser.prototype = Object.create(Stream.prototype, {
@@ -46,6 +48,30 @@ LogParser.prototype._process = function (chunk) {
if (chunk) for (var i = 0; i < chunk.length && !this._paused; i ++) {
var c = chunk[i]
+ if (this._sawLF) {
+ this._sawLF = false
+ if (c === space)
+ continue
+
+ var msg = this._buffer
+ this._buffer = []
+ // if it's [c,\n,x], then push [c], skip \n, continue with x
+ // if it's [\n,xxx], then just continue
+ // if it's [\n,\n,x], walk over the second one
+ if (i > 1) msg.push(chunk.slice(0, i - 1))
+ // slice off multiple \n chars
+ while ((c = chunk[i]) === lf && i < chunk.length) i++;
+ chunk = chunk.slice(i)
+ i = 0
+ this._handleMessage(msg)
+ continue
+ }
+
+ if (c === lf) {
+ this._sawLF = true
+ continue
+ }
+
// any special char can be escaped with \
if (this._escaped) {
this._escaped = false
@@ -72,47 +98,6 @@ LogParser.prototype._process = function (chunk) {
continue
}
- if (c === openCurly ||
- c === openSquare ||
- c === openWaka) {
- if (c === openCurly) {
- this._curly ++;
- this._type.push(closeCurly)
- } else if (c === openSquare) {
- this._square ++;
- this._type.push(closeSquare)
- } else {
- this._waka ++;
- this._type.push(closeWaka)
- }
- continue
- }
-
- if ((c === closeCurly ||
- c === closeSquare ||
- c === closeWaka) &&
- c === this._type[this._type.length-1]) {
- this._type.pop()
- if (c === closeCurly) {
- this._curly --;
- } else if (c === closeSquare) {
- this._square --;
- } else {
- this._waka --;
- }
- continue
- }
-
- if (c === lf && this._type.length === 0) {
- var msg = this._buffer
- this._buffer = []
- msg.push(chunk.slice(0, i))
- // slice off multiple \n chars
- while (chunk[i] === lf && i < chunk.length) i++;
- chunk = chunk.slice(i)
- i = 0
- this._handleMessage(msg)
- }
}
if (chunk && chunk.length !== 0) this._buffer.push(chunk);
if (this._ended && !this._paused) {
@@ -123,15 +108,15 @@ LogParser.prototype._process = function (chunk) {
// msg is an array of chunks representing a single message
LogParser.prototype._handleMessage = function (msg) {
- msg = Buffer.concat(msg)
- this.emit('data', msg + '\n')
+ msg = Buffer.concat(msg).toString().trim() + '\n'
+ this.emit('data', msg)
if (this.listeners('message')) {
this.emit('message', this.parseMessage(msg))
}
}
var messageExpr = /^\[([^\]]+)\] \[([^\]]+)\] \[([^\]]+)\] ([\s\S]*)$/
-var httpExpr = /^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+) - - ([A-Z]+) (.*) ([0-9]+)$/
+var httpExpr = /^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+) - - ([A-Z]+) (.*) ([0-9]+)\n?$/
var erlWtfExpr = /^([^{\[]+ )?([\s\S]*)/
LogParser.prototype.parseMessage = function (msg) {
@@ -144,6 +129,8 @@ LogParser.prototype.parseMessage = function (msg) {
result.level = parsed[2]
result.pid = parsed[3]
var more = parsed[4]
+ console.error(parsed, more, more.match(httpExpr))
+
var http, erlWtf
if (http = more.match(httpExpr)) {
result.type = 'http'
View
@@ -20,7 +20,8 @@ tap.test('basic parsing test', function (t) {
var i = 0
parser.on('message', function (c) {
- t.same(c, expect[i], 'item '+i)
+ var r = t.same(c, expect[i], 'item '+i)
+ if (!r.ok) console.error(c)
i++
})
View
@@ -230,4 +230,5 @@
[Thu, 12 Jul 2012 08:51:48 GMT] [info] [<0.21644.104>] 10.115.51.95 - - GET /registry/_design/app/_show/package/log4js?pkg=log4js&version=0.3.8 304
[Thu, 12 Jul 2012 08:51:48 GMT] [info] [<0.21503.104>] 10.115.51.95 - - GET /registry/nodist 200
[Thu, 12 Jul 2012 08:51:48 GMT] [info] [<0.21633.104>] 10.115.51.95 - - GET /registry/_design/app/_show/package/wordwrap?pkg=wordwrap 304
-
+[Tue, 31 Jul 2012 06:32:59 GMT] [error] [emulator] Error in process <0.10422.45> with exit value: {{nocatch,{os_process_error,"OS process timed out."}},[{couch_os_process,prompt,2,[{file,"/home/ubuntu/hosting/build-couchdb/git-build/git@github.com%3Airiscouch%2Fcouchdb%3Ahosting_v1.2.0r8/src/couchdb/co...
+[Thu, 12 Jul 2012 08:51:48 GMT] [info] [<0.21633.104>] 10.115.51.95 - - GET /registry/_design/app/_show/package/wordwrap?pkg=wordwrap 304
@@ -593,4 +593,21 @@ module.exports =
ip: '10.115.51.95',
method: 'GET',
url: '/registry/_design/app/_show/package/wordwrap?pkg=wordwrap',
- statusCode: 304 } ]
+ statusCode: 304 },
+
+ { date: new Date('Mon Jul 30 2012 23:32:59 GMT-0700 (PDT)'),
+ level: 'error',
+ pid: 'emulator',
+ type: 'erl',
+ message: 'Error in process <0.10422.45> with exit value: ',
+ dump: '{{nocatch,{os_process_error,"OS process timed out."}},[{couch_os_process,prompt,2,[{file,"/home/ubuntu/hosting/build-couchdb/git-build/git@github.com%3Airiscouch%2Fcouchdb%3Ahosting_v1.2.0r8/src/couchdb/co...' },
+
+ { date: new Date('Thu Jul 12 2012 01:51:48 GMT-0700 (PDT)'),
+ level: 'info',
+ pid: '<0.21633.104>',
+ type: 'http',
+ ip: '10.115.51.95',
+ method: 'GET',
+ url: '/registry/_design/app/_show/package/wordwrap?pkg=wordwrap',
+ statusCode: 304 }
+]

0 comments on commit 0fd6191

Please sign in to comment.