Parse numerical data

bpaquet · Oct 8, 2012 · 8690017 · 8690017
1 parent 6966c18
commit 8690017
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 5 deletions.
diff --git a/lib/filters/filter_regex.js b/lib/filters/filter_regex.js
@@ -54,13 +54,20 @@ FilterRegex.prototype.process = function(data) {
       data['@fields'] = {};
     }
     for(var i = 0; i < this.fields.length; i ++) {
-      if (result[i + 1]) {
+      var v = result[i + 1];
+      if (v) {
         if (this.date_format && this.fields[i] == 'timestamp') {
-          data['@timestamp'] = moment(result[i + 1], this.date_format).format();
+          data['@timestamp'] = moment(v, this.date_format).format();
           logger.debug('Event timestamp modified to', data['@timestamp']);
         }
         else {
-          data['@fields'][this.fields[i]] = result[i + 1];
+          if (v.match(/^[0-9]+$/)) {
+            v = parseInt(v);
+          }
+          else if (v.match(/^[0-9]+[\.,][0-9]+$/)) {
+            v = parseFloat(v.replace(',', '.'));
+          }
+          data['@fields'][this.fields[i]] = v;
         }
       }
     }

diff --git a/test/filter_helper.js b/test/filter_helper.js
@@ -1,8 +1,11 @@
 var assert = require('assert');
 
-function create(filter_name, filter_config, inputs, outputs) {
+function create(filter_name, filter_config, inputs, outputs, check_callback) {
   return createWithCallback(filter_name, filter_config, inputs, outputs.length, function(r) {
     assert.deepEqual(r, outputs);
+    if (check_callback) {
+      check_callback(r);
+    }
   });
 }
 

diff --git a/test/test_22_filter_regex.js b/test/test_22_filter_regex.js
@@ -18,6 +18,27 @@ vows.describe('Filter regex ').addBatch({
     {'@message': 'abcd efgh ijk', '@fields': {fa: 'abcd', fb: 'efgh', fc: 'toto'}},
     {'@message': 'abcdefghijk'},
   ]),
+  'number management': filter_helper.create('regex', '?regex=^(\\S+)$&fields=a', [
+    {'@message': '12'},
+    {'@message': '90'},
+    {'@message': '12.3'},
+    {'@message': '11,67'},
+    {'@message': 'aa'},
+    {'@message': ''},
+  ], [
+    {'@message': '12', '@fields': {a: 12}},
+    {'@message': '90', '@fields': {a: 90}},
+    {'@message': '12.3', '@fields': {a: 12.3}},
+    {'@message': '11,67', '@fields': {a: 11.67}},
+    {'@message': 'aa', '@fields': {a: 'aa'}},
+    {'@message': ''},
+  ], function(r) {
+    assert.equal(typeof(r[0]['@fields'].a), 'number');
+    assert.equal(typeof(r[1]['@fields'].a), 'number');
+    assert.equal(typeof(r[2]['@fields'].a), 'number');
+    assert.equal(typeof(r[3]['@fields'].a), 'number');
+    assert.equal(typeof(r[4]['@fields'].a), 'string');
+  }),
   'with star': filter_helper.create('regex', '?regex=^(\\S*) (\\S+)&fields=fa,fb', [
     {'@message': ' efgh ijk'},
   ], [
@@ -79,7 +100,11 @@ vows.describe('Filter regex ').addBatch({
       },
       '@timestamp': '2012-07-31T16:02:48+00:00'
     },
-  ]),
+  ], function(r) {
+    assert.equal(typeof(r[0]['@fields'].status), 'number');
+    assert.equal(typeof(r[0]['@fields'].bytes_sent), 'number');
+    assert.equal(typeof(r[0]['@fields'].referer), 'string');
+  }),
   'nginx parsing with predefined type': filter_helper.create('regex', 'nginx_combined', [
     {'@message': '127.0.0.1 - - [31/Jul/2012:18:02:28 +0200] "GET /favicon.ico HTTP/1.1" 502 574 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1215.0 Safari/537.2"'},
   ],[