Permalink
Browse files

Add filter only_field_exist and only_field_equal

  • Loading branch information...
1 parent bcb4f01 commit b5e5013c307e85dda71dee7926e8a0dff4e735b3 Bertrand Paquet committed Oct 14, 2012
Showing with 101 additions and 2 deletions.
  1. +12 −1 Readme.markdown
  2. +47 −1 lib/lib/base_component.js
  3. +2 −0 test/test_25_filter_compute_field.js
  4. +40 −0 test/test_26_message_filtering.js
View
@@ -118,6 +118,8 @@ Outputs and filter, commons parameters
===
* ``only_type``: execute the filter / output plugin only on lines with specified type. Example: ``only_type=nginx``
+* ``only_field_exist_toto``: execute the filter / output plugin only on lines with a field ``toto``. You can specify it multiple times, all fields have to exist.
+* ``only_field_equal_toto=aaa``: execute the filter / output plugin only on lines with a field ``toto``, with value ``aaa``. You can specify it multiple times, all fields have to exist and have the specified value.
Ouputs plugins
===
@@ -132,10 +134,19 @@ Example: ``output://zeromq://tcp://192.168.1.1:5555``, to send logs to 192.168.1
Elastic search
---
-This plugin is used on log server to send logs to elastic search.
+This plugin is used on log server to send logs to elastic search, using HTTP REST interface.
Example: ``output://elasticsearch://localhost:9001`` to send to the HTTP interface of an elastic search server listening on port 9001.
+Elastic search ZeroMQ
+---
+
+This plugin is used on log server to send logs to elastic search, using ZeroMQ transport.
+You can find the ZeroMQ transport here: https://github.com/bpaquet/transport-zeromq.
+
+Example: ``output://elasticsearch_zeromq://tcp://localhost:9700`` to send to the zeromq transport of an elastic search server listening on port 9700.
+
+
Statsd
---
View
@@ -14,6 +14,8 @@ BaseComponent.prototype.extendedLoadConfig = function(callback) {
}
BaseComponent.prototype.loadConfig = function(url, callback) {
+ this.message_filtering = {};
+
if (this.config.host_field || this.config.port_field || this.config.required_params || this.config.optional_params) {
if (url.length == 0) {
return this.emit('init_error', 'No empty url for ' + this.config.name);
@@ -69,6 +71,27 @@ BaseComponent.prototype.loadConfig = function(url, callback) {
}
}
+ if (this.parsed_url.params.only_type) {
+ this.message_filtering.only_type = this.parsed_url.params.only_type;
+ }
+
+ for(var i in this.parsed_url.params) {
+ var res = i.match(/^only_field_exist_(.+)$/);
+ if (res) {
+ if (!this.message_filtering.only_field_exist) {
+ this.message_filtering.only_field_exist = [];
+ }
+ this.message_filtering.only_field_exist.push(res[1]);
+ }
+ var res = i.match(/^only_field_equal_(.+)$/);
+ if (res) {
+ if (!this.message_filtering.only_field_equal) {
+ this.message_filtering.only_field_equal = {};
+ }
+ this.message_filtering.only_field_equal[res[1]] = this.parsed_url.params[i];
+ }
+ }
+
this.extendedLoadConfig(function(err) {
if (err) {
return callback(err);
@@ -92,9 +115,32 @@ BaseComponent.prototype.loadConfig = function(url, callback) {
}
BaseComponent.prototype.processMessage = function(data) {
- if (this.parsed_url.params.only_type && this.parsed_url.params.only_type != data['@type']) {
+ if (this.message_filtering.only_type && this.message_filtering.only_type != data['@type']) {
return false;
}
+
+ if (this.message_filtering.only_field_exist) {
+ if (! data['@fields']) {
+ return false;
+ }
+ for(var i = 0; i < this.message_filtering.only_field_exist.length; i ++) {
+ if (! data['@fields'][this.message_filtering.only_field_exist[i]]) {
+ return false;
+ }
+ }
+ }
+
+ if (this.message_filtering.only_field_equal) {
+ if (! data['@fields']) {
+ return false;
+ }
+ for(var i in this.message_filtering.only_field_equal) {
+ if (! data['@fields'][i] || data['@fields'][i] != this.message_filtering.only_field_equal[i]) {
+ return false;
+ }
+ }
+ }
+
return true;
}
@@ -15,9 +15,11 @@ vows.describe('Filter compute field ').addBatch({
{'@message': 'toto'},
{'@message': 'toto', '@fields': {'bouh': 'tata'}},
{'@message': 'toto', '@fields': {'bouh': 42}},
+ {'@message': 'toto', '@fields': {'bouh': 42, 'titi': 'abcdef'}},
], [
{'@message': 'toto'},
{'@message': 'toto', '@fields': {'bouh': 'tata', 'titi': 'abtata'}},
{'@message': 'toto', '@fields': {'bouh': 42, 'titi': 'ab42'}},
+ {'@message': 'toto', '@fields': {'bouh': 42, 'titi': 'ab42'}},
]),
}).export(module);
@@ -18,4 +18,44 @@ vows.describe('Message filtering ').addBatch({
{'@message': 'toto', '@type': 'tata'},
{'@message': 'toto', '@type': 'z', '@fields': {'titi': 'a'}},
]),
+ 'only field exist': filter_helper.create('compute_field', 'titi?value=a&only_field_exist_titi', [
+ {'@message': 'toto'},
+ {'@message': 'toto', '@fields': {'toto': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'b'}},
+ ], [
+ {'@message': 'toto'},
+ {'@message': 'toto', '@fields': {'toto': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'a'}},
+ ]),
+ 'multiple only field exist': filter_helper.create('compute_field', 'titi?value=a&only_field_exist_titi&only_field_exist_toto', [
+ {'@message': 'toto'},
+ {'@message': 'toto', '@fields': {'toto': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'b', 'toto': 'b'}},
+ ], [
+ {'@message': 'toto'},
+ {'@message': 'toto', '@fields': {'toto': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'a', 'toto': 'b'}},
+ ]),
+ 'only field equal': filter_helper.create('compute_field', 'titi?value=a&only_field_equal_titi=z', [
+ {'@message': 'toto'},
+ {'@message': 'toto', '@fields': {'titi': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'z'}},
+ ], [
+ {'@message': 'toto'},
+ {'@message': 'toto', '@fields': {'titi': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'a'}},
+ ]),
+ 'multiple only field equal': filter_helper.create('compute_field', 'titi?value=aa&only_field_equal_titi=a&only_field_equal_toto=b', [
+ {'@message': 'toto'},
+ {'@message': 'toto', '@fields': {'toto': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'a'}},
+ {'@message': 'toto', '@fields': {'titi': 'a', 'toto': 'b'}},
+ ], [
+ {'@message': 'toto'},
+ {'@message': 'toto', '@fields': {'toto': 'b'}},
+ {'@message': 'toto', '@fields': {'titi': 'a'}},
+ {'@message': 'toto', '@fields': {'titi': 'aa', 'toto': 'b'}},
+ ]),
}).export(module);

0 comments on commit b5e5013

Please sign in to comment.