ncm-metaconfig: logstash: deprecate grep filter, replace with conditi…

…onal drop
quattor · Dec 7, 2015 · 99a5a46 · 99a5a46
1 parent e052cd3
commit 99a5a46
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 1 deletion.
diff --git a/ncm-metaconfig/src/main/metaconfig/logstash/2.0/tests/profiles/config.pan b/ncm-metaconfig/src/main/metaconfig/logstash/2.0/tests/profiles/config.pan
@@ -95,6 +95,20 @@ prefix "/software/components/metaconfig/services/{/etc/logstash/conf.d/logstash.
             "patterns_dir", list("/usr/share/grok"),
             "add_field", nlist("program", "gpfs"),
             )),
+        nlist('drop', nlist(
+            "_conditional", nlist("expr", list(nlist(
+                "left", "[sometag]",
+                "test", "!~",
+                "right", "'^SOME_STRING'",
+                ))),
+            "percentage", 80,
+            "periodic_flush", true,
+            )),
+        nlist('drop', nlist("_conditional", nlist("expr", list(nlist(
+            "left", "[someothertag]",
+            "test", "=~",
+            "right", "'^SOME_OTHERSTRING'",
+            ))))),
         nlist("date", nlist(
             "match", nlist(
                 "name", "timestamp", 

diff --git a/ncm-metaconfig/src/main/metaconfig/logstash/2.0/tests/regexps/config/base b/ncm-metaconfig/src/main/metaconfig/logstash/2.0/tests/regexps/config/base
@@ -36,6 +36,16 @@ filter \{$
 ^\s{16}"message" => "%\{GPFSLOG\}"$
 ^\s{12}\}$
 ^\s{8}\}$
+^\s{8}if \(\[sometag\] !~ '\^SOME_STRING'\) {
+^\s{12}drop \{$
+^\s{16}periodic_flush => true$
+^\s{16}percentage => 80$
+^\s{12}\}$
+^\s{8}\}$
+^\s{8}if \(\[someothertag\] =~ '\^SOME_OTHERSTRING'\) \{$
+^\s{12}drop \{$
+^\s{12}\}$
+^\s{8}\}$
 ^\s{8}date \{$
 ^\s{12}match => \[ "timestamp", "E MMM dd HH:mm:ss.SSS yyyy", "E MMM  d HH:mm:ss.SSS yyyy" \]$
 ^\s{8}\}$

diff --git a/ncm-metaconfig/src/main/metaconfig/logstash/config/filter/drop.tt b/ncm-metaconfig/src/main/metaconfig/logstash/config/filter/drop.tt
@@ -0,0 +1,2 @@
+[%- INCLUDE "metaconfig/logstash/config/type.tt" type="boolean" names=['periodic_flush'] -%]
+[%- INCLUDE "metaconfig/logstash/config/type.tt" type="exact" names=['percentage'] -%]
diff --git a/ncm-metaconfig/src/main/metaconfig/logstash/pan/schema.pan b/ncm-metaconfig/src/main/metaconfig/logstash/pan/schema.pan
@@ -173,6 +173,12 @@ type logstash_filter_grep = {
     "negate" : boolean = false
 };
 
+type logstash_filter_drop = {
+    include logstash_filter_plugin_common
+    "percentage" ? long(0..100)
+    "periodic_flush" ? boolean
+};
+
 type logstash_filter_mutate = {
     include logstash_filter_plugin_common
     "convert" ? logstash_name_pattern[]
@@ -198,7 +204,8 @@ type logstash_filter_kv = {
 type logstash_filter_plugin = {
     "grok" ? logstash_filter_grok
     "date" ? logstash_filter_date
-    "grep" ? logstash_filter_grep
+    "grep" ? logstash_filter_grep with {deprecated(0, 'grep filter is removed from 2.0, use e.g. conditional drop'); true;}
+    "drop" ? logstash_filter_drop
     "mutate" ? logstash_filter_mutate
     "kv" ? logstash_filter_kv
     "bytes2human" ? logstash_filter_bytes2human