Add new pipeline function "grok_exists" (#5699)

Prior to this change, a missing grok pattern would
raise a error in the pipeline processor when using the "grok"
function. But the user would like to able to make one rule
which uses a grok pattern dynamically depending on if
a grok pattern exists or not.

This change adds a new function "grok_exists" which will
return true or false depending if a grok pattern exists.
Additionally it will make a entry to the graylog-server.log
if the second argument of the function is true and
the pattern was not found.

Fixes #5689

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/GrokExists.java

@@ -0,0 +1,79 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog.plugins.pipelineprocessor.functions;
+
+import org.graylog.plugins.pipelineprocessor.EvaluationContext;
+import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction;
+import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
+import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
+import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
+import org.graylog2.grok.GrokPatternRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.collect.ImmutableList.of;
+
+import javax.inject.Inject;
+
+public class GrokExists extends AbstractFunction<Boolean> {
+
+    private static final Logger log = LoggerFactory.getLogger(GrokExists.class);
+    public static final String NAME = "grok_exists";
+
+    private final ParameterDescriptor<String, String> patternParam;
+    private final ParameterDescriptor<Boolean, Boolean> doLog;
+
+    private final GrokPatternRegistry grokPatternRegistry;
+
+    @Inject
+    public GrokExists(GrokPatternRegistry grokPatternRegistry) {
+        this.grokPatternRegistry = grokPatternRegistry;
+
+        patternParam = ParameterDescriptor.string("pattern")
+                .description("The Grok Pattern which is to be tested for existance.").build();
+        doLog = ParameterDescriptor.bool("log_missing").optional()
+                .description("Log if the Grok Pattern is missing. Warning: Switching on this flag can lead" +
+                        " to a high volume of logs.").build();
+    }
+
+    @Override
+    public Boolean evaluate(FunctionArgs args, EvaluationContext context) {
+        final String pattern = patternParam.required(args, context);
+        final boolean logWhenNotFound = doLog.optional(args, context).orElse(false);
+
+        if (pattern == null) {
+            return null;
+        }
+
+        final boolean patternExists = grokPatternRegistry.grokPatternExists(pattern);
+        if (!patternExists && logWhenNotFound) {
+           log.info("Grok Pattern " + pattern + " does not exists.");
+        }
+
+        return patternExists;
+    }
+
+    @Override
+    public FunctionDescriptor<Boolean> descriptor() {
+       return FunctionDescriptor.<Boolean>builder()
+               .name(NAME)
+               .returnType(Boolean.class)
+               .params(of(patternParam, doLog))
+               .description("Checks if the given Grok pattern exists.")
+               .build();
+    }
+}

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/ProcessorFunctionsModule.java

@@ -164,6 +164,7 @@ protected void configure() {
         addMessageProcessorFunction(RegexMatch.NAME, RegexMatch.class);
         addMessageProcessorFunction(RegexReplace.NAME, RegexReplace.class);
         addMessageProcessorFunction(GrokMatch.NAME, GrokMatch.class);
+        addMessageProcessorFunction(GrokExists.NAME, GrokExists.class);
 
         // string functions
         addMessageProcessorFunction(Abbreviate.NAME, Abbreviate.class);

graylog2-server/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/GrokMatch.java

@@ -92,6 +92,4 @@ public boolean isMatches() {
             return captures.size() > 0;
         }
     }
-
-
 }

graylog2-server/src/main/java/org/graylog2/grok/GrokPatternRegistry.java

@@ -83,6 +83,10 @@ public void grokPatternsDeleted(GrokPatternsDeletedEvent event) {
         reload();
     }
 
+    public boolean grokPatternExists(String patternName) {
+        return patterns.get().stream().anyMatch(pattern -> pattern.name().contains(patternName));
+    }
+
     public Grok cachedGrokForPattern(String pattern) {
         return cachedGrokForPattern(pattern, false);
     }

graylog2-server/src/test/java/org/graylog/plugins/pipelineprocessor/functions/FunctionsSnippetsTest.java

@@ -34,6 +34,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Executors;
 import javax.inject.Provider;
@@ -301,19 +302,23 @@ public static void registerFunctions() {
         functions.put(IsUrl.NAME, new IsUrl());
 
         final GrokPatternService grokPatternService = mock(GrokPatternService.class);
+        final GrokPattern greedyPattern = GrokPattern.create("GREEDY", ".*");
         Set<GrokPattern> patterns = Sets.newHashSet(
+                greedyPattern,
                 GrokPattern.create("GREEDY", ".*"),
                 GrokPattern.create("BASE10NUM", "(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))"),
                 GrokPattern.create("NUMBER", "(?:%{BASE10NUM:UNWANTED})"),
                 GrokPattern.create("UNDERSCORE", "(?<test_field>test)"),
                 GrokPattern.create("NUM", "%{BASE10NUM}")
         );
         when(grokPatternService.loadAll()).thenReturn(patterns);
+        when(grokPatternService.loadByName("GREEDY")).thenReturn(Optional.of(greedyPattern));
         final EventBus clusterBus = new EventBus();
         final GrokPatternRegistry grokPatternRegistry = new GrokPatternRegistry(clusterBus,
                                                                                 grokPatternService,
                                                                                 Executors.newScheduledThreadPool(1));
         functions.put(GrokMatch.NAME, new GrokMatch(grokPatternRegistry));
+        functions.put(GrokExists.NAME, new GrokExists(grokPatternRegistry));
 
         functionRegistry = new FunctionRegistry(functions);
     }
@@ -469,6 +474,22 @@ public void digests() {
         assertThat(actionsTriggered.get()).isTrue();
     }
 
+    @Test
+    public void grok_exists() {
+        final Rule rule = parser.parseRule(ruleForTest(), false);
+        evaluateRule(rule);
+
+        assertThat(actionsTriggered.get()).isTrue();
+    }
+
+    @Test
+    public void grok_exists_not() {
+        final Rule rule = parser.parseRule(ruleForTest(), false);
+        evaluateRule(rule);
+
+        assertThat(actionsTriggered.get()).isFalse();
+    }
+
     @Test
     public void encodings() {
         final Rule rule = parser.parseRule(ruleForTest(), false);

graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok_exists.txt

@@ -0,0 +1,6 @@
+rule "grok_exists"
+when
+    grok_exists("GREEDY")
+then
+    trigger_test();
+end

graylog2-server/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok_exists_not.txt

@@ -0,0 +1,6 @@
+rule "grok_exists_not"
+when
+    grok_exists("DOESNOTEXISTS")
+then
+    trigger_test();
+end