initial support for grok patterns in extractors

* change the internal extractor interfaces to support multiple results, and make the original behavior a special case of that
* no outside visible changes are made, all existing extractors will just work as before
* opportunity to support multiple matcher groups is there, but needs significant ui work
* the grok pattern list is static right now
* missing tests for corner cases

graylog2-plugin-interfaces/src/main/java/org/graylog2/plugin/inputs/Extractor.java

@@ -24,13 +24,18 @@
 
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.Timer;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.database.EmbeddedPersistable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
@@ -55,12 +60,14 @@ public abstract class Extractor implements EmbeddedPersistable {
     public static final String FIELD_CONVERTERS = "converters";
     public static final String FIELD_CONVERTER_TYPE = "type";
     public static final String FIELD_CONVERTER_CONFIG = "config";
+    public static final ResultPredicate VALUE_NULL_PREDICATE = new ResultPredicate();
 
     public enum Type {
         SUBSTRING,
         REGEX,
         SPLIT_AND_INDEX,
-        COPY_INPUT
+        COPY_INPUT,
+        GROK
     }
 
     public enum CursorStrategy {
@@ -96,7 +103,7 @@ public enum ConditionType {
     private final String totalTimerName;
     private final String converterTimerName;
 
-    protected abstract Result run(String field);
+    protected abstract Result[] run(String field);
 
     protected final MetricRegistry metricRegistry;
 
@@ -148,7 +155,7 @@ public void runExtractor(Message msg) {
             return;
         }
 
-        String field = (String) msg.getField(sourceField);
+        final String field = (String) msg.getField(sourceField);
 
         // Decide if to extract at all.
         if (conditionType.equals(ConditionType.STRING)) {
@@ -163,28 +170,44 @@ public void runExtractor(Message msg) {
 
         final Timer.Context timerContext = metricRegistry.timer(getTotalTimerName()).time();
 
-        Result result = run(field);
+        final Result[] results = run(field);
 
-        if (result == null || result.getValue() == null) {
+        if (results == null || results.length == 0 || FluentIterable.of(results).anyMatch(VALUE_NULL_PREDICATE)) {
             timerContext.close();
             return;
+        } else if (results.length == 1) {
+            msg.addField(targetField, results[0].getValue());
         } else {
-            msg.addField(targetField, result.getValue());
+            for (final Result result : results) {
+                msg.addField(result.getTarget(), result.getValue());
+            }
         }
 
         // Remove original from message?
-        if (cursorStrategy.equals(CursorStrategy.CUT) && !targetField.equals(sourceField) && !Message.RESERVED_FIELDS.contains(sourceField)) {
-            StringBuilder sb = new StringBuilder(field);
-
-            sb.delete(result.getBeginIndex(), result.getEndIndex());
+        if (cursorStrategy.equals(CursorStrategy.CUT) && !targetField.equals(sourceField) && !Message.RESERVED_FIELDS.contains(sourceField) && results[0].beginIndex != -1) {
+            final StringBuilder sb = new StringBuilder(field);
+
+            final ImmutableList<Result> reverseList = FluentIterable.from(Arrays.asList(results)).toSortedList(new Comparator<Result>() {
+                @Override
+                public int compare(Result left, Result right) {
+                    // reversed!
+                    return -1 * ComparisonChain.start().compare(left.endIndex, right.endIndex).result();
+                }
+            });
+            // remove all from reverse so that the indices still match
+            for (final Result result : reverseList) {
+                sb.delete(result.getBeginIndex(), result.getEndIndex());
+            }
 
             String finalResult = sb.toString();
 
-            if (finalResult.isEmpty()) {
+            // also ignore pure whitespace
+            if (finalResult.trim().isEmpty()) {
                 finalResult = "fullyCutByExtractor";
             }
 
             msg.removeField(sourceField);
+            // TODO don't add an empty field back, or rather don't add fullyCutByExtractor
             msg.addField(sourceField, finalResult);
         }
 
@@ -212,7 +235,7 @@ public void runConverters(Message msg) {
                     @SuppressWarnings("unchecked")
                     final Map<String, Object> convert =
                             (Map<String, Object>) converter.convert((String) msg.getField(targetField));
-                    for (String reservedField : Message.RESERVED_FIELDS) {
+                    for (final String reservedField : Message.RESERVED_FIELDS) {
                         if (convert.containsKey(reservedField)) {
                             if (LOG.isDebugEnabled()) {
                                 LOG.debug(
@@ -347,11 +370,17 @@ public void incrementExceptions() {
     public static class Result {
 
         private final String value;
+        private final String target;
         private final int beginIndex;
         private final int endIndex;
 
         public Result(String value, int beginIndex, int endIndex) {
+            this(value, null, beginIndex, endIndex);
+        }
+
+        public Result(String value, String target, int beginIndex, int endIndex) {
             this.value = value;
+            this.target = target;
             this.beginIndex = beginIndex;
             this.endIndex = endIndex;
         }
@@ -360,6 +389,10 @@ public String getValue() {
             return value;
         }
 
+        public String getTarget() {
+            return target;
+        }
+
         public int getBeginIndex() {
             return beginIndex;
         }
@@ -369,4 +402,9 @@ public int getEndIndex() {
         }
 
     }
+
+    private static class ResultPredicate implements Predicate<Result> {
+        @Override
+        public boolean apply(Result input) { return input.getValue() == null; }
+    }
 }

graylog2-rest-client/src/main/java/org/graylog2/restclient/models/Extractor.java

@@ -55,7 +55,8 @@ public enum Type {
         SUBSTRING("Substring"),
         REGEX("Regular expression"),
         SPLIT_AND_INDEX("Split & Index"),
-        COPY_INPUT("Copy Input");
+        COPY_INPUT("Copy Input"),
+        GROK("Grok pattern");
         private final String description;
 
         Type(String description) {
@@ -201,6 +202,8 @@ public void loadConfigFromForm(Type extractorType, Map<String,String[]> form) {
                 break;
             case SPLIT_AND_INDEX:
                 loadSplitAndIndexConfig(form);
+            case GROK:
+                loadGrokConfig(form);
                 break;
         }
     }
@@ -333,6 +336,14 @@ private void loadSplitAndIndexConfig(Map<String,String[]> form) {
         extractorConfig.put("split_by", form.get("split_by")[0]);
         extractorConfig.put("index", Integer.parseInt(form.get("index")[0]));
     }
+
+    private void loadGrokConfig(Map<String,String[]> form) {
+        if (!formFieldSet(form, "grok_pattern")) {
+            throw new RuntimeException("Missing extractor config: grok_pattern");
+        }
+
+        extractorConfig.put("grok_pattern", form.get("grok_pattern")[0]);
+    }
 
     private boolean formFieldSet(Map<String,String[]> form, String key) {
         return form.get(key) != null && form.get(key)[0] != null && !form.get(key)[0].isEmpty();

graylog2-rest-client/src/main/java/org/graylog2/restclient/models/api/responses/GrokTestResponse.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2015 TORCH GmbH
+ *
+ * This file is part of Graylog2.
+ *
+ * Graylog2 is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog2 is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog2.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.restclient.models.api.responses;
+
+import java.util.List;
+
+public class GrokTestResponse {
+    public String pattern;
+    public boolean matched;
+    public String string;
+    public List<Match> matches;
+
+    // must be static for jackson deser
+    public static class Match {
+        public String name;
+        public String match;
+    }
+}

graylog2-server/pom.xml

@@ -277,6 +277,10 @@
             <artifactId>commons-codec</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>io.thekraken</groupId>
+            <artifactId>grok</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.atlassian.ip</groupId>
             <artifactId>atlassian-ip</artifactId>

graylog2-server/src/main/java/org/graylog2/inputs/extractors/CopyInputExtractor.java

@@ -41,7 +41,7 @@ public CopyInputExtractor(MetricRegistry metricRegistry, String id, String title
     }
 
     @Override
-    protected Result run(String value) {
-        return new Result(value, 0, value.length());
+    protected Result[] run(String value) {
+        return new Result[] { new Result(value, 0, value.length())};
     }
 }

graylog2-server/src/main/java/org/graylog2/inputs/extractors/ExtractorFactory.java

@@ -17,11 +17,11 @@
 package org.graylog2.inputs.extractors;
 
 import com.codahale.metrics.MetricRegistry;
-import javax.inject.Inject;
 import org.graylog2.ConfigurationException;
 import org.graylog2.plugin.inputs.Converter;
 import org.graylog2.plugin.inputs.Extractor;
 
+import javax.inject.Inject;
 import java.util.List;
 import java.util.Map;
 
@@ -36,7 +36,6 @@ public ExtractorFactory(MetricRegistry metricRegistry) {
         this.metricRegistry = metricRegistry;
     }
 
-    // TODO: This parameter list is growing a bit out of control.
     public Extractor factory(String id,
                                     String title,
                                     int order,
@@ -50,6 +49,7 @@ public Extractor factory(String id,
                                     String conditionValue)
             throws NoSuchExtractorException, Extractor.ReservedFieldException, ConfigurationException {
 
+        // TODO convert to guice factory
         switch (type) {
             case REGEX:
                 return new RegexExtractor(metricRegistry, id, title, order, cursorStrategy, sourceField, targetField, extractorConfig, creatorUserId, converters, conditionType, conditionValue);
@@ -59,6 +59,8 @@ public Extractor factory(String id,
                 return new SplitAndIndexExtractor(metricRegistry, id, title, order, cursorStrategy, sourceField, targetField, extractorConfig, creatorUserId, converters, conditionType, conditionValue);
             case COPY_INPUT:
                 return new CopyInputExtractor(metricRegistry, id, title, order, cursorStrategy, sourceField, targetField, extractorConfig, creatorUserId, converters, conditionType, conditionValue);
+            case GROK:
+                return new GrokExtrator(metricRegistry, id, title, order, cursorStrategy, sourceField, targetField, extractorConfig, creatorUserId, converters, conditionType, conditionValue);
             default:
                 throw new NoSuchExtractorException();
         }