Overhaul function/param descriptors and precompute constant function …

…arguments

To avoid reevaluating function arguments which never change for each message and are potentially expensive to calculate (either because their expression tree is deep or the function needs to construct other objects)  the Function interface now contains a callback to provide precomputed values for each constant argument.

This callback is only invoked for provably constant expressions, so it is safe to implement for any function.
The recommended way is to inherit from AbstractFunction<T> which implements the straightforward precompute function by simply memoizing the evaluated value.
For special cases, like the RegexMatch function, which precomputes a Pattern.compile from a constant parameter descriptors take a java.util.function.Function now, which gets invoked with the computed value. The default is the identity function.

Parameter descriptors are now responsible for evaluating argument expressions, and are fully generic.

src/main/java/org/graylog/plugins/pipelineprocessor/EvaluationContext.java

@@ -21,16 +21,33 @@
 import org.graylog2.plugin.Message;
 import org.graylog2.plugin.MessageCollection;
 import org.graylog2.plugin.Messages;
+import org.joda.time.DateTime;
 
 import java.util.List;
 import java.util.Map;
 
 public class EvaluationContext {
 
+    private static final EvaluationContext EMPTY_CONTEXT = new EvaluationContext() {
+        @Override
+        public void addCreatedMessage(Message newMessage) {
+            // cannot add messages to empty context
+        }
+
+        @Override
+        public void define(String identifier, Class type, Object value) {
+            // cannot define any variables in empty context
+        }
+    };
+
     private final Message message;
     private Map<String, TypedValue> ruleVars;
     private List<Message> createdMessages = Lists.newArrayList();
 
+    private EvaluationContext() {
+        this(new Message("__dummy", "__dummy", DateTime.parse("2010-07-30T16:03:25Z"))); // first Graylog release
+    }
+
     public EvaluationContext(Message message) {
         this.message = message;
         ruleVars = Maps.newHashMap();
@@ -60,6 +77,10 @@ public void clearCreatedMessages() {
         createdMessages.clear();
     }
 
+    public static EvaluationContext emptyContext() {
+        return EMPTY_CONTEXT;
+    }
+
     public class TypedValue {
         private final Class type;
         private final Object value;

src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorModule.java

@@ -24,6 +24,7 @@
 import org.graylog.plugins.pipelineprocessor.functions.DoubleCoercion;
 import org.graylog.plugins.pipelineprocessor.functions.FromInput;
 import org.graylog.plugins.pipelineprocessor.functions.LongCoercion;
+import org.graylog.plugins.pipelineprocessor.functions.RegexMatch;
 import org.graylog.plugins.pipelineprocessor.functions.StringCoercion;
 import org.graylog.plugins.pipelineprocessor.functions.messages.CreateMessage;
 import org.graylog.plugins.pipelineprocessor.functions.messages.DropMessage;
@@ -73,6 +74,9 @@ protected void configure() {
 
         // input related functions
         addMessageProcessorFunction(FromInput.NAME, FromInput.class);
+
+        // generic functions
+        addMessageProcessorFunction(RegexMatch.NAME, RegexMatch.class);
     }
 
     protected void addMessageProcessorFunction(String name, Class<? extends Function<?>> functionClass) {

src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java

@@ -31,7 +31,7 @@ public ArrayLiteralExpression(List<Expression> elements) {
 
     @Override
     public boolean isConstant() {
-        return false;
+        return elements.stream().allMatch(Expression::isConstant);
     }
 
     @Override

src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/EqualityExpression.java

@@ -38,7 +38,13 @@ public Class getType() {
 
     @Override
     public boolean evaluateBool(EvaluationContext context) {
-        final boolean equals = left.evaluate(context).equals(right.evaluate(context));
+        final Object left = this.left.evaluate(context);
+        final Object right = this.right.evaluate(context);
+        if (left == null) {
+            // TODO log error
+            return false;
+        }
+        final boolean equals = left.equals(right);
         if (checkEquality) {
             return equals;
         }

src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldAccessExpression.java

@@ -44,7 +44,11 @@ public Object evaluate(EvaluationContext context) {
         final Object bean = this.object.evaluate(context);
         final String fieldName = field.evaluate(context).toString();
         try {
-            final Object property = PropertyUtils.getProperty(bean, fieldName);
+            Object property = PropertyUtils.getProperty(bean, fieldName);
+            if (property == null) {
+                // in case the bean is a Map, try again with a simple property, it might be masked by the Map
+                property = PropertyUtils.getSimpleProperty(bean, fieldName);
+            }
             log.debug("[field access] property {} of bean {}: {}", fieldName, bean.getClass().getTypeName(), property);
             return property;
         } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {

src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FunctionExpression.java

@@ -27,10 +27,13 @@ public class FunctionExpression implements Expression {
     private final Function<?> function;
     private final FunctionDescriptor descriptor;
 
-    public FunctionExpression(Function<?> function, FunctionArgs args) {
+    public FunctionExpression(FunctionArgs args) {
         this.args = args;
-        this.function = function;
-        this.descriptor = function.descriptor();
+        this.function = args.getFunction();
+        this.descriptor = this.function.descriptor();
+
+        // precomputes all constant arguments to avoid dynamically recomputing trees on every invocation
+        this.function.preprocessArgs(args);
     }
 
     public Function<?> getFunction() {

src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MapLiteralExpression.java

@@ -33,7 +33,7 @@ public MapLiteralExpression(HashMap<String, Expression> map) {
 
     @Override
     public boolean isConstant() {
-        return false;
+        return map.values().stream().allMatch(Expression::isConstant);
     }
 
     @Override

src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java

@@ -0,0 +1,17 @@
+package org.graylog.plugins.pipelineprocessor.ast.functions;
+
+import org.graylog.plugins.pipelineprocessor.EvaluationContext;
+import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression;
+
+/**
+ * Helper Function implementation which evaluates and memoizes all constant FunctionArgs.
+ *
+ * @param <T> the return type
+ */
+public abstract class AbstractFunction<T> implements Function<T> {
+
+    @Override
+    public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) {
+        return arg.evaluate(EvaluationContext.emptyContext());
+    }
+}

src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/Function.java

@@ -18,10 +18,17 @@
 
 import com.google.common.collect.ImmutableList;
 import org.graylog.plugins.pipelineprocessor.EvaluationContext;
+import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
 
 public interface Function<T> {
 
-    Function ERROR_FUNCTION = new Function<Void>() {
+    Logger log = LoggerFactory.getLogger(Function.class);
+
+    Function ERROR_FUNCTION = new AbstractFunction<Void>() {
         @Override
         public Void evaluate(FunctionArgs args, EvaluationContext context) {
             return null;
@@ -37,6 +44,32 @@ public FunctionDescriptor<Void> descriptor() {
         }
     };
 
+    default void preprocessArgs(FunctionArgs args) {
+        for (Map.Entry<String, Expression> e : args.getConstantArgs().entrySet()) {
+            try {
+                final Object value = preComputeConstantArgument(args, e.getKey(), e.getValue());
+                if (value != null) {
+                    args.setPreComputedValue(e.getKey(), value);
+                }
+            } catch (Exception exception) {
+                log.warn("Unable to precompute argument value for " + e.getKey(), exception);
+            }
+        }
+
+    }
+
+    /**
+     * Implementations should provide a non-null value for each argument they wish to pre-compute.
+     * <br/>
+     * Examples include compile a Pattern from a regex string, which will never change during the lifetime of the function.
+     * If any part of the expression tree depends on external values this method will not be called, e.g. if the regex depends on a message field.
+     * @param args the function args for this functions, usually you don't need this
+     * @param name the name of the argument to potentially precompute
+     * @param arg the expression tree for the argument
+     * @return the precomputed value for the argument or <code>null</code> if the value should be dynamically calculated for each invocation
+     */
+    Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg);
+
     T evaluate(FunctionArgs args, EvaluationContext context);
 
     FunctionDescriptor<T> descriptor();

src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionArgs.java

@@ -16,14 +16,17 @@
  */
 package org.graylog.plugins.pipelineprocessor.ast.functions;
 
+import com.google.common.collect.Maps;
 import org.graylog.plugins.pipelineprocessor.EvaluationContext;
 import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.Collections;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 
@@ -32,7 +35,13 @@ public class FunctionArgs {
     @Nonnull
     private final Map<String, Expression> args;
 
-    public FunctionArgs(Map<String, Expression> args) {
+    private final Map<String, Object> constantValues = Maps.newHashMap();
+    private final Function function;
+    private final FunctionDescriptor descriptor;
+
+    public FunctionArgs(Function func, Map<String, Expression> args) {
+        function = func;
+        descriptor = function.descriptor();
         this.args = firstNonNull(args, Collections.emptyMap());
     }
 
@@ -42,13 +51,34 @@ public Map<String, Expression> getArgs() {
     }
 
     @Nonnull
+    public Map<String, Expression> getConstantArgs() {
+        return args.entrySet().stream()
+                .filter(e -> e.getValue().isConstant())
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+    }
+
+    @Nonnull
+    @Deprecated
     public <T> Optional<T> evaluated(String name, EvaluationContext context, Class<T> argumentType) {
+        return Optional.ofNullable(required(name, context, argumentType));
+    }
+
+    @Nullable
+    @Deprecated
+    public <T> T required(String name, EvaluationContext context, Class<T> argumentType) {
+        final ParameterDescriptor param = descriptor.param(name);
+
+        final Object precomputedValue = constantValues.get(name);
+        if (precomputedValue != null) {
+            return (T)param.transformedType().cast(precomputedValue);
+        }
         final Expression valueExpr = expression(name);
         if (valueExpr == null) {
-            return Optional.empty();
+            return null;
         }
         final Object value = valueExpr.evaluate(context);
-        return Optional.ofNullable(argumentType.cast(value));
+        final Object transformed = param.transform().apply(value);
+        return (T)param.transformedType().cast(transformed);
     }
 
     public boolean isPresent(String key) {
@@ -60,4 +90,20 @@ public Expression expression(String key) {
         return args.get(key);
     }
 
+    public Object getPreComputedValue(String name) {
+        return constantValues.get(name);
+    }
+
+    public void setPreComputedValue(@Nonnull String name, @Nonnull Object value) {
+        Objects.requireNonNull(value);
+        constantValues.put(name, value);
+    }
+
+    public Function<?> getFunction() {
+        return function;
+    }
+
+    public ParameterDescriptor<?, ?> param(String name) {
+        return descriptor.param(name);
+    }
 }

src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/FunctionDescriptor.java

@@ -18,6 +18,8 @@
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 
 @AutoValue
 public abstract class FunctionDescriptor<T> {
@@ -26,22 +28,35 @@ public abstract class FunctionDescriptor<T> {
 
     public abstract boolean pure();
 
-    public abstract Class<T> returnType();
+    public abstract Class<? extends T> returnType();
 
     public abstract ImmutableList<ParameterDescriptor> params();
 
+    public abstract ImmutableMap<String, ParameterDescriptor> paramMap();
+
+    public ParameterDescriptor param(String name) {
+        return paramMap().get(name);
+    }
+
     public static <T> Builder<T> builder() {
         //noinspection unchecked
         return new AutoValue_FunctionDescriptor.Builder().pure(false);
     }
 
     @AutoValue.Builder
     public static abstract class Builder<T> {
-        public abstract FunctionDescriptor<T> build();
+        public abstract FunctionDescriptor<T> autoBuild();
+
+        public FunctionDescriptor<T> build() {
+            return paramMap(Maps.uniqueIndex(params(), ParameterDescriptor::name))
+                    .autoBuild();
+        }
 
         public abstract Builder<T> name(String name);
         public abstract Builder<T> pure(boolean pure);
-        public abstract Builder<T> returnType(Class<T> type);
+        public abstract Builder<T> returnType(Class<? extends T> type);
         public abstract Builder<T> params(ImmutableList<ParameterDescriptor> params);
+        public abstract Builder<T> paramMap(ImmutableMap<String, ParameterDescriptor> map);
+        public abstract ImmutableList<ParameterDescriptor> params();
     }
 }