Add ordering of decorators and decoration stats to UI

graylog2-server/src/main/java/org/graylog2/decorators/Decorator.java

@@ -24,4 +24,5 @@ public interface Decorator {
     String type();
     Optional<String> stream();
     Map<String, Object> config();
+    int order();
 }

graylog2-server/src/main/java/org/graylog2/decorators/DecoratorImpl.java

@@ -31,11 +31,21 @@
 @AutoValue
 @JsonAutoDetect
 @CollectionName("decorators")
-public abstract class DecoratorImpl implements Decorator {
+public abstract class DecoratorImpl implements Decorator, Comparable {
     static final String FIELD_ID = "_id";
     static final String FIELD_TYPE = "type";
     static final String FIELD_CONFIG = "config";
     static final String FIELD_STREAM = "stream";
+    static final String FIELD_ORDER = "order";
+
+    @Override
+    public int compareTo(Object o) {
+        if (o instanceof Decorator) {
+            Decorator decorator = (Decorator)o;
+            return order() - decorator.order();
+        }
+        return 0;
+    }
 
     @JsonProperty(FIELD_ID)
     @ObjectId
@@ -55,30 +65,38 @@ public abstract class DecoratorImpl implements Decorator {
     @Override
     public abstract Optional<String> stream();
 
+    @JsonProperty(FIELD_ORDER)
+    @Override
+    public abstract int order();
+
     public abstract Builder toBuilder();
 
     @JsonCreator
     public static DecoratorImpl create(@JsonProperty(FIELD_ID) @Nullable String id,
                                        @JsonProperty(FIELD_TYPE) String type,
                                        @JsonProperty(FIELD_CONFIG) Map<String, Object> config,
-                                       @JsonProperty(FIELD_STREAM) Optional<String> stream) {
+                                       @JsonProperty(FIELD_STREAM) Optional<String> stream,
+                                       @JsonProperty(FIELD_ORDER) int order) {
         return new AutoValue_DecoratorImpl.Builder()
             .id(id)
             .type(type)
             .config(config)
             .stream(stream)
+            .order(order)
             .build();
     }
 
     public static Decorator create(@JsonProperty(FIELD_TYPE) String type,
                                    @JsonProperty(FIELD_CONFIG) Map<String, Object> config,
-                                   @JsonProperty(FIELD_STREAM) Optional<String> stream) {
-        return create(null, type, config, stream);
+                                   @JsonProperty(FIELD_STREAM) Optional<String> stream,
+                                   @JsonProperty(FIELD_ORDER) int order) {
+        return create(null, type, config, stream, order);
     }
 
     public static Decorator create(@JsonProperty(FIELD_TYPE) String type,
-                                   @JsonProperty(FIELD_CONFIG) Map<String, Object> config) {
-        return create(type, config, Optional.empty());
+                                   @JsonProperty(FIELD_CONFIG) Map<String, Object> config,
+                                   @JsonProperty(FIELD_ORDER) int order) {
+        return create(type, config, Optional.empty(), order);
     }
 
     @AutoValue.Builder
@@ -87,6 +105,7 @@ public abstract static class Builder {
         abstract Builder type(String type);
         abstract Builder config(Map<String, Object> config);
         abstract Builder stream(Optional<String> stream);
+        abstract Builder order(int order);
         public abstract DecoratorImpl build();
     }
 }

graylog2-server/src/main/java/org/graylog2/decorators/DecoratorProcessorImpl.java

@@ -16,12 +16,22 @@
  */
 package org.graylog2.decorators;
 
+import com.google.common.collect.Sets;
+import org.graylog2.plugin.Message;
 import org.graylog2.plugin.decorators.SearchResponseDecorator;
+import org.graylog2.rest.models.messages.responses.DecorationStats;
+import org.graylog2.rest.models.messages.responses.ResultMessageSummary;
+import org.graylog2.rest.resources.search.responses.SearchDecorationStats;
 import org.graylog2.rest.resources.search.responses.SearchResponse;
 
 import javax.inject.Inject;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class DecoratorProcessorImpl implements DecoratorProcessor {
     private final DecoratorResolver decoratorResolver;
@@ -36,11 +46,45 @@ public SearchResponse decorate(SearchResponse searchResponse, Optional<String> s
         final List<SearchResponseDecorator> searchResponseDecorators = streamId.isPresent() ?
             decoratorResolver.searchResponseDecoratorsForStream(streamId.get()) : decoratorResolver.searchResponseDecoratorsForGlobal();
         final Optional<SearchResponseDecorator> metaDecorator = searchResponseDecorators.stream()
-            .reduce((f, g) -> (v) -> f.apply(g.apply(v)));
+            .reduce((f, g) -> (v) -> g.apply(f.apply(v)));
         if (metaDecorator.isPresent()) {
-            return metaDecorator.get().apply(searchResponse);
+            final Map<String, ResultMessageSummary> originalMessages = searchResponse.messages()
+                .stream()
+                .collect(Collectors.toMap(message -> message.message().get("_id").toString(), Function.identity()));
+            final SearchResponse newSearchResponse = metaDecorator.get().apply(searchResponse);
+            final Set<String> newFields = extractFields(newSearchResponse.messages());
+            final Set<String> addedFields = Sets.difference(newFields, searchResponse.fields())
+                .stream()
+                .filter(field -> !Message.RESERVED_FIELDS.contains(field) && !field.equals("streams"))
+                .collect(Collectors.toSet());
+
+            final List<ResultMessageSummary> decoratedMessages = newSearchResponse.messages()
+                .stream()
+                .map(resultMessage -> {
+                    final ResultMessageSummary originalMessage = originalMessages.get(resultMessage.message().get("_id").toString());
+                    if (originalMessage != null) {
+                        return resultMessage
+                            .toBuilder()
+                            .decorationStats(DecorationStats.create(originalMessage.message(), resultMessage.message()))
+                            .build();
+                    }
+                    return resultMessage;
+                })
+                .collect(Collectors.toList());
+            return newSearchResponse
+                .toBuilder()
+                .messages(decoratedMessages)
+                .fields(newFields)
+                .decorationStats(SearchDecorationStats.create(addedFields))
+                .build();
         }
 
         return searchResponse;
     }
+
+    private Set<String> extractFields(List<ResultMessageSummary> messages) {
+        return messages.stream()
+            .map(message -> message.message().keySet())
+            .reduce(new HashSet<>(), (set1, set2) -> { set1.addAll(set2); return set1; });
+    }
 }

graylog2-server/src/main/java/org/graylog2/decorators/DecoratorResolver.java

@@ -39,14 +39,18 @@ public DecoratorResolver(DecoratorService decoratorService,
     }
 
     public List<SearchResponseDecorator> searchResponseDecoratorsForStream(String streamId) {
-        return this.decoratorService.findForStream(streamId).stream()
+        return this.decoratorService.findForStream(streamId)
+            .stream()
+            .sorted()
             .map(this::instantiateSearchResponseDecorator)
             .filter(Objects::nonNull)
             .collect(Collectors.toList());
     }
 
     public List<SearchResponseDecorator> searchResponseDecoratorsForGlobal() {
-        return this.decoratorService.findForGlobal().stream()
+        return this.decoratorService.findForGlobal()
+            .stream()
+            .sorted()
             .map(this::instantiateSearchResponseDecorator)
             .filter(Objects::nonNull)
             .collect(Collectors.toList());

graylog2-server/src/main/java/org/graylog2/decorators/DecoratorService.java

@@ -28,8 +28,8 @@ public interface DecoratorService {
     List<Decorator> findForGlobal();
     List<Decorator> findAll();
     Decorator findById(String decoratorId) throws NotFoundException;
-    Decorator create(String type, Map<String, Object> config, String stream);
-    Decorator create(String type, Map<String, Object> config);
+    Decorator create(String type, Map<String, Object> config, String stream, int order);
+    Decorator create(String type, Map<String, Object> config, int order);
     Decorator save(Decorator decorator);
     int delete(String decoratorId);
 }

graylog2-server/src/main/java/org/graylog2/decorators/DecoratorServiceImpl.java

@@ -72,13 +72,13 @@ public List<Decorator> findAll() {
     }
 
     @Override
-    public Decorator create(String type, Map<String, Object> config, String stream) {
-        return DecoratorImpl.create(type, config, Optional.of(stream));
+    public Decorator create(String type, Map<String, Object> config, String stream, int order) {
+        return DecoratorImpl.create(type, config, Optional.of(stream), order);
     }
 
     @Override
-    public Decorator create(String type, Map<String, Object> config) {
-        return DecoratorImpl.create(type, config);
+    public Decorator create(String type, Map<String, Object> config, int order) {
+        return DecoratorImpl.create(type, config, order);
     }
 
     @Override

graylog2-server/src/main/java/org/graylog2/plugin/decorators/SearchResponseDecorator.java

@@ -42,8 +42,8 @@ protected Descriptor() {
             throw new IllegalStateException("This class should not be instantiated directly, this is a bug.");
         }
 
-        public Descriptor(String name, boolean exclusive, String linkToDocs, String humanName) {
-            super(name, exclusive, linkToDocs);
+        public Descriptor(String name, String linkToDocs, String humanName) {
+            super(name, false, linkToDocs);
             this.humanName = humanName;
         }
 

graylog2-server/src/main/java/org/graylog2/rest/models/messages/responses/DecorationStats.java

@@ -0,0 +1,73 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest.models.messages.responses;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.Sets;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class DecorationStats {
+    private static final String FIELD_ADDED_FIELDS = "added_fields";
+    private static final String FIELD_CHANGED_FIELDS = "changed_fields";
+    private static final String FIELD_REMOVED_FIELDS = "removed_fields";
+
+    @JsonIgnore
+    public abstract Map<String, Object> originalMessage();
+
+    @JsonIgnore
+    public abstract Map<String, Object> decoratedMessage();
+
+    @SuppressWarnings("unused")
+    @JsonProperty(FIELD_ADDED_FIELDS)
+    public Map<String, Object> addedFields() {
+        return Sets.difference(decoratedMessage().keySet(), originalMessage().keySet())
+            .stream()
+            .collect(Collectors.toMap(Function.identity(), key -> decoratedMessage().get(key)));
+    }
+
+    @SuppressWarnings("unused")
+    @JsonProperty(FIELD_CHANGED_FIELDS)
+    public Map<String, Object> changedFields() {
+        return Sets.intersection(originalMessage().keySet(), decoratedMessage().keySet())
+            .stream()
+            .filter(key -> !originalMessage().get(key).equals(decoratedMessage().get(key)))
+            .collect(Collectors.toMap(Function.identity(), key -> originalMessage().get(key)));
+    }
+
+    @SuppressWarnings("unused")
+    @JsonProperty(FIELD_REMOVED_FIELDS)
+    public Map<String, Object> removedFields() {
+        return Sets.difference(originalMessage().keySet(), decoratedMessage().keySet())
+            .stream()
+            .collect(Collectors.toMap(Function.identity(), key -> originalMessage().get(key)));
+    }
+
+    public static DecorationStats create(Map<String, Object> originalMessage,
+                                         Map<String, Object> decoratedMessage) {
+        return new AutoValue_DecorationStats(originalMessage, decoratedMessage);
+    }
+}

graylog2-server/src/main/java/org/graylog2/rest/models/messages/responses/ResultMessageSummary.java

@@ -29,20 +29,64 @@
 @AutoValue
 @JsonAutoDetect
 public abstract class ResultMessageSummary {
-    @JsonProperty("highlight_ranges")
+    private static final String FIELD_HIGHLIGHT_RANGES = "highlight_ranges";
+    private static final String FIELD_MESSAGE = "message";
+    private static final String FIELD_INDEX = "index";
+    private static final String FIELD_DECORATION_STATS = "decoration_stats";
+
+    @JsonProperty(FIELD_HIGHLIGHT_RANGES)
     @Nullable
     public abstract Multimap<String, Range<Integer>> highlightRanges();
 
-    @JsonProperty
+    @JsonProperty(FIELD_MESSAGE)
     public abstract Map<String, Object> message();
 
-    @JsonProperty
+    @JsonProperty(FIELD_INDEX)
     public abstract String index();
 
+    @JsonProperty(FIELD_DECORATION_STATS)
+    @Nullable
+    public abstract DecorationStats decorationStats();
+
+    private static Builder builder() {
+        return new AutoValue_ResultMessageSummary.Builder();
+    }
+
+    public abstract Builder toBuilder();
+
     @JsonCreator
-    public static ResultMessageSummary create(@Nullable @JsonProperty("highlight_ranges") Multimap<String, Range<Integer>> highlightRanges,
-                                              @JsonProperty("message") Map<String, Object> message,
-                                              @JsonProperty("index") String index) {
-        return new AutoValue_ResultMessageSummary(highlightRanges, message, index);
+    public static ResultMessageSummary create(@Nullable @JsonProperty(FIELD_HIGHLIGHT_RANGES) Multimap<String, Range<Integer>> highlightRanges,
+                                              @JsonProperty(FIELD_MESSAGE) Map<String, Object> message,
+                                              @JsonProperty(FIELD_INDEX) String index,
+                                              @JsonProperty(FIELD_DECORATION_STATS) DecorationStats decorationStats) {
+        return builder()
+            .decorationStats(decorationStats)
+            .highlightRanges(highlightRanges)
+            .index(index)
+            .message(message)
+            .build();
+    }
+
+    public static ResultMessageSummary create(@Nullable @JsonProperty(FIELD_HIGHLIGHT_RANGES) Multimap<String, Range<Integer>> highlightRanges,
+                                              @JsonProperty(FIELD_MESSAGE) Map<String, Object> message,
+                                              @JsonProperty(FIELD_INDEX) String index) {
+        return builder()
+            .highlightRanges(highlightRanges)
+            .index(index)
+            .message(message)
+            .build();
+    }
+
+    @AutoValue.Builder
+    public static abstract class Builder {
+        public abstract Builder highlightRanges(Multimap<String, Range<Integer>> highlightRanges);
+
+        public abstract Builder message(Map<String, Object> message);
+
+        public abstract Builder index(String index);
+
+        public abstract Builder decorationStats(DecorationStats decorationStats);
+
+        public abstract ResultMessageSummary build();
     }
 }

graylog2-server/src/main/java/org/graylog2/rest/resources/search/responses/SearchDecorationStats.java

@@ -0,0 +1,39 @@
+/**
+ * This file is part of Graylog.
+ *
+ * Graylog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Graylog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.graylog2.rest.resources.search.responses;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+
+import java.util.Set;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class SearchDecorationStats {
+    private static final String FIELD_ADDED_FIELDS = "added_fields";
+
+    @SuppressWarnings("unused")
+    @JsonProperty(FIELD_ADDED_FIELDS)
+    public abstract Set<String> addedFields();
+
+    @JsonCreator
+    public static SearchDecorationStats create(@JsonProperty(FIELD_ADDED_FIELDS) Set<String> addedFields) {
+        return new AutoValue_SearchDecorationStats(addedFields);
+    }
+}