Merge pull request #1342 from Graylog2/issue-1335

Store internal message fields ("gl2_*") as doc values

graylog2-server/src/main/java/org/graylog2/configuration/ElasticsearchConfiguration.java

@@ -120,9 +120,6 @@ public class ElasticsearchConfiguration {
     @Parameter(value = "index_optimization_max_num_segments", validator = PositiveIntegerValidator.class)
     private int indexOptimizationMaxNumSegments = 1;
 
-    @Parameter(value = "elasticsearch_store_timestamps_as_doc_values")
-    private boolean storeTimestampsAsDocValues = true;
-
     @Parameter(value = "elasticsearch_request_timeout", validator = PositiveDurationValidator.class)
     private Duration requestTimeout = Duration.minutes(1L);
 
@@ -254,10 +251,6 @@ public String getPathData() {
         return pathData;
     }
 
-    public boolean isStoreTimestampsAsDocValues() {
-        return storeTimestampsAsDocValues;
-    }
-
     public Duration getRequestTimeout() {
         return requestTimeout;
     }

graylog2-server/src/main/java/org/graylog2/indexer/IndexMapping.java

@@ -90,9 +90,9 @@ public Map<String, Object> metaMapping() {
                         "format", "date_time"));
     }
 
-    public Map<String, Object> messageMapping(final String analyzer, boolean storeTimestampsAsDocValues) {
+    public Map<String, Object> messageMapping(final String analyzer) {
         return ImmutableMap.of(
-                "properties", partFieldProperties(analyzer, storeTimestampsAsDocValues),
+                "properties", partFieldProperties(analyzer),
                 "dynamic_templates", partDefaultAllInDynamicTemplate(),
                 // Compress source field
                 "_source", enabledAndCompressed(),
@@ -104,28 +104,35 @@ public Map<String, Object> messageMapping(final String analyzer, boolean storeTi
      * Disable analyzing for every field by default.
      */
     private List<Map<String, Map<String, Object>>> partDefaultAllInDynamicTemplate() {
-        final Map<String, String> notAnalyzed = ImmutableMap.of("index", "not_analyzed");
+        final Map<String, Serializable> mappingInternal = ImmutableMap.<String, Serializable>of(
+                "index", "not_analyzed",
+                "doc_values", true);
+        final Map<String, Object> defaultInternal = ImmutableMap.of(
+                "match", "gl2_*",
+                "mapping", mappingInternal);
+        final Map<String, Map<String, Object>> templateInternal = ImmutableMap.of("internal_fields", defaultInternal);
+
+        final Map<String, String> mappingAll = ImmutableMap.of("index", "not_analyzed");
         final Map<String, Object> defaultAll = ImmutableMap.of(
                 // Match all
                 "match", "*",
                 // Analyze nothing by default
-                "mapping", notAnalyzed);
-        final Map<String, Map<String, Object>> template = ImmutableMap.of("store_generic", defaultAll);
+                "mapping", mappingAll);
+        final Map<String, Map<String, Object>> templateAll = ImmutableMap.of("store_generic", defaultAll);
 
-        return ImmutableList.of(template);
+        return ImmutableList.of(templateInternal, templateAll);
     }
 
     /*
      * Enable analyzing for some fields again. Like for message and full_message.
      */
-    private Map<String, Map<String, ? extends Serializable>> partFieldProperties(String analyzer,
-                                                                                 boolean storeTimestampsAsDocValues) {
+    private Map<String, Map<String, ? extends Serializable>> partFieldProperties(String analyzer) {
         return ImmutableMap.of(
                 "message", analyzedString(analyzer),
                 "full_message", analyzedString(analyzer),
                 // http://joda-time.sourceforge.net/api-release/org/joda/time/format/DateTimeFormat.html
                 // http://www.elasticsearch.org/guide/reference/mapping/date-format.html
-                "timestamp", typeTimeWithMillis(storeTimestampsAsDocValues),
+                "timestamp", typeTimeWithMillis(true),
                 // to support wildcard searches in source we need to lowercase the content (wildcard search lowercases search term)
                 "source", analyzedString("analyzer_keyword"));
     }
@@ -137,16 +144,11 @@ private Map<String, String> analyzedString(String analyzer) {
                 "analyzer", analyzer);
     }
 
-    private Map<String, Serializable> typeTimeWithMillis(boolean storeTimestampsAsDocValues) {
-        final ImmutableMap.Builder<String, Serializable> builder = ImmutableMap.<String, Serializable>builder()
-                .put("type", "date")
-                .put("format", Tools.ES_DATE_FORMAT);
-
-        if (storeTimestampsAsDocValues) {
-            builder.put("doc_values", true);
-        }
-
-        return builder.build();
+    private Map<String, Serializable> typeTimeWithMillis(boolean storeAsDocValues) {
+        return ImmutableMap.<String, Serializable>of(
+                "type", "date",
+                "format", Tools.ES_DATE_FORMAT,
+                "doc_values", storeAsDocValues);
     }
 
     private Map<String, Boolean> enabled() {

graylog2-server/src/main/java/org/graylog2/indexer/indices/Indices.java

@@ -199,8 +199,7 @@ public boolean create(String indexName) {
             return false;
         }
 
-        final Map<String, Object> messageMapping = indexMapping.messageMapping(configuration.getAnalyzer(),
-                configuration.isStoreTimestampsAsDocValues());
+        final Map<String, Object> messageMapping = indexMapping.messageMapping(configuration.getAnalyzer());
         final PutMappingResponse messageMappingResponse =
                 indexMapping.createMapping(indexName, IndexMapping.TYPE_MESSAGE, messageMapping).actionGet();
         final Map<String, Object> metaMapping = indexMapping.metaMapping();

misc/graylog2.conf

@@ -191,11 +191,6 @@ allow_highlighting = false
 # Note that this setting only takes effect on newly created indices.
 elasticsearch_analyzer = standard
 
-# Store message timestamps as doc values in Elasticsearch. This will improve memory the consumption of
-# Elasticsearch at the cost of some performance at indexing time and increased index size.
-# See http://www.elastic.co/guide/en/elasticsearch/guide/master/doc-values.html for details.
-#elasticsearch_store_timestamps_as_doc_values = true
-
 # Global request timeout for Elasticsearch requests (e. g. during search, index creation, or index time-range
 # calculations) based on a best-effort to restrict the runtime of Elasticsearch operations.
 # Default: 1m