[7.x] Composable templates: add a default mapping for @timestamp (#59244

) (#59510) This adds a low precendece mapping for the `@timestamp` field with type `date`. This will aid with the bootstrapping of data streams as a timestamp mapping can be omitted when nanos precision is not needed. (cherry picked from commit 4e72f43) Signed-off-by: Andrei Dan <andrei.dan@elastic.co>
elastic · Jul 14, 2020 · 4180333 · 4180333
1 parent 5580eb6
commit 4180333
Show file tree

Hide file tree

Showing 20 changed files with 387 additions and 265 deletions.
diff --git a/docs/reference/data-streams/set-up-a-data-stream.asciidoc b/docs/reference/data-streams/set-up-a-data-stream.asciidoc
@@ -99,7 +99,7 @@ PUT /_ilm/policy/logs_policy
 Each data stream requires an <<indices-templates,index template>>. The data
 stream uses this template to create its backing indices.
 
-Index templates for data streams must contain:
+An index template for a data stream must contain:
 
 * A name or wildcard (`*`) pattern for the data stream in the `index_patterns`
 property.
@@ -138,16 +138,22 @@ this pattern.
   This timestamp field must be included in every document indexed to the data
   stream.
 
-* A <<date,`date`>> or <<date_nanos,`date_nanos`>> field mapping for the
-timestamp field specified in the `timestamp_field` property.
+The template can also contain:
+
+* An optional field mapping for the `@timestamp` field. Both the <<date,`date`>> and
+<<date_nanos,`date_nanos`>> field data types are supported. If no mapping is specified,
+a <<date,`date`>> field data type with default options is used.
++
+This mapping can include other <<mapping-params,mapping parameters>>, such as
+<<mapping-date-format,`format`>>.
 +
-IMPORTANT: Carefully consider the timestamp field's mapping, including
-<<mapping-params,mapping parameters>> such as <<mapping-date-format,`format`>>.
-Once the stream is created, you can only update the timestamp field's mapping by
-reindexing the data stream. See
+IMPORTANT: Carefully consider the `@timestamp` field's mapping, including
+its <<mapping-params,mapping parameters>>.
+Once the stream is created, you can only update the `@timestamp` field's mapping
+by reindexing the data stream. See
 <<data-streams-use-reindex-to-change-mappings-settings>>.
 
-* If you intend to use {ilm-init}, you must specify the
+* If you intend to use {ilm-init}, the
   <<configure-a-data-stream-ilm-policy,lifecycle policy>> in the
   `index.lifecycle.name` setting.
 
@@ -174,13 +180,6 @@ PUT /_index_template/logs_data_stream
     "timestamp_field": "@timestamp"
   },
   "template": {
-    "mappings": {
-      "properties": {
-        "@timestamp": {
-          "type": "date"
-        }
-      }
-    },
     "settings": {
       "index.lifecycle.name": "logs_policy"
     }

diff --git a/docs/reference/data-streams/use-a-data-stream.asciidoc b/docs/reference/data-streams/use-a-data-stream.asciidoc
@@ -21,15 +21,6 @@ PUT /_index_template/logs_data_stream
   "index_patterns": [ "logs*" ],
   "data_stream": {
     "timestamp_field": "@timestamp"
-  },
-  "template": {
-    "mappings": {
-      "properties": {
-        "@timestamp": {
-          "type": "date"
-        }
-      }
-    }
   }
 }
 

diff --git a/...ec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml b/...ec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml
@@ -35,8 +35,8 @@ setup:
 ---
 "Resolve index with indices and aliases":
   - skip:
-      version: " - 7.8.99"
-      reason: "resolve index api only supported in 7.9+"
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.resolve_index:
@@ -63,8 +63,8 @@ setup:
 ---
 "Resolve index with hidden and closed indices":
   - skip:
-      version: " - 7.8.99"
-      reason: change after backporting
+      version: " - 7.9.99"
+      reason: "enable in 7.9+ when backported"
 
   - do:
       indices.resolve_index:

diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
@@ -68,6 +68,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -87,6 +88,16 @@
  */
 public class MetadataIndexTemplateService {
 
+    public static final String DEFAULT_TIMESTAMP_FIELD = "@timestamp";
+    public static final String DEFAULT_TIMESTAMP_MAPPING = "{\n" +
+        "   \"_doc\": {\n" +
+        "     \"properties\": {\n" +
+        "       \"@timestamp\": {\n" +
+        "         \"type\": \"date\"\n" +
+        "       }\n" +
+        "     }\n" +
+        "   }\n" +
+        " }";
     private static final Logger logger = LogManager.getLogger(MetadataIndexTemplateService.class);
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
@@ -916,7 +927,7 @@ public static String findV2Template(Metadata metadata, String indexName, boolean
      */
     public static List<CompressedXContent> collectMappings(final ClusterState state,
                                                            final String templateName,
-                                                           final String indexName) {
+                                                           final String indexName) throws Exception {
         final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName);
         assert template != null : "attempted to resolve mappings for a template [" + templateName +
             "] that did not exist in the cluster state";
@@ -931,11 +942,16 @@ public static List<CompressedXContent> collectMappings(final ClusterState state,
             .map(ComponentTemplate::template)
             .map(Template::mappings)
             .filter(Objects::nonNull)
-            .collect(Collectors.toList());
+            .collect(Collectors.toCollection(LinkedList::new));
         // Add the actual index template's mappings, since it takes the highest precedence
         Optional.ofNullable(template.template())
             .map(Template::mappings)
             .ifPresent(mappings::add);
+        if (template.getDataStreamTemplate() != null && indexName.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
+            // add a default mapping for the `@timestamp` field, at the lowest precedence, to make bootstrapping data streams more
+            // straightforward as all backing indices are required to have a timestamp field
+            mappings.add(0, new CompressedXContent(DEFAULT_TIMESTAMP_MAPPING));
+        }
 
         // Only include _timestamp mapping snippet if creating backing index.
         if (indexName.startsWith(DataStream.BACKING_INDEX_PREFIX)) {
@@ -1112,7 +1128,7 @@ private static void validateCompositeTemplate(final ClusterState state,
                     }
                 }
 
-                List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName, indexName );
+                List<CompressedXContent> mappings = collectMappings(stateWithIndex, templateName, indexName);
                 try {
                     MapperService mapperService = tempIndexService.mapperService();
                     for (CompressedXContent mapping : mappings) {