Default to @timestamp in composable template datastream definition (#…

…59317) (#59516)

This makes the data_stream timestamp field specification optional when
defining a composable template.
When there isn't one specified it will default to `@timestamp`.

(cherry picked from commit 5609353)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -2037,7 +2037,7 @@ public void testDataStreams() throws Exception {
         CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}");
         Template template = new Template(null, mappings, null);
         ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template,
-            Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate());
         PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
             new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate);
         AcknowledgedResponse response = execute(putComposableIndexTemplateRequest,

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java

@@ -86,7 +86,7 @@ private static ComposableIndexTemplate randomIndexTemplate() {
         Long priority = randomBoolean() ? null : randomNonNegativeLong();
         Long version = randomBoolean() ? null : randomNonNegativeLong();
         if (randomBoolean()) {
-            dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate("@timestamp");
+            dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate();
         }
         return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate);
     }

docs/reference/data-streams/change-mappings-and-settings.asciidoc

@@ -52,9 +52,7 @@ PUT /_ilm/policy/logs_policy
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {
@@ -69,9 +67,7 @@ PUT /_index_template/logs_data_stream
 PUT /_index_template/new_logs_data_stream
 {
   "index_patterns": [ "new_logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {
@@ -124,9 +120,7 @@ for a new field, `message`, to the template.
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {
@@ -193,9 +187,7 @@ mapping parameter to `true`.
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {
@@ -279,9 +271,7 @@ template's `index.refresh_interval` index setting to `30s` (30 seconds).
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {
@@ -345,9 +335,7 @@ The following <<indices-templates,put index template API>> requests adds new
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {
@@ -460,9 +448,7 @@ uses the `logs_data_stream` template as its basis, with the following changes:
 PUT /_index_template/new_logs_data_stream
 {
   "index_patterns": [ "new_logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {

docs/reference/data-streams/set-up-a-data-stream.asciidoc

@@ -134,9 +134,7 @@ this pattern.
 ----
 ====
 
-* A `data_stream` definition containing the `timestamp_field` property.
-  This timestamp field must be included in every document indexed to the data
-  stream.
+* A `data_stream` object with an empty body (`{ }`).
 
 The template can also contain:
 
@@ -176,9 +174,7 @@ configures the `logs_data_stream` template.
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "settings": {
       "index.lifecycle.name": "logs_policy"

docs/reference/data-streams/use-a-data-stream.asciidoc

@@ -19,8 +19,15 @@ the following:
 PUT /_index_template/logs_data_stream
 {
   "index_patterns": [ "logs*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
+  "data_stream": {},
+  "template": {
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        }
+      }
+    }
   }
 }
 

docs/reference/ilm/ilm-tutorial.asciidoc

@@ -121,19 +121,17 @@ with the options you specify.
 PUT _index_template/timeseries_template
 {
   "index_patterns": ["timeseries"],                   <1>
-  "data_stream": {
-    "timestamp_field": "@timestamp"                   <2>
-  },
+  "data_stream": {},
   "template": {
     "settings": {
       "number_of_shards": 1,
       "number_of_replicas": 1,
-      "index.lifecycle.name": "timeseries_policy"     <3>
+      "index.lifecycle.name": "timeseries_policy"     <2>
     },
     "mappings": {
       "properties": {
         "@timestamp": {
-          "type": "date"                              <4>
+          "type": "date"                              <3>
         }
       }
     }
@@ -143,11 +141,9 @@ PUT _index_template/timeseries_template
 // TEST[continued]
 
 <1> Apply the template when a document is indexed into the `timeseries` target.
-<2> Identifies the timestamp field for the data source. This field must be present
-in all documents indexed into the `timeseries` data stream.
-<3> The name of the {ilm-init} policy used to manage the data stream.
-<4> A <<date,`date`>> or <<date_nanos,`date_nanos`>> field mapping for the
-timestamp field specified in the `timestamp_field` property
+<2> The name of the {ilm-init} policy used to manage the data stream.
+<3> A <<date,`date`>> or <<date_nanos,`date_nanos`>> field mapping for the
+"@timestamp" field the data stream will use.
 ====
 
 [discrete]

docs/reference/indices/create-data-stream.asciidoc

@@ -25,9 +25,7 @@ PUT /_index_template/template
       }
     }
   },
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  }
+  "data_stream": {}
 }
 ----
 ////

docs/reference/indices/delete-data-stream.asciidoc

@@ -23,9 +23,7 @@ PUT /_index_template/template
       }
     }
   },
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  }
+  "data_stream": {}
 }
 
 PUT /_data_stream/my-data-stream

docs/reference/indices/get-data-stream.asciidoc

@@ -35,9 +35,7 @@ PUT /_ilm/policy/my-lifecycle-policy
 PUT /_index_template/my-index-template
 {
   "index_patterns": [ "my-data-stream*" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {

docs/reference/indices/index-templates.asciidoc

@@ -493,19 +493,10 @@ PUT /_index_template/template_1
       }
     }
   },
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  }
+  "data_stream": {}
 }
 --------------------------------------------------
 
-Required properties of a data stream definition:
-
-`timestamp_field`::
-(Required, string) The name of the timestamp field. This field must be present
-in all documents indexed into the data stream and must be of type
-<<date, `date`>> or <<date_nanos, `date_nanos`>>.
-
 [[multiple-component-templates]]
 ===== Composing multiple component templates
 

docs/reference/indices/resolve.asciidoc

@@ -30,9 +30,7 @@ PUT /my-index/_alias/f-alias
 PUT /_index_template/foo_data_stream
 {
   "index_patterns": [ "foo" ],
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  },
+  "data_stream": {},
   "template": {
     "mappings": {
       "properties": {

docs/reference/indices/rollover-index.asciidoc

@@ -241,9 +241,7 @@ PUT _index_template/template
       }
     }
   },
-  "data_stream": {
-    "timestamp_field": "@timestamp"
-  }
+  "data_stream": {}
 }
 -----------------------------------
 // TEST

rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml

@@ -1,8 +1,8 @@
 ---
 setup:
   - skip:
-      version: "7.8.99 - "
-      reason: "resolve index api only supported in 7.9+"
+      version: "7.9.99 - "
+      reason: "enable in 7.9+ when backported"
       features: allowed_warnings
 
   - do:

server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java

@@ -102,7 +102,7 @@ public void testUsageOfDataStreamFails() throws IOException {
         Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser));
 
         ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null,
-            null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, new ComposableIndexTemplate.DataStreamTemplate());
         Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE,
             new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet());
         Exception actualException = (Exception) e.getCause();

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -249,65 +250,50 @@ public String toString() {
 
     public static class DataStreamTemplate implements Writeable, ToXContentObject {
 
-        private static final ConstructingObjectParser<DataStreamTemplate, Void> PARSER = new ConstructingObjectParser<>(
+        private static final ObjectParser<DataStreamTemplate, Void> PARSER = new ObjectParser<>(
             "data_stream_template",
-            args -> new DataStreamTemplate((String) args[0])
+            DataStreamTemplate::new
         );
 
-        static {
-            PARSER.declareString(ConstructingObjectParser.constructorArg(), DataStream.TIMESTAMP_FIELD_FIELD);
-        }
-
-        private final String timestampField;
-
-        public DataStreamTemplate(String timestampField) {
-            if (FIXED_TIMESTAMP_FIELD.equals(timestampField) == false) {
-                throw new IllegalArgumentException("unexpected timestamp field [" + timestampField + "]");
-            }
-
-            this.timestampField = timestampField;
+        public DataStreamTemplate() {
         }
 
         public String getTimestampField() {
-            return timestampField;
+            return FIXED_TIMESTAMP_FIELD;
         }
 
-        DataStreamTemplate(StreamInput in) throws IOException {
-            this(in.readString());
+        DataStreamTemplate(StreamInput in) {
+            this();
         }
 
         /**
          * @return a mapping snippet for a backing index with `_data_stream_timestamp` meta field mapper properly configured.
          */
         public Map<String, Object> getDataSteamMappingSnippet() {
             return singletonMap(MapperService.SINGLE_MAPPING_NAME, singletonMap("_data_stream_timestamp",
-                singletonMap("path", timestampField)));
+                singletonMap("path", FIXED_TIMESTAMP_FIELD)));
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(timestampField);
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field(DataStream.TIMESTAMP_FIELD_FIELD.getPreferredName(), timestampField);
             builder.endObject();
             return builder;
         }
 
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            DataStreamTemplate that = (DataStreamTemplate) o;
-            return timestampField.equals(that.timestampField);
+            return o != null && getClass() == o.getClass();
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(timestampField);
+            return DataStreamTemplate.class.hashCode();
         }
     }
 }

server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java

@@ -36,7 +36,7 @@ public void testResolveAutoCreateDataStreams() {
         Metadata metadata;
         {
             Metadata.Builder mdBuilder = new Metadata.Builder();
-            DataStreamTemplate dataStreamTemplate = new DataStreamTemplate("@timestamp");
+            DataStreamTemplate dataStreamTemplate = new DataStreamTemplate();
             mdBuilder.put("1", new ComposableIndexTemplate(Collections.singletonList("legacy-logs-*"), null, null, 10L, null, null, null));
             mdBuilder.put("2", new ComposableIndexTemplate(Collections.singletonList("logs-*"),
                 null, null, 20L, null, null, dataStreamTemplate));

server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

@@ -48,8 +48,8 @@
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -539,7 +539,7 @@ public void testRolloverClusterState() throws Exception {
     public void testRolloverClusterStateForDataStream() throws Exception {
         final DataStream dataStream = DataStreamTests.randomInstance();
         ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"),
-            null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+            null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
         Metadata.Builder builder = Metadata.builder();
         builder.put("template", template);
         for (Index index : dataStream.getIndices()) {
@@ -638,7 +638,7 @@ public void testValidation() throws Exception {
             sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName();
             defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
             ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList(dataStream.getName() + "*"),
-                null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp"));
+                null, null, null, null, null, new ComposableIndexTemplate.DataStreamTemplate());
             builder.put("template", template);
             for (Index index : dataStream.getIndices()) {
                 builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index));

server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java

@@ -148,7 +148,7 @@ private static ComposableIndexTemplate.DataStreamTemplate randomDataStreamTempla
         if (randomBoolean()) {
             return null;
         } else {
-            return new ComposableIndexTemplate.DataStreamTemplate("@timestamp");
+            return new ComposableIndexTemplate.DataStreamTemplate();
         }
     }