Remove index_mode property from index templates (#85985)

This PR removes the `index_mode` field from index templates (that was part of `data_stream` json object).
This field was used to determine whether a tsdb data stream should be created or a regular data stream.
As part of this change, a tsdb data stream is now created when the template that matches with the data stream to be created has the following properties:

* A `index.routing_path` setting has been defined in the composable index template.
* The fields mentioned in the index.routing_path setting refers to fields in the mapping of this template that are of type keyword and the `time_series_dimension` attribute is set to true.

Relates to #74660

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -1154,7 +1154,7 @@ public void testIndexDocsWithCustomRoutingAllowed() throws Exception {
             null,
             null,
             null,
-            new ComposableIndexTemplate.DataStreamTemplate(false, true, null)
+            new ComposableIndexTemplate.DataStreamTemplate(false, true)
         );
         client().execute(
             PutComposableIndexTemplateAction.INSTANCE,
@@ -1804,7 +1804,7 @@ public void testPartitionedTemplate() throws IOException {
             null,
             null,
             null,
-            new ComposableIndexTemplate.DataStreamTemplate(false, true, null)
+            new ComposableIndexTemplate.DataStreamTemplate(false, true)
         );
         ComposableIndexTemplate finalTemplate = template;
         client().execute(
@@ -1830,7 +1830,7 @@ public void testPartitionedTemplate() throws IOException {
             null,
             null,
             null,
-            new ComposableIndexTemplate.DataStreamTemplate(false, true, null)
+            new ComposableIndexTemplate.DataStreamTemplate(false, true)
         );
         client().execute(
             PutComposableIndexTemplateAction.INSTANCE,
@@ -1856,7 +1856,7 @@ public void testPartitionedTemplate() throws IOException {
             null,
             null,
             null,
-            new ComposableIndexTemplate.DataStreamTemplate(false, false, null)
+            new ComposableIndexTemplate.DataStreamTemplate(false, false)
         );
         ComposableIndexTemplate finalTemplate1 = template;
         Exception e = expectThrows(
@@ -1898,7 +1898,7 @@ public void testSearchWithRouting() throws IOException, ExecutionException, Inte
             null,
             null,
             null,
-            new ComposableIndexTemplate.DataStreamTemplate(false, true, null)
+            new ComposableIndexTemplate.DataStreamTemplate(false, true)
         );
         client().execute(
             PutComposableIndexTemplateAction.INSTANCE,

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -10,15 +10,16 @@
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -87,7 +88,7 @@ public void testTimeRanges() throws Exception {
                 null,
                 null,
                 null,
-                new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES),
+                new ComposableIndexTemplate.DataStreamTemplate(false, false),
                 null
             )
         );
@@ -166,6 +167,122 @@ public void testTimeRanges() throws Exception {
         }
     }
 
+    public void testTsdbTemplatesWithComponentTemplate() throws Exception {
+        var mappingTemplate = """
+            {
+              "_doc":{
+                "properties": {
+                  "metricset": {
+                    "type": "keyword",
+                    "time_series_dimension": true
+                  }
+                }
+              }
+            }""";
+        var putComponentTemplateRequest = new PutComponentTemplateAction.Request("id");
+        putComponentTemplateRequest.componentTemplate(
+            new ComponentTemplate(
+                new Template(
+                    Settings.builder().put("index.routing_path", "metricset").build(),
+                    new CompressedXContent(mappingTemplate),
+                    null
+                ),
+                null,
+                null
+            )
+        );
+        Exception e = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().execute(PutComponentTemplateAction.INSTANCE, putComponentTemplateRequest).actionGet()
+        );
+        // We can't know whether the index template that uses this component template has data streams enabled:
+        // (The generation of certain settings index.mode, index.time_series_start_time etc. only doesn't happen)
+        assertThat(e.getMessage(), equalTo("[index.routing_path] requires [index.mode=time_series]"));
+    }
+
+    public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exception {
+        var mappingTemplate = """
+            {
+              "_doc":{
+                "properties": {
+                  "metricset": {
+                    "type": "keyword"
+                  }
+                }
+              }
+            }""";
+        var request = new PutComposableIndexTemplateAction.Request("id");
+        request.indexTemplate(
+            new ComposableIndexTemplate(
+                List.of("k8s*"),
+                new Template(
+                    Settings.builder().put("index.routing_path", "metricset").build(),
+                    new CompressedXContent(mappingTemplate),
+                    null
+                ),
+                null,
+                null,
+                null,
+                null,
+                new ComposableIndexTemplate.DataStreamTemplate(false, false),
+                null
+            )
+        );
+        Exception e = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet()
+        );
+        assertThat(
+            e.getCause().getCause().getMessage(),
+            equalTo(
+                "All fields that match routing_path must be keywords with [time_series_dimension: true] and "
+                    + "without the [script] parameter. [metricset] was not [time_series_dimension: true]."
+            )
+        );
+    }
+
+    public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception {
+        var mappingTemplate = """
+            {
+              "_doc":{
+                "properties": {
+                  "metricset": {
+                    "type": "long",
+                    "time_series_dimension": true
+                  }
+                }
+              }
+            }""";
+        var request = new PutComposableIndexTemplateAction.Request("id");
+        request.indexTemplate(
+            new ComposableIndexTemplate(
+                List.of("k8s*"),
+                new Template(
+                    Settings.builder().put("index.routing_path", "metricset").build(),
+                    new CompressedXContent(mappingTemplate),
+                    null
+                ),
+                null,
+                null,
+                null,
+                null,
+                new ComposableIndexTemplate.DataStreamTemplate(false, false),
+                null
+            )
+        );
+        Exception e = expectThrows(
+            IllegalArgumentException.class,
+            () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet()
+        );
+        assertThat(
+            e.getCause().getCause().getMessage(),
+            equalTo(
+                "All fields that match routing_path must be keywords with [time_series_dimension: true] and "
+                    + "without the [script] parameter. [metricset] was [long]."
+            )
+        );
+    }
+
     static String formatInstant(Instant instant) {
         return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant);
     }

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java

@@ -86,7 +86,6 @@ public class TsdbDataStreamRestIT extends ESRestTestCase {
                 }
             },
             "data_stream": {
-                "index_mode": "time_series"
             }
         }""";
 
@@ -463,7 +462,7 @@ public void testChangeTemplateIndexMode() throws Exception {
                 e.getMessage(),
                 containsString(
                     "composable template [1] with index patterns [k8s*], priority [null],"
-                        + " index_mode [null] would cause tsdb data streams [k8s] to no longer match a data stream template"
+                        + " index.routing_path [] would cause tsdb data streams [k8s] to no longer match a data stream template"
                         + " with a time_series index_mode"
                 )
             );

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java

@@ -30,7 +30,7 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider {
     public Settings getAdditionalIndexSettings(
         String indexName,
         String dataStreamName,
-        IndexMode templateIndexMode,
+        boolean timeSeries,
         Metadata metadata,
         Instant resolvedAt,
         Settings allSettings
@@ -42,14 +42,16 @@ public Settings getAdditionalIndexSettings(
             // so checking that index_mode==null|standard and templateIndexMode == TIME_SERIES
             boolean migrating = dataStream != null
                 && (dataStream.getIndexMode() == null || dataStream.getIndexMode() == IndexMode.STANDARD)
-                && templateIndexMode == IndexMode.TIME_SERIES;
+                && timeSeries;
             IndexMode indexMode;
             if (migrating) {
                 indexMode = IndexMode.TIME_SERIES;
             } else if (dataStream != null) {
                 indexMode = dataStream.getIndexMode();
+            } else if (timeSeries) {
+                indexMode = IndexMode.TIME_SERIES;
             } else {
-                indexMode = templateIndexMode;
+                indexMode = null;
             }
             if (indexMode != null) {
                 Settings.Builder builder = Settings.builder();

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java

@@ -34,7 +34,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProviders;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
@@ -287,7 +286,7 @@ public void cleanup() {
     private ClusterState createInitialState() {
         ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of("logs-*"))
             .template(new Template(Settings.builder().put("index.routing_path", "uid").build(), null, null))
-            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES))
+            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
             .build();
         Metadata.Builder builder = Metadata.builder();
         builder.put("template", template);

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java

@@ -39,7 +39,7 @@ public void testGetAdditionalIndexSettings() {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            true,
             metadata,
             now,
             settings
@@ -61,7 +61,7 @@ public void testGetAdditionalIndexSettingsLookAheadTime() {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            true,
             metadata,
             now,
             settings
@@ -89,7 +89,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() {
         var result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            IndexMode.TIME_SERIES,
+            true,
             metadata,
             now,
             settings
@@ -139,7 +139,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi
             () -> provider.getAdditionalIndexSettings(
                 DataStream.getDefaultBackingIndexName(dataStreamName, 1),
                 dataStreamName,
-                IndexMode.TIME_SERIES,
+                true,
                 metadata,
                 now,
                 settings
@@ -155,7 +155,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi
         );
     }
 
-    public void testGetAdditionalIndexSettingsIndexModeNotSpecified() {
+    public void testGetAdditionalIndexSettingsNonTsdbTemplate() {
         Metadata metadata = Metadata.EMPTY_METADATA;
         String dataStreamName = "logs-app1";
 
@@ -164,30 +164,12 @@ public void testGetAdditionalIndexSettingsIndexModeNotSpecified() {
         Settings result = provider.getAdditionalIndexSettings(
             DataStream.getDefaultBackingIndexName(dataStreamName, 1),
             dataStreamName,
-            null,
+            false,
             metadata,
             Instant.ofEpochMilli(1L),
             settings
         );
         assertThat(result.size(), equalTo(0));
     }
 
-    public void testGetAdditionalIndexSettingsIndexModeStandardSpecified() {
-        Metadata metadata = Metadata.EMPTY_METADATA;
-        String dataStreamName = "logs-app1";
-
-        Settings settings = Settings.EMPTY;
-        var provider = new DataStreamIndexSettingsProvider();
-        Settings result = provider.getAdditionalIndexSettings(
-            DataStream.getDefaultBackingIndexName(dataStreamName, 1),
-            dataStreamName,
-            IndexMode.STANDARD,
-            metadata,
-            Instant.ofEpochMilli(1L),
-            settings
-        );
-        assertThat(result.size(), equalTo(1));
-        assertThat(result.get(IndexSettings.MODE.getKey()), equalTo(IndexMode.STANDARD.name().toLowerCase(Locale.ROOT)));
-    }
-
 }

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java

@@ -47,7 +47,6 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return List.of(DataStreamsPlugin.class);
     }
 
-    private String timestampFieldName = "@timestamp";
     private final Set<String> createdDataStreams = new HashSet<>();
 
     @Override
@@ -231,16 +230,16 @@ private String createDataStream() throws Exception {
     private String createDataStream(boolean hidden) throws Exception {
         String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault());
         Template idxTemplate = new Template(null, new CompressedXContent("""
-            {"properties":{"%s":{"type":"date"},"data":{"type":"keyword"}}}
-            """.formatted(timestampFieldName)), null);
+            {"properties":{"@timestamp":{"type":"date"},"data":{"type":"keyword"}}}
+            """), null);
         ComposableIndexTemplate template = new ComposableIndexTemplate(
             List.of(dataStreamName + "*"),
             idxTemplate,
             null,
             null,
             null,
             null,
-            new ComposableIndexTemplate.DataStreamTemplate(hidden, false, null),
+            new ComposableIndexTemplate.DataStreamTemplate(hidden, false),
             null
         );
         assertTrue(
@@ -265,7 +264,7 @@ private long createDocument(String dataStreamName) throws Exception {
                 .source(
                     JsonXContent.contentBuilder()
                         .startObject()
-                        .field(timestampFieldName, timestamp)
+                        .field("@timestamp", timestamp)
                         .field("data", randomAlphaOfLength(25))
                         .endObject()
                 )

modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java

@@ -64,7 +64,7 @@ public void testRolloverClusterStateForDataStream() throws Exception {
         );
         ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStream.getName() + "*"))
             .template(new Template(Settings.builder().put("index.routing_path", "uid").build(), null, null))
-            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES))
+            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
             .build();
         Metadata.Builder builder = Metadata.builder();
         builder.put("template", template);
@@ -162,7 +162,7 @@ public void testRolloverAndMigrateDataStream() throws Exception {
         );
         ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStream.getName() + "*"))
             .template(new Template(Settings.builder().put("index.routing_path", "uid").build(), null, null))
-            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, IndexMode.TIME_SERIES))
+            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
             .build();
         Metadata.Builder builder = Metadata.builder();
         builder.put("template", template);
@@ -246,7 +246,7 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting
         );
         ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStream.getName() + "*"))
             .template(new Template(Settings.builder().put("index.routing_path", "uid").build(), null, null))
-            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, randomBoolean() ? IndexMode.STANDARD : null))
+            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false))
             .build();
         Metadata.Builder builder = Metadata.builder();
         builder.put("template", template);