[8.18] System data streams incorrectly show up in the list of template validation problems (#128161)

docs/changelog/128161.yaml

@@ -0,0 +1,6 @@
+pr: 128161
+summary: Fix system data streams incorrectly showing up in the list of template validation
+  problems
+area: Data streams
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -838,7 +838,13 @@ static void validateDataStreamOptions(Metadata metadata, String indexTemplateNam
      * addition/update time
      */
     private static void validateDataStreamsStillReferenced(ClusterState state, String templateName, ComposableIndexTemplate newTemplate) {
-        final Set<String> dataStreams = state.metadata().dataStreams().keySet();
+        final Set<String> dataStreams = state.metadata()
+            .dataStreams()
+            .entrySet()
+            .stream()
+            .filter(entry -> entry.getValue().isSystem() == false)
+            .map(Map.Entry::getKey)
+            .collect(Collectors.toSet());
 
         Function<Metadata, Set<String>> findUnreferencedDataStreams = meta -> {
             final Set<String> unreferenced = new HashSet<>();

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.health.node.selection.HealthNodeTaskExecutor;
@@ -1693,6 +1694,81 @@ public void testInvalidNonDataStreamTemplateWithDataStreamOptions() throws Excep
         );
     }
 
+    public void testSystemDataStreamsIgnoredByValidateIndexTemplateV2() throws Exception {
+        /*
+         * This test makes sure that system data streams (which do not have named templates) do not appear in the list of data streams
+         * without named templates when validateIndexTemplateV2 fails due to another non-system data stream not having a named template.
+         */
+        MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
+        final String dataStreamTemplateName = "data_stream_template";
+        final String indexTemplateName = "index_template";
+        final String systemDataStreamName = "system_ds";
+        final String ordinaryDataStreamName = "my_ds";
+        final String ordinaryDataStreamIndexPattern = "my_ds*";
+        ComposableIndexTemplate highPriorityDataStreamTemplate = ComposableIndexTemplate.builder()
+            .indexPatterns(List.of(ordinaryDataStreamIndexPattern))
+            .priority(275L)
+            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean()))
+            .build();
+        ComposableIndexTemplate highPriorityIndexTemplate = ComposableIndexTemplate.builder()
+            .indexPatterns(List.of(ordinaryDataStreamIndexPattern))
+            .priority(200L)
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
+            .metadata(
+                Metadata.builder()
+                    .dataStreams(
+                        Map.of(
+                            systemDataStreamName,
+                            DataStreamTestHelper.randomInstance(systemDataStreamName, System::currentTimeMillis, randomBoolean(), true),
+                            ordinaryDataStreamName,
+                            DataStreamTestHelper.randomInstance(ordinaryDataStreamName, System::currentTimeMillis, randomBoolean(), false)
+                        ),
+                        Map.of()
+                    )
+                    .indexTemplates(
+                        Map.of(dataStreamTemplateName, highPriorityDataStreamTemplate, indexTemplateName, highPriorityIndexTemplate)
+                    )
+            )
+            .build();
+        ComposableIndexTemplate lowPriorityDataStreamTemplate = ComposableIndexTemplate.builder()
+            .indexPatterns(List.of(ordinaryDataStreamIndexPattern))
+            .priority(1L)
+            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean()))
+            .build();
+        /*
+         * Here we attempt to change the priority of a template that matches an existing non-system data stream so that it is so low that
+         * the data stream matches the index (non data-stream) template instead. We expect an error, but that the error only mentions the
+         * non-system data stream.
+         */
+        Exception exception = expectThrows(
+            Exception.class,
+            () -> metadataIndexTemplateService.validateIndexTemplateV2(dataStreamTemplateName, lowPriorityDataStreamTemplate, clusterState)
+        );
+        assertThat(
+            exception.getMessage(),
+            containsString(
+                Strings.format(
+                    "composable template [%s] with index patterns [%s], priority [1] would cause data streams [%s] to no longer "
+                        + "match a data stream template",
+                    dataStreamTemplateName,
+                    ordinaryDataStreamIndexPattern,
+                    ordinaryDataStreamName
+                )
+            )
+        );
+        ComposableIndexTemplate mediumPriorityDataStreamTemplate = ComposableIndexTemplate.builder()
+            .indexPatterns(List.of(ordinaryDataStreamIndexPattern))
+            .priority(201L)
+            .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean()))
+            .build();
+        /*
+         * We have now corrected the problem -- the priority of the new template is lower than the old data stream template but still higher
+         * than the non-data-stream index template. So we expect no validation errors.
+         */
+        metadataIndexTemplateService.validateIndexTemplateV2(dataStreamTemplateName, mediumPriorityDataStreamTemplate, clusterState);
+    }
+
     private ClusterState addComponentTemplate(
         MetadataIndexTemplateService service,
         ClusterState state,

test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java

@@ -339,6 +339,11 @@ public static DataStream randomInstance(LongSupplier timeProvider, boolean failu
     }
 
     public static DataStream randomInstance(String dataStreamName, LongSupplier timeProvider, boolean failureStore) {
+        // Some tests don't work well with system data streams, since these data streams require special handling
+        return randomInstance(dataStreamName, timeProvider, failureStore, false);
+    }
+
+    public static DataStream randomInstance(String dataStreamName, LongSupplier timeProvider, boolean failureStore, boolean system) {
         List<Index> indices = randomIndexInstances();
         long generation = indices.size() + ESTestCase.randomLongBetween(1, 128);
         indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(LuceneTestCase.random())));
@@ -363,9 +368,9 @@ public static DataStream randomInstance(String dataStreamName, LongSupplier time
             dataStreamName,
             generation,
             metadata,
-            randomBoolean(),
+            system ? true : randomBoolean(),
             replicated,
-            false, // Some tests don't work well with system data streams, since these data streams require special handling
+            system,
             timeProvider,
             randomBoolean(),
             randomBoolean() ? IndexMode.STANDARD : null, // IndexMode.TIME_SERIES triggers validation that many unit tests doesn't pass