diff --git a/docs/changelog/128161.yaml b/docs/changelog/128161.yaml new file mode 100644 index 0000000000000..92044a34a9938 --- /dev/null +++ b/docs/changelog/128161.yaml @@ -0,0 +1,6 @@ +pr: 128161 +summary: Fix system data streams incorrectly showing up in the list of template validation + problems +area: Data streams +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index cc5be49d7fe63..1ef46049808ff 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -838,7 +838,13 @@ static void validateDataStreamOptions(Metadata metadata, String indexTemplateNam * addition/update time */ private static void validateDataStreamsStillReferenced(ClusterState state, String templateName, ComposableIndexTemplate newTemplate) { - final Set dataStreams = state.metadata().dataStreams().keySet(); + final Set dataStreams = state.metadata() + .dataStreams() + .entrySet() + .stream() + .filter(entry -> entry.getValue().isSystem() == false) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); Function> findUnreferencedDataStreams = meta -> { final Set unreferenced = new HashSet<>(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 1365f56b50e5f..51db7ecef8989 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.Environment; import org.elasticsearch.health.node.selection.HealthNodeTaskExecutor; @@ -1693,6 +1694,81 @@ public void testInvalidNonDataStreamTemplateWithDataStreamOptions() throws Excep ); } + public void testSystemDataStreamsIgnoredByValidateIndexTemplateV2() throws Exception { + /* + * This test makes sure that system data streams (which do not have named templates) do not appear in the list of data streams + * without named templates when validateIndexTemplateV2 fails due to another non-system data stream not having a named template. + */ + MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); + final String dataStreamTemplateName = "data_stream_template"; + final String indexTemplateName = "index_template"; + final String systemDataStreamName = "system_ds"; + final String ordinaryDataStreamName = "my_ds"; + final String ordinaryDataStreamIndexPattern = "my_ds*"; + ComposableIndexTemplate highPriorityDataStreamTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(ordinaryDataStreamIndexPattern)) + .priority(275L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean())) + .build(); + ComposableIndexTemplate highPriorityIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(ordinaryDataStreamIndexPattern)) + .priority(200L) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata( + Metadata.builder() + .dataStreams( + Map.of( + systemDataStreamName, + DataStreamTestHelper.randomInstance(systemDataStreamName, System::currentTimeMillis, randomBoolean(), true), + ordinaryDataStreamName, + DataStreamTestHelper.randomInstance(ordinaryDataStreamName, System::currentTimeMillis, randomBoolean(), false) + ), + Map.of() + ) + .indexTemplates( + Map.of(dataStreamTemplateName, highPriorityDataStreamTemplate, indexTemplateName, highPriorityIndexTemplate) + ) + ) + .build(); + ComposableIndexTemplate lowPriorityDataStreamTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(ordinaryDataStreamIndexPattern)) + .priority(1L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean())) + .build(); + /* + * Here we attempt to change the priority of a template that matches an existing non-system data stream so that it is so low that + * the data stream matches the index (non data-stream) template instead. We expect an error, but that the error only mentions the + * non-system data stream. + */ + Exception exception = expectThrows( + Exception.class, + () -> metadataIndexTemplateService.validateIndexTemplateV2(dataStreamTemplateName, lowPriorityDataStreamTemplate, clusterState) + ); + assertThat( + exception.getMessage(), + containsString( + Strings.format( + "composable template [%s] with index patterns [%s], priority [1] would cause data streams [%s] to no longer " + + "match a data stream template", + dataStreamTemplateName, + ordinaryDataStreamIndexPattern, + ordinaryDataStreamName + ) + ) + ); + ComposableIndexTemplate mediumPriorityDataStreamTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of(ordinaryDataStreamIndexPattern)) + .priority(201L) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(randomBoolean(), randomBoolean())) + .build(); + /* + * We have now corrected the problem -- the priority of the new template is lower than the old data stream template but still higher + * than the non-data-stream index template. So we expect no validation errors. + */ + metadataIndexTemplateService.validateIndexTemplateV2(dataStreamTemplateName, mediumPriorityDataStreamTemplate, clusterState); + } + private ClusterState addComponentTemplate( MetadataIndexTemplateService service, ClusterState state, diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java index d54a66661b764..2a1e84442600b 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/metadata/DataStreamTestHelper.java @@ -339,6 +339,11 @@ public static DataStream randomInstance(LongSupplier timeProvider, boolean failu } public static DataStream randomInstance(String dataStreamName, LongSupplier timeProvider, boolean failureStore) { + // Some tests don't work well with system data streams, since these data streams require special handling + return randomInstance(dataStreamName, timeProvider, failureStore, false); + } + + public static DataStream randomInstance(String dataStreamName, LongSupplier timeProvider, boolean failureStore, boolean system) { List indices = randomIndexInstances(); long generation = indices.size() + ESTestCase.randomLongBetween(1, 128); indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(LuceneTestCase.random()))); @@ -363,9 +368,9 @@ public static DataStream randomInstance(String dataStreamName, LongSupplier time dataStreamName, generation, metadata, - randomBoolean(), + system ? true : randomBoolean(), replicated, - false, // Some tests don't work well with system data streams, since these data streams require special handling + system, timeProvider, randomBoolean(), randomBoolean() ? IndexMode.STANDARD : null, // IndexMode.TIME_SERIES triggers validation that many unit tests doesn't pass