diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index e5abf16a74016..1d62054b737c7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -35,6 +35,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; @@ -157,12 +158,11 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn } static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIndexService, - ClusterState currentState, - String dataStreamName, - List backingIndices, - IndexMetadata writeIndex, - SystemDataStreamDescriptor systemDataStreamDescriptor) throws Exception - { + ClusterState currentState, + String dataStreamName, + List backingIndices, + IndexMetadata writeIndex, + SystemDataStreamDescriptor systemDataStreamDescriptor) throws Exception { Objects.requireNonNull(metadataCreateIndexService); Objects.requireNonNull(currentState); Objects.requireNonNull(backingIndices); @@ -177,8 +177,8 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must be lowercase"); } if (dataStreamName.startsWith(DataStream.BACKING_INDEX_PREFIX)) { - throw new IllegalArgumentException("data_stream [" + dataStreamName + "] must not start with '" - + DataStream.BACKING_INDEX_PREFIX + "'"); + throw new IllegalArgumentException( + "data_stream [" + dataStreamName + "] must not start with '" + DataStream.BACKING_INDEX_PREFIX + "'"); } final boolean isSystem = systemDataStreamDescriptor != null; @@ -219,9 +219,23 @@ static ClusterState createDataStream(MetadataCreateIndexService metadataCreateIn DataStream newDataStream = new DataStream(dataStreamName, timestampField, dsBackingIndices, 1L, template.metadata() != null ? Map.copyOf(template.metadata()) : null, hidden, false, isSystem); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); - logger.info("adding data stream [{}] with write index [{}] and backing indices [{}]", dataStreamName, + + List aliases = new ArrayList<>(); + if (template.template() != null && template.template().aliases() != null) { + for (var alias : template.template().aliases().values()) { + aliases.add(alias.getAlias()); + builder.put(alias.getAlias(), dataStreamName, alias.writeIndex(), alias.filter() == null ? null : alias.filter().string()); + } + } + + logger.info( + "adding data stream [{}] with write index [{}], backing indices [{}], and aliases [{}]", + dataStreamName, writeIndex.getIndex().getName(), - Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray())); + Strings.arrayToCommaDelimitedString(backingIndices.stream().map(i -> i.getIndex().getName()).toArray()), + Strings.collectionToCommaDelimitedString(aliases) + ); + return ClusterState.builder(currentState).metadata(builder).build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 1c0adeb62199a..c63d29eb2ce07 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -504,7 +504,8 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu logger.debug("applying create index request using composable template [{}]", templateName); ComposableIndexTemplate template = currentState.getMetadata().templatesV2().get(templateName); - if (request.dataStreamName() == null && template.getDataStreamTemplate() != null) { + final boolean isDataStream = template.getDataStreamTemplate() != null; + if (isDataStream && request.dataStreamName() == null) { throw new IllegalArgumentException("cannot create index with name [" + request.index() + "], because it matches with template [" + templateName + "] that creates data streams only, " + "use create data stream api instead"); @@ -519,14 +520,28 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); - return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, - indexService -> resolveAndValidateAliases(request.index(), request.aliases(), - MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), - // the context is only used for validation so it's fine to pass fake values for the - // shard id and the current timestamp - aliasValidator, xContentRegistry, indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), - indexService.dateMathExpressionResolverAt(request.getNameResolvedAt())), - Collections.singletonList(templateName), metadataTransformer); + return applyCreateIndexWithTemporaryService( + currentState, + request, + silent, + null, + tmpImd, + mappings, + indexService -> resolveAndValidateAliases( + request.index(), + // data stream aliases are created separately in MetadataCreateDataStreamService::createDataStream + isDataStream ? Set.of() : request.aliases(), + isDataStream ? List.of() : MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), + currentState.metadata(), + aliasValidator, + xContentRegistry, + // the context is used ony for validation so it's fine to pass fake values for the shard id and the current timestamp + indexService.newSearchExecutionContext(0, 0, null, () -> 0L, null, emptyMap()), + indexService.dateMathExpressionResolverAt(request.getNameResolvedAt()) + ), + Collections.singletonList(templateName), + metadataTransformer + ); } private ClusterState applyCreateIndexRequestForSystemDataStream(final ClusterState currentState, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index bd34d5adf9d24..de9201a0f839c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1146,12 +1146,6 @@ static List> resolveAliases(final ComposableIndexTemp .map(Template::aliases) .ifPresent(aliases::add); - // A template that creates data streams can't also create aliases. - // (otherwise we end up with aliases pointing to backing indices of data streams) - if (aliases.size() > 0 && template.getDataStreamTemplate() != null) { - throw new IllegalArgumentException("template [" + templateName + "] has alias and data stream definitions"); - } - // Aliases are applied in order, but subsequent alias configuration from the same name is // ignored, so in order for the order to be correct, alias configuration should be in order // of precedence (with the index template first) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 6bc2e5f0efc68..c2bf804d21b10 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -635,82 +635,6 @@ protected String contentType() { } } - public void testRolloverDataStreamWorksWithTemplateThatAlsoCreatesAliases() throws Exception { - final DataStream dataStream = DataStreamTestHelper.randomInstance() - // ensure no replicate data stream - .promoteDataStream(); - ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStream.getName() + "*")) - .template(new Template(null, null, Map.of("my-alias", AliasMetadata.newAliasMetadataBuilder("my-alias").build()))) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()).build(); - Metadata.Builder builder = Metadata.builder(); - builder.put("template", template); - for (Index index : dataStream.getIndices()) { - builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); - } - builder.put(dataStream); - final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build(); - - ThreadPool testThreadPool = new TestThreadPool(getTestName()); - try { - DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder( - "@timestamp", - DateFieldMapper.Resolution.MILLISECONDS, - null, - ScriptCompiler.NONE, - false, - Version.CURRENT).build(new ContentPath()); - MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class); - when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp"); - MetadataFieldMapper mockedTimestampField = new MetadataFieldMapper(mockedTimestampFieldType) { - @Override - protected String contentType() { - return null; - } - }; - MetadataFieldMapper[] metadataFieldMappers = {new MetadataIndexTemplateServiceTests.MetadataTimestampFieldMapper(true)}; - RootObjectMapper.Builder root = new RootObjectMapper.Builder("_doc"); - root.add(new DateFieldMapper.Builder(dataStream.getTimeStampField().getName(), DateFieldMapper.Resolution.MILLISECONDS, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, ScriptCompiler.NONE, true, Version.CURRENT)); - Mapping mapping = new Mapping(root.build(new ContentPath("")), metadataFieldMappers, Collections.emptyMap()); - MappingLookup mappingLookup = MappingLookup.fromMappers( - mapping, - List.of(mockedTimestampField, dateFieldMapper), - List.of(), - List.of()); - ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); - Environment env = mock(Environment.class); - when(env.sharedDataFile()).thenReturn(null); - AllocationService allocationService = mock(AllocationService.class); - when(allocationService.reroute(any(ClusterState.class), any(String.class))).then(i -> i.getArguments()[0]); - IndicesService indicesService = mockIndicesServices(mappingLookup); - IndexNameExpressionResolver mockIndexNameExpressionResolver = mock(IndexNameExpressionResolver.class); - when(mockIndexNameExpressionResolver.resolveDateMathExpression(any())).then(returnsFirstArg()); - - ShardLimitValidator shardLimitValidator = new ShardLimitValidator(Settings.EMPTY, clusterService); - MetadataCreateIndexService createIndexService = new MetadataCreateIndexService(Settings.EMPTY, - clusterService, indicesService, allocationService, new AliasValidator(), shardLimitValidator, env, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, testThreadPool, null, EmptySystemIndices.INSTANCE, false); - MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(clusterService, indicesService, - new AliasValidator(), null, xContentRegistry()); - MetadataRolloverService rolloverService = new MetadataRolloverService(testThreadPool, createIndexService, indexAliasesService, - mockIndexNameExpressionResolver, EmptySystemIndices.INSTANCE); - - MaxDocsCondition condition = new MaxDocsCondition(randomNonNegativeLong()); - List> metConditions = Collections.singletonList(condition); - CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); - - // Ensure that a warning header is emitted - Exception e = expectThrows( - IllegalArgumentException.class, - () -> rolloverService.rolloverClusterState(clusterState, dataStream.getName(), null, createIndexRequest, metConditions, - randomBoolean(), false) - ); - assertThat(e.getMessage(), equalTo("template [template] has alias and data stream definitions")); - } finally { - testThreadPool.shutdown(); - } - } - public void testValidation() throws Exception { final String rolloverTarget; final String sourceIndexName; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index a369f011ae88e..3edadb8c35637 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -23,12 +23,15 @@ import org.elasticsearch.indices.SystemIndices.Feature; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createFirstBackingIndex; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createTimestampField; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateMapping; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -66,6 +69,55 @@ public void testCreateDataStream() throws Exception { assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).isSystem(), is(false)); } + public void testCreateDataStreamWithAliasFromTemplate() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "my-data-stream"; + final int aliasCount = randomIntBetween(0, 3); + Map aliases = new HashMap<>(aliasCount); + for (int k = 0; k < aliasCount; k++) { + final String aliasName = randomAlphaOfLength(6); + var builder = AliasMetadata.newAliasMetadataBuilder(aliasName); + if (randomBoolean()) { + builder.filter(Map.of("term", Map.of("user", Map.of("value", randomAlphaOfLength(5))))); + } + builder.writeIndex(randomBoolean()); + aliases.put(aliasName, builder.build()); + } + ComposableIndexTemplate template = new ComposableIndexTemplate.Builder() + .indexPatterns(List.of(dataStreamName + "*")) + .dataStreamTemplate(new DataStreamTemplate()) + .template(new Template(null, null, aliases)) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metadata(Metadata.builder().put("template", template).build()) + .build(); + CreateDataStreamClusterStateUpdateRequest req = + new CreateDataStreamClusterStateUpdateRequest(dataStreamName, TimeValue.ZERO, TimeValue.ZERO); + ClusterState newState = MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); + assertThat(newState.metadata().dataStreams().size(), equalTo(1)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false)); + assertThat(newState.metadata().dataStreamAliases().size(), is(aliasCount)); + for (String aliasName : aliases.keySet()) { + var expectedAlias = aliases.get(aliasName); + var actualAlias = newState.metadata().dataStreamAliases().get(aliasName); + assertThat(actualAlias, is(notNullValue())); + assertThat(actualAlias.getName(), equalTo(expectedAlias.alias())); + assertThat(actualAlias.getFilter(), equalTo(expectedAlias.filter())); + assertThat(actualAlias.getWriteDataStream(), equalTo(expectedAlias.writeIndex() ? dataStreamName : null)); + } + + assertThat(newState.metadata().dataStreamAliases().values().stream().map(DataStreamAlias::getName).toArray(), + arrayContainingInAnyOrder (new ArrayList<>(aliases.keySet()).toArray())); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue()); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getAliases().size(), is(0)); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"), + equalTo("true")); + assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).isSystem(), is(false)); + } + public void testCreateSystemDataStream() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = ".system-data-stream"; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index ab0402d6bc906..651f341c3acda 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1207,31 +1207,6 @@ public void testResolveAliases() throws Exception { assertThat(resolvedAliases, equalTo(List.of(a3, a1, a2))); } - public void testResolveAliasesDataStreams() throws Exception { - Map a1 = new HashMap<>(); - a1.put("logs", AliasMetadata.newAliasMetadataBuilder("logs").build()); - - // index template can't have data streams and aliases - ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("logs-*"), - new Template(null, null, a1), null, 0L, 1L, null, new ComposableIndexTemplate.DataStreamTemplate(), null); - ClusterState state1 = ClusterState.builder(ClusterState.EMPTY_STATE) - .metadata(Metadata.builder().put("1", it).build()) - .build(); - Exception e = - expectThrows(IllegalArgumentException.class, () -> MetadataIndexTemplateService.resolveAliases(state1.metadata(), "1")); - assertThat(e.getMessage(), equalTo("template [1] has alias and data stream definitions")); - - // index template can't have data streams and a component template with an aliases - ComponentTemplate componentTemplate = new ComponentTemplate(new Template(null, null, a1), null, null); - it = new ComposableIndexTemplate(List.of("logs-*"), null, List.of("c1"), 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate(), null); - ClusterState state2 = ClusterState.builder(ClusterState.EMPTY_STATE) - .metadata(Metadata.builder().put("1", it).put("c1", componentTemplate).build()) - .build(); - e = expectThrows(IllegalArgumentException.class, () -> MetadataIndexTemplateService.resolveAliases(state2.metadata(), "1")); - assertThat(e.getMessage(), equalTo("template [1] has alias and data stream definitions")); - } - public void testAddInvalidTemplate() throws Exception { ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("a"), null, Arrays.asList("good", "bad"), null, null, null); diff --git a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java index 47c80dba46775..142b3241d637e 100644 --- a/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java +++ b/x-pack/plugin/data-streams/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; public class DataStreamsRestIT extends ESRestTestCase { @@ -85,14 +86,6 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException { assertEquals(1, XContentMapValues.extractValue("hits.total.value", results)); } - public void testAddingIndexTemplateWithAliasesAndDataStream() { - Request putComposableIndexTemplateRequest = new Request("PUT", "/_index_template/my-template"); - String body = "{\"index_patterns\":[\"mypattern*\"],\"data_stream\":{},\"template\":{\"aliases\":{\"my-alias\":{}}}}"; - putComposableIndexTemplateRequest.setJsonEntity(body); - Exception e = expectThrows(ResponseException.class, () -> client().performRequest(putComposableIndexTemplateRequest)); - assertThat(e.getMessage(), containsString("template [my-template] has alias and data stream definitions")); - } - public void testDataStreamAliases() throws Exception { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); @@ -237,6 +230,32 @@ public void testGetAliasApiFilterByDataStreamAlias() throws Exception { assertEquals(404, getAliasesResponse.get("status")); } + public void testDataStreamWithAliasFromTemplate() throws IOException { + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity( + "{\"index_patterns\": [\"logs-*\"], \"template\": { \"aliases\": { \"logs\": {} } }, \"data_stream\": {}}"); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + Request createDocRequest = new Request("POST", "/logs-emea/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + createDocRequest = new Request("POST", "/logs-nasa/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + Request getAliasesRequest = new Request("GET", "/_aliases"); + Map getAliasesResponse = entityAsMap(client().performRequest(getAliasesRequest)); + assertThat(getAliasesResponse.size(), is(2)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-emea.aliases", getAliasesResponse)); + assertEquals(Map.of("logs", Map.of()), XContentMapValues.extractValue("logs-nasa.aliases", getAliasesResponse)); + + Request searchRequest = new Request("GET", "/logs/_search"); + Map searchResponse = entityAsMap(client().performRequest(searchRequest)); + assertEquals(2, XContentMapValues.extractValue("hits.total.value", searchResponse)); + } + public void testDataStreamWriteAlias() throws IOException { // Create a template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1");