From 1729237bfa5723aa578675d37ab107eeddcd672c Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 23 Sep 2025 08:50:10 +0200 Subject: [PATCH 1/6] Cleanup of IndexSettingProvider In https://github.com/elastic/elasticsearch/pull/133232, we've added the ability to provide index metadata with an IndexSettingProvider. It turned out that we don't need that functionality as we ended up using a private index setting in https://github.com/elastic/elasticsearch/pull/132566. This also adds the `IndexVersion` as another parameter. This is in preparation for [this](https://github.com/elastic/elasticsearch/pull/132566#discussion_r2366143253) suggestion to conditionally set one or another setting, depending on the index version. --- .../DataStreamIndexSettingsProvider.java | 15 +- .../action/TransportGetDataStreamsAction.java | 7 +- .../DataStreamIndexSettingsProviderTests.java | 74 ++++----- .../TransportSimulateIndexTemplateAction.java | 6 +- .../metadata/MetadataCreateIndexService.java | 81 +++------- .../MetadataIndexTemplateService.java | 20 +-- .../metadata/MetadataMappingService.java | 5 +- .../cluster/routing/allocation/DataTier.java | 8 +- .../org/elasticsearch/index/IndexMode.java | 7 +- .../index/IndexSettingProvider.java | 29 ++-- .../index/IndexSettingProviders.java | 4 +- ...sportSimulateIndexTemplateActionTests.java | 14 +- .../MetadataCreateIndexServiceTests.java | 34 ++-- .../metadata/MetadataMappingServiceTests.java | 13 +- .../index/IndexSettingProviderTests.java | 8 +- .../LogsdbIndexModeSettingsProvider.java | 8 +- .../LogsdbIndexModeSettingsProviderTests.java | 150 +++++++++--------- ...dexSettingsProviderLegacyLicenseTests.java | 50 ++++-- 18 files changed, 242 insertions(+), 291 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java index 20c36bb4188c0..e66dc8f970819 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProvider.java @@ -38,8 +38,6 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; -import java.util.Map; -import java.util.function.BiConsumer; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DIMENSIONS; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; @@ -62,7 +60,7 @@ public class DataStreamIndexSettingsProvider implements IndexSettingProvider { } @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, @Nullable String dataStreamName, @Nullable IndexMode templateIndexMode, @@ -70,8 +68,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { if (dataStreamName != null) { DataStream dataStream = projectMetadata.dataStreams().get(dataStreamName); @@ -157,12 +155,7 @@ public void provideAdditionalMetadata( * dimension fields via a wildcard pattern. */ @Override - public void onUpdateMappings( - IndexMetadata indexMetadata, - DocumentMapper documentMapper, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata - ) { + public void onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documentMapper, Settings.Builder additionalSettings) { List indexDimensions = indexMetadata.getTimeSeriesDimensions(); if (indexDimensions.isEmpty()) { return; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 41129a207c131..79db9b4c8ada0 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; @@ -199,7 +200,7 @@ static IndexMode resolveMode( IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate); for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) { Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( MetadataIndexTemplateService.VALIDATE_INDEX_NAME, dataStream.getName(), indexMode, @@ -207,8 +208,8 @@ static IndexMode resolveMode( Instant.now(), settings, List.of(), - builder, - (k, v) -> {} + IndexVersion.current(), + builder ); Settings addlSettings = builder.build(); var rawMode = addlSettings.get(IndexSettings.MODE.getKey()); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java index ac398c67130f6..a55ccd5298304 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java @@ -93,7 +93,7 @@ public void testGetAdditionalIndexSettings() throws Exception { } """; Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -101,8 +101,8 @@ public void testGetAdditionalIndexSettings() throws Exception { now, settings, List.of(new CompressedXContent(mapping)), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: @@ -147,7 +147,7 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw } """; Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -155,8 +155,8 @@ public void testGetAdditionalIndexSettingsIndexRoutingPathAlreadyDefined() throw now, settings, List.of(new CompressedXContent(mapping)), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: @@ -221,7 +221,7 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { } """; Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -229,8 +229,8 @@ public void testGetAdditionalIndexSettingsMappingsMerging() throws Exception { now, settings, List.of(new CompressedXContent(mapping1), new CompressedXContent(mapping2), new CompressedXContent(mapping3)), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: @@ -255,7 +255,7 @@ public void testGetAdditionalIndexSettingsNoMappings() { Instant now = Instant.now().truncatedTo(ChronoUnit.SECONDS); Settings settings = Settings.EMPTY; Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -263,8 +263,8 @@ public void testGetAdditionalIndexSettingsNoMappings() { now, settings, List.of(), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: @@ -284,7 +284,7 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception { TimeValue lookAheadTime = TimeValue.timeValueMinutes(30); Settings settings = builder().put("index.mode", "time_series").put("index.look_ahead_time", lookAheadTime.getStringRep()).build(); Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -292,8 +292,8 @@ public void testGetAdditionalIndexSettingsLookAheadTime() throws Exception { now, settings, List.of(new CompressedXContent("{}")), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: @@ -313,7 +313,7 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception { TimeValue lookBackTime = TimeValue.timeValueHours(12); Settings settings = builder().put("index.mode", "time_series").put("index.look_back_time", lookBackTime.getStringRep()).build(); Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -321,8 +321,8 @@ public void testGetAdditionalIndexSettingsLookBackTime() throws Exception { now, settings, List.of(new CompressedXContent("{}")), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: @@ -349,7 +349,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exce Instant now = sixHoursAgo.plus(6, ChronoUnit.HOURS); Settings settings = Settings.EMPTY; Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -357,8 +357,8 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreated() throws Exce now, settings, List.of(new CompressedXContent("{}")), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); var result = additionalSettings.build(); assertThat(result.size(), equalTo(2)); @@ -389,7 +389,7 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi Settings settings = Settings.EMPTY; Exception e = expectThrows( IllegalStateException.class, - () -> provider.provideAdditionalMetadata( + () -> provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -397,8 +397,8 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi now, settings, null, - builder(), - (k, v) -> {} + IndexVersion.current(), + builder() ) ); assertThat( @@ -418,7 +418,7 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() { Settings settings = Settings.EMPTY; Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, null, @@ -426,8 +426,8 @@ public void testGetAdditionalIndexSettingsNonTsdbTemplate() { Instant.ofEpochMilli(1L), settings, null, - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); assertThat(result.size(), equalTo(0)); @@ -444,7 +444,7 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { Settings settings = Settings.EMPTY; Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, IndexMode.TIME_SERIES, @@ -452,8 +452,8 @@ public void testGetAdditionalIndexSettingsMigrateToTsdb() { now, settings, List.of(), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: @@ -477,7 +477,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() { ); Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, null, @@ -485,8 +485,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromTsdb() { Instant.ofEpochMilli(1L), Settings.EMPTY, List.of(), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); Settings result = additionalSettings.build(); assertThat(result.size(), equalTo(0)); @@ -900,7 +900,7 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce Settings settings = Settings.EMPTY; Settings.Builder additionalSettings = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 1), dataStreamName, IndexMode.TIME_SERIES, @@ -908,8 +908,8 @@ private Settings generateTsdbSettings(String mapping, Instant now) throws IOExce now, settings, List.of(new CompressedXContent(mapping)), - additionalSettings, - (k, v) -> {} + IndexVersion.current(), + additionalSettings ); var result = additionalSettings.build(); // The index.time_series.end_time setting requires index.mode to be set to time_series adding it here so that we read this setting: @@ -948,7 +948,7 @@ private Settings onUpdateMappings(String routingPath, String dimensions, String documentMapper = mapperService.documentMapper(); } Settings.Builder additionalSettings = builder(); - provider.onUpdateMappings(im, documentMapper, additionalSettings, (k, v) -> {}); + provider.onUpdateMappings(im, documentMapper, additionalSettings); return additionalSettings.build(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 5ab28d192adb0..26039be93fdd6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -279,7 +279,7 @@ public static Template resolveTemplate( Set overrulingSettings = new HashSet<>(); for (var provider : indexSettingProviders) { Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( indexName, template.getDataStreamTemplate() != null ? indexName : null, simulatedProject.retrieveIndexModeFromTemplate(template), @@ -287,8 +287,8 @@ public static Template resolveTemplate( now, templateSettings, mappings, - builder, - (k, v) -> {} + IndexVersion.current(), + builder ); Settings result = builder.build(); MetadataCreateIndexService.validateAdditionalSettings(provider, result, additionalSettings); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 8b42fd66c60ef..2d30dc13b9fa9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -45,7 +45,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -610,8 +609,7 @@ private ClusterState applyCreateIndexWithTemporaryService( private IndexMetadata buildAndValidateTemporaryIndexMetadata( final Settings aggregatedIndexSettings, final CreateIndexClusterStateUpdateRequest request, - final int routingNumShards, - ImmutableOpenMap.Builder> customMetadataBuilder + final int routingNumShards ) { final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings); @@ -626,7 +624,6 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata( tmpImdBuilder.setRoutingNumShards(routingNumShards); tmpImdBuilder.settings(indexSettings); tmpImdBuilder.system(isSystem); - tmpImdBuilder.putCustom(customMetadataBuilder.build()); // Set up everything, now locally create the index to see that things are ok, and apply IndexMetadata tempMetadata = tmpImdBuilder.build(); @@ -666,7 +663,6 @@ private ClusterState applyCreateIndexRequestWithV1Templates( final ProjectMetadata projectMetadata = metadata.getProject(request.projectId()); final RoutingTable routingTable = currentState.routingTable(request.projectId()); - ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); final Settings aggregatedIndexSettings = aggregateIndexSettings( metadata, projectMetadata, @@ -680,16 +676,10 @@ private ClusterState applyCreateIndexRequestWithV1Templates( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - customMetadataBuilder + indexSettingProviders ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata( - aggregatedIndexSettings, - request, - routingNumShards, - customMetadataBuilder - ); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); return applyCreateIndexWithTemporaryService( currentState, @@ -766,7 +756,6 @@ private ClusterState applyCreateIndexRequestWithV2Template( xContentRegistry, request.index() ); - ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); final Settings aggregatedIndexSettings = aggregateIndexSettings( metadata, projectMetadata, @@ -780,16 +769,10 @@ private ClusterState applyCreateIndexRequestWithV2Template( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - customMetadataBuilder + indexSettingProviders ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata( - aggregatedIndexSettings, - request, - routingNumShards, - customMetadataBuilder - ); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); return applyCreateIndexWithTemporaryService( currentState, @@ -828,7 +811,6 @@ private ClusterState applyCreateIndexRequestForSystemIndex( final ProjectMetadata projectMetadata = metadata.getProject(request.projectId()); final RoutingTable routingTable = currentState.routingTable(request.projectId()); - ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); final Settings aggregatedIndexSettings = aggregateIndexSettings( metadata, projectMetadata, @@ -842,16 +824,10 @@ private ClusterState applyCreateIndexRequestForSystemIndex( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - customMetadataBuilder + indexSettingProviders ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); - final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata( - aggregatedIndexSettings, - request, - routingNumShards, - customMetadataBuilder - ); + final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); return applyCreateIndexWithTemporaryService( currentState, @@ -902,7 +878,6 @@ private ClusterState applyCreateIndexRequestForSystemDataStream( final ProjectMetadata projectMetadata = metadata.getProject(request.projectId()); final RoutingTable routingTable = currentState.routingTable(request.projectId()); - ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); final Settings aggregatedIndexSettings = aggregateIndexSettings( metadata, projectMetadata, @@ -916,16 +891,10 @@ private ClusterState applyCreateIndexRequestForSystemDataStream( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - customMetadataBuilder + indexSettingProviders ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); - final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata( - aggregatedIndexSettings, - request, - routingNumShards, - customMetadataBuilder - ); + final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); return applyCreateIndexWithTemporaryService( currentState, @@ -1018,7 +987,6 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( final ProjectMetadata projectMetadata = metadata.getProject(request.projectId()); final RoutingTable routingTable = currentState.routingTable(request.projectId()); - ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); final Settings aggregatedIndexSettings = aggregateIndexSettings( metadata, projectMetadata, @@ -1032,16 +1000,10 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - customMetadataBuilder + indexSettingProviders ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata( - aggregatedIndexSettings, - request, - routingNumShards, - customMetadataBuilder - ); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); return applyCreateIndexWithTemporaryService( currentState, @@ -1128,8 +1090,7 @@ static Settings aggregateIndexSettings( Settings settings, IndexScopedSettings indexScopedSettings, ShardLimitValidator shardLimitValidator, - Set indexSettingProviders, - ImmutableOpenMap.Builder> customMetadataBuilder + Set indexSettingProviders ) { final boolean isDataStreamIndex = request.dataStreamName() != null; @@ -1138,6 +1099,13 @@ static Settings aggregateIndexSettings( // index (see more comments below) final Settings.Builder templateSettings = Settings.builder().put(combinedTemplateSettings); final Settings.Builder requestSettings = Settings.builder().put(request.settings()); + IndexVersion createdVersion; + IndexVersion requestSettingsIndexVersion = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(request.settings()); + if (requestSettingsIndexVersion.after(IndexVersions.ZERO)) { + createdVersion = requestSettingsIndexVersion; + } else { + createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion()); + } final Settings.Builder indexSettingsBuilder = Settings.builder(); if (sourceMetadata == null) { @@ -1156,7 +1124,7 @@ static Settings aggregateIndexSettings( Set overrulingSettings = new HashSet<>(); for (IndexSettingProvider provider : indexSettingProviders) { Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( request.index(), request.dataStreamName(), templateIndexMode, @@ -1164,8 +1132,8 @@ static Settings aggregateIndexSettings( resolvedAt, templateAndRequestSettings, combinedTemplateMappings, - builder, - customMetadataBuilder::put + createdVersion, + builder ); var newAdditionalSettings = builder.build(); validateAdditionalSettings(provider, newAdditionalSettings, additionalIndexSettings); @@ -1241,10 +1209,7 @@ static Settings aggregateIndexSettings( } } - if (indexSettingsBuilder.get(IndexMetadata.SETTING_VERSION_CREATED) == null) { - IndexVersion createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion()); - indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion); - } + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion); if (INDEX_NUMBER_OF_SHARDS_SETTING.exists(indexSettingsBuilder) == false) { indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, INDEX_NUMBER_OF_SHARDS_SETTING.get(settings)); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 3b94c4ed14458..2233d30709cd9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -788,10 +787,9 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo final var combinedMappings = collectMappings(indexTemplate, componentTemplates, "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, componentTemplates); var additionalSettingsBuilder = Settings.builder(); - ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); for (var provider : indexSettingProviders) { Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( VALIDATE_INDEX_NAME, indexTemplate.getDataStreamTemplate() != null ? VALIDATE_DATA_STREAM_NAME : null, projectMetadata.retrieveIndexModeFromTemplate(indexTemplate), @@ -799,8 +797,8 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo now, combinedSettings, combinedMappings, - builder, - customMetadataBuilder::put + IndexVersion.current(), + builder ); var newAdditionalSettings = builder.build(); MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalSettingsBuilder); @@ -834,15 +832,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo // Finally, right before adding the template, we need to ensure that the composite settings, // mappings, and aliases are valid after it's been composed with the component templates try { - validateCompositeTemplate( - projectMetadata, - name, - templateToValidate, - customMetadataBuilder.build(), - indicesService, - xContentRegistry, - systemIndices - ); + validateCompositeTemplate(projectMetadata, name, templateToValidate, indicesService, xContentRegistry, systemIndices); } catch (Exception e) { throw new IllegalArgumentException( "composable template [" @@ -1958,7 +1948,6 @@ private static void validateCompositeTemplate( final ProjectMetadata project, final String templateName, final ComposableIndexTemplate template, - final ImmutableOpenMap> customMetadata, final IndicesService indicesService, final NamedXContentRegistry xContentRegistry, final SystemIndices systemIndices @@ -1986,7 +1975,6 @@ private static void validateCompositeTemplate( // Validate index metadata (settings) final IndexMetadata tmpIndexMetadata = IndexMetadata.builder(temporaryIndexName) .settings(finalResolvedSettings) - .putCustom(customMetadata) .build(); indicesService.withTempIndexService(tmpIndexMetadata, tempIndexService -> { // Validate aliases diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java index 017bb9883cc08..f0ae537855a27 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.IOUtils; @@ -219,14 +218,13 @@ private ClusterState applyRequest( indexMetadataBuilder.putInferenceFields(docMapper.mappers().inferenceFields()); } boolean updatedSettings = false; - ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); final Settings.Builder additionalIndexSettings = Settings.builder(); if (updatedMapping) { indexMetadataBuilder.mappingVersion(1 + indexMetadataBuilder.mappingVersion()) .mappingsUpdatedVersion(IndexVersion.current()); for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) { Settings.Builder newAdditionalSettingsBuilder = Settings.builder(); - provider.onUpdateMappings(indexMetadata, docMapper, newAdditionalSettingsBuilder, customMetadataBuilder::put); + provider.onUpdateMappings(indexMetadata, docMapper, newAdditionalSettingsBuilder); if (newAdditionalSettingsBuilder.keys().isEmpty() == false) { Settings newAdditionalSettings = newAdditionalSettingsBuilder.build(); MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalIndexSettings); @@ -242,7 +240,6 @@ private ClusterState applyRequest( indexMetadataBuilder.settings(indexSettingsBuilder.build()); indexMetadataBuilder.settingsVersion(1 + indexMetadata.getSettingsVersion()); } - indexMetadataBuilder.putCustom(customMetadataBuilder.build()); /* * This implicitly increments the index metadata version and builds the index metadata. This means that we need to have * already incremented the mapping version if necessary. Therefore, the mapping version increment must remain before this diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java index 8878af8a02c52..8f4d92bcc8133 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import java.time.Instant; @@ -33,7 +34,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.BiConsumer; import java.util.function.Function; /** @@ -219,7 +219,7 @@ public static class DefaultHotAllocationSettingProvider implements IndexSettingP private static final Logger logger = LogManager.getLogger(DefaultHotAllocationSettingProvider.class); @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, @Nullable String dataStreamName, IndexMode templateIndexMode, @@ -227,8 +227,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { Set settings = indexTemplateAndCreateRequestSettings.keySet(); if (settings.contains(TIER_PREFERENCE)) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 40231ac9e3d6c..687d48ef1cc25 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -47,7 +47,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.function.BiConsumer; import java.util.function.BooleanSupplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -619,7 +618,7 @@ public String toString() { */ public static final class IndexModeSettingsProvider implements IndexSettingProvider { @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -627,8 +626,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { IndexMode indexMode = templateIndexMode; if (indexMode == null) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java index 8ecc3f60ce7d8..5d4794e7f80df 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java @@ -22,8 +22,6 @@ import java.io.IOException; import java.time.Instant; import java.util.List; -import java.util.Map; -import java.util.function.BiConsumer; /** * An {@link IndexSettingProvider} is a provider for index level settings and custom index metadata that can be set @@ -43,12 +41,11 @@ public interface IndexSettingProvider { * @param indexTemplateAndCreateRequestSettings All the settings resolved from the template that matches and any settings * defined on the create index request * @param combinedTemplateMappings All the mappings resolved from the template that matches + * @param indexVersion The index version to be used for the new index. + * Always {@link IndexVersion#current()} when invoked during validation. * @param additionalSettings A settings builder to which additional settings can be added - * @param additionalCustomMetadata A consumer to which additional - * {@linkplain IndexMetadata.Builder#putCustom(String, Map) custom index metadata} - * can be added */ - void provideAdditionalMetadata( + void provideAdditionalSettings( String indexName, @Nullable String dataStreamName, @Nullable IndexMode templateIndexMode, @@ -56,27 +53,19 @@ void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ); /** * Called when the mappings for an existing index are updated, before the new index metadata is created. * This method can be used to update index settings and to provide custom metadata based on the new mappings. * - * @param indexMetadata The index metadata for the index being updated - * @param documentMapper The document mapper containing the updated mappings - * @param additionalSettings A settings builder to which additional settings can be added - * @param additionalCustomMetadata A consumer to which additional - * {@linkplain IndexMetadata.Builder#putCustom(String, Map) custom index metadata} - * can be added + * @param indexMetadata The index metadata for the index being updated + * @param documentMapper The document mapper containing the updated mappings + * @param additionalSettings A settings builder to which additional settings can be added */ - default void onUpdateMappings( - IndexMetadata indexMetadata, - DocumentMapper documentMapper, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata - ) {} + default void onUpdateMappings(IndexMetadata indexMetadata, DocumentMapper documentMapper, Settings.Builder additionalSettings) {} /** * Infrastructure class that holds services that can be used by {@link IndexSettingProvider} instances. diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProviders.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProviders.java index 062bb391bca9b..386f4b7abc1e6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProviders.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProviders.java @@ -44,8 +44,8 @@ public static IndexSettingProviders of(Consumer settingsBuilde resolvedAt, indexTemplateAndCreateRequestSettings, combinedTemplateMappings, - additionalSettings, - additionalCustomMetadata) -> settingsBuilderConsumer.accept(additionalSettings) + indexVersion, + additionalSettings) -> settingsBuilderConsumer.accept(additionalSettings) ) ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 8b4805f675224..29ec3c22ed1e0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.test.ESTestCase; @@ -25,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.BiConsumer; import static org.hamcrest.CoreMatchers.is; import static org.mockito.ArgumentMatchers.any; @@ -63,7 +63,7 @@ public void testSettingsProviderIsOverridden() throws Exception { // Create a setting provider that sets the test-setting to 0 Set indexSettingsProviders = Set.of(new IndexSettingProvider() { @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -71,14 +71,14 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings allSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { additionalSettings.put("test-setting", 0); } }, new IndexSettingProvider() { @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -86,8 +86,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { additionalSettings.put("test-setting-2", 10); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index c57dba8a29aee..f5a85558d2b0e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -42,7 +42,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.version.CompatibilityVersions; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -774,7 +773,7 @@ public void testAggregateSettingsProviderOverrulesSettingsFromRequest() { randomShardLimitService(), Set.of(new IndexSettingProvider() { @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -782,8 +781,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { additionalSettings.put("request_setting", "overrule_value").put("other_setting", "other_value"); } @@ -826,7 +825,7 @@ public void testAggregateSettingsProviderIsNotFilteredOnFailureStore() { randomShardLimitService(), Set.of(new IndexSettingProvider() { @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -834,8 +833,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { additionalSettings.put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING.getKey(), "override"); } @@ -870,7 +869,7 @@ public void testAggregateSettingsProviderOverrulesNullFromRequest() { randomShardLimitService(), Set.of(new IndexSettingProvider() { @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -878,8 +877,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { additionalSettings.put("request_setting", "overrule_value").put("other_setting", "other_value"); } @@ -914,7 +913,7 @@ public void testAggregateSettingsProviderOverrulesSettingsFromTemplates() { IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), Set.of(new IndexSettingProvider() { - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -922,8 +921,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { additionalSettings.put("template_setting", "overrule_value").put("other_setting", "other_value"); } @@ -959,7 +958,7 @@ public void testAggregateSettingsProviderOverrulesNullFromTemplates() { randomShardLimitService(), Set.of(new IndexSettingProvider() { @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -967,8 +966,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { additionalSettings.put("template_setting", "overrule_value").put("other_setting", "other_value"); } @@ -1935,8 +1934,7 @@ private Settings aggregateIndexSettings( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - ImmutableOpenMap.builder() + indexSettingProviders ); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMappingServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMappingServiceTests.java index 3a41a959d290e..b6e0831b27d1e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMappingServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMappingServiceTests.java @@ -29,9 +29,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.function.BiConsumer; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @@ -151,7 +149,7 @@ public void testUpdateSettings() throws Exception { final MetadataMappingService.PutMappingExecutor putMappingExecutor = mappingService.new PutMappingExecutor( new IndexSettingProviders(Set.of(new IndexSettingProvider() { @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -159,19 +157,17 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) {} @Override public void onUpdateMappings( IndexMetadata indexMetadata, DocumentMapper documentMapper, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + Settings.Builder additionalSettings ) { additionalSettings.put("index.mapping.total_fields.limit", 42); - additionalCustomMetadata.accept("foo", Map.of("bar", "baz")); } })) ); @@ -193,7 +189,6 @@ public void onUpdateMappings( IndexMetadata indexMetadata = resultingState.metadata().indexMetadata(indexService.index()); assertThat(indexMetadata.getSettingsVersion(), equalTo(1 + previousVersion)); assertThat(indexMetadata.getSettings().get("index.mapping.total_fields.limit"), equalTo("42")); - assertThat(indexMetadata.getCustomData("foo"), equalTo(Map.of("bar", "baz"))); } private static List singleTask(PutMappingClusterStateUpdateRequest request) { diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java index f920d0c5bc7bc..c06fa774e95b3 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java @@ -18,9 +18,7 @@ import java.time.Instant; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiConsumer; public class IndexSettingProviderTests extends ESSingleNodeTestCase { @@ -87,7 +85,7 @@ static class TestIndexSettingsProvider implements IndexSettingProvider { } @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( String indexName, String dataStreamName, IndexMode templateIndexMode, @@ -95,8 +93,8 @@ public void provideAdditionalMetadata( Instant resolvedAt, Settings indexTemplateAndCreateRequestSettings, List combinedTemplateMappings, - Settings.Builder additionalSettings, - BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + Settings.Builder additionalSettings ) { if (enabled.get()) { additionalSettings.put("index.refresh_interval", intervalValue); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index caa4210fd6fcb..a63845bcb13f0 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -41,9 +41,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; -import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; @@ -98,7 +96,7 @@ public boolean overrulesTemplateAndRequestSettings() { } @Override - public void provideAdditionalMetadata( + public void provideAdditionalSettings( final String indexName, final String dataStreamName, IndexMode templateIndexMode, @@ -106,8 +104,8 @@ public void provideAdditionalMetadata( final Instant resolvedAt, Settings settings, final List combinedTemplateMappings, - final Settings.Builder additionalSettings, - final BiConsumer> additionalCustomMetadata + IndexVersion indexVersion, + final Settings.Builder additionalSettings ) { boolean isLogsDB = templateIndexMode == IndexMode.LOGSDB; // This index name is used when validating component and index templates, we should skip this check in that case. diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index ed754766b3c72..84f2d269f9cc7 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -130,7 +130,7 @@ private Settings generateLogsdbSettings(Settings settings, String mapping, Versi return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); }, IndexVersion::current, () -> version, true, true); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 0), DATA_STREAM_NAME, IndexMode.LOGSDB, @@ -138,8 +138,8 @@ private Settings generateLogsdbSettings(Settings settings, String mapping, Versi Instant.now(), settings, mapping == null ? List.of() : List.of(new CompressedXContent(mapping)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); var result = settingsBuilder.build(); return builder().put(result).build(); @@ -152,7 +152,7 @@ public void testDisabled() throws IOException { ); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -160,8 +160,8 @@ public void testDisabled() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -175,7 +175,7 @@ public void testOnIndexCreation() throws IOException { ); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( "logs-apache-production", null, null, @@ -183,8 +183,8 @@ public void testOnIndexCreation() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -198,7 +198,7 @@ public void testOnExplicitStandardIndex() throws IOException { ); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -206,8 +206,8 @@ public void testOnExplicitStandardIndex() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -221,7 +221,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { ); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -229,8 +229,8 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -244,7 +244,7 @@ public void testNonLogsDataStream() throws IOException { ); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs", null, @@ -252,8 +252,8 @@ public void testNonLogsDataStream() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -263,7 +263,7 @@ public void testNonLogsDataStream() throws IOException { public void testWithoutLogsComponentTemplate() throws IOException { final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -271,8 +271,8 @@ public void testWithoutLogsComponentTemplate() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -282,7 +282,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { public void testWithLogsComponentTemplate() throws IOException { final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -290,8 +290,8 @@ public void testWithLogsComponentTemplate() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -301,7 +301,7 @@ public void testWithLogsComponentTemplate() throws IOException { public void testWithMultipleComponentTemplates() throws IOException { final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -309,8 +309,8 @@ public void testWithMultipleComponentTemplates() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -320,7 +320,7 @@ public void testWithMultipleComponentTemplates() throws IOException { public void testWithCustomComponentTemplatesOnly() throws IOException { final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -328,8 +328,8 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -339,7 +339,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { public void testNonMatchingTemplateIndexPattern() throws IOException { final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -347,8 +347,8 @@ public void testNonMatchingTemplateIndexPattern() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -362,7 +362,7 @@ public void testCaseSensitivity() throws IOException { ); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "LOGS-apache-production", null, @@ -370,8 +370,8 @@ public void testCaseSensitivity() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -382,7 +382,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException { final LogsdbIndexModeSettingsProvider provider = withoutMapperService(true); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production-eu", null, @@ -390,8 +390,8 @@ public void testMultipleHyphensInDataStreamName() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings additionalIndexSettings = settingsBuilder.build(); @@ -401,7 +401,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException { public void testBeforeAndAfterSettingUpdate() throws IOException { final LogsdbIndexModeSettingsProvider provider = withoutMapperService(false); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -409,8 +409,8 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k2, v2) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings beforeSettings = settingsBuilder.build(); @@ -419,7 +419,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { provider.updateClusterIndexModeLogsdbEnabled(true); settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -427,8 +427,8 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k1, v1) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings afterSettings = settingsBuilder.build(); @@ -437,7 +437,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { provider.updateClusterIndexModeLogsdbEnabled(false); settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( null, "logs-apache-production", null, @@ -445,8 +445,8 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, List.of(new CompressedXContent(DEFAULT_MAPPING)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); final Settings laterSettings = settingsBuilder.build(); @@ -699,7 +699,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { .build(); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, null, @@ -707,8 +707,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { Instant.ofEpochMilli(1L), settings, List.of(), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); Settings result = settingsBuilder.build(); assertThat(result.size(), equalTo(0)); @@ -716,7 +716,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { logsdbLicenseService.setSyntheticSourceFallback(true); settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, null, @@ -724,8 +724,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { Instant.ofEpochMilli(1L), settings, List.of(), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); result = settingsBuilder.build(); assertThat(result.size(), equalTo(1)); @@ -733,7 +733,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { assertThat(newMapperServiceCounter.get(), equalTo(2)); settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, IndexMode.TIME_SERIES, @@ -741,8 +741,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { Instant.ofEpochMilli(1L), settings, List.of(), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); result = settingsBuilder.build(); assertThat(result.size(), equalTo(1)); @@ -750,7 +750,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { assertThat(newMapperServiceCounter.get(), equalTo(3)); settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, IndexMode.LOGSDB, @@ -758,8 +758,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { Instant.ofEpochMilli(1L), settings, List.of(), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); result = settingsBuilder.build(); assertThat(result.size(), equalTo(3)); @@ -782,7 +782,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceOldNode() Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(DATA_STREAM_NAME, 2), DATA_STREAM_NAME, null, @@ -790,8 +790,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceOldNode() Instant.ofEpochMilli(1L), settings, List.of(), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); var result = settingsBuilder.build(); assertTrue(result.isEmpty()); @@ -811,7 +811,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( 1 ); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, null, @@ -819,8 +819,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( Instant.ofEpochMilli(1L), settings, List.of(), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); Settings result = settingsBuilder.build(); assertThat(result.size(), equalTo(0)); @@ -835,7 +835,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( ); settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, null, @@ -843,8 +843,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( Instant.ofEpochMilli(1L), settings, List.of(), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); result = settingsBuilder.build(); assertThat(result.size(), equalTo(4)); @@ -854,7 +854,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(dataStreamName, 2), dataStreamName, null, @@ -862,8 +862,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( Instant.ofEpochMilli(1L), builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.toString()).build(), List.of(), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); result = settingsBuilder.build(); assertThat(result.size(), equalTo(0)); @@ -1306,7 +1306,7 @@ public void testSortFastRefresh() throws Exception { true ); Settings.Builder settingsBuilder = builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( DataStream.getDefaultBackingIndexName(systemIndex, 0), systemIndex, IndexMode.LOGSDB, @@ -1314,8 +1314,8 @@ public void testSortFastRefresh() throws Exception { Instant.now(), settings, List.of(new CompressedXContent(mappings)), - settingsBuilder, - (k, v) -> {} + IndexVersion.current(), + settingsBuilder ); var additionalIndexSettings = settingsBuilder.build(); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 61c360371c8a9..e8fb07b8be376 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -65,7 +65,17 @@ public void testGetAdditionalIndexSettingsDefault() { String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata(indexName, dataStreamName, null, null, null, settings, List.of(), builder, (k, v) -> {}); + provider.provideAdditionalSettings( + indexName, + dataStreamName, + null, + null, + null, + settings, + List.of(), + IndexVersion.current(), + builder + ); var result = builder.build(); var expectedBuilder = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED"); if (PatternTextFieldMapper.PATTERN_TEXT_MAPPER.isEnabled()) { @@ -79,7 +89,17 @@ public void testGetAdditionalIndexSettingsApm() throws IOException { String dataStreamName = "metrics-apm.app.test"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata(indexName, dataStreamName, null, null, null, settings, List.of(), builder, (k, v) -> {}); + provider.provideAdditionalSettings( + indexName, + dataStreamName, + null, + null, + null, + settings, + List.of(), + IndexVersion.current(), + builder + ); var result = builder.build(); Settings expectedAdditionalSettings = PatternTextFieldMapper.PATTERN_TEXT_MAPPER.isEnabled() ? Settings.builder().put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true).build() @@ -95,14 +115,24 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException { for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata(indexName, dataStreamName, null, null, null, settings, List.of(), builder, (k, v) -> {}); + provider.provideAdditionalSettings( + indexName, + dataStreamName, + null, + null, + null, + settings, + List.of(), + IndexVersion.current(), + builder + ); var result = builder.build(); assertEquals(expectedAdditionalSettings, result); } for (String indexName : new String[] { ".profiling-sq-executables", ".profiling-sq-leafframes", ".profiling-stacktraces" }) { Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata(indexName, null, null, null, null, settings, List.of(), builder, (k, v) -> {}); + provider.provideAdditionalSettings(indexName, null, null, null, null, settings, List.of(), IndexVersion.current(), builder); var result = builder.build(); assertEquals(expectedAdditionalSettings, result); } @@ -113,7 +143,7 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( indexName, dataStreamName, IndexMode.TIME_SERIES, @@ -121,8 +151,8 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { null, settings, List.of(), - builder, - (k, v) -> {} + IndexVersion.current(), + builder ); var result = builder.build(); Settings expectedAdditionalSettings = PatternTextFieldMapper.PATTERN_TEXT_MAPPER.isEnabled() @@ -159,7 +189,7 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); Settings.Builder builder = Settings.builder(); - provider.provideAdditionalMetadata( + provider.provideAdditionalSettings( indexName, dataStreamName, IndexMode.TIME_SERIES, @@ -167,8 +197,8 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception null, settings, List.of(), - builder, - (k, v) -> {} + IndexVersion.current(), + builder ); var result = builder.build(); From 2b9be20743ad13aea489a1fc2aec9d0ef9b472db Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 23 Sep 2025 06:59:49 +0000 Subject: [PATCH 2/6] [CI] Auto commit changes from spotless --- .../cluster/metadata/MetadataIndexTemplateService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 2233d30709cd9..4aa8e5dfa551e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1973,9 +1973,7 @@ private static void validateCompositeTemplate( .build(); // Validate index metadata (settings) - final IndexMetadata tmpIndexMetadata = IndexMetadata.builder(temporaryIndexName) - .settings(finalResolvedSettings) - .build(); + final IndexMetadata tmpIndexMetadata = IndexMetadata.builder(temporaryIndexName).settings(finalResolvedSettings).build(); indicesService.withTempIndexService(tmpIndexMetadata, tempIndexService -> { // Validate aliases MetadataCreateIndexService.resolveAndValidateAliases( From 7a15254ae1826d7aa691d4400009e5df72f4ae1a Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 23 Sep 2025 19:02:28 +0200 Subject: [PATCH 3/6] Use index version of write index --- .../datastreams/action/TransportGetDataStreamsAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 79db9b4c8ada0..820f7178de44a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -198,6 +198,7 @@ static IndexMode resolveMode( ComposableIndexTemplate indexTemplate ) { IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate); + IndexVersion indexVersion = state.metadata().index(dataStream.getWriteIndex()).getCreationVersion(); for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) { Settings.Builder builder = Settings.builder(); provider.provideAdditionalSettings( @@ -208,7 +209,7 @@ static IndexMode resolveMode( Instant.now(), settings, List.of(), - IndexVersion.current(), + indexVersion, builder ); Settings addlSettings = builder.build(); From 1667c93e0bd18d50fcc34372e98cf44767947b89 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 24 Sep 2025 08:14:32 +0200 Subject: [PATCH 4/6] Fix resolution of IndexVersion --- .../metadata/MetadataCreateIndexService.java | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 2d30dc13b9fa9..c511eecda5b2d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1099,18 +1099,26 @@ static Settings aggregateIndexSettings( // index (see more comments below) final Settings.Builder templateSettings = Settings.builder().put(combinedTemplateSettings); final Settings.Builder requestSettings = Settings.builder().put(request.settings()); - IndexVersion createdVersion; - IndexVersion requestSettingsIndexVersion = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(request.settings()); - if (requestSettingsIndexVersion.after(IndexVersions.ZERO)) { - createdVersion = requestSettingsIndexVersion; - } else { - createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion()); - } final Settings.Builder indexSettingsBuilder = Settings.builder(); if (sourceMetadata == null) { - final Settings templateAndRequestSettings = Settings.builder().put(combinedTemplateSettings).put(request.settings()).build(); - + // Create a combined builder that serves two purposes: + // 1) It is used to pass to the IndexSettingProviders so they can see the combined settings + // that will be applied to the new index + // 2) It is used to create the IndexVersion to be passed in to the IndexSettingProviders. + // After the IndexSettingProviders have added their settings, the version is re-calculated + // in case one of the providers set a specific version. + Settings.Builder templateAndRequestSettingsBuilder = Settings.builder().put(combinedTemplateSettings).put(request.settings()); + if (request.isFailureIndex()) { + DataStreamFailureStoreDefinition.filterUserDefinedSettings(templateAndRequestSettingsBuilder); + } + final Settings templateAndRequestSettings = templateAndRequestSettingsBuilder.build(); + IndexVersion createdVersion; + if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.exists(templateAndRequestSettings)) { + createdVersion = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(templateAndRequestSettings); + } else { + createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion()); + } final IndexMode templateIndexMode = Optional.of(request) .filter(r -> r.isFailureIndex() == false) .map(CreateIndexClusterStateUpdateRequest::matchingTemplate) @@ -1209,7 +1217,10 @@ static Settings aggregateIndexSettings( } } - indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion); + if (indexSettingsBuilder.get(IndexMetadata.SETTING_VERSION_CREATED) == null) { + IndexVersion createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion()); + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion); + } if (INDEX_NUMBER_OF_SHARDS_SETTING.exists(indexSettingsBuilder) == false) { indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, INDEX_NUMBER_OF_SHARDS_SETTING.get(settings)); } From 929fe93199066b73b2873aefa453bc88d66f5b0a Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 24 Sep 2025 08:27:16 +0200 Subject: [PATCH 5/6] Adjust JavaDoc of IndexSettingProvider It can't provide custom metadata anymore --- .../cluster/metadata/MetadataCreateIndexService.java | 2 +- .../java/org/elasticsearch/index/IndexSettingProvider.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index c511eecda5b2d..02bebc75c3832 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1107,7 +1107,7 @@ static Settings aggregateIndexSettings( // that will be applied to the new index // 2) It is used to create the IndexVersion to be passed in to the IndexSettingProviders. // After the IndexSettingProviders have added their settings, the version is re-calculated - // in case one of the providers set a specific version. + // in case one of the providers sets a specific version. Settings.Builder templateAndRequestSettingsBuilder = Settings.builder().put(combinedTemplateSettings).put(request.settings()); if (request.isFailureIndex()) { DataStreamFailureStoreDefinition.filterUserDefinedSettings(templateAndRequestSettingsBuilder); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java index 5d4794e7f80df..6b79e331d30e8 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java @@ -24,12 +24,12 @@ import java.util.List; /** - * An {@link IndexSettingProvider} is a provider for index level settings and custom index metadata that can be set + * An {@link IndexSettingProvider} is a provider for index level settings that can be set * explicitly as a default value (so they show up as "set" for newly created indices) */ public interface IndexSettingProvider { /** - * Allows to provide default index {@link Settings} and custom index metadata for a newly created index. + * Allows to provide default index {@link Settings} for a newly created index. * * @param indexName The name of the new index being created * @param dataStreamName The name of the data stream if the index being created is part of a data stream @@ -59,7 +59,7 @@ void provideAdditionalSettings( /** * Called when the mappings for an existing index are updated, before the new index metadata is created. - * This method can be used to update index settings and to provide custom metadata based on the new mappings. + * This method can be used to provide additional index settings based on the new mappings. * * @param indexMetadata The index metadata for the index being updated * @param documentMapper The document mapper containing the updated mappings From dcc5af91277ae45dd483d889d1d7a3a6a3fe8729 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 24 Sep 2025 11:52:34 +0200 Subject: [PATCH 6/6] Disallow providers to set the index version Otherwise, they can't rely on the provided version to be the one that will be actually used for the created index. --- .../metadata/MetadataCreateIndexService.java | 54 +++++++++++-------- .../index/IndexSettingProvider.java | 4 +- .../index/IndexSettingProviderTests.java | 12 +++++ 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 02bebc75c3832..eeae0fbb5ef8b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1100,25 +1100,26 @@ static Settings aggregateIndexSettings( final Settings.Builder templateSettings = Settings.builder().put(combinedTemplateSettings); final Settings.Builder requestSettings = Settings.builder().put(request.settings()); + // Create a combined builder that serves two purposes: + // 1) It is used to pass to the IndexSettingProviders so they can see the combined settings + // that will be applied to the new index + // 2) It is used to create the IndexVersion for the new index which will be passed in to the IndexSettingProviders. + // IndexSettingProviders are not allowed to set the index version. + // Otherwise, they wouldn't be able to rely on the version they receive because another provider would be able to change it. + Settings.Builder templateAndRequestSettingsBuilder = Settings.builder().put(combinedTemplateSettings).put(request.settings()); + if (request.isFailureIndex()) { + DataStreamFailureStoreDefinition.filterUserDefinedSettings(templateAndRequestSettingsBuilder); + } + final Settings templateAndRequestSettings = templateAndRequestSettingsBuilder.build(); + final IndexVersion createdVersion; + if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.exists(templateAndRequestSettings)) { + createdVersion = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(templateAndRequestSettings); + } else { + createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion()); + } + final Settings.Builder indexSettingsBuilder = Settings.builder(); if (sourceMetadata == null) { - // Create a combined builder that serves two purposes: - // 1) It is used to pass to the IndexSettingProviders so they can see the combined settings - // that will be applied to the new index - // 2) It is used to create the IndexVersion to be passed in to the IndexSettingProviders. - // After the IndexSettingProviders have added their settings, the version is re-calculated - // in case one of the providers sets a specific version. - Settings.Builder templateAndRequestSettingsBuilder = Settings.builder().put(combinedTemplateSettings).put(request.settings()); - if (request.isFailureIndex()) { - DataStreamFailureStoreDefinition.filterUserDefinedSettings(templateAndRequestSettingsBuilder); - } - final Settings templateAndRequestSettings = templateAndRequestSettingsBuilder.build(); - IndexVersion createdVersion; - if (IndexMetadata.SETTING_INDEX_VERSION_CREATED.exists(templateAndRequestSettings)) { - createdVersion = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(templateAndRequestSettings); - } else { - createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion()); - } final IndexMode templateIndexMode = Optional.of(request) .filter(r -> r.isFailureIndex() == false) .map(CreateIndexClusterStateUpdateRequest::matchingTemplate) @@ -1217,10 +1218,7 @@ static Settings aggregateIndexSettings( } } - if (indexSettingsBuilder.get(IndexMetadata.SETTING_VERSION_CREATED) == null) { - IndexVersion createdVersion = IndexVersion.min(IndexVersion.current(), nodes.getMaxDataNodeCompatibleIndexVersion()); - indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion); - } + indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion); if (INDEX_NUMBER_OF_SHARDS_SETTING.exists(indexSettingsBuilder) == false) { indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, INDEX_NUMBER_OF_SHARDS_SETTING.get(settings)); } @@ -1265,13 +1263,14 @@ static Settings aggregateIndexSettings( } /** - * Validates whether additional settings don't have keys that are already defined in all additional settings. + * Validates that additional settings don't have keys that are already defined in all additional settings + * and that they don't try to set {@link IndexMetadata#SETTING_VERSION_CREATED}. * * @param provider The {@link IndexSettingProvider} that produced additionalSettings * @param additionalSettings The settings produced by the specified provider * @param allAdditionalSettings A settings builder containing all additional settings produced by any {@link IndexSettingProvider} * that already executed - * @throws IllegalArgumentException If keys in additionalSettings are already defined in allAdditionalSettings + * @throws IllegalArgumentException If any of the validations fail */ public static void validateAdditionalSettings( IndexSettingProvider provider, @@ -1284,6 +1283,15 @@ public static void validateAdditionalSettings( var message = Strings.format("additional index setting [%s] added by [%s] is already present", settingName, name); throw new IllegalArgumentException(message); } + if (IndexMetadata.SETTING_VERSION_CREATED.equals(settingName)) { + var name = provider.getClass().getSimpleName(); + var message = Strings.format( + "setting [%s] added by [%s] is not allowed to be set via an IndexSettingProvider", + settingName, + name + ); + throw new IllegalArgumentException(message); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java index 6b79e331d30e8..260b96f0b5bb3 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java @@ -43,7 +43,9 @@ public interface IndexSettingProvider { * @param combinedTemplateMappings All the mappings resolved from the template that matches * @param indexVersion The index version to be used for the new index. * Always {@link IndexVersion#current()} when invoked during validation. - * @param additionalSettings A settings builder to which additional settings can be added + * @param additionalSettings A settings builder to which additional settings can be added. + * Providing {@link IndexMetadata#SETTING_VERSION_CREATED} is disallowed and leads to + * an {@link IllegalArgumentException} during validation of the additional settings. */ void provideAdditionalSettings( String indexName, diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java index c06fa774e95b3..a9af807890f55 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java @@ -45,6 +45,14 @@ public void testIndexCreation() throws Exception { "additional index setting [index.refresh_interval] added by [TestIndexSettingsProvider] is already present", e.getMessage() ); + + INDEX_SETTING_VERSION_ENABLED.set(true); + INDEX_SETTING_PROVIDER2_ENABLED.set(false); + e = expectThrows(IllegalArgumentException.class, () -> createIndex("my-index5", Settings.builder().build())); + assertEquals( + "setting [index.version.created] added by [TestIndexSettingsProvider] is not allowed to be set via an IndexSettingProvider", + e.getMessage() + ); } @Override @@ -72,6 +80,7 @@ public Collection getAdditionalIndexSettingProviders(Index private static final AtomicBoolean INDEX_SETTING_PROVIDER1_ENABLED = new AtomicBoolean(false); private static final AtomicBoolean INDEX_SETTING_PROVIDER2_ENABLED = new AtomicBoolean(false); private static final AtomicBoolean INDEX_SETTING_DEPTH_ENABLED = new AtomicBoolean(true); + private static final AtomicBoolean INDEX_SETTING_VERSION_ENABLED = new AtomicBoolean(false); private static final AtomicBoolean INDEX_SETTING_OVERRULING = new AtomicBoolean(false); static class TestIndexSettingsProvider implements IndexSettingProvider { @@ -101,6 +110,9 @@ public void provideAdditionalSettings( if (INDEX_SETTING_DEPTH_ENABLED.get()) { additionalSettings.put("index.mapping.depth.limit", 100); } + if (INDEX_SETTING_VERSION_ENABLED.get()) { + additionalSettings.put("index.version.created", IndexVersion.current()); + } } }