diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 94d9b87467ea8..c6365aeabad75 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -285,7 +285,7 @@ public static Template resolveTemplate( MetadataCreateIndexService.validateAdditionalSettings(provider, result, additionalSettings); dummySettings.put(result); additionalSettings.put(result); - if (provider.overrulesTemplateAndRequestSettings()) { + if (provider.overrulesSettings()) { overrulingSettings.addAll(result.keySet()); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 69e3b7b70ff82..764c5a974cca9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -980,7 +980,8 @@ static Settings aggregateIndexSettings( final Settings.Builder indexSettingsBuilder = Settings.builder(); if (sourceMetadata == null) { - final Settings templateAndRequestSettings = Settings.builder().put(combinedTemplateSettings).put(request.settings()).build(); + final Settings.Builder settingsBuilder = Settings.builder().put(combinedTemplateSettings).put(request.settings()); + final Settings templateAndRequestSettings = settingsBuilder.build(); final IndexMode templateIndexMode = Optional.of(request) .filter(r -> r.isFailureIndex() == false) @@ -988,26 +989,46 @@ static Settings aggregateIndexSettings( .map(metadata::retrieveIndexModeFromTemplate) .orElse(null); - // Loop through all the explicit index setting providers, adding them to the + // Loop through non-overruling setting providers, adding them to the // additionalIndexSettings map final Settings.Builder additionalIndexSettings = Settings.builder(); final var resolvedAt = Instant.ofEpochMilli(request.getNameResolvedAt()); - Set overrulingSettings = new HashSet<>(); for (IndexSettingProvider provider : indexSettingProviders) { + if (provider.overrulesSettings()) { + continue; + } var newAdditionalSettings = provider.getAdditionalIndexSettings( request.index(), request.dataStreamName(), templateIndexMode, - currentState.getMetadata(), + metadata, resolvedAt, templateAndRequestSettings, combinedTemplateMappings ); validateAdditionalSettings(provider, newAdditionalSettings, additionalIndexSettings); additionalIndexSettings.put(newAdditionalSettings); - if (provider.overrulesTemplateAndRequestSettings()) { - overrulingSettings.addAll(newAdditionalSettings.keySet()); + } + + // Feed the settings generated by non-overruling to overruling setting providers, + // along with template and request settings. + final Settings templateAndRequestAndProviderSettings = settingsBuilder.put(additionalIndexSettings.build()).build(); + Set overrulingSettings = new HashSet<>(); + for (IndexSettingProvider provider : indexSettingProviders) { + if (provider.overrulesSettings() == false) { + continue; } + var newAdditionalSettings = provider.getAdditionalIndexSettings( + request.index(), + request.dataStreamName(), + templateIndexMode, + metadata, + resolvedAt, + templateAndRequestAndProviderSettings, + combinedTemplateMappings + ); + additionalIndexSettings.put(newAdditionalSettings); + overrulingSettings.addAll(newAdditionalSettings.keySet()); } for (String explicitSetting : additionalIndexSettings.keys()) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java index 6a553d5dc5440..3fef3d85fca4b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java @@ -37,8 +37,9 @@ public interface IndexSettingProvider { * otherwise null is returned. * @param metadata The current metadata instance that doesn't yet contain the index to be created * @param resolvedAt The time the request to create this new index was accepted. - * @param indexTemplateAndCreateRequestSettings All the settings resolved from the template that matches and any settings - * defined on the create index request + * @param incomingSettings All the settings resolved from the templates that match, the create index request, + * and from non-overruling providers if this is an overruling one + * per {@link #overrulesSettings()} * @param combinedTemplateMappings All the mappings resolved from the template that matches */ Settings getAdditionalIndexSettings( @@ -47,7 +48,7 @@ Settings getAdditionalIndexSettings( @Nullable IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, - Settings indexTemplateAndCreateRequestSettings, + Settings incomingSettings, List combinedTemplateMappings ); @@ -59,13 +60,16 @@ record Parameters(CheckedFunction map } /** - * Indicates whether the additional settings that this provider returns can overrule the settings defined in matching template - * or in create index request. + * Indicates whether the additional settings that this provider returns can overrule the settings defined in + * matching templates, create index request, and settings returned by non-overruling providers. All these + * settings are included as input to + * {@link #getAdditionalIndexSettings(String, String, IndexMode, Metadata, Instant, Settings, List)} + * and can therefore be used to decide what settings to provide/overwrite. * * Note that this is not used during index template validation, to avoid overruling template settings that may apply to * different contexts (e.g. the provider is not used, or it returns different setting values). */ - default boolean overrulesTemplateAndRequestSettings() { + default boolean overrulesSettings() { return false; } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 95446149f026b..4f46d0b7ca70f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -95,7 +95,7 @@ public Settings getAdditionalIndexSettings( } @Override - public boolean overrulesTemplateAndRequestSettings() { + public boolean overrulesSettings() { return true; } }); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 96a74d2e23aad..390c3a59540b9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -726,7 +726,7 @@ public Settings getAdditionalIndexSettings( } @Override - public boolean overrulesTemplateAndRequestSettings() { + public boolean overrulesSettings() { return true; } }) @@ -769,7 +769,7 @@ public Settings getAdditionalIndexSettings( } @Override - public boolean overrulesTemplateAndRequestSettings() { + public boolean overrulesSettings() { return true; } }) @@ -812,7 +812,7 @@ public Settings getAdditionalIndexSettings( } @Override - public boolean overrulesTemplateAndRequestSettings() { + public boolean overrulesSettings() { return true; } }) @@ -855,7 +855,7 @@ public Settings getAdditionalIndexSettings( } @Override - public boolean overrulesTemplateAndRequestSettings() { + public boolean overrulesSettings() { return true; } }) @@ -866,6 +866,66 @@ public boolean overrulesTemplateAndRequestSettings() { assertThat(aggregatedIndexSettings.get("other_setting"), equalTo("other_value")); } + public void testAggregateSettingsProviderUsesSettingsFromProvider() { + IndexTemplateMetadata templateMetadata = addMatchingTemplate(builder -> { + builder.settings(Settings.builder().put("template_setting", "value1")); + }); + Metadata metadata = new Metadata.Builder().templates(Map.of("template_1", templateMetadata)).build(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + request.settings(Settings.builder().put("request_setting", "value2").build()); + + Settings aggregatedIndexSettings = aggregateIndexSettings( + clusterState, + request, + templateMetadata.settings(), + null, + null, + Settings.EMPTY, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Set.of(new IndexSettingProvider() { + @Override + public Settings getAdditionalIndexSettings( + String indexName, + String dataStreamName, + IndexMode templateIndexMode, + Metadata metadata, + Instant resolvedAt, + Settings incomingSettings, + List combinedTemplateMappings + ) { + return Settings.builder().put("provided_setting_1", "value_3").put("provided_setting_2", "value_4").build(); + } + }, new IndexSettingProvider() { + @Override + public Settings getAdditionalIndexSettings( + String indexName, + String dataStreamName, + IndexMode templateIndexMode, + Metadata metadata, + Instant resolvedAt, + Settings incomingSettings, + List combinedTemplateMappings + ) { + if (incomingSettings.hasValue("provided_setting_1")) { + return Settings.builder().put("provided_setting_2", "value_6").put("request_setting", "value_5").build(); + } + return Settings.builder().put("request_setting", "value_5").build(); + } + + @Override + public boolean overrulesSettings() { + return true; + } + }) + ); + + assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); + assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("value_5")); + assertThat(aggregatedIndexSettings.get("provided_setting_1"), equalTo("value_3")); + assertThat(aggregatedIndexSettings.get("provided_setting_2"), equalTo("value_6")); + } + public void testInvalidAliasName() { final String[] invalidAliasNames = new String[] { "-alias1", "+alias2", "_alias3", "a#lias", "al:ias", ".", ".." }; String aliasName = randomFrom(invalidAliasNames); diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java index adac8bf204f3e..3314eff359857 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingProviderTests.java @@ -33,13 +33,14 @@ public void testIndexCreation() throws Exception { assertTrue(indexService.getIndexSettings().getSettings().hasValue("index.refresh_interval")); assertEquals("10", indexService.getIndexSettings().getSettings().get("index.mapping.depth.limit")); - INDEX_SETTING_OVERRULING.set(true); + INDEX_SETTING_PROVIDER2_ENABLED.set(true); indexService = createIndex("my-index3", settings); assertTrue(indexService.getIndexSettings().getSettings().hasValue("index.refresh_interval")); assertEquals("100", indexService.getIndexSettings().getSettings().get("index.mapping.depth.limit")); INDEX_SETTING_DEPTH_ENABLED.set(false); - INDEX_SETTING_PROVIDER2_ENABLED.set(true); + INDEX_SETTING_PROVIDER2_ENABLED.set(false); + INDEX_SETTING_PROVIDER3_ENABLED.set(true); var e = expectThrows(IllegalArgumentException.class, () -> createIndex("my-index4", settings)); assertEquals( "additional index setting [index.refresh_interval] added by [TestIndexSettingsProvider] is already present", @@ -49,39 +50,48 @@ public void testIndexCreation() throws Exception { @Override protected Collection> getPlugins() { - return List.of(Plugin1.class, Plugin2.class); + return List.of(Plugin1.class, Plugin2.class, Plugin3.class); } public static class Plugin1 extends Plugin { @Override public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { - return List.of(new TestIndexSettingsProvider("-1", INDEX_SETTING_PROVIDER1_ENABLED)); + return List.of(new TestIndexSettingsProvider("1s", INDEX_SETTING_PROVIDER1_ENABLED, false)); } - } public static class Plugin2 extends Plugin { @Override public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { - return List.of(new TestIndexSettingsProvider("100s", INDEX_SETTING_PROVIDER2_ENABLED)); + return List.of(new TestIndexSettingsProvider("1s", INDEX_SETTING_PROVIDER2_ENABLED, true)); + } + } + + public static class Plugin3 extends Plugin { + + @Override + public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { + return List.of(new TestIndexSettingsProvider("100s", INDEX_SETTING_PROVIDER3_ENABLED, false)); } } private static final AtomicBoolean INDEX_SETTING_PROVIDER1_ENABLED = new AtomicBoolean(false); private static final AtomicBoolean INDEX_SETTING_PROVIDER2_ENABLED = new AtomicBoolean(false); + private static final AtomicBoolean INDEX_SETTING_PROVIDER3_ENABLED = new AtomicBoolean(false); private static final AtomicBoolean INDEX_SETTING_DEPTH_ENABLED = new AtomicBoolean(true); - private static final AtomicBoolean INDEX_SETTING_OVERRULING = new AtomicBoolean(false); static class TestIndexSettingsProvider implements IndexSettingProvider { private final String intervalValue; private final AtomicBoolean enabled; + private final boolean overruling; - TestIndexSettingsProvider(String intervalValue, AtomicBoolean enabled) { + TestIndexSettingsProvider(String intervalValue, AtomicBoolean enabled, boolean overruling) { this.intervalValue = intervalValue; this.enabled = enabled; + this.overruling = overruling; } @Override @@ -91,13 +101,14 @@ public Settings getAdditionalIndexSettings( IndexMode templateIndexMode, Metadata metadata, Instant resolvedAt, - Settings indexTemplateAndCreateRequestSettings, + Settings incomingSettings, List combinedTemplateMappings ) { if (enabled.get()) { var builder = Settings.builder().put("index.refresh_interval", intervalValue); if (INDEX_SETTING_DEPTH_ENABLED.get()) { - builder.put("index.mapping.depth.limit", 100); + // Verify that the value can be passed from non-overruling to overruling instance. + builder.put("index.mapping.depth.limit", incomingSettings.hasValue("index.refresh_interval") ? 100 : 1); } return builder.build(); } else { @@ -106,8 +117,8 @@ public Settings getAdditionalIndexSettings( } @Override - public boolean overrulesTemplateAndRequestSettings() { - return INDEX_SETTING_OVERRULING.get(); + public boolean overrulesSettings() { + return overruling; } } } diff --git a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java index f5ac107628d1a..381c83ceee289 100644 --- a/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java +++ b/x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.logsdb; import org.elasticsearch.client.Request; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.cluster.ElasticsearchCluster; @@ -171,4 +172,35 @@ public void testLogsdbOverrideNullInTemplate() throws IOException { assertEquals("logsdb", settings.get("index.mode")); assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode")); } + + public void testLogsdbOverrideDefaultModeForLogsIndex() throws IOException { + Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity("{ \"transient\": { \"cluster.logsdb.enabled\": true } }"); + assertOK(client().performRequest(request)); + + request = new Request("POST", "/_index_template/1"); + request.setJsonEntity(""" + { + "index_patterns": ["logs-test-*"], + "data_stream": { + } + } + """); + assertOK(client().performRequest(request)); + + request = new Request("POST", "/logs-test-foo/_doc"); + request.setJsonEntity(""" + { + "@timestamp": "2020-01-01T00:00:00.000Z", + "host.name": "foo", + "message": "bar" + } + """); + assertOK(client().performRequest(request)); + + String index = DataStream.getDefaultBackingIndexName("logs-test-foo", 1); + var settings = (Map) ((Map) getIndexSettings(index).get(index)).get("settings"); + assertEquals("logsdb", settings.get("index.mode")); + assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode")); + } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java index 16759c3292f7a..2bf8b00cf551c 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.logsdb; +import org.elasticsearch.client.Request; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; @@ -35,12 +37,6 @@ protected String getTestRestCluster() { } public void testFeatureUsageWithLogsdbIndex() throws IOException { - { - var response = getAsMap("/_license/feature_usage"); - @SuppressWarnings("unchecked") - List> features = (List>) response.get("features"); - assertThat(features, Matchers.empty()); - } { if (randomBoolean()) { createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build()); @@ -81,4 +77,35 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException { } } + public void testLogsdbSourceModeForLogsIndex() throws IOException { + Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity("{ \"transient\": { \"cluster.logsdb.enabled\": true } }"); + assertOK(client().performRequest(request)); + + request = new Request("POST", "/_index_template/1"); + request.setJsonEntity(""" + { + "index_patterns": ["logs-test-*"], + "data_stream": { + } + } + """); + assertOK(client().performRequest(request)); + + request = new Request("POST", "/logs-test-foo/_doc"); + request.setJsonEntity(""" + { + "@timestamp": "2020-01-01T00:00:00.000Z", + "host.name": "foo", + "message": "bar" + } + """); + assertOK(client().performRequest(request)); + + String index = DataStream.getDefaultBackingIndexName("logs-test-foo", 1); + var settings = (Map) ((Map) getIndexSettings(index).get(index)).get("settings"); + assertEquals("logsdb", settings.get("index.mode")); + assertNull(settings.get("index.mapping.source.mode")); + } + } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java index 4625fe91294d7..4184727e5ea5c 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java @@ -48,7 +48,7 @@ final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider } @Override - public boolean overrulesTemplateAndRequestSettings() { + public boolean overrulesSettings() { // Indicates that the provider value takes precedence over any user setting. return true; }