diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index cd4d1bebc94be..1dc62a180ec98 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -129,8 +129,6 @@ public class MetadataCreateIndexService { private final boolean forbidPrivateIndexSettings; private final Set indexSettingProviders = new HashSet<>(); - private volatile boolean enforceDefaultTierPreference; - public MetadataCreateIndexService( final Settings settings, final ClusterService clusterService, @@ -156,14 +154,6 @@ public MetadataCreateIndexService( this.systemIndices = systemIndices; this.forbidPrivateIndexSettings = forbidPrivateIndexSettings; this.shardLimitValidator = shardLimitValidator; - - enforceDefaultTierPreference = DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.get(settings); - clusterService.getClusterSettings().addSettingsUpdateConsumer(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING, - this::setEnforceDefaultTierPreference); - } - - public void setEnforceDefaultTierPreference(boolean enforceDefaultTierPreference) { - this.enforceDefaultTierPreference = enforceDefaultTierPreference; } /** @@ -492,8 +482,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, resolveSettings(templates), - null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders, - this.enforceDefaultTierPreference); + null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -528,8 +517,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, resolveSettings(currentState.metadata(), templateName), - null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders, - this.enforceDefaultTierPreference); + null, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -583,8 +571,7 @@ private ClusterState applyCreateIndexRequestForSystemDataStream(final ClusterSta settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders, - this.enforceDefaultTierPreference + indexSettingProviders ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); final IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -648,7 +635,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt } final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, Settings.EMPTY, - sourceMetadata, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders, this.enforceDefaultTierPreference); + sourceMetadata, settings, indexScopedSettings, shardLimitValidator, indexSettingProviders); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards); @@ -710,7 +697,7 @@ static Map parseV1Mappings(String mappingsJson, List indexSettingProviders, boolean enforceDefaultTierPreference) { + Set indexSettingProviders) { final boolean isDataStreamIndex = request.dataStreamName() != null; // Create builders for the template and request settings. We transform these into builders @@ -770,16 +757,14 @@ static Settings aggregateIndexSettings(ClusterState currentState, CreateIndexClu indexSettingsBuilder.put(requestSettings.build()); if (sourceMetadata == null) { // not for shrink/split/clone - if (enforceDefaultTierPreference) { - // regardless of any previous logic, we're going to force there - // to be an appropriate non-empty value for the tier preference - String currentTierPreference = indexSettingsBuilder.get(DataTier.TIER_PREFERENCE); - if (DataTier.parseTierList(currentTierPreference).isEmpty()) { - String newTierPreference = isDataStreamIndex ? DataTier.DATA_HOT : DataTier.DATA_CONTENT; - logger.debug("enforcing default [{}] setting for [{}] creation, replacing [{}] with [{}]", - DataTier.TIER_PREFERENCE, request.index(), currentTierPreference, newTierPreference); - indexSettingsBuilder.put(DataTier.TIER_PREFERENCE, newTierPreference); - } + // regardless of any previous logic, we're going to force there + // to be an appropriate non-empty value for the tier preference + String currentTierPreference = indexSettingsBuilder.get(DataTier.TIER_PREFERENCE); + if (DataTier.parseTierList(currentTierPreference).isEmpty()) { + String newTierPreference = isDataStreamIndex ? DataTier.DATA_HOT : DataTier.DATA_CONTENT; + logger.debug("enforcing default [{}] setting for [{}] creation, replacing [{}] with [{}]", + DataTier.TIER_PREFERENCE, request.index(), currentTierPreference, newTierPreference); + indexSettingsBuilder.put(DataTier.TIER_PREFERENCE, newTierPreference); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java index 5ecd0866bc7d3..4e09bec981540 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/DataTier.java @@ -45,12 +45,13 @@ public class DataTier { public static final Set ALL_DATA_TIERS = Set.of(DATA_CONTENT, DATA_HOT, DATA_WARM, DATA_COLD, DATA_FROZEN); - // this setting is for migrating from 7.x (where a tier preference was not required, and did not necessarily + // deprecated setting for migrating from 7.x (where a tier preference was not required, and did not necessarily // have a default value), to 8.x (where a tier preference will be required, and a default value will be injected). - // it will be removed as a breaking change in some future version, likely 9.0. + // in version 8.0 and onward, this setting doesn't control any logic anymore, and it will be removed as a breaking change in + // some future version, likely 9.0. public static final String ENFORCE_DEFAULT_TIER_PREFERENCE = "cluster.routing.allocation.enforce_default_tier_preference"; public static final Setting ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING = - Setting.boolSetting(ENFORCE_DEFAULT_TIER_PREFERENCE, true, Property.Dynamic, Property.NodeScope); + Setting.boolSetting(ENFORCE_DEFAULT_TIER_PREFERENCE, true, Property.Dynamic, Property.NodeScope, Property.Deprecated); public static final String TIER_PREFERENCE = "index.routing.allocation.include._tier_preference"; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 97b7d3a24d21f..824bf5bd41644 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -66,7 +65,6 @@ public class DiskThresholdDecider extends AllocationDecider { private static final Logger logger = LogManager.getLogger(DiskThresholdDecider.class); - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DiskThresholdDecider.class); public static final String NAME = "disk_threshold"; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index b5dcd023a2f9a..2927f5416de93 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -581,7 +581,7 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), randomBoolean()); + Collections.emptySet()); assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); assertThat(aggregatedIndexSettings.get("request_setting"), equalTo("value2")); @@ -634,7 +634,7 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateMetadata.settings(), null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), randomBoolean()); + Collections.emptySet()); assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest")); assertThat(aggregatedIndexSettings.get("key1"), equalTo("requestValue")); @@ -649,7 +649,7 @@ null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLi public void testDefaultSettings() { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), randomBoolean()); + Collections.emptySet()); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1")); } @@ -657,7 +657,7 @@ null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLi public void testSettingsFromClusterState() { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), Collections.emptySet(), randomBoolean()); + randomShardLimitService(), Collections.emptySet()); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15")); } @@ -682,7 +682,7 @@ public void testTemplateOrder() throws Exception { Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, MetadataIndexTemplateService.resolveSettings(templates), null, Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), Collections.emptySet(), randomBoolean()); + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), Collections.emptySet()); List resolvedAliases = resolveAndValidateAliases(request.index(), request.aliases(), MetadataIndexTemplateService.resolveAliases(templates), Metadata.builder().build(), aliasValidator, xContentRegistry(), searchExecutionContext, @@ -740,7 +740,7 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() Settings aggregatedIndexSettings = aggregateIndexSettings(clusterState, request, templateMetadata.settings(), clusterState.metadata().index("sourceIndex"), Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), Collections.emptySet(), randomBoolean()); + randomShardLimitService(), Collections.emptySet()); assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue())); assertThat(aggregatedIndexSettings.get("requestSetting"), is("requestValue")); @@ -943,8 +943,7 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { assertThat(targetRoutingNumberOfShards, is(6)); } - private Optional aggregatedTierPreference(Settings settings, boolean isDataStream, - boolean enforceDefaultTierPreference) { + private Optional aggregatedTierPreference(Settings settings, boolean isDataStream) { Settings templateSettings = Settings.EMPTY; request.settings(Settings.EMPTY); @@ -961,7 +960,7 @@ private Optional aggregatedTierPreference(Settings settings, boolean isD } Settings aggregatedIndexSettings = aggregateIndexSettings(ClusterState.EMPTY_STATE, request, templateSettings, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Set.of(new DataTier.DefaultHotAllocationSettingProvider()), enforceDefaultTierPreference); + Set.of(new DataTier.DefaultHotAllocationSettingProvider())); if (aggregatedIndexSettings.keySet().contains(DataTier.TIER_PREFERENCE)) { return Optional.of(aggregatedIndexSettings.get(DataTier.TIER_PREFERENCE)); @@ -976,20 +975,19 @@ public void testEnforceDefaultTierPreference() { // empty settings gets the appropriate tier settings = Settings.EMPTY; - tier = aggregatedTierPreference(settings, false, randomBoolean()); + tier = aggregatedTierPreference(settings, false); assertEquals(DataTier.DATA_CONTENT, tier.get()); settings = Settings.EMPTY; - tier = aggregatedTierPreference(settings, true, randomBoolean()); + tier = aggregatedTierPreference(settings, true); assertEquals(DataTier.DATA_HOT, tier.get()); // an explicit tier is respected settings = Settings.builder().put(DataTier.TIER_PREFERENCE, DataTier.DATA_COLD).build(); - tier = aggregatedTierPreference(settings, randomBoolean(), randomBoolean()); + tier = aggregatedTierPreference(settings, randomBoolean()); assertEquals(DataTier.DATA_COLD, tier.get()); - // any of the INDEX_ROUTING_.*_GROUP_PREFIX settings still result in a default if - // we're enforcing + // any of the INDEX_ROUTING_.*_GROUP_PREFIX settings still result in a default String includeRoutingSetting = randomFrom( IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX, IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX, @@ -997,24 +995,16 @@ public void testEnforceDefaultTierPreference() { settings = Settings.builder() .put(includeRoutingSetting, randomAlphaOfLength(10)) .build(); - tier = aggregatedTierPreference(settings, false, true); + tier = aggregatedTierPreference(settings, false); assertEquals(DataTier.DATA_CONTENT, tier.get()); - // (continued from above) but not if we aren't - tier = aggregatedTierPreference(settings, false, false); - assertTrue(tier.isEmpty()); - // an explicit null gets an empty tier if we're not enforcing + // an explicit null gets the appropriate tier settings = Settings.builder().putNull(DataTier.TIER_PREFERENCE).build(); - tier = aggregatedTierPreference(settings, randomBoolean(), false); - assertTrue(tier.isEmpty()); - - // an explicit null gets the appropriate tier if we are enforcing - settings = Settings.builder().putNull(DataTier.TIER_PREFERENCE).build(); - tier = aggregatedTierPreference(settings, false, true); + tier = aggregatedTierPreference(settings, false); assertEquals(DataTier.DATA_CONTENT, tier.get()); settings = Settings.builder().putNull(DataTier.TIER_PREFERENCE).build(); - tier = aggregatedTierPreference(settings, true, true); + tier = aggregatedTierPreference(settings, true); assertEquals(DataTier.DATA_HOT, tier.get()); } @@ -1024,7 +1014,7 @@ public void testRejectWithSoftDeletesDisabled() { request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), randomBoolean()); + Collections.emptySet()); }); assertThat(error.getMessage(), equalTo("Creating indices with soft-deletes disabled is no longer supported. " + "Please do not specify a value for setting [index.soft_deletes.enabled].")); @@ -1046,7 +1036,7 @@ public void testRejectTranslogRetentionSettings() { IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), randomBoolean())); + Collections.emptySet())); assertThat(error.getMessage(), equalTo("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are no longer supported. Please do not specify values for these settings")); } @@ -1063,7 +1053,7 @@ public void testDeprecateTranslogRetentionSettings() { request.settings(settings.build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), randomBoolean()); + Collections.emptySet()); assertWarnings("Translog retention settings [index.translog.retention.age] " + "and [index.translog.retention.size] are deprecated and effectively ignored. They will be removed in a future version."); } @@ -1076,7 +1066,7 @@ public void testDeprecateSimpleFS() { request.settings(settings.build()); aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet(), randomBoolean()); + Collections.emptySet()); assertWarnings("[simplefs] is deprecated and will be removed in 8.0. Use [niofs] or other file systems instead. " + "Elasticsearch 7.15 or later uses [niofs] for the [simplefs] store type " + "as it offers superior or equivalent performance to [simplefs]."); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 7ac86d9179db3..2a3a5444f22b1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -37,6 +37,7 @@ import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; import org.elasticsearch.action.admin.indices.segments.ShardSegments; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -68,7 +69,6 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; -import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.service.ClusterService; @@ -781,6 +781,15 @@ public CreateIndexRequestBuilder prepareCreate(String index, int numNodes, Setti return client().admin().indices().prepareCreate(index).setSettings(builder.build()); } + /** + * updates the settings for an index + */ + public void updateIndexSettings(String index, Settings.Builder settingsBuilder) { + UpdateSettingsRequestBuilder settingsRequest = client().admin().indices().prepareUpdateSettings(index); + settingsRequest.setSettings(settingsBuilder); + assertAcked(settingsRequest.execute().actionGet()); + } + private Settings.Builder getExcludeSettings(int num, Settings.Builder builder) { String exclude = String.join(",", internalCluster().allDataNodesButN(num)); builder.put("index.routing.allocation.exclude._name", exclude); @@ -1756,8 +1765,6 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS)) // randomly enable low-level search cancellation to make sure it does not alter results .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), randomBoolean()) - // randomly enable enforcing a default tier_preference to make sure it does not alter results - .put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.getKey(), randomBoolean()) .putList(DISCOVERY_SEED_HOSTS_SETTING.getKey()) // empty list disables a port scan for other nodes .putList(DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), "file") .put(TransportSearchAction.DEFAULT_PRE_FILTER_SHARD_SIZE.getKey(), randomFrom(1, 2, diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java index 3e11fc0c6d287..653608b731d38 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.autoscaling.storage; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -152,10 +151,6 @@ public void testScaleFromEmptyLegacy() { ) ); - // if we inject a default tier preference, then this test wouldn't be valid anymore, - // so let's turn that off - enforceDefaultTierPreference(false); - putAutoscalingPolicy("hot", DataTier.DATA_HOT); putAutoscalingPolicy("warm", DataTier.DATA_WARM); putAutoscalingPolicy("cold", DataTier.DATA_COLD); @@ -185,6 +180,10 @@ public void testScaleFromEmptyLegacy() { .build() ) ); + + // the tier preference will have defaulted to data_content, set it back to null + updateIndexSettings(indexName, Settings.builder().putNull(DataTier.TIER_PREFERENCE)); + refresh(indexName); assertThat(capacity().results().get("warm").requiredCapacity().total().storage().getBytes(), Matchers.equalTo(0L)); assertThat(capacity().results().get("cold").requiredCapacity().total().storage().getBytes(), Matchers.equalTo(0L)); @@ -249,10 +248,4 @@ private void putAutoscalingPolicy(String policyName, String role) { ); assertAcked(client().execute(PutAutoscalingPolicyAction.INSTANCE, request).actionGet()); } - - public void enforceDefaultTierPreference(boolean enforceDefaultTierPreference) { - ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); - request.transientSettings(Settings.builder().put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE, enforceDefaultTierPreference).build()); - assertAcked(client().admin().cluster().updateSettings(request).actionGet()); - } } diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java index df3195b5afbb1..070b1ebc2c934 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.cluster.routing.allocation; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; @@ -15,10 +14,10 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.xpack.core.DataTiersFeatureSetUsage; import org.elasticsearch.xpack.core.action.XPackUsageRequestBuilder; import org.elasticsearch.xpack.core.action.XPackUsageResponse; @@ -28,7 +27,6 @@ import java.util.Collections; import java.util.Map; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -86,51 +84,9 @@ public void testOverrideDefaultAllocation() { ensureYellow(index); } - public void testRequestSettingOverridesAllocation() { - startWarmOnlyNode(); - startColdOnlyNode(); - ensureGreen(); - enforceDefaultTierPreference(false); - - client().admin().indices().prepareCreate(index) - .setWaitForActiveShards(0) - .setSettings(Settings.builder() - .putNull(DataTier.TIER_PREFERENCE)) - .get(); - - Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); - assertThat(DataTier.TIER_PREFERENCE_SETTING.get(idxSettings), equalTo("")); - // Even the key shouldn't exist if it has been nulled out - assertFalse(idxSettings.keySet().toString(), idxSettings.keySet().contains(DataTier.TIER_PREFERENCE)); - - // index should be yellow - logger.info("--> waiting for {} to be yellow", index); - ensureYellow(index); - - client().admin().indices().prepareDelete(index).get(); - - // Now test it overriding the "require" setting, in which case the preference should be skipped - client().admin().indices().prepareCreate(index) - .setWaitForActiveShards(0) - .setSettings(Settings.builder() - .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".box", "cold")) - .get(); - - idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); - assertThat(DataTier.TIER_PREFERENCE_SETTING.get(idxSettings), equalTo("")); - // The key should not be put in place since it was overridden - assertFalse(idxSettings.keySet().contains(DataTier.TIER_PREFERENCE)); - assertThat(idxSettings.get(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".box"), equalTo("cold")); - - // index should be yellow - logger.info("--> waiting for {} to be yellow", index); - ensureYellow(index); - } - - public void testRequestSettingOverriddenIfEnforced() { + public void testRequestSettingOverridden() { startContentOnlyNode(); ensureGreen(); - enforceDefaultTierPreference(true); client().admin().indices().prepareCreate(index) .setWaitForActiveShards(0) @@ -184,48 +140,8 @@ public void testShrinkStaysOnTier() { .get(); } - public void testTemplateOverridesDefaults() { - startWarmOnlyNode(); - enforceDefaultTierPreference(false); - - Template t = new Template(Settings.builder() - .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + ".box", "warm") - .build(), null, null); - ComposableIndexTemplate ct = new ComposableIndexTemplate.Builder() - .indexPatterns(Collections.singletonList(index)) - .template(t).build(); - client().execute(PutComposableIndexTemplateAction.INSTANCE, - new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); - - client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get(); - - Settings idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); - assertThat(idxSettings.keySet().contains(DataTier.TIER_PREFERENCE), equalTo(false)); - - // index should be yellow - ensureYellow(index); - - client().admin().indices().prepareDelete(index).get(); - - t = new Template(Settings.builder() - .putNull(DataTier.TIER_PREFERENCE) - .build(), null, null); - ct = new ComposableIndexTemplate.Builder().indexPatterns(Collections.singletonList(index)) - .template(t).build(); - client().execute(PutComposableIndexTemplateAction.INSTANCE, - new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); - - client().admin().indices().prepareCreate(index).setWaitForActiveShards(0).get(); - - idxSettings = client().admin().indices().prepareGetIndex().addIndices(index).get().getSettings().get(index); - assertThat(idxSettings.keySet().contains(DataTier.TIER_PREFERENCE), equalTo(false)); - - ensureYellow(index); - } - - public void testTemplateOverriddenIfEnforced() { + public void testTemplateOverridden() { startContentOnlyNode(); - enforceDefaultTierPreference(true); Template t = new Template(Settings.builder() .putNull(DataTier.TIER_PREFERENCE) @@ -376,10 +292,4 @@ public void startFrozenOnlyNode() { .build(); internalCluster().startNode(nodeSettings); } - - public void enforceDefaultTierPreference(boolean enforceDefaultTierPreference) { - ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); - request.transientSettings(Settings.builder().put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE, enforceDefaultTierPreference).build()); - assertAcked(client().admin().cluster().updateSettings(request).actionGet()); - } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index c0274c85d6354..a028e1558c5c4 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -83,7 +83,8 @@ private DeprecationChecks() { NodeDeprecationChecks::checkScriptContextCache, NodeDeprecationChecks::checkScriptContextCompilationsRateLimitSetting, NodeDeprecationChecks::checkScriptContextCacheSizeSetting, - NodeDeprecationChecks::checkScriptContextCacheExpirationSetting + NodeDeprecationChecks::checkScriptContextCacheExpirationSetting, + NodeDeprecationChecks::checkEnforceDefaultTierPreferenceSetting ); static List> INDEX_SETTINGS_CHECKS = List.of( diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java index 355fc21efd60d..5d69db09bc595 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.deprecation; import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.Setting; @@ -441,4 +442,18 @@ static DeprecationIssue checkScriptContextCacheExpirationSetting(final Settings } return null; } + + static DeprecationIssue checkEnforceDefaultTierPreferenceSetting(final Settings settings, final PluginsAndModules pluginsAndModules) { + if (DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.exists(settings)) { + String key = DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.getKey(); + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + String.format(Locale.ROOT, "setting [%s] is deprecated and will not be available in a future version", key), + "https://www.elastic.co/guide/en/elasticsearch/reference/current/data-tiers.html", + String.format(Locale.ROOT, "found [%s] configured. Discontinue use of this setting.", key), + false, null); + } + + return null; + } + } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java index 310470e2ae639..157742e0d13a3 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; @@ -459,4 +460,23 @@ public void testScriptContextCacheExpirationSetting() { "[script.context.moving-function.cache_expire] setting was deprecated in Elasticsearch and will be removed in a future" + " release! See the breaking changes documentation for the next major version."); } + + public void testEnforceDefaultTierPreferenceSetting() { + Settings settings = Settings.builder() + .put(DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING.getKey(), randomBoolean()) + .build(); + + List issues = DeprecationChecks.filterChecks(NODE_SETTINGS_CHECKS, c -> c.apply(settings, null)); + + final String expectedUrl = "https://www.elastic.co/guide/en/elasticsearch/reference/current/data-tiers.html"; + assertThat(issues, hasItem( + new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "setting [cluster.routing.allocation.enforce_default_tier_preference] is deprecated and" + + " will not be available in a future version", + expectedUrl, + "found [cluster.routing.allocation.enforce_default_tier_preference] configured." + + " Discontinue use of this setting.", + false, null))); + } + } diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java index 2fe831273c36f..5aeba982a135a 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/MigrateToDataTiersIT.java @@ -72,10 +72,6 @@ public void refreshIndexAndStartILM() throws IOException { policy = "policy-" + randomAlphaOfLength(5); alias = "alias-" + randomAlphaOfLength(5); assertOK(client().performRequest(new Request("POST", "_ilm/start"))); - - // we can't have the pre-migration indices getting tier preferences auto-assigned, - // if we did, then we wouldn't really be testing the migration *to* data tiers anymore :D - enforceDefaultTierPreference(false); } @AfterClass @@ -141,12 +137,16 @@ public void testMigrateToDataTiersAction() throws Exception { for (int i = 1; i <= 2; i++) { // assign the rollover-only policy to a few other indices - these indices will end up getting caught by the catch-all // tier preference migration - createIndexWithSettings(client(), rolloverIndexPrefix + "-00000" + i, alias + i, Settings.builder() + String rolloverIndex = rolloverIndexPrefix + "-00000" + i; + createIndexWithSettings(client(), rolloverIndex, alias + i, Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .putNull(DataTier.TIER_PREFERENCE) + .putNull(DataTier.TIER_PREFERENCE) // since we always enforce a tier preference, this will be ignored (i.e. data_content) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias + i) ); + + // the tier preference will have defaulted to data_content, set it back to null + updateIndexSettings(rolloverIndex, Settings.builder().putNull(DataTier.TIER_PREFERENCE)); } // let's stop ILM so we can perform the migration @@ -163,6 +163,9 @@ public void testMigrateToDataTiersAction() throws Exception { .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "data", "warm"); createIndex(indexWithDataWarmRouting, settings.build()); + // the tier preference will have defaulted to data_content, set it back to null + updateIndexSettings(indexWithDataWarmRouting, Settings.builder().putNull(DataTier.TIER_PREFERENCE)); + Request migrateRequest = new Request("POST", "_ilm/migrate_to_data_tiers"); migrateRequest.setJsonEntity( "{\"legacy_template_to_delete\": \"" + templateName + "\", \"node_attribute\": \"data\"}" @@ -214,12 +217,13 @@ public void testMigrateToDataTiersAction() throws Exception { assertThat(cachedPhaseDefinition, containsString(SetPriorityAction.NAME)); assertThat(cachedPhaseDefinition, containsString(ForceMergeAction.NAME)); - // ENFORCE_DEFAULT_TIER_PREFERENCE has been set to true - Request getSettingsRequest = new Request("GET", "_cluster/settings"); + // ENFORCE_DEFAULT_TIER_PREFERENCE is not mentioned (and defaults to true) + Request getSettingsRequest = new Request("GET", "_cluster/settings?include_defaults"); Response getSettingsResponse = client().performRequest(getSettingsRequest); ObjectMapper mapper = new ObjectMapper(); JsonNode json = mapper.readTree(getSettingsResponse.getEntity().getContent()); - assertTrue(json.at("/persistent/cluster/routing/allocation/enforce_default_tier_preference").asBoolean()); + assertTrue(json.at("/persistent/cluster/routing/allocation/enforce_default_tier_preference").isMissingNode()); + assertTrue(json.at("/defaults/cluster/routing/allocation/enforce_default_tier_preference").asBoolean()); } @SuppressWarnings("unchecked") @@ -250,7 +254,7 @@ public void testMigrationDryRun() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(LifecycleSettings.LIFECYCLE_NAME, policy) - .putNull(DataTier.TIER_PREFERENCE) + .putNull(DataTier.TIER_PREFERENCE) // since we always enforce a tier preference, this will be ignored (i.e. data_content) .put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, alias) ); @@ -275,6 +279,9 @@ public void testMigrationDryRun() throws Exception { .put(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "data", "warm"); createIndex(indexWithDataWarmRouting, settings.build()); + // the tier preference will have defaulted to data_content, set it back to null + updateIndexSettings(indexWithDataWarmRouting, Settings.builder().putNull(DataTier.TIER_PREFERENCE)); + Request migrateRequest = new Request("POST", "_ilm/migrate_to_data_tiers"); migrateRequest.addParameter("dry_run", "true"); migrateRequest.setJsonEntity( @@ -343,14 +350,4 @@ private void createLegacyTemplate(String templateName) throws IOException { templateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); client().performRequest(templateRequest); } - - public void enforceDefaultTierPreference(boolean enforceDefaultTierPreference) throws IOException { - Request request = new Request("PUT", "_cluster/settings"); - request.setJsonEntity("{\n" + - " \"persistent\": {\n" + - " \"" + DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE + "\" : " + enforceDefaultTierPreference + "\n" + - " }\n" + - "}"); - assertOK(client().performRequest(request)); - } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index 9d91edde74e38..d7bab2749f69c 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -129,11 +129,10 @@ public static Tuple migrateToDataTiersRouting(Cl Metadata.Builder mb = Metadata.builder(currentState.metadata()); - // set ENFORCE_DEFAULT_TIER_PREFERENCE to true (in the persistent settings) - mb.persistentSettings(Settings.builder() - .put(mb.persistentSettings()) - .put(ENFORCE_DEFAULT_TIER_PREFERENCE, true) - .build()); + // remove ENFORCE_DEFAULT_TIER_PREFERENCE from the persistent settings + Settings.Builder persistentSettingsBuilder = Settings.builder().put(mb.persistentSettings()); + persistentSettingsBuilder.remove(ENFORCE_DEFAULT_TIER_PREFERENCE); + mb.persistentSettings(persistentSettingsBuilder.build()); // and remove it from the transient settings, just in case it was there Settings.Builder transientSettingsBuilder = Settings.builder().put(mb.transientSettings());