Skip to content

Commit

Permalink
[7.17] Migrate to data tiers routing configures correct default for m…
Browse files Browse the repository at this point in the history 
…ounted indices (#97950)

* Migrate to data tiers routing configures correct default for mounted indices

(cherry picked from commit eaec9602a9c4a4b092f36fd35f58d1e4b8faca52)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>

* Add isExplicitDataTier method back as it's used in 7.17

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
  • Loading branch information
@andreidan @elasticmachine
andreidan and elasticmachine committed Jul 26, 2023
1 parent 0c21da5 commit f0b7122
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 0 deletions.
11 changes: 11 additions & 0 deletions ...va/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.elasticsearch.core.Nullable;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xpack.core.ilm.AllocateAction;
import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
Expand All @@ -37,6 +38,7 @@
import org.elasticsearch.xpack.core.ilm.Phase;
import org.elasticsearch.xpack.core.ilm.PhaseExecutionInfo;
import org.elasticsearch.xpack.core.ilm.Step;
import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;

import java.time.Instant;
import java.util.ArrayList;
Expand Down Expand Up @@ -756,6 +758,15 @@ private static Settings migrateToDefaultTierPreference(ClusterState currentState

boolean isDataStream = currentState.metadata().findDataStreams(indexName).isEmpty() == false;
String convertedTierPreference = isDataStream ? DataTier.DATA_HOT : DataTier.DATA_CONTENT;
if (SearchableSnapshotsSettings.isSearchableSnapshotStore(indexMetadata.getSettings())) {
if (SearchableSnapshotsSettings.isPartialSearchableSnapshotIndex(indexMetadata.getSettings())) {
// partially mounted index
convertedTierPreference = MountSearchableSnapshotRequest.Storage.SHARED_CACHE.defaultDataTiersPreference();
} else {
// fully mounted index
convertedTierPreference = MountSearchableSnapshotRequest.Storage.FULL_COPY.defaultDataTiersPreference();
}
}
newSettingsBuilder.put(TIER_PREFERENCE, convertedTierPreference);
logger.debug("index [{}]: configured setting [{}] to [{}]", indexName, TIER_PREFERENCE, convertedTierPreference);
return newSettingsBuilder.build();
Expand Down
81 changes: 81 additions & 0 deletions ...g/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingServiceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.ParseField;
Expand All @@ -40,6 +42,7 @@
import org.elasticsearch.xpack.core.ilm.Phase;
import org.elasticsearch.xpack.core.ilm.SetPriorityAction;
import org.elasticsearch.xpack.core.ilm.ShrinkAction;
import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
import org.junit.Before;

import java.io.ByteArrayInputStream;
Expand All @@ -54,6 +57,7 @@
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING;
import static org.elasticsearch.cluster.routing.allocation.DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE;
import static org.elasticsearch.cluster.routing.allocation.DataTier.TIER_PREFERENCE;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE;
import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.allocateActionDefinesRoutingRules;
import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.convertAttributeValueToTierPreference;
import static org.elasticsearch.xpack.cluster.metadata.MetadataMigrateToDataTiersRoutingService.migrateIlmPolicies;
Expand Down Expand Up @@ -865,6 +869,83 @@ public void testMigrateIndices() {
}
}

public void testMigrateMountedIndices() {
{
// migrate "cold" custom routing attribute for fully mounted index
IndexMetadata.Builder partiallyMountedIndex = IndexMetadata.builder("foo")
.settings(
getBaseIndexSettings().put(DATA_ROUTING_REQUIRE_SETTING, "cold")
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE)
.put(SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING.getKey(), false)
);
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(partiallyMountedIndex)).build();

Metadata.Builder mb = Metadata.builder(state.metadata());

List<String> migratedIndices = migrateIndices(mb, state, "data");
assertThat(migratedIndices.size(), is(1));

ClusterState migratedState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build();
IndexMetadata migratedIndex = migratedState.metadata().index("foo");
assertThat(migratedIndex.getSettings().get(DATA_ROUTING_INCLUDE_SETTING), nullValue());
assertThat(
migratedIndex.getSettings().get(TIER_PREFERENCE),
is(MountSearchableSnapshotRequest.Storage.FULL_COPY.defaultDataTiersPreference())
);
}

{
// migrate the _wrong_ custom routing attribute for partially mounted index without any tier preference will keep data_frozen
IndexMetadata.Builder partiallyMountedIndex = IndexMetadata.builder("foo")
.settings(
getBaseIndexSettings().put(BOX_ROUTING_REQUIRE_SETTING, "cold")
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE)
.put(SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING.getKey(), true)
);
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(partiallyMountedIndex)).build();

Metadata.Builder mb = Metadata.builder(state.metadata());

List<String> migratedIndices = migrateIndices(mb, state, "data");
assertThat(migratedIndices.size(), is(1));

ClusterState migratedState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build();
IndexMetadata migratedIndex = migratedState.metadata().index("foo");
assertThat(migratedIndex.getSettings().get(BOX_ROUTING_REQUIRE_SETTING), is("cold"));
// partially mounted index must remain in `data_frozen`
assertThat(
migratedIndex.getSettings().get(TIER_PREFERENCE),
is(MountSearchableSnapshotRequest.Storage.SHARED_CACHE.defaultDataTiersPreference())
);
}

{
// migrate the _wrong_ custom routing attribute for fully mounted index without any tier preference will configure
// MountSearchableSnapshotRequest.Storage.FULL_COPY.defaultDataTiersPreference()
IndexMetadata.Builder partiallyMountedIndex = IndexMetadata.builder("foo")
.settings(
getBaseIndexSettings().put(BOX_ROUTING_REQUIRE_SETTING, "cold")
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE)
.put(SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING.getKey(), false)
);
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(Metadata.builder().put(partiallyMountedIndex)).build();

Metadata.Builder mb = Metadata.builder(state.metadata());

List<String> migratedIndices = migrateIndices(mb, state, "data");
assertThat(migratedIndices.size(), is(1));

ClusterState migratedState = ClusterState.builder(ClusterName.DEFAULT).metadata(mb).build();
IndexMetadata migratedIndex = migratedState.metadata().index("foo");
assertThat(migratedIndex.getSettings().get(BOX_ROUTING_REQUIRE_SETTING), is("cold"));

assertThat(
migratedIndex.getSettings().get(TIER_PREFERENCE),
is(MountSearchableSnapshotRequest.Storage.FULL_COPY.defaultDataTiersPreference())
);
}
}

public void testColdestAttributeIsConvertedToTierPreference() {
// `include` is colder than `require`
{
Expand Down

0 comments on commit f0b7122

Please sign in to comment.