From 3129f5b42ed490e0d285295e2d6c18b33d132759 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 8 Jul 2019 10:31:15 +0100 Subject: [PATCH] Do not copy initial recovery filter during split (#44053) If an index is the result of a shrink then it will have a value set for `index.routing.allocation.initial_recovery._id`. If this index is subsequently split then this value will be copied over, forcing the initial allocation of the split shards to occur on the node on which the shrink took place. Moreover if this node no longer exists then the split will fail. This commit suppresses the copying of this setting when splitting an index. Fixes #43955 --- .../metadata/MetaDataCreateIndexService.java | 11 +++-- .../admin/indices/create/ShrinkIndexIT.java | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 838b1e2547204..26a97d40d3377 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -761,19 +761,22 @@ static void prepareResizeIndexSettings( final ResizeType type, final boolean copySettings, final IndexScopedSettings indexScopedSettings) { + + // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away + // once we are allocated. + final String initialRecoveryIdFilter = IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey() + "_id"; + final IndexMetaData sourceMetaData = currentState.metaData().index(resizeSourceIndex.getName()); if (type == ResizeType.SHRINK) { final List nodesToAllocateOn = validateShrinkIndex(currentState, resizeSourceIndex.getName(), mappingKeys, resizeIntoName, indexSettingsBuilder.build()); indexSettingsBuilder - // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away - // once we are allocated. - .put(IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey() + "_id", - Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray())) + .put(initialRecoveryIdFilter, Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray())) // we only try once and then give up with a shrink index .put("index.allocation.max_retries", 1); } else if (type == ResizeType.SPLIT) { validateSplitIndex(currentState, resizeSourceIndex.getName(), mappingKeys, resizeIntoName, indexSettingsBuilder.build()); + indexSettingsBuilder.putNull(initialRecoveryIdFilter); } else { throw new IllegalStateException("unknown resize type is " + type); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index 32cc6564e5ca0..c7cc99d93e685 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; @@ -61,6 +62,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.VersionUtils; import java.util.Arrays; @@ -554,4 +556,42 @@ public void testShrinkCommitsMergeOnIdle() throws Exception { EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String)null )).get(); } + + public void testShrinkThenSplitWithFailedNode() throws Exception { + internalCluster().ensureAtLeastNumDataNodes(3); + + final int shardCount = between(2, 5); + prepareCreate("original").setSettings(Settings.builder().put(indexSettings()) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shardCount)).get(); + client().admin().indices().prepareFlush("original").get(); + ensureGreen(); + final String shrinkNode + = client().admin().cluster().prepareNodesInfo("data:true").clear().get().getNodes().get(0).getNode().getName(); + client().admin().indices().prepareUpdateSettings("original") + .setSettings(Settings.builder() + .put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), shrinkNode) + .put(IndexMetaData.SETTING_BLOCKS_WRITE, true)).get(); + ensureGreen(); + + assertAcked(client().admin().indices().prepareResizeIndex("original", "shrunk").setSettings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .putNull(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey()) + .build()).setResizeType(ResizeType.SHRINK).get()); + ensureGreen(); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(shrinkNode)); + + // demonstrate that the index.routing.allocation.initial_recovery setting from the shrink doesn't carry over into the split index, + // because this would cause the shrink to fail as the initial_recovery node is no longer present. + + logger.info("--> executing split"); + assertAcked(client().admin().indices().prepareResizeIndex("shrunk", "splitagain").setSettings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shardCount) + .putNull(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey()) + .build()).setResizeType(ResizeType.SPLIT)); + ensureGreen("splitagain"); + } }