Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix ignoring existed cluster settings in DataTierAllocationDecider #67137

Merged
merged 1 commit into from Jan 7, 2021
+74 −16
Merged

Fix ignoring existed cluster settings in DataTierAllocationDecider #67137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java
View file
Open in desktop
Expand Up @@ -365,7 +365,7 @@ private static AllocationDeciders createAllocationDeciders(AllocationDecider...
return new AllocationDeciders(
Stream.of(
Stream.of(extraDeciders),
Stream.of(new DataTierAllocationDecider(clusterSettings)),
Stream.of(new DataTierAllocationDecider(Settings.EMPTY, clusterSettings)),
systemAllocationDeciders.stream()
).flatMap(s -> s).collect(Collectors.toList())
);
Expand Down
13 changes: 8 additions & 5 deletions ...in/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java
View file
Open in desktop
Expand Up @@ -70,11 +70,14 @@ private static void validateTierSetting(String setting) {
}
}

private volatile String clusterRequire = null;
private volatile String clusterInclude = null;
private volatile String clusterExclude = null;

public DataTierAllocationDecider(ClusterSettings clusterSettings) {
private volatile String clusterRequire;
private volatile String clusterInclude;
private volatile String clusterExclude;

public DataTierAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
clusterRequire = CLUSTER_ROUTING_REQUIRE_SETTING.get(settings);
clusterInclude = CLUSTER_ROUTING_INCLUDE_SETTING.get(settings);
clusterExclude = CLUSTER_ROUTING_EXCLUDE_SETTING.get(settings);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_REQUIRE_SETTING, s -> this.clusterRequire = s);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_INCLUDE_SETTING, s -> this.clusterInclude = s);
clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_EXCLUDE_SETTING, s -> this.clusterExclude = s);
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java
View file
Open in desktop
Expand Up @@ -394,7 +394,7 @@ public Set<DiscoveryNodeRole> getRoles() {

@Override
public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
return Collections.singleton(new DataTierAllocationDecider(clusterSettings));
return Collections.singleton(new DataTierAllocationDecider(settings, clusterSettings));
}

@Override
Expand Down
14 changes: 7 additions & 7 deletions ...ugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStep.java
View file
Open in desktop
Expand Up @@ -48,12 +48,6 @@ public class DataTierMigrationRoutedStep extends ClusterStateWaitStep {
ALL_CLUSTER_SETTINGS = allSettings;
}

private static final AllocationDeciders ALLOCATION_DECIDERS = new AllocationDeciders(
List.of(
new DataTierAllocationDecider(new ClusterSettings(Settings.EMPTY, ALL_CLUSTER_SETTINGS))
)
);

DataTierMigrationRoutedStep(StepKey key, StepKey nextStepKey) {
super(key, nextStepKey);
}
Expand All @@ -65,6 +59,12 @@ public boolean isRetryable() {

@Override
public Result isConditionMet(Index index, ClusterState clusterState) {
AllocationDeciders allocationDeciders = new AllocationDeciders(
List.of(
new DataTierAllocationDecider(clusterState.getMetadata().settings(),
new ClusterSettings(Settings.EMPTY, ALL_CLUSTER_SETTINGS))
)
);
IndexMetadata idxMeta = clusterState.metadata().index(index);
if (idxMeta == null) {
// Index must have been since deleted, ignore it
Expand Down Expand Up @@ -99,7 +99,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) {
return new Result(true, null);
}

int allocationPendingAllShards = getPendingAllocations(index, ALLOCATION_DECIDERS, clusterState);
int allocationPendingAllShards = getPendingAllocations(index, allocationDeciders, clusterState);

if (allocationPendingAllShards > 0) {
String statusMessage = availableDestinationTier.map(
Expand Down
2 changes: 1 addition & 1 deletion ...plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SetSingleNodeAllocateStep.java
View file
Open in desktop
Expand Up @@ -74,7 +74,7 @@ public void performAction(IndexMetadata indexMetadata, ClusterState clusterState
AllocationDeciders allocationDeciders = new AllocationDeciders(List.of(
new FilterAllocationDecider(clusterState.getMetadata().settings(),
new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
new DataTierAllocationDecider(new ClusterSettings(Settings.EMPTY, ALL_CLUSTER_SETTINGS)),
new DataTierAllocationDecider(clusterState.getMetadata().settings(), new ClusterSettings(Settings.EMPTY, ALL_CLUSTER_SETTINGS)),
new NodeVersionAllocationDecider()
));
final RoutingNodes routingNodes = clusterState.getRoutingNodes();
Expand Down
57 changes: 56 additions & 1 deletion ...va/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java
View file
Open in desktop
Expand Up @@ -37,6 +37,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;

Expand All @@ -53,7 +54,7 @@ public class DataTierAllocationDeciderTests extends ESAllocationTestCase {
private static final DiscoveryNode DATA_NODE = newNode("node-data", Collections.singleton(DiscoveryNodeRole.DATA_ROLE));

private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ALL_SETTINGS);
private final DataTierAllocationDecider decider = new DataTierAllocationDecider(clusterSettings);
private final DataTierAllocationDecider decider = new DataTierAllocationDecider(Settings.EMPTY, clusterSettings);
private final AllocationDeciders allocationDeciders = new AllocationDeciders(
Arrays.asList(decider,
new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
Expand Down Expand Up @@ -648,6 +649,60 @@ public void testPreferredTierAvailable() {
equalTo(Optional.of("data_warm")));
}

public void testExistedClusterFilters() {
Settings existedSettings = Settings.builder()
.put("cluster.routing.allocation.include._tier", "data_hot,data_warm")
.put("cluster.routing.allocation.exclude._tier", "data_cold")
.build();
ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ALL_SETTINGS);
DataTierAllocationDecider dataTierAllocationDecider = new DataTierAllocationDecider(existedSettings, clusterSettings);
AllocationDeciders allocationDeciders = new AllocationDeciders(
List.of(dataTierAllocationDecider));
AllocationService service = new AllocationService(allocationDeciders,
new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE,
EmptySnapshotsInfoService.INSTANCE);

ClusterState clusterState = prepareState(service.reroute(ClusterState.EMPTY_STATE, "initial state"));

RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, clusterState.getRoutingNodes(), clusterState,
null, null, 0);
allocation.debugDecision(true);
Decision d;
RoutingNode node;

for (DiscoveryNode n : Arrays.asList(HOT_NODE, WARM_NODE)) {
node = new RoutingNode(n.getId(), n, shard);
d = dataTierAllocationDecider.canAllocate(shard, node, allocation);
assertThat(d.type(), equalTo(Decision.Type.YES));
d = dataTierAllocationDecider.canRemain(shard, node, allocation);
assertThat(d.type(), equalTo(Decision.Type.YES));
}

node = new RoutingNode(DATA_NODE.getId(), DATA_NODE, shard);
d = dataTierAllocationDecider.canAllocate(shard, node, allocation);
assertThat(d.type(), equalTo(Decision.Type.NO));
assertThat(d.getExplanation(),
containsString("node matches any cluster setting [cluster.routing.allocation.exclude._tier] " +
"tier filters [data_cold]"));
d = dataTierAllocationDecider.canRemain(shard, node, allocation);
assertThat(d.type(), equalTo(Decision.Type.NO));
assertThat(d.getExplanation(),
containsString("node matches any cluster setting [cluster.routing.allocation.exclude._tier] " +
"tier filters [data_cold]"));

node = new RoutingNode(COLD_NODE.getId(), COLD_NODE, shard);
d = dataTierAllocationDecider.canAllocate(shard, node, allocation);
assertThat(d.type(), equalTo(Decision.Type.NO));
assertThat(d.getExplanation(),
containsString("node does not match any cluster setting [cluster.routing.allocation.include._tier] " +
"tier filters [data_hot,data_warm]"));
d = dataTierAllocationDecider.canRemain(shard, node, allocation);
assertThat(d.type(), equalTo(Decision.Type.NO));
assertThat(d.getExplanation(),
containsString("node does not match any cluster setting [cluster.routing.allocation.include._tier] " +
"tier filters [data_hot,data_warm]"));
}

private ClusterState prepareState(ClusterState initialState) {
return prepareState(initialState, Settings.EMPTY);
}
Expand Down