From f96a6eabe6272f30398d0aa4f7b1d17eba63b7a4 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 23 Sep 2025 13:51:34 +1000 Subject: [PATCH 1/3] Always allow allocation of unassigned shards --- .../decider/WriteLoadConstraintDecider.java | 5 ++ .../WriteLoadConstraintDeciderTests.java | 66 +++++++++++++++---- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java index e814f570a67bb..4c5ce44459157 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java @@ -43,6 +43,11 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return Decision.single(Decision.Type.YES, NAME, "Decider is disabled"); } + // Never reject allocation of an unassigned shard + if (shardRouting.assignedToNode() == false) { + return Decision.single(Decision.Type.YES, NAME, "Shard is unassigned. Decider takes no action."); + } + // Check whether the shard being relocated has any write load estimate. If it does not, then this decider has no opinion. var allShardWriteLoads = allocation.clusterInfo().getShardWriteLoads(); var shardWriteLoad = allShardWriteLoads.get(shardRouting.shardId()); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java index 12bfd8a0a4789..45c931d9f8be0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java @@ -25,6 +25,8 @@ import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.WriteLoadConstraintSettings; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; @@ -102,38 +104,65 @@ public void testWriteLoadDeciderCanAllocate() { ) .build() ); - assertEquals( + assertDecisionMatches( "Assigning a new shard to a node that is above the threshold should fail", - Decision.Type.NO, writeLoadDecider.canAllocate( testHarness.shardRouting2, testHarness.exceedingThresholdRoutingNode, testHarness.routingAllocation - ).type() + ), + Decision.Type.NO, + "Node [*] with write thread pool utilization [0.99] already exceeds the high utilization threshold of [0.900000]. " + + "Cannot allocate shard [[test-index][1]] to node without risking increased write latencies." ); - assertEquals( + assertDecisionMatches( + "Unassigned shard should always be accepted", + writeLoadDecider.canAllocate( + testHarness.unassignedShardRouting, + testHarness.exceedingThresholdRoutingNode, + testHarness.routingAllocation + ), + Decision.Type.YES, + "Shard is unassigned. Decider takes no action." + ); + assertDecisionMatches( "Assigning a new shard to a node that has capacity should succeed", + writeLoadDecider.canAllocate(testHarness.shardRouting1, testHarness.belowThresholdRoutingNode, testHarness.routingAllocation), Decision.Type.YES, - writeLoadDecider.canAllocate(testHarness.shardRouting1, testHarness.belowThresholdRoutingNode, testHarness.routingAllocation) - .type() + null ); - assertEquals( + assertDecisionMatches( "Assigning a new shard without a write load estimate should _not_ be blocked by lack of capacity", - Decision.Type.YES, writeLoadDecider.canAllocate( testHarness.thirdRoutingNoWriteLoad, testHarness.exceedingThresholdRoutingNode, testHarness.routingAllocation - ).type() + ), + Decision.Type.YES, + "Shard has no estimated write load. Decider takes no action." ); - assertEquals( + assertDecisionMatches( "Assigning a new shard that would cause the node to exceed capacity should fail", + writeLoadDecider.canAllocate(testHarness.shardRouting1, testHarness.nearThresholdRoutingNode, testHarness.routingAllocation), Decision.Type.NO, - writeLoadDecider.canAllocate(testHarness.shardRouting1, testHarness.nearThresholdRoutingNode, testHarness.routingAllocation) - .type() + "The high utilization threshold of [0.900000] would be exceeded on node [*] with utilization [0.89] " + + "if shard [[test-index][0]] with estimated additional utilisation [0.06250] (write load [0.50000] / threads [8]) were " + + "assigned to it. Cannot allocate shard to node without risking increased write latencies." ); } + private void assertDecisionMatches(String description, Decision decision, Decision.Type type, String explanationPattern) { + assertEquals(description, type, decision.type()); + if (explanationPattern == null) { + assertNull(decision.getExplanation()); + } else { + assertTrue( + Strings.format("Expected: \"%s\", got \"%s\"", explanationPattern, decision.getExplanation()), + Regex.simpleMatch(explanationPattern, decision.getExplanation()) + ); + } + } + /** * Carries all the cluster state objects needed for testing after {@link #createClusterStateAndRoutingAllocation} sets them up. */ @@ -145,7 +174,8 @@ private record TestHarness( RoutingNode nearThresholdRoutingNode, ShardRouting shardRouting1, ShardRouting shardRouting2, - ShardRouting thirdRoutingNoWriteLoad + ShardRouting thirdRoutingNoWriteLoad, + ShardRouting unassignedShardRouting ) {} /** @@ -183,6 +213,7 @@ private TestHarness createClusterStateAndRoutingAllocation(String indexName) { ShardId testShardId1 = new ShardId(testIndex, 0); ShardId testShardId2 = new ShardId(testIndex, 1); ShardId testShardId3NoWriteLoad = new ShardId(testIndex, 2); + ShardId testShardId4Unassigned = new ShardId(testIndex, 3); /** * Create a ClusterInfo that includes the node and shard level write load estimates for a variety of node capacity situations. @@ -248,6 +279,12 @@ private TestHarness createClusterStateAndRoutingAllocation(String indexName) { true, ShardRoutingState.STARTED ); + ShardRouting unassignedShardRouting = TestShardRouting.newShardRouting( + testShardId4Unassigned, + null, + true, + ShardRoutingState.UNASSIGNED + ); RoutingNode exceedingThresholdRoutingNode = RoutingNodesHelper.routingNode( exceedingThresholdDiscoveryNode.getId(), @@ -273,7 +310,8 @@ private TestHarness createClusterStateAndRoutingAllocation(String indexName) { nearThresholdRoutingNode, shardRouting1, shardRouting2, - thirdRoutingNoWriteLoad + thirdRoutingNoWriteLoad, + unassignedShardRouting ); } From 45279952c2624f14fcdcf3496f37ad7477c900f2 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 1 Oct 2025 11:15:21 +1000 Subject: [PATCH 2/3] Randomly include write loads when testing unassigned assignment --- .../decider/WriteLoadConstraintDeciderTests.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java index 45c931d9f8be0..abe2c727bcf54 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java @@ -119,7 +119,7 @@ public void testWriteLoadDeciderCanAllocate() { "Unassigned shard should always be accepted", writeLoadDecider.canAllocate( testHarness.unassignedShardRouting, - testHarness.exceedingThresholdRoutingNode, + randomFrom(testHarness.exceedingThresholdRoutingNode, testHarness.belowThresholdRoutingNode), testHarness.routingAllocation ), Decision.Type.YES, @@ -239,6 +239,9 @@ private TestHarness createClusterStateAndRoutingAllocation(String indexName) { shardIdToWriteLoadEstimate.put(testShardId1, 0.5); shardIdToWriteLoadEstimate.put(testShardId2, 0.5); shardIdToWriteLoadEstimate.put(testShardId3NoWriteLoad, 0d); + if (randomBoolean()) { + shardIdToWriteLoadEstimate.put(testShardId4Unassigned, randomDoubleBetween(0.0, 2.0, true)); + } ClusterInfo clusterInfo = ClusterInfo.builder() .nodeUsageStatsForThreadPools(nodeIdToNodeUsageStatsForThreadPools) @@ -298,8 +301,7 @@ private TestHarness createClusterStateAndRoutingAllocation(String indexName) { ); RoutingNode nearThresholdRoutingNode = RoutingNodesHelper.routingNode( nearThresholdDiscoveryNode3.getId(), - nearThresholdDiscoveryNode3, - new ShardRouting[] {} + nearThresholdDiscoveryNode3 ); return new TestHarness( From 21e737a3a60fdc54076f0a60e982ef4b4e815feb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 1 Oct 2025 01:30:23 +0000 Subject: [PATCH 3/3] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/8.18.csv | 2 +- server/src/main/resources/transport/upper_bounds/8.19.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.0.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.1.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/resources/transport/upper_bounds/8.18.csv b/server/src/main/resources/transport/upper_bounds/8.18.csv index ffc592e1809ee..266bfbbd3bf78 100644 --- a/server/src/main/resources/transport/upper_bounds/8.18.csv +++ b/server/src/main/resources/transport/upper_bounds/8.18.csv @@ -1 +1 @@ -initial_elasticsearch_8_18_8,8840010 +transform_check_for_dangling_tasks,8840011 diff --git a/server/src/main/resources/transport/upper_bounds/8.19.csv b/server/src/main/resources/transport/upper_bounds/8.19.csv index 3cc6f439c5ea5..3600b3f8c633a 100644 --- a/server/src/main/resources/transport/upper_bounds/8.19.csv +++ b/server/src/main/resources/transport/upper_bounds/8.19.csv @@ -1 +1 @@ -initial_elasticsearch_8_19_5,8841069 +transform_check_for_dangling_tasks,8841070 diff --git a/server/src/main/resources/transport/upper_bounds/9.0.csv b/server/src/main/resources/transport/upper_bounds/9.0.csv index 8ad2ed1a4cacf..c11e6837bb813 100644 --- a/server/src/main/resources/transport/upper_bounds/9.0.csv +++ b/server/src/main/resources/transport/upper_bounds/9.0.csv @@ -1 +1 @@ -initial_elasticsearch_9_0_8,9000017 +transform_check_for_dangling_tasks,9000018 diff --git a/server/src/main/resources/transport/upper_bounds/9.1.csv b/server/src/main/resources/transport/upper_bounds/9.1.csv index 1cea5dc4d929b..80b97d85f7511 100644 --- a/server/src/main/resources/transport/upper_bounds/9.1.csv +++ b/server/src/main/resources/transport/upper_bounds/9.1.csv @@ -1 +1 @@ -initial_elasticsearch_9_1_5,9112008 +transform_check_for_dangling_tasks,9112009 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 6e7d51d3d3020..058760544f9da 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -security_stats_endpoint,9168000 +esql_aggregate_metric_double_created_version,9184000