From 7365324f1c98f08fe25e492730b5e78ebe09f6b6 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 23 Sep 2025 14:44:07 +1000 Subject: [PATCH 01/11] Enable write load constraint decider by default --- .../cluster/routing/allocation/WriteLoadConstraintSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java index 21c1a7ba04f0f..9e8c42d1522ac 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java @@ -59,7 +59,7 @@ public boolean disabled() { public static final Setting WRITE_LOAD_DECIDER_ENABLED_SETTING = Setting.enumSetting( WriteLoadDeciderStatus.class, SETTING_PREFIX + "enabled", - WriteLoadDeciderStatus.DISABLED, + WriteLoadDeciderStatus.ENABLED, Setting.Property.Dynamic, Setting.Property.NodeScope ); From 7a8536d03a527c21ed22d5531bde32da35c8f105 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 23 Sep 2025 15:11:09 +1000 Subject: [PATCH 02/11] Fix disabled test --- .../decider/WriteLoadConstraintDeciderTests.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java index 12bfd8a0a4789..6e24354878c5c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDeciderTests.java @@ -43,9 +43,14 @@ public void testWriteLoadDeciderDisabled() { String indexName = "test-index"; var testHarness = createClusterStateAndRoutingAllocation(indexName); - // The write load decider is disabled by default. - - var writeLoadDecider = createWriteLoadConstraintDecider(Settings.builder().build()); + var writeLoadDecider = createWriteLoadConstraintDecider( + Settings.builder() + .put( + WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(), + WriteLoadConstraintSettings.WriteLoadDeciderStatus.DISABLED + ) + .build() + ); assertEquals( Decision.Type.YES, From 24020991aa0f9ea4a8bae25ced8666776ae3570e Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 24 Sep 2025 12:02:59 +1000 Subject: [PATCH 03/11] Add label/description for YES decision --- .../routing/allocation/decider/WriteLoadConstraintDecider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java index e814f570a67bb..070dc50739882 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java @@ -95,7 +95,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return Decision.single(Decision.Type.NO, NAME, explain); } - return Decision.YES; + return Decision.single(Decision.Type.YES, NAME, "There is sufficient write thread-pool capacity to accept this allocation"); } @Override From 14c936d054dd1cea2c51ba7c4f5ca89418d4c328 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 24 Sep 2025 02:12:18 +0000 Subject: [PATCH 04/11] [CI] Update transport version definitions --- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 6e7d51d3d3020..b1209b927d8a5 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -security_stats_endpoint,9168000 +inference_api_openai_embeddings_headers,9169000 From b4f08ddc5f43bfe752bcb4fcf46db6aff381d507 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 24 Sep 2025 12:10:02 +1000 Subject: [PATCH 05/11] Remove assumptions around write load decider default enablement --- .../index/shard/IndexShardIT.java | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index 058d56b9d5a29..b82046526448e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -305,6 +305,9 @@ public void testHeapUsageEstimateIsPresent() { } public void testNodeWriteLoadsArePresent() { + // Disable write load decider to begin with + setWriteLoadDeciderEnablement(WriteLoadConstraintSettings.WriteLoadDeciderStatus.DISABLED); + InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class); ClusterInfoServiceUtils.refresh(clusterInfoService); Map nodeThreadPoolStats = clusterInfoService.getClusterInfo() @@ -315,15 +318,10 @@ public void testNodeWriteLoadsArePresent() { assertTrue(nodeThreadPoolStats.isEmpty()); // Enable collection for node write loads. - updateClusterSettings( - Settings.builder() - .put( - WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(), - randomBoolean() - ? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED - : WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY - ) - .build() + setWriteLoadDeciderEnablement( + randomBoolean() + ? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED + : WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY ); try { // Force a ClusterInfo refresh to run collection of the node thread pool usage stats. @@ -352,6 +350,12 @@ public void testNodeWriteLoadsArePresent() { } } + private void setWriteLoadDeciderEnablement(WriteLoadConstraintSettings.WriteLoadDeciderStatus status) { + updateClusterSettings( + Settings.builder().put(WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(), status).build() + ); + } + public void testShardWriteLoadsArePresent() { // Create some indices and some write-load final int numIndices = randomIntBetween(1, 5); @@ -365,7 +369,10 @@ public void testShardWriteLoadsArePresent() { final InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class); - // Not collecting stats yet because allocation write load stats collection is disabled by default. + // Explicitly disable write load decider + setWriteLoadDeciderEnablement(WriteLoadConstraintSettings.WriteLoadDeciderStatus.DISABLED); + + // Stats should not be collected when the decider is disabled { ClusterInfoServiceUtils.refresh(clusterInfoService); final Map shardWriteLoads = clusterInfoService.getClusterInfo().getShardWriteLoads(); @@ -374,15 +381,10 @@ public void testShardWriteLoadsArePresent() { } // Turn on collection of write-load stats. - updateClusterSettings( - Settings.builder() - .put( - WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(), - randomBoolean() - ? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED - : WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY - ) - .build() + setWriteLoadDeciderEnablement( + randomBoolean() + ? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED + : WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY ); try { From 5068f10664dbe731e2c48f61a65656380846f169 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 24 Sep 2025 12:20:28 +1000 Subject: [PATCH 06/11] Only allow enablement of the write load decider when the feature flag is turned on --- .../allocation/WriteLoadConstraintSettings.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java index 9e8c42d1522ac..64fc1aee4585d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.unit.RatioValue; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.TimeValue; /** @@ -24,6 +25,10 @@ public class WriteLoadConstraintSettings { private static final String SETTING_PREFIX = "cluster.routing.allocation.write_load_decider."; + private static final FeatureFlag WRITE_LOAD_DECIDER_ENABLED_FF = new FeatureFlag( + "cluster.routing.allocation.write_load_decider_enabled" + ); + public enum WriteLoadDeciderStatus { /** * The decider is disabled @@ -59,7 +64,7 @@ public boolean disabled() { public static final Setting WRITE_LOAD_DECIDER_ENABLED_SETTING = Setting.enumSetting( WriteLoadDeciderStatus.class, SETTING_PREFIX + "enabled", - WriteLoadDeciderStatus.ENABLED, + WriteLoadDeciderStatus.DISABLED, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -113,7 +118,11 @@ public boolean disabled() { private volatile TimeValue queueLatencyThreshold; public WriteLoadConstraintSettings(ClusterSettings clusterSettings) { - clusterSettings.initializeAndWatch(WRITE_LOAD_DECIDER_ENABLED_SETTING, status -> this.writeLoadDeciderStatus = status); + if (WRITE_LOAD_DECIDER_ENABLED_FF.isEnabled()) { + clusterSettings.initializeAndWatch(WRITE_LOAD_DECIDER_ENABLED_SETTING, status -> this.writeLoadDeciderStatus = status); + } else { + writeLoadDeciderStatus = WriteLoadDeciderStatus.DISABLED; + } clusterSettings.initializeAndWatch( WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING, timeValue -> this.minimumRerouteInterval = timeValue From 6a93c563a93f2e03fa684ae5cb050eb299d45711 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 24 Sep 2025 12:37:19 +1000 Subject: [PATCH 07/11] Fix feature flag, enable decider by default --- .../routing/allocation/WriteLoadConstraintSettings.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java index 64fc1aee4585d..f2c18cb8d5b48 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java @@ -24,10 +24,7 @@ public class WriteLoadConstraintSettings { private static final String SETTING_PREFIX = "cluster.routing.allocation.write_load_decider."; - - private static final FeatureFlag WRITE_LOAD_DECIDER_ENABLED_FF = new FeatureFlag( - "cluster.routing.allocation.write_load_decider_enabled" - ); + private static final FeatureFlag WRITE_LOAD_DECIDER_ENABLED_FF = new FeatureFlag("write_load_decider_enabled"); public enum WriteLoadDeciderStatus { /** @@ -64,7 +61,7 @@ public boolean disabled() { public static final Setting WRITE_LOAD_DECIDER_ENABLED_SETTING = Setting.enumSetting( WriteLoadDeciderStatus.class, SETTING_PREFIX + "enabled", - WriteLoadDeciderStatus.DISABLED, + WriteLoadDeciderStatus.ENABLED, Setting.Property.Dynamic, Setting.Property.NodeScope ); From df433012f39c740588003371f3b7c28e32dec011 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 24 Sep 2025 13:43:30 +1000 Subject: [PATCH 08/11] Move utility method to bottom of class --- .../org/elasticsearch/index/shard/IndexShardIT.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index b82046526448e..ef036a1bb30b1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -350,12 +350,6 @@ public void testNodeWriteLoadsArePresent() { } } - private void setWriteLoadDeciderEnablement(WriteLoadConstraintSettings.WriteLoadDeciderStatus status) { - updateClusterSettings( - Settings.builder().put(WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(), status).build() - ); - } - public void testShardWriteLoadsArePresent() { // Create some indices and some write-load final int numIndices = randomIntBetween(1, 5); @@ -766,6 +760,12 @@ public void postDelete(ShardId shardId, Engine.Delete delete, Engine.DeleteResul } } + private void setWriteLoadDeciderEnablement(WriteLoadConstraintSettings.WriteLoadDeciderStatus status) { + updateClusterSettings( + Settings.builder().put(WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey(), status).build() + ); + } + public static final IndexShard recoverShard(IndexShard newShard) throws IOException { DiscoveryNode localNode = DiscoveryNodeUtils.builder("foo").roles(emptySet()).build(); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); From 149b2ea9caa671210fa6e3b74a4c64951382be6e Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Mon, 29 Sep 2025 17:18:26 +1000 Subject: [PATCH 09/11] Use feature flag to configure default --- .../routing/allocation/WriteLoadConstraintSettings.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java index f2c18cb8d5b48..f46289f1062f4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java @@ -61,7 +61,7 @@ public boolean disabled() { public static final Setting WRITE_LOAD_DECIDER_ENABLED_SETTING = Setting.enumSetting( WriteLoadDeciderStatus.class, SETTING_PREFIX + "enabled", - WriteLoadDeciderStatus.ENABLED, + WRITE_LOAD_DECIDER_ENABLED_FF.isEnabled() ? WriteLoadDeciderStatus.ENABLED : WriteLoadDeciderStatus.DISABLED, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -115,11 +115,7 @@ public boolean disabled() { private volatile TimeValue queueLatencyThreshold; public WriteLoadConstraintSettings(ClusterSettings clusterSettings) { - if (WRITE_LOAD_DECIDER_ENABLED_FF.isEnabled()) { - clusterSettings.initializeAndWatch(WRITE_LOAD_DECIDER_ENABLED_SETTING, status -> this.writeLoadDeciderStatus = status); - } else { - writeLoadDeciderStatus = WriteLoadDeciderStatus.DISABLED; - } + clusterSettings.initializeAndWatch(WRITE_LOAD_DECIDER_ENABLED_SETTING, status -> this.writeLoadDeciderStatus = status); clusterSettings.initializeAndWatch( WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING, timeValue -> this.minimumRerouteInterval = timeValue From 9845c3847c52786e4404d45cbf7316d38ca3baa8 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 30 Sep 2025 10:17:48 +1000 Subject: [PATCH 10/11] Tidy --- .../index/shard/IndexShardIT.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java index ef036a1bb30b1..f51e7209314ba 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -344,9 +344,7 @@ public void testNodeWriteLoadsArePresent() { assertThat(writeThreadPoolStats.maxThreadPoolQueueLatencyMillis(), greaterThanOrEqualTo(0L)); } } finally { - updateClusterSettings( - Settings.builder().putNull(WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey()).build() - ); + clearWriteLoadDeciderEnablementSetting(); } } @@ -366,22 +364,22 @@ public void testShardWriteLoadsArePresent() { // Explicitly disable write load decider setWriteLoadDeciderEnablement(WriteLoadConstraintSettings.WriteLoadDeciderStatus.DISABLED); - // Stats should not be collected when the decider is disabled - { - ClusterInfoServiceUtils.refresh(clusterInfoService); - final Map shardWriteLoads = clusterInfoService.getClusterInfo().getShardWriteLoads(); - assertNotNull(shardWriteLoads); - assertTrue(shardWriteLoads.isEmpty()); - } + try { + // Stats should not be collected when the decider is disabled + { + ClusterInfoServiceUtils.refresh(clusterInfoService); + final Map shardWriteLoads = clusterInfoService.getClusterInfo().getShardWriteLoads(); + assertNotNull(shardWriteLoads); + assertTrue(shardWriteLoads.isEmpty()); + } - // Turn on collection of write-load stats. - setWriteLoadDeciderEnablement( - randomBoolean() - ? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED - : WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY - ); + // Turn on collection of write-load stats. + setWriteLoadDeciderEnablement( + randomBoolean() + ? WriteLoadConstraintSettings.WriteLoadDeciderStatus.ENABLED + : WriteLoadConstraintSettings.WriteLoadDeciderStatus.LOW_THRESHOLD_ONLY + ); - try { // Force a ClusterInfo refresh to run collection of the write-load stats. ClusterInfoServiceUtils.refresh(clusterInfoService); final Map shardWriteLoads = clusterInfoService.getClusterInfo().getShardWriteLoads(); @@ -400,12 +398,14 @@ public void testShardWriteLoadsArePresent() { assertThat(maximumLoadRecorded, greaterThan(0.0)); } } finally { - updateClusterSettings( - Settings.builder().putNull(WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey()).build() - ); + clearWriteLoadDeciderEnablementSetting(); } } + private void clearWriteLoadDeciderEnablementSetting() { + updateClusterSettings(Settings.builder().putNull(WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING.getKey()).build()); + } + public void testMaxHeapPerNodeIsPresent() { InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) getInstanceFromNode(ClusterInfoService.class); ClusterInfoServiceUtils.refresh(clusterInfoService); From 917c617b2f78ec9764a8bad57935575dd4a924c9 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 30 Sep 2025 10:22:20 +1000 Subject: [PATCH 11/11] feature flag naming --- .../routing/allocation/WriteLoadConstraintSettings.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java index f46289f1062f4..156f1d30d66c0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java @@ -24,7 +24,7 @@ public class WriteLoadConstraintSettings { private static final String SETTING_PREFIX = "cluster.routing.allocation.write_load_decider."; - private static final FeatureFlag WRITE_LOAD_DECIDER_ENABLED_FF = new FeatureFlag("write_load_decider_enabled"); + private static final FeatureFlag WRITE_LOAD_DECIDER_FEATURE_FLAG = new FeatureFlag("write_load_decider"); public enum WriteLoadDeciderStatus { /** @@ -61,7 +61,7 @@ public boolean disabled() { public static final Setting WRITE_LOAD_DECIDER_ENABLED_SETTING = Setting.enumSetting( WriteLoadDeciderStatus.class, SETTING_PREFIX + "enabled", - WRITE_LOAD_DECIDER_ENABLED_FF.isEnabled() ? WriteLoadDeciderStatus.ENABLED : WriteLoadDeciderStatus.DISABLED, + WRITE_LOAD_DECIDER_FEATURE_FLAG.isEnabled() ? WriteLoadDeciderStatus.ENABLED : WriteLoadDeciderStatus.DISABLED, Setting.Property.Dynamic, Setting.Property.NodeScope );