From 8838355cf63638092b00cce23b6f50721331b091 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 19 Nov 2025 12:21:05 -0800 Subject: [PATCH] [SPARK-54422][K8S] Increase `spark.kubernetes.allocation.batch.size` to 20 --- docs/core-migration-guide.md | 4 ++++ docs/running-on-kubernetes.md | 2 +- .../src/main/scala/org/apache/spark/deploy/k8s/Config.scala | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md index 1d55c4c3e66d..d7ff6b1509ce 100644 --- a/docs/core-migration-guide.md +++ b/docs/core-migration-guide.md @@ -22,6 +22,10 @@ license: | * Table of contents {:toc} +## Upgrading from Core 4.1 to 4.2 + +- Since Spark 4.2, Spark will allocate executor pods with a batch size of `20`. To restore the legacy behavior, you can set `spark.kubernetes.allocation.batch.size` to `10`. + ## Upgrading from Core 4.0 to 4.1 - Since Spark 4.1, Spark Master deamon provides REST API by default. To restore the behavior before Spark 4.1, you can set `spark.master.rest.enabled` to `false`. diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index ababe38e4e25..eac8d6a5d1ec 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -685,7 +685,7 @@ See the [configuration page](configuration.html) for information on Spark config spark.kubernetes.allocation.batch.size - 10 + 20 Number of pods to launch at once in each round of executor pod allocation. diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index 1fb214c87c53..3f131874fc5e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -487,7 +487,7 @@ private[spark] object Config extends Logging { .version("2.3.0") .intConf .checkValue(value => value > 0, "Allocation batch size should be a positive integer") - .createWithDefault(10) + .createWithDefault(20) val KUBERNETES_ALLOCATION_BATCH_DELAY = ConfigBuilder("spark.kubernetes.allocation.batch.delay")