diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md index 1d55c4c3e66d..d7ff6b1509ce 100644 --- a/docs/core-migration-guide.md +++ b/docs/core-migration-guide.md @@ -22,6 +22,10 @@ license: | * Table of contents {:toc} +## Upgrading from Core 4.1 to 4.2 + +- Since Spark 4.2, Spark will allocate executor pods with a batch size of `20`. To restore the legacy behavior, you can set `spark.kubernetes.allocation.batch.size` to `10`. + ## Upgrading from Core 4.0 to 4.1 - Since Spark 4.1, Spark Master deamon provides REST API by default. To restore the behavior before Spark 4.1, you can set `spark.master.rest.enabled` to `false`. diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index ababe38e4e25..eac8d6a5d1ec 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -685,7 +685,7 @@ See the [configuration page](configuration.html) for information on Spark config spark.kubernetes.allocation.batch.size - 10 + 20 Number of pods to launch at once in each round of executor pod allocation. diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index 1fb214c87c53..3f131874fc5e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -487,7 +487,7 @@ private[spark] object Config extends Logging { .version("2.3.0") .intConf .checkValue(value => value > 0, "Allocation batch size should be a positive integer") - .createWithDefault(10) + .createWithDefault(20) val KUBERNETES_ALLOCATION_BATCH_DELAY = ConfigBuilder("spark.kubernetes.allocation.batch.delay")