From 503794d87f313bf818efa263c5cde931ad0b9eda Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 5 May 2026 19:51:20 -0700 Subject: [PATCH] [SPARK-56736] Add `sparkVersion` method to KubernetesConf abstract class --- .../spark/deploy/k8s/KubernetesConf.scala | 6 +++-- .../deploy/k8s/KubernetesConfSuite.scala | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala index 22a21c2ed4aaf..9070d6ce35ff6 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala @@ -49,6 +49,8 @@ private[spark] abstract class KubernetesConf(val sparkConf: SparkConf) { def appName: String = get("spark.app.name", "spark") + def sparkVersion: String = SPARK_VERSION + def namespace: String = get(KUBERNETES_NAMESPACE) def imagePullPolicy: String = get(CONTAINER_IMAGE_PULL_POLICY) @@ -122,7 +124,7 @@ class KubernetesDriverConf( override def labels: Map[String, String] = { val presetLabels = Map( - SPARK_VERSION_LABEL -> SPARK_VERSION, + SPARK_VERSION_LABEL -> sparkVersion, SPARK_APP_ID_LABEL -> appId, SPARK_APP_NAME_LABEL -> KubernetesConf.getAppNameLabel(appName), SPARK_ROLE_LABEL -> SPARK_POD_DRIVER_ROLE) @@ -199,7 +201,7 @@ private[spark] class KubernetesExecutorConf( override def labels: Map[String, String] = { val presetLabels = Map( - SPARK_VERSION_LABEL -> SPARK_VERSION, + SPARK_VERSION_LABEL -> sparkVersion, SPARK_EXECUTOR_ID_LABEL -> executorId, SPARK_APP_ID_LABEL -> appId, SPARK_APP_NAME_LABEL -> KubernetesConf.getAppNameLabel(appName), diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala index 37778ff1bf8f1..c1b3e8d6b252b 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala @@ -298,6 +298,31 @@ class KubernetesConfSuite extends SparkFunSuite { } } + test("SPARK-56736: sparkVersion returns the runtime Spark version for driver and executor") { + val sparkConf = new SparkConf(false) + val driverConf = KubernetesTestConf.createDriverConf(sparkConf) + val execConf = KubernetesTestConf.createExecutorConf(sparkConf) + assert(driverConf.sparkVersion === SPARK_VERSION) + assert(execConf.sparkVersion === SPARK_VERSION) + assert(driverConf.labels(SPARK_VERSION_LABEL) === SPARK_VERSION) + assert(execConf.labels(SPARK_VERSION_LABEL) === SPARK_VERSION) + } + + test("SPARK-56736: KubernetesDriverConf subclass can override sparkVersion") { + val customVersion = "9.9.9-custom" + val customConf = new KubernetesDriverConf( + new SparkConf(false), + KubernetesTestConf.APP_ID, + JavaMainAppResource(None), + KubernetesTestConf.MAIN_CLASS, + APP_ARGS, + None) { + override def sparkVersion: String = customVersion + } + assert(customConf.sparkVersion === customVersion) + assert(customConf.labels(SPARK_VERSION_LABEL) === customVersion) + } + test("SPARK-52902: K8s image configs support {{SPARK_VERSION}} placeholder") { val sparkConf = new SparkConf(false) sparkConf.set(CONTAINER_IMAGE, "apache/spark:{{SPARK_VERSION}}")