diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala index 19784e8a8074f..b76c77a656d2d 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStep.scala @@ -48,6 +48,9 @@ private[spark] class BaseDriverConfigurationStep( // Memory settings private val driverMemoryMiB = submissionSparkConf.get( org.apache.spark.internal.config.DRIVER_MEMORY) + private val driverMemoryString = submissionSparkConf.get( + org.apache.spark.internal.config.DRIVER_MEMORY.key, + org.apache.spark.internal.config.DRIVER_MEMORY.defaultValueString) private val memoryOverheadMiB = submissionSparkConf .get(KUBERNETES_DRIVER_MEMORY_OVERHEAD) .getOrElse(math.max((MEMORY_OVERHEAD_FACTOR * driverMemoryMiB).toInt, @@ -102,7 +105,7 @@ private[spark] class BaseDriverConfigurationStep( .addToEnv(driverExtraClasspathEnv.toSeq: _*) .addNewEnv() .withName(ENV_DRIVER_MEMORY) - .withValue(driverContainerMemoryWithOverheadMiB + "M") // JVM treats the "M" unit as "Mi" + .withValue(driverMemoryString) .endEnv() .addNewEnv() .withName(ENV_DRIVER_MAIN_CLASS) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala index 53c4ea60afacd..5e3718800d928 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/submitsteps/BaseDriverConfigurationStepSuite.scala @@ -42,10 +42,10 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite { test("Set all possible configurations from the user.") { val sparkConf = new SparkConf() .set(KUBERNETES_DRIVER_POD_NAME, "spark-driver-pod") - .set(org.apache.spark.internal.config.DRIVER_CLASS_PATH, "/opt/spark/spark-exmaples.jar") + .set(org.apache.spark.internal.config.DRIVER_CLASS_PATH, "/opt/spark/spark-examples.jar") .set("spark.driver.cores", "2") .set(KUBERNETES_DRIVER_LIMIT_CORES, "4") - .set(org.apache.spark.internal.config.DRIVER_MEMORY, 256L) + .set(org.apache.spark.internal.config.DRIVER_MEMORY.key, "256M") .set(KUBERNETES_DRIVER_MEMORY_OVERHEAD, 200L) .set(DRIVER_DOCKER_IMAGE, "spark-driver:latest") .set(s"spark.kubernetes.driver.annotation.$CUSTOM_ANNOTATION_KEY", CUSTOM_ANNOTATION_VALUE) @@ -80,8 +80,8 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite { .map(env => (env.getName, env.getValue)) .toMap assert(envs.size === 6) - assert(envs(ENV_SUBMIT_EXTRA_CLASSPATH) === "/opt/spark/spark-exmaples.jar") - assert(envs(ENV_DRIVER_MEMORY) === "456M") + assert(envs(ENV_SUBMIT_EXTRA_CLASSPATH) === "/opt/spark/spark-examples.jar") + assert(envs(ENV_DRIVER_MEMORY) === "256M") assert(envs(ENV_DRIVER_MAIN_CLASS) === MAIN_CLASS) assert(envs(ENV_DRIVER_ARGS) === "arg1 arg2") assert(envs(DRIVER_CUSTOM_ENV_KEY1) === "customDriverEnv1")