diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index ed766e7050d1c..76823e4d603ff 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -146,6 +146,20 @@ private[spark] object Config extends Logging { .checkValues(Set("Always", "Never", "IfNotPresent")) .createWithDefault("IfNotPresent") + val DRIVER_CONTAINER_NAME = + ConfigBuilder("spark.kubernetes.driver.container.name") + .doc("Name of the driver container.") + .version("3.4.0") + .stringConf + .createWithDefault(DEFAULT_DRIVER_CONTAINER_NAME) + + val EXECUTOR_CONTAINER_NAME = + ConfigBuilder("spark.kubernetes.executor.container.name") + .doc("Name of the executors containers.") + .version("3.4.0") + .stringConf + .createWithDefault(DEFAULT_EXECUTOR_CONTAINER_NAME) + val IMAGE_PULL_SECRETS = ConfigBuilder("spark.kubernetes.container.image.pullSecrets") .doc("Comma separated list of the Kubernetes secrets used " + diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala index 4b5eece54c8c5..5325d8601de7b 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala @@ -101,7 +101,7 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf) ) val driverUIPort = SparkUI.getUIPort(conf.sparkConf) val driverContainer = new ContainerBuilder(pod.container) - .withName(Option(pod.container.getName).getOrElse(DEFAULT_DRIVER_CONTAINER_NAME)) + .withName(Option(pod.container.getName).getOrElse(DRIVER_CONTAINER_NAME)) .withImage(driverContainerImage) .withImagePullPolicy(conf.imagePullPolicy) .addNewPort() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index 0b0bbc30ba41a..21600337f6196 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -187,7 +187,7 @@ private[spark] class BasicExecutorFeatureStep( } val executorContainer = new ContainerBuilder(pod.container) - .withName(Option(pod.container.getName).getOrElse(DEFAULT_EXECUTOR_CONTAINER_NAME)) + .withName(Option(pod.container.getName).getOrElse(EXECUTOR_CONTAINER_NAME)) .withImage(executorContainerImage) .withImagePullPolicy(kubernetesConf.imagePullPolicy) .editOrNewResources() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala index ff47c17148f35..f020f20cdb4d1 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala @@ -23,6 +23,7 @@ import scala.collection.JavaConverters._ import io.fabric8.kubernetes.api.model.ContainerStateTerminated import io.fabric8.kubernetes.api.model.Pod +import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.internal.Logging @@ -43,7 +44,7 @@ private[spark] case class ExecutorPodsSnapshot( object ExecutorPodsSnapshot extends Logging { private var shouldCheckAllContainers: Boolean = _ - private var sparkContainerName: String = DEFAULT_EXECUTOR_CONTAINER_NAME + private var sparkContainerName: String = EXECUTOR_CONTAINER_NAME def apply(executorPods: Seq[Pod], fullSnapshotTs: Long): ExecutorPodsSnapshot = { ExecutorPodsSnapshot(toStatesByExecutorId(executorPods), fullSnapshotTs) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala index 55d28dc5a739c..83a59b38ccb7b 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala @@ -24,7 +24,6 @@ import io.fabric8.kubernetes.client.KubernetesClient import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesUtils, SparkKubernetesClientFactory} import org.apache.spark.deploy.k8s.Config._ -import org.apache.spark.deploy.k8s.Constants.DEFAULT_EXECUTOR_CONTAINER_NAME import org.apache.spark.internal.Logging import org.apache.spark.scheduler.{ExternalClusterManager, SchedulerBackend, TaskScheduler, TaskSchedulerImpl} import org.apache.spark.util.{Clock, SystemClock, ThreadUtils, Utils} @@ -98,7 +97,7 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit ExecutorPodsSnapshot.setShouldCheckAllContainers( sc.conf.get(KUBERNETES_EXECUTOR_CHECK_ALL_CONTAINERS)) val sparkContainerName = sc.conf.get(KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME) - .getOrElse(DEFAULT_EXECUTOR_CONTAINER_NAME) + .getOrElse(EXECUTOR_CONTAINER_NAME) ExecutorPodsSnapshot.setSparkContainerName(sparkContainerName) val subscribersExecutor = ThreadUtils .newDaemonThreadPoolScheduledExecutor( diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala index dbe2f29872ad1..b2ccfc24d3abd 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala @@ -19,13 +19,13 @@ package org.apache.spark.scheduler.cluster.k8s import io.fabric8.kubernetes.api.model.Pod import scala.collection.mutable -import org.apache.spark.deploy.k8s.Constants.DEFAULT_EXECUTOR_CONTAINER_NAME +import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.util.ManualClock class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore { ExecutorPodsSnapshot.setShouldCheckAllContainers(false) - ExecutorPodsSnapshot.setSparkContainerName(DEFAULT_EXECUTOR_CONTAINER_NAME) + ExecutorPodsSnapshot.setSparkContainerName(EXECUTOR_CONTAINER_NAME) val clock = new ManualClock()