diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala index a2a46614fb8f8..09943b7974ed9 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala @@ -24,6 +24,7 @@ import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.deploy.k8s.Constants._ import org.apache.spark.deploy.k8s.submit._ +import org.apache.spark.internal.Logging import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.util.Utils @@ -123,7 +124,7 @@ private[spark] class KubernetesExecutorConf( val appId: String, val executorId: String, val driverPod: Option[Pod]) - extends KubernetesConf(sparkConf) { + extends KubernetesConf(sparkConf) with Logging { override val resourceNamePrefix: String = { get(KUBERNETES_EXECUTOR_POD_NAME_PREFIX).getOrElse( @@ -148,7 +149,8 @@ private[spark] class KubernetesExecutorConf( executorCustomLabels ++ presetLabels } - override def environment: Map[String, String] = sparkConf.getExecutorEnv.toMap + override def environment: Map[String, String] = sparkConf.getExecutorEnv.filter( + p => checkExecutorEnvKey(p._1)).toMap override def annotations: Map[String, String] = { KubernetesUtils.parsePrefixedKeyValuePairs(sparkConf, KUBERNETES_EXECUTOR_ANNOTATION_PREFIX) @@ -166,6 +168,20 @@ private[spark] class KubernetesExecutorConf( KubernetesVolumeUtils.parseVolumesWithPrefix(sparkConf, KUBERNETES_EXECUTOR_VOLUMES_PREFIX) } + private def checkExecutorEnvKey(key: String): Boolean = { + // Pattern for matching an executorEnv key, which meets certain naming rules. + val executorEnvRegex = "[-._a-zA-Z][-._a-zA-Z0-9]*".r + if (executorEnvRegex.pattern.matcher(key).matches()) { + true + } else { + logWarning(s"Invalid key: $key: " + + "a valid environment variable name must consist of alphabetic characters, " + + "digits, '_', '-', or '.', and must not start with a digit." + + s"Regex used for validation is '$executorEnvRegex')") + false + } + } + } private[spark] object KubernetesConf { diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala index d51b1e661bb1e..5591974c564e8 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala @@ -44,6 +44,12 @@ class KubernetesConfSuite extends SparkFunSuite { "customEnvKey2" -> "customEnvValue2") private val DRIVER_POD = new PodBuilder().build() private val EXECUTOR_ID = "executor-id" + private val EXECUTOR_ENV_VARS = Map( + "spark.executorEnv.1executorEnvVars1/var1" -> "executorEnvVars1", + "spark.executorEnv.executorEnvVars2*var2" -> "executorEnvVars2", + "spark.executorEnv.executorEnvVars3_var3" -> "executorEnvVars3", + "spark.executorEnv.executorEnvVars4-var4" -> "executorEnvVars4", + "spark.executorEnv.executorEnvVars5-var5" -> "executorEnvVars5/var5") test("Resolve driver labels, annotations, secret mount paths, envs, and memory overhead") { val sparkConf = new SparkConf(false) @@ -132,4 +138,22 @@ class KubernetesConfSuite extends SparkFunSuite { assert(conf.secretNamesToMountPaths === SECRET_NAMES_TO_MOUNT_PATHS) assert(conf.secretEnvNamesToKeyRefs === SECRET_ENV_VARS) } + + test("Verify that executorEnv key conforms to the regular specification") { + val sparkConf = new SparkConf(false) + EXECUTOR_ENV_VARS.foreach { case (key, value) => + sparkConf.set(key, value) + } + + val conf = KubernetesConf.createExecutorConf( + sparkConf, + EXECUTOR_ID, + KubernetesTestConf.APP_ID, + Some(DRIVER_POD)) + assert(conf.environment === + Map( + "executorEnvVars3_var3" -> "executorEnvVars3", + "executorEnvVars4-var4" -> "executorEnvVars4", + "executorEnvVars5-var5" -> "executorEnvVars5/var5")) + } }