diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index ceb5dfbe6a134..e261844f34b8b 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -655,6 +655,8 @@ private[spark] class SparkSubmit extends Logging { confKey = SGX_MEM_SIZE.key), OptionAssigner(args.sgxJvmMem, KUBERNETES, ALL_DEPLOY_MODES, confKey = SGX_JVM_MEM_SIZE.key), + OptionAssigner(args.sgxLogLevel, KUBERNETES, ALL_DEPLOY_MODES, + confKey = SGX_LOG_LEVEL.key), // An internal option used only for spark-shell to add user jars to repl's classloader, // previously it uses "spark.jars" or "spark.yarn.dist.jars" which now may be pointed to diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 24ec31cac8adf..426aab26079e7 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -222,7 +222,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S sgxJvmMem = Option(sgxJvmMem) .getOrElse(sparkProperties.get(config.SGX_JVM_MEM_SIZE.key).orNull) sgxLogLevel = Option(sgxLogLevel) - .getOrElse(sparkProperties.get(config.SGX_JVM_LOG_LEVEL.key).orNull) + .getOrElse(sparkProperties.get(config.SGX_LOG_LEVEL.key).orNull) // Global defaults. These should be keep to minimum to avoid confusing behavior. master = Option(master).getOrElse("local[*]") diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 846a99823c9d0..56b101bde484d 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -2045,7 +2045,7 @@ package object config { .bytesConf(ByteUnit.GiB) .createWithDefaultString("16g") - private[spark] val SGX_JVM_LOG_LEVEL = + private[spark] val SGX_LOG_LEVEL = ConfigBuilder("spark.kubernetes.sgx.log.level") .doc("the log level of sgx execuotrs, e.g. all, error, debug") .version("3.1.2") diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala index 7e15975a0c2ed..1b337469b7ddf 100644 --- a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala +++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala @@ -356,7 +356,7 @@ object ResourceProfile extends Logging { val sgxEnabled = conf.get(SGX_ENABLED) val sgxMem = conf.get(SGX_MEM_SIZE) val sgxJvmMem = conf.get(SGX_JVM_MEM_SIZE) - val sgxLogLevel = conf.get(SGX_JVM_LOG_LEVEL) + val sgxLogLevel = conf.get(SGX_LOG_LEVEL) defaultProfileExecutorResources = Some(DefaultProfileExecutorResources(cores, memory, offheapMem, pysparkMem, overheadMem, customResources, sgxEnabled, sgxMem, sgxJvmMem, sgxLogLevel)) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala index 9dfb3697f306c..dbdfab1f56e69 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala @@ -68,6 +68,7 @@ private[spark] object Constants { val ENV_SGX_ENABLED = "SGX_ENABLED" val ENV_SGX_MEM_SIZE = "SGX_MEM_SIZE" val ENV_SGX_JVM_MEM_SIZE = "SGX_JVM_MEM_SIZE" + val ENV_SGX_LOG_LEVEL = "SGX_LOG_LEVEL" // Spark app configs for containers val SPARK_CONF_VOLUME_DRIVER = "spark-conf-volume-driver" val SPARK_CONF_VOLUME_EXEC = "spark-conf-volume-exec" diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index 6040a421c1ad2..cd0cd3552f42e 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -124,7 +124,8 @@ private[spark] class BasicExecutorFeatureStep( (ENV_RESOURCE_PROFILE_ID, resourceProfile.id.toString), (ENV_SGX_ENABLED, execResources.sgxEnabled.toString), (ENV_SGX_MEM_SIZE, execResources.sgxMemGiB + "G"), - (ENV_SGX_JVM_MEM_SIZE, execResources.sgxJvmMemGiB + "G") + (ENV_SGX_JVM_MEM_SIZE, execResources.sgxJvmMemGiB + "G"), + (ENV_SGX_LOG_LEVEL, execResources.sgxLogLevel.toString) ) ++ kubernetesConf.environment).map { case (k, v) => new EnvVarBuilder() .withName(k)