MapR [SPARK-1040] Spark thrift server is started by passing daemon me…

…mory from config to overwrite the SPARK_DAEMON_MEMORY (apache#993) Co-authored-by: Tetiana Fioshkina <tetiana.fioshkina@hpe.com>
mapr · Nov 6, 2023 · 3011423 · 3011423
1 parent a78e295
commit 3011423
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java
@@ -73,6 +73,9 @@ public class SparkLauncher extends AbstractLauncher<SparkLauncher> {
   /** Configuration key for the number of executor CPU cores. */
   public static final String EXECUTOR_CORES = "spark.executor.cores";
 
+  /** Configuration key for the daemon memory used by thrift server. */
+  public static final String THRIFT_DAEMON_MEMORY = "spark.thirft.daemon.memory";
+
   static final String PYSPARK_DRIVER_PYTHON = "spark.pyspark.driver.python";
 
   static final String PYSPARK_PYTHON = "spark.pyspark.python";

diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
@@ -283,8 +283,8 @@ private List<String> buildSparkSubmitCommand(Map<String, String> env)
       // - SPARK_MEM env variable
       // - default value (1g)
       // Take Thrift Server as daemon
-      String tsMemory =
-        isThriftServer(mainClass) ? System.getenv("SPARK_DAEMON_MEMORY") : null;
+      String tsMemory = isThriftServer(mainClass) ?
+        firstNonEmpty(config.get(SparkLauncher.THRIFT_DAEMON_MEMORY), System.getenv("SPARK_DAEMON_MEMORY")) : null;
       String memory = firstNonEmpty(tsMemory, config.get(SparkLauncher.DRIVER_MEMORY),
         System.getenv("SPARK_DRIVER_MEMORY"), System.getenv("SPARK_MEM"), DEFAULT_MEM);
       cmd.add("-Xmx" + memory);