[SPARK-4616][Core] - SPARK_CONF_DIR is not effective in spark-submit

bin/load-spark-env.sh

@@ -27,12 +27,17 @@ if [ -z "$SPARK_ENV_LOADED" ]; then
   # Returns the parent of the directory this script lives in.
   parent_dir="$(cd "`dirname "$0"`"/..; pwd)"
 
-  user_conf_dir="${SPARK_CONF_DIR:-"$parent_dir"/conf}"
-
-  if [ -f "${user_conf_dir}/spark-env.sh" ]; then
+  # SPARK-4616
+  # Check if an environment file was passed in through `spark-submit` else provide a healthy default
+  # - note: we cannot use SPARK_CONF_DIR here as it could be set within the `spark-env.sh` file
+  #         so rather check specifically for the `spark-env.sh` file in a default location if one
+  #         wasn't provided
+  environment_file="${SPARK_SUBMIT_ENVIRONMENT_FILE:-"$parent_dir/conf/spark-env.sh"}"
+
+  if [ -f "${environment_file}" ]; then
     # Promote all variable declarations to environment (exported) variables
     set -a
-    . "${user_conf_dir}/spark-env.sh"
+    . "${environment_file}"
     set +a
   fi
 fi
@@ -43,7 +48,7 @@ if [ -z "$SPARK_SCALA_VERSION" ]; then
 
     ASSEMBLY_DIR2="$FWDIR/assembly/target/scala-2.11"
     ASSEMBLY_DIR1="$FWDIR/assembly/target/scala-2.10"
-    
+
     if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
         echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2
         echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2
@@ -54,5 +59,5 @@ if [ -z "$SPARK_SCALA_VERSION" ]; then
         export SPARK_SCALA_VERSION="2.11"
     else
         export SPARK_SCALA_VERSION="2.10"
-    fi        
+    fi
 fi

bin/spark-class

@@ -30,7 +30,7 @@ FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
 # Export this as SPARK_HOME
 export SPARK_HOME="$FWDIR"
 
-. "$FWDIR"/bin/load-spark-env.sh
+. "$SPARK_HOME"/bin/load-spark-env.sh
 
 if [ -z "$1" ]; then
   echo "Usage: spark-class <class> [<args>]" 1>&2

bin/spark-submit

@@ -30,6 +30,8 @@ while (($#)); do
     SPARK_SUBMIT_DEPLOY_MODE=$2
   elif [ "$1" = "--properties-file" ]; then
     SPARK_SUBMIT_PROPERTIES_FILE=$2
+  elif [ "$1" = "--environment-file" ]; then
+    SPARK_SUBMIT_ENVIRONMENT_FILE=$2
   elif [ "$1" = "--driver-memory" ]; then
     export SPARK_SUBMIT_DRIVER_MEMORY=$2
   elif [ "$1" = "--driver-library-path" ]; then
@@ -42,10 +44,20 @@ while (($#)); do
   shift
 done
 
-DEFAULT_PROPERTIES_FILE="$SPARK_HOME/conf/spark-defaults.conf"
-export SPARK_SUBMIT_DEPLOY_MODE=${SPARK_SUBMIT_DEPLOY_MODE:-"client"}
+# SPARK-4616 - Add environment file to source in possible configuration options
+DEFAULT_ENVIRONMENT_FILE="$SPARK_HOME/conf/spark-env.sh"
+export SPARK_SUBMIT_ENVIRONMENT_FILE=${SPARK_SUBMIT_ENVIRONMENT_FILE:-"$DEFAULT_ENVIRONMENT_FILE"}
+
+# Source in the environment specifically to ensure whether SPARK_CONF_DIR has been set or not
+. "$SPARK_HOME"/bin/load-spark-env.sh
+
+# If the SPARK_CONF_DIR wasn't set from spark-env.sh then provide healthy defaults
+SPARK_CONF_DIR=${SPARK_CONF_DIR:-"$SPARK_HOME/conf"}
+DEFAULT_PROPERTIES_FILE="$SPARK_CONF_DIR/spark-defaults.conf"
 export SPARK_SUBMIT_PROPERTIES_FILE=${SPARK_SUBMIT_PROPERTIES_FILE:-"$DEFAULT_PROPERTIES_FILE"}
 
+export SPARK_SUBMIT_DEPLOY_MODE=${SPARK_SUBMIT_DEPLOY_MODE:-"client"}
+
 # For client mode, the driver will be launched in the same JVM that launches
 # SparkSubmit, so we may need to read the properties file for any extra class
 # paths, library paths, java options and memory early on. Otherwise, it will

bin/utils.sh

@@ -35,7 +35,8 @@ function gatherSparkSubmitOpts() {
       --master | --deploy-mode | --class | --name | --jars | --py-files | --files | \
       --conf | --properties-file | --driver-memory | --driver-java-options | \
       --driver-library-path | --driver-class-path | --executor-memory | --driver-cores | \
-      --total-executor-cores | --executor-cores | --queue | --num-executors | --archives)
+      --total-executor-cores | --executor-cores | --queue | --num-executors | \
+      --archives | --environment-file )
         if [[ $# -lt 2 ]]; then
           "$SUBMIT_USAGE_FUNCTION"
           exit 1;

core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala

@@ -34,6 +34,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String], env: Map[String, St
   var executorCores: String = null
   var totalExecutorCores: String = null
   var propertiesFile: String = null
+  var environmentFile: String = null
   var driverMemory: String = null
   var driverExtraClassPath: String = null
   var driverExtraLibraryPath: String = null
@@ -188,6 +189,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String], env: Map[String, St
     |  executorCores           $executorCores
     |  totalExecutorCores      $totalExecutorCores
     |  propertiesFile          $propertiesFile
+    |  environmentFile         $environmentFile
     |  driverMemory            $driverMemory
     |  driverCores             $driverCores
     |  driverExtraClassPath    $driverExtraClassPath
@@ -283,6 +285,10 @@ private[spark] class SparkSubmitArguments(args: Seq[String], env: Map[String, St
         propertiesFile = value
         parse(tail)
 
+      case ("--environment-file") :: value :: tail =>
+        environmentFile = value
+        parse(tail)
+
       case ("--supervise") :: tail =>
         supervise = true
         parse(tail)
@@ -365,6 +371,9 @@ private[spark] class SparkSubmitArguments(args: Seq[String], env: Map[String, St
         |  --conf PROP=VALUE           Arbitrary Spark configuration property.
         |  --properties-file FILE      Path to a file from which to load extra properties. If not
         |                              specified, this will look for conf/spark-defaults.conf.
+        |  --environment-file FILE     Path to a file from which to load various Spark environment
+        |                              options (e.g. SPARK_CONF_DIR, etc.). If not specified, this
+        |                              will look for conf/spark-env.sh.
         |
         |  --driver-memory MEM         Memory for driver (e.g. 1000M, 2G) (Default: 512M).
         |  --driver-java-options       Extra Java options to pass to the driver.