Merge pull request #303 from navinrathore/ZinggShellPythonPhases

env variables can be defined in zingg.conf in addition to spark properties

config/zingg.conf

@@ -9,6 +9,11 @@
 # Leading or trailing spaces could be fine.
 # Please note that any key or value already comprising spaces or double quotes must be enclosed with single quotes ('')
 
+### Environment variables
+# env var names should not include dot('.'), underscore('_') is fine
+EMAIL=xxx@yyy.com
+LICENSE=test
+SPARK_MASTER=local[*]
 
 ### General properties
 spark.serializer=org.apache.spark.serializer.KryoSerializer

scripts/zingg.sh

@@ -14,7 +14,7 @@ else
 fi
 
 function read_zingg_conf() {
-	local CONF_PROPS=""
+	CONF_PROPS=""
 
 	ZINGG_CONF_DIR="$(cd "`dirname "$0"`"/../config; pwd)"
 
@@ -26,11 +26,14 @@ function read_zingg_conf() {
 		# Trim leading and trailing spaces
 		key=$(echo $key | sed 's/^[[:blank:]]*//;s/[[:blank:]]*$//;')
 		value=$(echo $value | sed 's/^[[:blank:]]*//;s/[[:blank:]]*$//;')
-		# Append to conf variable
-		CONF_PROPS+=" --conf ${key}=${value}"
+		if [[ $key == spark* ]]; then
+			# Append to conf variable
+			CONF_PROPS+=" --conf ${key}=${value}"
+		else
+			# Add to env var list
+			ENVIRONMENT_VARS[${key}]=${value}
+		fi;
 	done <<< "$(echo -e "$PROPERTIES")"
-
-	echo $CONF_PROPS
 }
 
 function is_python_phase() {
@@ -74,7 +77,19 @@ while [[ $# -gt 0 ]]; do
 done
 set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
 
-OPTION_SPARK_CONF+=$(read_zingg_conf)
+declare -A ENVIRONMENT_VARS
+read_zingg_conf
+
+# echo "${!ENVIRONMENT_VARS[@]}"
+# echo "${ENVIRONMENT_VARS[@]}"
+# echo ${CONF_PROPS}
+
+for x in ${!ENVIRONMENT_VARS[@]}; do
+	eval ${x}=${ENVIRONMENT_VARS[${x}]}
+done
+
+OPTION_SPARK_CONF=${CONF_PROPS}
+
 
 # if it is a python phase
 if [[ $RUN_PYTHON_PHASE -eq 0 ]]; then