[ADAM-543][ADAM-544] Fix issues with ADAM scripts and classpath

bin/adam-pyspark

@@ -23,10 +23,15 @@ ADAM_REPO="$(cd `dirname $0`/..; pwd)"
 CLASSPATH=$("$ADAM_REPO"/bin/compute-adam-classpath.sh)
 ADAM_JARS=$("$ADAM_REPO"/bin/compute-adam-jars.sh)
 
-if [ ! -z "$ADDL_JARS"]; then
+if [ ! -z "$ADDL_JARS" ]; then
   ADAM_JARS="$ADAM_JARS","$ADDL_JARS"
 fi
 
+# append ADAM_JARS to the --jars option, if any
+NEW_OPTIONS=$("$ADAM_REPO"/bin/append_to_option.py , --jars "$ADAM_JARS" "$@")
+# append CLASSPATH to the --driver-class-path
+NEW_OPTIONS=$("$ADAM_REPO"/bin/append_to_option.py ":" --driver-class-path "$CLASSPATH" "$NEW_OPTIONS")
+
 if [ -z "$SPARK_HOME" ]; then
   echo "Attempting to use 'pyspark' on default path; you might need to set SPARK_HOME"
   echo ""
@@ -41,5 +46,4 @@ fi
   --conf spark.kryo.registrator=org.bdgenomics.adam.serialization.ADAMKryoRegistrator \
   --conf spark.kryoserializer.buffer.mb=4 \
   --conf spark.kryo.referenceTracking=true \
-  --jars "$ADAM_JARS" \
-  "$@"
+  $NEW_OPTIONS

bin/adam-shell

@@ -20,13 +20,15 @@
 # Figure out where ADAM is installed
 ADAM_REPO="$(cd `dirname $0`/..; pwd)"
 
-CLASSPATH=$("$ADAM_REPO"/bin/compute-adam-classpath.sh)
 ADAM_JARS=$("$ADAM_REPO"/bin/compute-adam-jars.sh)
 
 if [ ! -z "$ADDL_JARS" ]; then
   ADAM_JARS="$ADAM_JARS","$ADDL_JARS"
 fi
 
+# append ADAM_JARS to the --jars option, if any
+NEW_OPTIONS=$("$ADAM_REPO"/bin/append_to_option.py , --jars "$ADAM_JARS" "$@")
+
 if [ -z "$SPARK_HOME" ]; then
   echo "Attempting to use 'spark-shell' on default path; you might need to set SPARK_HOME"
   echo ""
@@ -44,5 +46,4 @@ fi
   ${ADAM_OPTS:- } \
   --conf spark.executor.memory=${ADAM_EXECUTOR_MEMORY:-"4g"} \
   --driver-memory ${ADAM_DRIVER_MEMORY:-"4g"} \
-  --jars "$ADAM_JARS" \
-  "$@"
+  $NEW_OPTIONS

bin/adam-submit

@@ -27,6 +27,9 @@ if [ ! -z "$ADDL_JARS" ]; then
   ADAM_JARS="$ADAM_JARS","$ADDL_JARS"
 fi
 
+# append ADAM_JARS to the --jars option, if any
+NEW_OPTIONS=$("$SCRIPT_DIR"/bin/append_to_option.py , --jars "$ADAM_JARS" "$@")
+
 # Binary distribution
 REPO_DIR="$SCRIPT_DIR/repo"
 if [ ! -d "$REPO_DIR" ]; then
@@ -67,7 +70,7 @@ function usage() {
 }
 source "$SPARK_HOME"/bin/utils.sh
 SUBMIT_USAGE_FUNCTION=usage
-gatherSparkSubmitOpts "$@"
+gatherSparkSubmitOpts $NEW_OPTIONS
 
 # submit the job to Spark
 "$SPARK_SUBMIT" \
@@ -79,7 +82,6 @@ gatherSparkSubmitOpts "$@"
   ${ADAM_OPTS:- } \
   --conf spark.executor.memory=${ADAM_EXECUTOR_MEMORY:-"4g"} \
   --driver-memory ${ADAM_DRIVER_MEMORY:-"4g"} \
-  --jars "$ADAM_JARS" \
   "${SUBMISSION_OPTS[@]}" \
   "$ADAM_CLI_JAR" \
   "${APPLICATION_OPTS[@]}"

bin/append_to_option.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+#
+# Licensed to Big Data Genomics (BDG) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The BDG licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Append text to an option.
+
+Usage:
+
+    append_to_option.py DELIMITER OPTION APPEND_STRING OPTION_STRING
+
+For example, running
+
+    append_to_option.py , --jars myproject.jar --option1 value1 --jars otherproject.jar --option2 value2
+
+will write to stdout
+
+    --option1 value1 --jars otherproject.jar,myproject.jar --option2 value2
+"""
+
+import sys
+
+delimiter = sys.argv[1]
+target = sys.argv[2]
+append = sys.argv[3]
+original = sys.argv[4:]
+
+if original.count(target) > 1:
+    sys.stderr.write("Found multiple %s in the option list." % target)
+    sys.exit(1)
+
+if original.count(target) == 0:
+    original.extend([target, append])
+else:  # original.count(target) == 1
+    idx = original.index(target)
+    new_value = delimiter.join([original[idx + 1], append])
+    original[idx + 1] = new_value
+sys.stdout.write(' '.join(original))