From 4bd448971a2a9feab634eab9cde2bb18dd2c69d1 Mon Sep 17 00:00:00 2001 From: nishkamravi2 Date: Thu, 19 Mar 2015 08:38:30 -0700 Subject: [PATCH 1/3] Update AbstractCommandBuilder.java --- .../java/org/apache/spark/launcher/AbstractCommandBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index 59d826cba1e2d..3d0edb3202cc2 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -341,7 +341,7 @@ private String findAssembly(String scalaVersion) { libdir = new File(sparkHome, String.format("assembly/target/scala-%s", scalaVersion)); } - final Pattern re = Pattern.compile("spark-assembly.*[hadoop]*.*\\.jar"); + final Pattern re = Pattern.compile("spark-assembly.*(hadoop)?.*\\.jar"); FileFilter filter = new FileFilter() { @Override public boolean accept(File file) { From 32c3ab3af6c3689225c73b12c945f3f589022242 Mon Sep 17 00:00:00 2001 From: nishkamravi2 Date: Thu, 19 Mar 2015 10:04:43 -0700 Subject: [PATCH 2/3] Update AbstractCommandBuilder.java --- .../java/org/apache/spark/launcher/AbstractCommandBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index 3d0edb3202cc2..dc8ab3cdfd0f0 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -341,7 +341,7 @@ private String findAssembly(String scalaVersion) { libdir = new File(sparkHome, String.format("assembly/target/scala-%s", scalaVersion)); } - final Pattern re = Pattern.compile("spark-assembly.*(hadoop)?.*\\.jar"); + final Pattern re = Pattern.compile("spark-assembly.*\\.jar"); FileFilter filter = new FileFilter() { @Override public boolean accept(File file) { From 06bfeb02173caf8babd75c3bf7f88b28ac8d7a0b Mon Sep 17 00:00:00 2001 From: nishkamravi2 Date: Thu, 19 Mar 2015 23:04:46 -0700 Subject: [PATCH 3/3] Update spark-class --- bin/spark-class | 48 ++++++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/bin/spark-class b/bin/spark-class index e29b234afaf96..7119883c57baa 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -40,36 +40,24 @@ else fi fi -# Look for the launcher. In non-release mode, add the compiled classes directly to the classpath -# instead of looking for a jar file. -SPARK_LAUNCHER_CP= -if [ -f $SPARK_HOME/RELEASE ]; then - LAUNCHER_DIR="$SPARK_HOME/lib" - num_jars="$(ls -1 "$LAUNCHER_DIR" | grep "^spark-launcher.*\.jar$" | wc -l)" - if [ "$num_jars" -eq "0" -a -z "$SPARK_LAUNCHER_CP" ]; then - echo "Failed to find Spark launcher in $LAUNCHER_DIR." 1>&2 - echo "You need to build Spark before running this program." 1>&2 - exit 1 - fi - - LAUNCHER_JARS="$(ls -1 "$LAUNCHER_DIR" | grep "^spark-launcher.*\.jar$" || true)" - if [ "$num_jars" -gt "1" ]; then - echo "Found multiple Spark launcher jars in $LAUNCHER_DIR:" 1>&2 - echo "$LAUNCHER_JARS" 1>&2 - echo "Please remove all but one jar." 1>&2 - exit 1 - fi - - SPARK_LAUNCHER_CP="${LAUNCHER_DIR}/${LAUNCHER_JARS}" -else - LAUNCHER_DIR="$SPARK_HOME/launcher/target/scala-$SPARK_SCALA_VERSION" - if [ ! -d "$LAUNCHER_DIR/classes" ]; then - echo "Failed to find Spark launcher classes in $LAUNCHER_DIR." 1>&2 - echo "You need to build Spark before running this program." 1>&2 - exit 1 - fi - SPARK_LAUNCHER_CP="$LAUNCHER_DIR/classes" +# Find assembly jar +SPARK_ASSEMBLY_JAR= +ASSEMBLY_DIR="$SPARK_HOME/lib" +num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^spark-assembly.*\.jar$" | wc -l)" +if [ "$num_jars" -eq "0" -a -z "$SPARK_ASSEMBLY_JAR" ]; then + echo "Failed to find Spark assembly in $ASSEMBLY_DIR." 1>&2 + echo "You need to build Spark before running this program." 1>&2 + exit 1 fi +ASSEMBLY_JARS="$(ls -1 "$ASSEMBLY_DIR" | grep "^spark-assembly.*\.jar$" || true)" +if [ "$num_jars" -gt "1" ]; then + echo "Found multiple Spark assembly jars in $ASSEMBLY_DIR:" 1>&2 + echo "$ASSEMBLY_JARS" 1>&2 + echo "Please remove all but one jar." 1>&2 + exit 1 +fi + +SPARK_ASSEMBLY_JAR="${ASSEMBLY_DIR}/${ASSEMBLY_JARS}" # The launcher library will print arguments separated by a NULL character, to allow arguments with # characters that would be otherwise interpreted by the shell. Read that in a while loop, populating @@ -77,7 +65,7 @@ fi CMD=() while IFS= read -d '' -r ARG; do CMD+=("$ARG") -done < <("$RUNNER" -cp "$SPARK_LAUNCHER_CP" org.apache.spark.launcher.Main "$@") +done < <("$RUNNER" -cp "$SPARK_ASSEMBLY_JAR" org.apache.spark.launcher.Main "$@") if [ "${CMD[0]}" = "usage" ]; then "${CMD[@]}"