From 94e9f84668b40edf539b0f5c0ffa271b0090c2a7 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sat, 3 May 2014 21:52:44 -0700 Subject: [PATCH 1/2] SPARK-1703 Warn users if Spark is run on JRE6 but compiled with JDK7. This add some guards and good warning messages if users hit this issue. --- bin/compute-classpath.sh | 14 ++++++++++++++ bin/spark-class | 9 ++++++++- make-distribution.sh | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index b0218531e9eb8..171096fbec178 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -32,6 +32,12 @@ CLASSPATH="$SPARK_CLASSPATH:$SPARK_SUBMIT_CLASSPATH:$FWDIR/conf" ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SCALA_VERSION" +if [ -n "${JAVA_HOME}" ]; then + JAR_CMD="${JAVA_HOME}/bin/jar" +else + JAR_CMD="jar" +fi + # First check if we have a dependencies jar. If so, include binary classes with the deps jar if [ -f "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar ]; then CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SCALA_VERSION/classes" @@ -54,6 +60,14 @@ else else ASSEMBLY_JAR=`ls "$ASSEMBLY_DIR"/spark-assembly*hadoop*.jar` fi + jar_error_check=$($JAR_CMD -tf $ASSEMBLY_JAR org/apache/spark/SparkContext 2>&1) + if [[ "$jar_error_check" =~ "invalid CEN header" ]]; then + echo "Loading Spark jar with '$JAR_CMD' failed. " + echo "This is likely because Spark was compiled with Java 7 and run " + echo "with Java 6. (see SPARK-1703). Please use Java 7 to run Spark " + echo "or build Spark with Java 6." + exit 1 + fi CLASSPATH="$CLASSPATH:$ASSEMBLY_JAR" fi diff --git a/bin/spark-class b/bin/spark-class index e8160c8af64c1..8b97bc1fe88cb 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -138,7 +138,14 @@ if [ -e "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar ]; then fi # Compute classpath using external script -CLASSPATH=`$FWDIR/bin/compute-classpath.sh` +CLASSPATH_OUTPUT=$($FWDIR/bin/compute-classpath.sh) +if [[ "$?" != "0" ]]; then + echo "$CLASSPATH_OUTPUT" + exit 1 +else + CLASSPATH=$CLASSPATH_OUTPUT +fi + if [[ "$1" =~ org.apache.spark.tools.* ]]; then CLASSPATH="$CLASSPATH:$SPARK_TOOLS_JAR" fi diff --git a/make-distribution.sh b/make-distribution.sh index c05dcd89d90a7..5cec06ef16582 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -50,6 +50,20 @@ if [ $? == -1 ] ;then exit -1; fi +if [ -z "${JAVA_HOME}" ]; then + echo "Error: JAVA_HOME is not set, cannot proceed." + exit -1 +fi + +JAVA_CMD=$JAVA_HOME/bin/java +JAVA_VERSION=$($JAVA_CMD -version 2>&1) +if ! [[ "$JAVA_VERSION" =~ "1.6" ]]; then + echo "Error: Java version was not 1.6. Spark must be compiled with JDK 1.6 " + echo "(see SPARK-1703). Output from 'java -version' was:" + echo "$JAVA_VERSION" + exit -1 +fi + # Initialize defaults SPARK_HADOOP_VERSION=1.0.4 SPARK_YARN=false From a38a9581bc820759c15526f9bccc3a96fa6579e2 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 4 May 2014 11:33:18 -0700 Subject: [PATCH 2/2] Code review feedback --- bin/spark-class | 6 +++--- make-distribution.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/spark-class b/bin/spark-class index 8b97bc1fe88cb..72f8b9bf9a495 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -138,12 +138,12 @@ if [ -e "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar ]; then fi # Compute classpath using external script -CLASSPATH_OUTPUT=$($FWDIR/bin/compute-classpath.sh) +classpath_output=$($FWDIR/bin/compute-classpath.sh) if [[ "$?" != "0" ]]; then - echo "$CLASSPATH_OUTPUT" + echo "$classpath_output" exit 1 else - CLASSPATH=$CLASSPATH_OUTPUT + CLASSPATH=$classpath_output fi if [[ "$1" =~ org.apache.spark.tools.* ]]; then diff --git a/make-distribution.sh b/make-distribution.sh index 5cec06ef16582..b21cc4ae607be 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -58,8 +58,8 @@ fi JAVA_CMD=$JAVA_HOME/bin/java JAVA_VERSION=$($JAVA_CMD -version 2>&1) if ! [[ "$JAVA_VERSION" =~ "1.6" ]]; then - echo "Error: Java version was not 1.6. Spark must be compiled with JDK 1.6 " - echo "(see SPARK-1703). Output from 'java -version' was:" + echo "Error: JAVA_HOME must point to a JDK 6 installation (see SPARK-1703)." + echo "Output from 'java -version' was:" echo "$JAVA_VERSION" exit -1 fi