Fix INSTANCE errors by stabilizing the sort order of "lib/spark/*" …

…JARs in classpath
apache · May 23, 2017 · 052f5d8 · 052f5d8
1 parent d86ef0d
commit 052f5d8
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 5 deletions.
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
@@ -37,14 +37,20 @@ else
   JAR_CMD="jar"
 fi
 
+lib_spark_classpath=''
 # Use pio-assembly JAR from either RELEASE or assembly directory
 if [ -f "${FWDIR}/RELEASE" ]; then
   assembly_folder="${FWDIR}"/lib
-  CLASSPATH="$CLASSPATH:${FWDIR}/lib/spark/*"
+  lib_spark_jars=`ls "${FWDIR}"/lib/spark/*.jar`
 else
   assembly_folder="${ASSEMBLY_DIR}"
-  CLASSPATH="$CLASSPATH:${FWDIR}/assembly/src/universal/lib/spark/*"
+  lib_spark_jars=`ls "${FWDIR}"/assembly/src/universal/lib/spark/*.jar`
 fi
+# stabile classpath for Spark JARs
+for J in $lib_spark_jars; do
+  lib_spark_classpath="${lib_spark_classpath}:${J}"
+done
+CLASSPATH="${CLASSPATH}${lib_spark_classpath}"
 
 ASSEMBLY_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null)
 

diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
@@ -94,9 +94,9 @@ object Common extends EitherLogging {
   }
 
   def jarFilesForSpark(pioHome: String): Array[File] = {
-    def jarFilesAt(path: File): Array[File] = path.listFiles filter {
-      _.getName.toLowerCase.endsWith(".jar")
-    }
+    def jarFilesAt(path: File): Array[File] = path.listFiles
+      .filter { _.getName.toLowerCase.endsWith(".jar") }
+      .sortBy( f => f.getName ) // stable sorting to avoid classpath variability
     jarFilesAt(new File(pioHome, "lib/spark"))
   }