From ab89fbc10e72887b4a5abc0914e1ed414c5c3f1f Mon Sep 17 00:00:00 2001 From: Chan Lee Date: Thu, 25 May 2017 23:22:47 -0700 Subject: [PATCH 1/4] [PIO-72] Fix class loading for pio-shell --- bin/compute-classpath.sh | 21 ++++++++++----------- bin/pio-class | 2 -- bin/pio-shell | 5 ++--- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index 032d111619..78532933db 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -17,20 +17,11 @@ # limitations under the License. # -SCALA_VERSION=2.10 - # Figure out where PredictionIO is installed FWDIR="$(cd `dirname $0`/..; pwd)" . ${FWDIR}/bin/load-pio-env.sh -# Build up classpath -CLASSPATH="${PIO_CONF_DIR}" - -CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*:${FWDIR}/lib/spark/*" - -ASSEMBLY_DIR="${FWDIR}/assembly" - if [ -n "$JAVA_HOME" ]; then JAR_CMD="$JAVA_HOME/bin/jar" else @@ -38,15 +29,23 @@ else fi # Use pio-assembly JAR from either RELEASE or assembly directory +ASSEMBLY_DIR="${FWDIR}/assembly" if [ -f "${FWDIR}/RELEASE" ]; then assembly_folder="${FWDIR}"/lib else assembly_folder="${ASSEMBLY_DIR}" fi -ASSEMBLY_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null) +MAIN_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null) +DATA_JARS=$(ls "${assembly_folder}/spark"/pio-data-*assembly*.jar 2>/dev/null) +# Comma-separated list of assembly jars for submitting to spark-shell +ASSEMBLY_JARS=$(printf "${MAIN_JAR}\n${DATA_JARS}" | paste -sd "," -) -CLASSPATH="$CLASSPATH:${ASSEMBLY_JAR}" +# Build up classpath +CLASSPATH="${PIO_CONF_DIR}" +CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*" +CLASSPATH="$CLASSPATH:${MAIN_JAR}" +CLASSPATH="$CLASSPATH:${assembly_folder}/spark/*" # Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail ! Note, this # assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts diff --git a/bin/pio-class b/bin/pio-class index 8bde407fd8..94cc5204f4 100755 --- a/bin/pio-class +++ b/bin/pio-class @@ -22,8 +22,6 @@ case "`uname`" in CYGWIN*) cygwin=true;; esac -SCALA_VERSION=2.10 - # Figure out where PredictionIO is installed FWDIR="$(cd `dirname $0`/..; pwd)" diff --git a/bin/pio-shell b/bin/pio-shell index d47008ff35..5a5745a362 100755 --- a/bin/pio-shell +++ b/bin/pio-shell @@ -55,11 +55,10 @@ fi if [[ "$1" == "--with-spark" ]] then echo "Starting the PIO shell with the Apache Spark Shell." - # compute the $ASSEMPLY_JAR, the location of the assemply jar, with - # bin/compute-classpath.sh + # Get paths of assembly jars to pass to spark-shell . ${PIO_HOME}/bin/compute-classpath.sh shift - $SPARK_HOME/bin/spark-shell --jars $ASSEMBLY_JAR $@ + ${SPARK_HOME}/bin/spark-shell --jars ${ASSEMBLY_JARS} $@ else echo -e "\033[0;33mStarting the PIO shell without Apache Spark.\033[0m" echo -e "\033[0;33mIf you need the Apache Spark library, run 'pio-shell --with-spark [spark-submit arguments...]'.\033[0m" From b4576130d4bad07c83fa4926cc1b48f43c2247a9 Mon Sep 17 00:00:00 2001 From: Chan Lee Date: Fri, 26 May 2017 03:04:44 -0700 Subject: [PATCH 2/4] Fix error for loading classpath on travis --- bin/compute-classpath.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index 78532933db..e234725d8c 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -37,15 +37,15 @@ else fi MAIN_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null) -DATA_JARS=$(ls "${assembly_folder}/spark"/pio-data-*assembly*.jar 2>/dev/null) +DATA_JARS=$(ls "${FWDIR}"/lib/spark/pio-data-*assembly*.jar 2>/dev/null) # Comma-separated list of assembly jars for submitting to spark-shell ASSEMBLY_JARS=$(printf "${MAIN_JAR}\n${DATA_JARS}" | paste -sd "," -) # Build up classpath CLASSPATH="${PIO_CONF_DIR}" CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*" +CLASSPATH="$CLASSPATH:${FWDIR}/lib/spark/*" CLASSPATH="$CLASSPATH:${MAIN_JAR}" -CLASSPATH="$CLASSPATH:${assembly_folder}/spark/*" # Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail ! Note, this # assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts @@ -70,4 +70,4 @@ if [ -n "$MYSQL_JDBC_DRIVER" ]; then fi -echo "$CLASSPATH" +echo "$CLASSPATH" \ No newline at end of file From b5d334056af3397984985d2f88a53da1944662a1 Mon Sep 17 00:00:00 2001 From: Chan Lee Date: Tue, 27 Jun 2017 12:45:56 -0700 Subject: [PATCH 3/4] Fix bug on class loading for pio --- bin/compute-classpath.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index e234725d8c..cd0527f188 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -29,22 +29,21 @@ else fi # Use pio-assembly JAR from either RELEASE or assembly directory -ASSEMBLY_DIR="${FWDIR}/assembly" if [ -f "${FWDIR}/RELEASE" ]; then assembly_folder="${FWDIR}"/lib else - assembly_folder="${ASSEMBLY_DIR}" + assembly_folder="${FWDIR}"/assembly/src/universal/lib fi MAIN_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null) -DATA_JARS=$(ls "${FWDIR}"/lib/spark/pio-data-*assembly*.jar 2>/dev/null) +DATA_JARS=$(ls "${assembly_folder}"/spark/pio-data-*assembly*.jar 2>/dev/null) # Comma-separated list of assembly jars for submitting to spark-shell ASSEMBLY_JARS=$(printf "${MAIN_JAR}\n${DATA_JARS}" | paste -sd "," -) # Build up classpath CLASSPATH="${PIO_CONF_DIR}" CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*" -CLASSPATH="$CLASSPATH:${FWDIR}/lib/spark/*" +CLASSPATH="$CLASSPATH:${assembly_folder}/spark/*" CLASSPATH="$CLASSPATH:${MAIN_JAR}" # Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail ! Note, this @@ -69,5 +68,4 @@ if [ -n "$MYSQL_JDBC_DRIVER" ]; then CLASSPATH="$CLASSPATH:$MYSQL_JDBC_DRIVER" fi - -echo "$CLASSPATH" \ No newline at end of file +echo "$CLASSPATH" From c71eb37988c825ae4758e34a3816e404c01771c7 Mon Sep 17 00:00:00 2001 From: Chan Lee Date: Tue, 27 Jun 2017 13:50:28 -0700 Subject: [PATCH 4/4] Fix classloading bug for travis --- tests/build_docker.sh | 13 ++++++++----- .../apache/predictionio/tools/Common.scala | 19 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/build_docker.sh b/tests/build_docker.sh index d467e97a29..b17a304e63 100755 --- a/tests/build_docker.sh +++ b/tests/build_docker.sh @@ -38,11 +38,14 @@ fi -Dhadoop.version=$PIO_HADOOP_VERSION \ -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION sbt/sbt clean storage/clean -rm -rf assembly/*.jar -cp dist/lib/*.jar assembly/ -rm -rf lib/spark -mkdir -p lib/spark -cp dist/lib/spark/*.jar lib/spark + +assembly_folder=assembly/src/universal/lib +rm -rf ${assembly_folder}/*.jar +rm -rf ${assembly_folder}/spark +mkdir -p ${assembly_folder}/spark + +cp dist/lib/*.jar ${assembly_folder} +cp dist/lib/spark/*.jar ${assembly_folder}/spark rm *.tar.gz docker build -t predictionio/pio . popd diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala index 0ddd644981..047b8ddcf9 100644 --- a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala +++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala @@ -65,6 +65,15 @@ object Common extends EitherLogging { } } + def getCoreDir(pioHome: String): String = { + if (new File(pioHome + File.separator + "RELEASE").exists) { + pioHome + File.separator + "lib" + } else { + Array(pioHome, "assembly", "src", "universal", "lib") + .mkString(File.separator) + } + } + def getEngineDirPath(directory: Option[String]): String = { new File(directory.getOrElse(".")).getCanonicalPath } @@ -97,18 +106,12 @@ object Common extends EitherLogging { def jarFilesAt(path: File): Array[File] = path.listFiles filter { _.getName.toLowerCase.endsWith(".jar") } - jarFilesAt(new File(pioHome, "lib/spark")) + jarFilesAt(new File(getCoreDir(pioHome) + File.separator + "spark")) } def coreAssembly(pioHome: String): Expected[File] = { val core = s"pio-assembly-${BuildInfo.version}.jar" - val coreDir = - if (new File(pioHome + File.separator + "RELEASE").exists) { - new File(pioHome + File.separator + "lib") - } else { - new File(pioHome + File.separator + "assembly") - } - val coreFile = new File(coreDir, core) + val coreFile = new File(getCoreDir(pioHome), core) if (coreFile.exists) { Right(coreFile) } else {