diff --git a/assembly/pom.xml b/assembly/pom.xml index 53059eaa34faf..764eb44d27a7b 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -217,19 +217,6 @@ maven-assembly-plugin 2.4 - dist package @@ -242,7 +229,7 @@ - + diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 60cea3f48f841..3effbca1bbcd5 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -350,13 +350,13 @@ object SparkSubmit { printErrorAndExit("py4j-0.8.2.1-src.zip does not exist for python application " + "in yarn mode.") } - pythonPath += Seq(pyLibPath, "pyspark.zip").mkString(File.separator) - pythonPath += Seq(pyLibPath, "py4j-0.8.2.1-src.zip").mkString(File.separator) + pythonPath += pyArchivesFile.getAbsolutePath() + pythonPath += py4jFile.getAbsolutePath() } pyArchives = pythonPath.mkString(",") } - pyArchives = pyArchives.split(",").map( localPath=> { + pyArchives = pyArchives.split(",").map { localPath=> val localURI = Utils.resolveURI(localPath) if (localURI.getScheme != "local") { args.files = mergeFileLists(args.files, localURI.toString) @@ -364,7 +364,7 @@ object SparkSubmit { } else { localURI.getPath.toString } - }).mkString(File.pathSeparator) + }.mkString(File.pathSeparator) sysProps("spark.submit.pyArchives") = pyArchives }