address vanzin's comments

apache · Apr 29, 2015 · e6b573b · e6b573b
1 parent f11f84a
commit e6b573b
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 18 deletions.
diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -217,19 +217,6 @@
             <artifactId>maven-assembly-plugin</artifactId>
             <version>2.4</version>
             <executions>
-              <!--execution>
-                    <id>pyspark-zip</id>
-                    <phase>package</phase>
-                    <goals>
-                        <goal>single</goal>
-                    </goals>
-                    <configuration>
-                        <skipAssembly>true</skipAssembly>
-                        <descriptors>
-                            <descriptor>src/main/assembly/pyspark-assembly.xml</descriptor>
-                        </descriptors>
-                    </configuration>
-                </execution-->
               <execution>
                 <id>dist</id>
                 <phase>package</phase>
@@ -242,7 +229,7 @@
                   </descriptors>
                 </configuration>
               </execution>
-          </executions>
+            </executions>
           </plugin>
         </plugins>
       </build>

diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -350,21 +350,21 @@ object SparkSubmit {
             printErrorAndExit("py4j-0.8.2.1-src.zip does not exist for python application " +
               "in yarn mode.")
           }
-          pythonPath += Seq(pyLibPath, "pyspark.zip").mkString(File.separator)
-          pythonPath += Seq(pyLibPath, "py4j-0.8.2.1-src.zip").mkString(File.separator)
+          pythonPath += pyArchivesFile.getAbsolutePath()
+          pythonPath += py4jFile.getAbsolutePath()
         }
         pyArchives = pythonPath.mkString(",")
       }
 
-      pyArchives = pyArchives.split(",").map( localPath=> {
+      pyArchives = pyArchives.split(",").map { localPath=>
         val localURI = Utils.resolveURI(localPath)
         if (localURI.getScheme != "local") {
           args.files = mergeFileLists(args.files, localURI.toString)
           (new Path(localPath)).getName
         } else {
           localURI.getPath.toString
         }
-      }).mkString(File.pathSeparator)
+      }.mkString(File.pathSeparator)
       sysProps("spark.submit.pyArchives") = pyArchives
     }