From 2163647684d0ee79741b167f0f7c2cb378d0d561 Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Fri, 25 Mar 2016 16:33:26 -0700 Subject: [PATCH 1/2] [SPARK-13955] [yarn] Also look for Spark jars in the build directory. Move the logic to find Spark jars to CommandBuilderUtils and make it available for YARN code, so that it's possible to easily launche Spark on YARN from a build directory. Tested by running SparkPi from the build directory on YARN. --- .../launcher/AbstractCommandBuilder.java | 23 +---------------- .../spark/launcher/CommandBuilderUtils.java | 25 +++++++++++++++++++ .../org/apache/spark/deploy/yarn/Client.scala | 11 ++++---- .../launcher/YarnCommandBuilderUtils.scala | 9 +++++++ 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index 587fda7a3c1da..d02b2a499455e 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -174,7 +174,7 @@ List buildClassPath(String appClassPath) throws IOException { // Add Spark jars to the classpath. For the testing case, we rely on the test code to set and // propagate the test classpath appropriately. For normal invocation, look for the jars // directory under SPARK_HOME. - String jarsDir = findJarsDir(!isTesting); + String jarsDir = findJarsDir(getSparkHome(), getScalaVersion(), !isTesting); if (jarsDir != null) { addToClassPath(cp, join(File.separator, jarsDir, "*")); } @@ -311,27 +311,6 @@ private Properties loadPropertiesFile() throws IOException { return props; } - private String findJarsDir(boolean failIfNotFound) { - // TODO: change to the correct directory once the assembly build is changed. - String sparkHome = getSparkHome(); - File libdir; - if (new File(sparkHome, "RELEASE").isFile()) { - libdir = new File(sparkHome, "lib"); - checkState(!failIfNotFound || libdir.isDirectory(), - "Library directory '%s' does not exist.", - libdir.getAbsolutePath()); - } else { - libdir = new File(sparkHome, String.format("assembly/target/scala-%s", getScalaVersion())); - if (!libdir.isDirectory()) { - checkState(!failIfNotFound, - "Library directory '%s' does not exist; make sure Spark is built.", - libdir.getAbsolutePath()); - libdir = null; - } - } - return libdir != null ? libdir.getAbsolutePath() : null; - } - private String getConfDir() { String confDir = getenv("SPARK_CONF_DIR"); return confDir != null ? confDir : join(File.separator, getSparkHome(), "conf"); diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java index 39fdf300e26cd..1e55aad5c918b 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java +++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java @@ -349,4 +349,29 @@ static int javaMajorVersion(String javaVersion) { return Integer.parseInt(version[1]); } } + + /** + * Find the location of the Spark jars dir, depending on whether we're looking at a build + * or a distribution directory. + */ + static String findJarsDir(String sparkHome, String scalaVersion, boolean failIfNotFound) { + // TODO: change to the correct directory once the assembly build is changed. + File libdir; + if (new File(sparkHome, "RELEASE").isFile()) { + libdir = new File(sparkHome, "lib"); + checkState(!failIfNotFound || libdir.isDirectory(), + "Library directory '%s' does not exist.", + libdir.getAbsolutePath()); + } else { + libdir = new File(sparkHome, String.format("assembly/target/scala-%s", scalaVersion)); + if (!libdir.isDirectory()) { + checkState(!failIfNotFound, + "Library directory '%s' does not exist; make sure Spark is built.", + libdir.getAbsolutePath()); + libdir = null; + } + } + return libdir != null ? libdir.getAbsolutePath() : null; + } + } diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 6bbc8c2dfa19a..7b29c1ae4d888 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -468,12 +468,11 @@ private[spark] class Client( // No configuration, so fall back to uploading local jar files. logWarning(s"Neither ${SPARK_JARS.key} nor ${SPARK_ARCHIVE.key} is set, falling back " + "to uploading libraries under SPARK_HOME.") - val jarsDir = new File(sparkConf.getenv("SPARK_HOME"), "lib") - if (jarsDir.isDirectory()) { - jarsDir.listFiles().foreach { f => - if (f.isFile() && f.getName().toLowerCase().endsWith(".jar")) { - distribute(f.getAbsolutePath(), targetDir = Some(LOCALIZED_LIB_DIR)) - } + val jarsDir = new File(YarnCommandBuilderUtils.findJarsDir( + sparkConf.getenv("SPARK_HOME"))) + jarsDir.listFiles().foreach { f => + if (f.isFile() && f.getName().toLowerCase().endsWith(".jar")) { + distribute(f.getAbsolutePath(), targetDir = Some(LOCALIZED_LIB_DIR)) } } } diff --git a/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala b/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala index 7d246bf407121..6c3556a2ee43e 100644 --- a/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala +++ b/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala @@ -19,6 +19,7 @@ package org.apache.spark.launcher import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer +import scala.util.Properties /** * Exposes methods from the launcher library that are used by the YARN backend. @@ -29,6 +30,14 @@ private[spark] object YarnCommandBuilderUtils { CommandBuilderUtils.quoteForBatchScript(arg) } + def findJarsDir(sparkHome: String): String = { + val scalaVer = Properties.versionNumberString + .split("\\.") + .take(2) + .mkString(".") + CommandBuilderUtils.findJarsDir(sparkHome, scalaVer, true) + } + /** * Adds the perm gen configuration to the list of java options if needed and not yet added. * From cdb233d087ad08f74ea1e908406a79f2ba5a8ba2 Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Wed, 30 Mar 2016 11:31:07 -0700 Subject: [PATCH 2/2] Fix test. --- .../test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala index 24472e006b875..e3613a93ed05f 100644 --- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala +++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.deploy.yarn -import java.io.File +import java.io.{File, FileOutputStream} import java.net.URI import java.util.Properties @@ -274,6 +274,7 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll val jarsDir = new File(temp, "lib") assert(jarsDir.mkdir()) val jar = TestUtils.createJarWithFiles(Map(), jarsDir) + new FileOutputStream(new File(temp, "RELEASE")).close() val sparkConf = new SparkConfWithEnv(Map("SPARK_HOME" -> temp.getAbsolutePath())) val client = createClient(sparkConf)