Skip to content

[SPARK-29472][YARN] Mechanism for Excluding Jars at Launch for YARN #26122

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
modi95 wants to merge 2 commits into from
Closed

[SPARK-29472][YARN] Mechanism for Excluding Jars at Launch for YARN #26122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1d80278
Merge pull request #1 from apache/master
modi95 Oct 15, 2019
9c3ca05
Conf to Exclude Spark Jars on YARN
modi95 Oct 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,11 @@ private[spark] class Client(
try {
jarsStream.setLevel(0)
jarsDir.listFiles().foreach { f =>
if (f.isFile && f.getName.toLowerCase(Locale.ROOT).endsWith(".jar") && f.canRead) {
if (f.isFile
&& f.getName.toLowerCase(Locale.ROOT).endsWith(".jar")
&& f.canRead
&& !f.getName.matches(sparkConf.get("spark.yarn.jars.exclusionRegex", "")))
{
jarsStream.putNextEntry(new ZipEntry(f.getName))
Files.copy(f, jarsStream)
jarsStream.closeEntry()
Expand Down
27 changes: 27 additions & 0 deletions resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package org.apache.spark.deploy.yarn
import java.io.{File, FileInputStream, FileOutputStream}
import java.net.URI
import java.util.Properties
import java.util.zip.ZipFile

import scala.collection.JavaConverters._
import scala.collection.mutable.{HashMap => MutableHashMap}
Expand Down Expand Up @@ -331,6 +332,32 @@ class ClientSuite extends SparkFunSuite with Matchers {
classpath(client) should contain (buildPath(PWD, LOCALIZED_LIB_DIR, "*"))
}

test("exclude local spark jars when specified") {
val temp = Utils.createTempDir()
val jarsDir = new File(temp, "jars")
assert(jarsDir.mkdir())

val jarInclude = new File(jarsDir, "include.jar")
val jarExclude = new File(jarsDir, "exclude.jar")
new FileOutputStream(jarInclude).close()
new FileOutputStream(jarExclude).close()

val sparkConf = new SparkConfWithEnv(Map("SPARK_HOME" -> temp.getAbsolutePath()))
sparkConf.set("spark.yarn.jars.exclusionRegex", ".*exclude.*")

val client = createClient(sparkConf)
client.prepareLocalResources(new Path(temp.getAbsolutePath()), Nil)

val outputZipFile = new ZipFile(
temp
.listFiles()
.filter(fl => fl.getName.matches("__spark_libs__.*zip"))(0)
)

outputZipFile.getEntry(jarInclude.getName) should not be (null)
outputZipFile.getEntry(jarExclude.getName) should be (null)
}

test("ignore same name jars") {
val libs = Utils.createTempDir()
val jarsDir = new File(libs, "jars")
Expand Down