Skip to content

Commit

Permalink
Refector code and expand comment.
Browse files Browse the repository at this point in the history
  • Loading branch information
sarutak committed Aug 13, 2020
1 parent 2693d2f commit 5d65caf
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 10 deletions.
17 changes: 8 additions & 9 deletions core/src/main/scala/org/apache/spark/executor/Executor.scala
Original file line number Diff line number Diff line change
Expand Up @@ -222,17 +222,16 @@ private[spark] class Executor(

private val appStartTime = conf.getLong("spark.app.startTime", 0)

// Jars and files specified by spark.jars and spark.files.
private val initialUserJars =
conf.getOption("spark.app.initial.jar.urls").map { urls =>
// To allow users to distribute plugins and their required files
// specified by --jars and --files on application submission, those jars/files should be
// downloaded and added to the class loader via updateDependencies.
// This should be done before plugin initialization below
// because executors search plugins from the class loader and initialize them.
private val Seq(initialUserJars, initialUserFiles) = Seq("jar", "file").map { key =>
conf.getOption(s"spark.app.initial.$key.urls").map { urls =>
Map(urls.split(",").map(url => (url, appStartTime)): _*)
}.getOrElse(Map.empty)

private val initialUserFiles =
conf.getOption("spark.app.initial.file.urls").map { urls =>
Map(urls.split(",").map(url => (url, appStartTime)): _*)
}.getOrElse(Map.empty)

}
updateDependencies(initialUserFiles, initialUserJars)

// Plugins need to load using a class loader that includes the executor's user classpath.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1221,7 +1221,6 @@ class SparkSubmitSuite
testRemoteResources(enableHttpFs = true, forceDownloadSchemes = Seq("*"))
}


test("SPARK-32119: Jars and files should be loaded when Executors launch for plugins") {
val tempDir = Utils.createTempDir()
val tempFileName = "test.txt"
Expand Down

0 comments on commit 5d65caf

Please sign in to comment.