From 5eb089203fb859cebb9564d881e001472d31ffec Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 27 Oct 2015 14:19:17 -0700 Subject: [PATCH 1/9] Add addJar to PySpark API but it isn't very useful at present --- python/pyspark/context.py | 8 ++++++++ python/pyspark/tests.py | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/python/pyspark/context.py b/python/pyspark/context.py index afd74d937a41..4b441d82e45a 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -806,6 +806,14 @@ def addPyFile(self, path): import importlib importlib.invalidate_caches() + def addJar(self, path): + """ + Adds a JAR dependency for all tasks to be executed on this SparkContext in the future. + The `path` passed can be either a local file, a file in HDFS (or other Hadoop-supported + filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. + """ + self._jsc.sc().addJar(path) + def setCheckpointDir(self, dirName): """ Set the directory under which RDDs are going to be checkpointed. The diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 5bd94476597a..93791e5dd622 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -410,6 +410,10 @@ def test_add_file_locally(self): with open(download_path) as test_file: self.assertEqual("Hello World!\n", test_file.readline()) + def test_add_jar(self): + path = os.path.join(SPARK_HOME, "./R/pkg/inst/test_support/sparktestjar_2.10-1.0.jar") + self.assertTrue(self._jsc.sc().addedJars().toString().find("sparktestjar") != -1) + def test_add_py_file_locally(self): # To ensure that we're actually testing addPyFile's effects, check that # this fails due to `userlibrary` not being on the Python path: From 3b8b1b086d486c86c0028c19f1875f9e14ea7a53 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 27 Oct 2015 15:47:34 -0700 Subject: [PATCH 2/9] Add a helper to update the running class loader when adding classes from python + add a util function to simplify. Follow up should make Kafakutils & related use the util function --- .../apache/spark/api/python/PythonUtils.scala | 30 +++++++++++++++++-- python/pyspark/context.py | 10 +++++++ python/pyspark/tests.py | 7 +++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala index 292ac4cfc35b..cfc7fb0d6178 100644 --- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala +++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala @@ -18,15 +18,17 @@ package org.apache.spark.api.python import java.io.File +import java.net.{URL, URI} import java.util.{List => JList} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -import org.apache.spark.SparkContext +import org.apache.spark.{Logging, SparkContext} import org.apache.spark.api.java.{JavaSparkContext, JavaRDD} +import org.apache.spark.util.{MutableURLClassLoader, Utils} -private[spark] object PythonUtils { +private[spark] object PythonUtils extends Logging { /** Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from our JAR */ def sparkPythonPath: String = { val pythonPath = new ArrayBuffer[String] @@ -74,4 +76,28 @@ private[spark] object PythonUtils { def toScalaMap[K, V](jm: java.util.Map[K, V]): Map[K, V] = { jm.asScala.toMap } + + /** + * Update the current threads class loader. + * Requires the current class loader is a MutableURLClassLoader, otherwise skips updating with a + * warning. Intended for use by addJar(), although constructing an instance of the class will + * still require: + * sc._jvm.java.lang.Thread.currentThread().getContextClassLoader().loadClass("class name") + * as described in SPARK-5185. + */ + def updatePrimaryClassLoader(jsc: JavaSparkContext) { + val sc = jsc.sc + val jars = sc.addedJars.keys + val currentCL = Utils.getContextOrSparkClassLoader + currentCL match { + case cl: MutableURLClassLoader => { + val existingJars = cl.getURLs().map(_.toString).toSet + val newJars = (jars.toSet -- existingJars) + logDebug(s"Adding jars ${newJars} to ${existingJars}") + val newJarURLs = newJars.map(new URI(_).toURL()) + newJarURLs.foreach(cl.addURL(_)) + } + case _ => logWarning(s"Unsupported class loader $currentCL will not update jars") + } + } } diff --git a/python/pyspark/context.py b/python/pyspark/context.py index 4b441d82e45a..3113f2182d18 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -813,6 +813,16 @@ def addJar(self, path): filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. """ self._jsc.sc().addJar(path) + self._jvm.PythonUtils.updatePrimaryClassLoader(self._jsc) + + def _loadClass(self, className): + """ + .. note:: Experimental + + Loads a JVM class using the MutableClass loader used by spark. + This function exists because Py4J uses a different class loader. + """ + self._jvm.java.lang.Thread.currentThread().getContextClassLoader().loadClass(className) def setCheckpointDir(self, dirName): """ diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 93791e5dd622..925ed64b2783 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -411,8 +411,15 @@ def test_add_file_locally(self): self.assertEqual("Hello World!\n", test_file.readline()) def test_add_jar(self): + # We shouldn't be able to load anything from the package before it is added + self.assertRaises(Exception, + lambda: sc._loadClass("sparkR.test.hello")) + # Load the new jar path = os.path.join(SPARK_HOME, "./R/pkg/inst/test_support/sparktestjar_2.10-1.0.jar") + sc.addJar(path) self.assertTrue(self._jsc.sc().addedJars().toString().find("sparktestjar") != -1) + # Try and load a different one of the classes + cls = sc._loadClass("sparkR.test.basicFunction") def test_add_py_file_locally(self): # To ensure that we're actually testing addPyFile's effects, check that From b8b8d72c26401a9d26a86f4829f8720b3192c1cc Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 27 Oct 2015 22:23:13 -0700 Subject: [PATCH 3/9] Finish up test --- python/pyspark/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 925ed64b2783..baf0fe7cf3f3 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -416,10 +416,10 @@ def test_add_jar(self): lambda: sc._loadClass("sparkR.test.hello")) # Load the new jar path = os.path.join(SPARK_HOME, "./R/pkg/inst/test_support/sparktestjar_2.10-1.0.jar") - sc.addJar(path) - self.assertTrue(self._jsc.sc().addedJars().toString().find("sparktestjar") != -1) + self.sc.addJar(path) + self.assertTrue(self.sc._jsc.sc().addedJars().toString().find("sparktestjar") != -1) # Try and load a different one of the classes - cls = sc._loadClass("sparkR.test.basicFunction") + cls = self.sc._loadClass("sparkR.test.basicFunction") def test_add_py_file_locally(self): # To ensure that we're actually testing addPyFile's effects, check that From a0fdddb883818aaa1c89b6f014bc731ecd3111e5 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 28 Oct 2015 22:34:39 -0700 Subject: [PATCH 4/9] Use the context or spark class loader since sometimes (very rarely) we can have a null context class loader (see SPARK-1403) --- python/pyspark/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/context.py b/python/pyspark/context.py index 3113f2182d18..4542ecf1bdf9 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -822,7 +822,7 @@ def _loadClass(self, className): Loads a JVM class using the MutableClass loader used by spark. This function exists because Py4J uses a different class loader. """ - self._jvm.java.lang.Thread.currentThread().getContextClassLoader().loadClass(className) + self._jvm.org.apache.spark.util.Utils.getContextOrSparkClassLoader().loadClass(className) def setCheckpointDir(self, dirName): """ From 371fc7dc54d68f1cb7e8aa7d3d8d4bcad8838dda Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Fri, 30 Oct 2015 00:41:13 -0700 Subject: [PATCH 5/9] Switch the addToCurrentThread functionality for addJar into a bool and expose it through the general spark context --- .../scala/org/apache/spark/SparkContext.scala | 14 ++++++++++ .../apache/spark/api/python/PythonUtils.scala | 28 ++----------------- .../scala/org/apache/spark/util/Utils.scala | 23 +++++++++++++++ python/pyspark/context.py | 7 +++-- python/pyspark/tests.py | 2 +- 5 files changed, 44 insertions(+), 30 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index a6857b4c7d88..43bdb864af8a 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1672,6 +1672,20 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli postEnvironmentUpdate() } + /** + * Adds a JAR dependency for all tasks to be executed on this SparkContext in the future. + * The `path` passed can be either a local file, a file in HDFS (or other Hadoop-supported + * filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. + * If addToCurrentThread is true, attempt to add the new class to the current threads class + * loader. + */ + def addJar(path: String, addCurrent: Boolean) { + addJar(path) + if (addCurrent) { + Utils.updatePrimaryClassLoader(this) + } + } + /** * Clear the job's list of JARs added by `addJar` so that they do not get downloaded to * any new nodes. diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala index cfc7fb0d6178..bc415280e414 100644 --- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala +++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala @@ -24,11 +24,11 @@ import java.util.{List => JList} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -import org.apache.spark.{Logging, SparkContext} +import org.apache.spark.SparkContext import org.apache.spark.api.java.{JavaSparkContext, JavaRDD} import org.apache.spark.util.{MutableURLClassLoader, Utils} -private[spark] object PythonUtils extends Logging { +private[spark] object PythonUtils { /** Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from our JAR */ def sparkPythonPath: String = { val pythonPath = new ArrayBuffer[String] @@ -76,28 +76,4 @@ private[spark] object PythonUtils extends Logging { def toScalaMap[K, V](jm: java.util.Map[K, V]): Map[K, V] = { jm.asScala.toMap } - - /** - * Update the current threads class loader. - * Requires the current class loader is a MutableURLClassLoader, otherwise skips updating with a - * warning. Intended for use by addJar(), although constructing an instance of the class will - * still require: - * sc._jvm.java.lang.Thread.currentThread().getContextClassLoader().loadClass("class name") - * as described in SPARK-5185. - */ - def updatePrimaryClassLoader(jsc: JavaSparkContext) { - val sc = jsc.sc - val jars = sc.addedJars.keys - val currentCL = Utils.getContextOrSparkClassLoader - currentCL match { - case cl: MutableURLClassLoader => { - val existingJars = cl.getURLs().map(_.toString).toSet - val newJars = (jars.toSet -- existingJars) - logDebug(s"Adding jars ${newJars} to ${existingJars}") - val newJarURLs = newJars.map(new URI(_).toURL()) - newJarURLs.foreach(cl.addURL(_)) - } - case _ => logWarning(s"Unsupported class loader $currentCL will not update jars") - } - } } diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 5a976ee839b1..f86d8c513e55 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -174,6 +174,29 @@ private[spark] object Utils extends Logging { // scalastyle:on classforname } + /** + * Update the current threads class loader. + * Requires the current class loader is a MutableURLClassLoader, otherwise skips updating with a + * warning. Intended for use by addJar(), although constructing an instance of the class will + * still require: + * sc._jvm.java.lang.Thread.currentThread().getContextClassLoader().loadClass("class name") + * as described in SPARK-5185. + */ + def updatePrimaryClassLoader(sc: SparkContext) { + val jars = sc.addedJars.keys + val currentCL = Utils.getContextOrSparkClassLoader + currentCL match { + case cl: MutableURLClassLoader => { + val existingJars = cl.getURLs().map(_.toString).toSet + val newJars = (jars.toSet -- existingJars) + logDebug(s"Adding jars ${newJars} to ${existingJars}") + val newJarURLs = newJars.map(new URI(_).toURL()) + newJarURLs.foreach(cl.addURL(_)) + } + case _ => logWarning(s"Unsupported cl $currentCL will not update jars for current thread") + } + } + /** * Primitive often used when writing [[java.nio.ByteBuffer]] to [[java.io.DataOutput]] */ diff --git a/python/pyspark/context.py b/python/pyspark/context.py index 4542ecf1bdf9..639a81c39543 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -806,14 +806,15 @@ def addPyFile(self, path): import importlib importlib.invalidate_caches() - def addJar(self, path): + def addJar(self, path, addToCurrentThread=False): """ Adds a JAR dependency for all tasks to be executed on this SparkContext in the future. The `path` passed can be either a local file, a file in HDFS (or other Hadoop-supported filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. + If addToCurrentThread is true, attempt to add the new class to the current threads class + loader. """ - self._jsc.sc().addJar(path) - self._jvm.PythonUtils.updatePrimaryClassLoader(self._jsc) + self._jsc.sc().addJar(path, addToCurrentThread) def _loadClass(self, className): """ diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index baf0fe7cf3f3..36336ff18eb7 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -416,7 +416,7 @@ def test_add_jar(self): lambda: sc._loadClass("sparkR.test.hello")) # Load the new jar path = os.path.join(SPARK_HOME, "./R/pkg/inst/test_support/sparktestjar_2.10-1.0.jar") - self.sc.addJar(path) + self.sc.addJar(path, True) self.assertTrue(self.sc._jsc.sc().addedJars().toString().find("sparktestjar") != -1) # Try and load a different one of the classes cls = self.sc._loadClass("sparkR.test.basicFunction") From 72b9f36aa5c195cc3071aa040542527c863dd1a6 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 2 Nov 2015 15:09:41 -0800 Subject: [PATCH 6/9] CR feedback: only add the class we are asked to add to the class loader, not all classes --- .../scala/org/apache/spark/SparkContext.scala | 27 +++++++++---------- .../scala/org/apache/spark/util/Utils.scala | 23 ---------------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 43bdb864af8a..8731c56fc9a8 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1611,8 +1611,14 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli * Adds a JAR dependency for all tasks to be executed on this SparkContext in the future. * The `path` passed can be either a local file, a file in HDFS (or other Hadoop-supported * filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. + * If addToCurrentThread is true, attempt to add the new class to the current threads class + * loader. */ def addJar(path: String) { + addJar(path, false) + } + + def addJar(path: String, addToCurrentThread: Boolean) { if (path == null) { logWarning("null specified as parameter to addJar") } else { @@ -1667,25 +1673,18 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli if (key != null) { addedJars(key) = System.currentTimeMillis logInfo("Added JAR " + path + " at " + key + " with timestamp " + addedJars(key)) + if (addToCurrentThread) { + val currentCL = Utils.getContextOrSparkClassLoader + currentCL match { + case cl: MutableURLClassLoader => cl.addURL(new URI(key).toURL()) + case _ => logWarning(s"Unsupported cl $currentCL will not update jars for current thread") + } + } } } postEnvironmentUpdate() } - /** - * Adds a JAR dependency for all tasks to be executed on this SparkContext in the future. - * The `path` passed can be either a local file, a file in HDFS (or other Hadoop-supported - * filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. - * If addToCurrentThread is true, attempt to add the new class to the current threads class - * loader. - */ - def addJar(path: String, addCurrent: Boolean) { - addJar(path) - if (addCurrent) { - Utils.updatePrimaryClassLoader(this) - } - } - /** * Clear the job's list of JARs added by `addJar` so that they do not get downloaded to * any new nodes. diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index f86d8c513e55..5a976ee839b1 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -174,29 +174,6 @@ private[spark] object Utils extends Logging { // scalastyle:on classforname } - /** - * Update the current threads class loader. - * Requires the current class loader is a MutableURLClassLoader, otherwise skips updating with a - * warning. Intended for use by addJar(), although constructing an instance of the class will - * still require: - * sc._jvm.java.lang.Thread.currentThread().getContextClassLoader().loadClass("class name") - * as described in SPARK-5185. - */ - def updatePrimaryClassLoader(sc: SparkContext) { - val jars = sc.addedJars.keys - val currentCL = Utils.getContextOrSparkClassLoader - currentCL match { - case cl: MutableURLClassLoader => { - val existingJars = cl.getURLs().map(_.toString).toSet - val newJars = (jars.toSet -- existingJars) - logDebug(s"Adding jars ${newJars} to ${existingJars}") - val newJarURLs = newJars.map(new URI(_).toURL()) - newJarURLs.foreach(cl.addURL(_)) - } - case _ => logWarning(s"Unsupported cl $currentCL will not update jars for current thread") - } - } - /** * Primitive often used when writing [[java.nio.ByteBuffer]] to [[java.io.DataOutput]] */ From 2467ce59b7a2cf7af71ef896752e88c148116c8e Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 3 Nov 2015 12:08:28 -0800 Subject: [PATCH 7/9] Style fix --- core/src/main/scala/org/apache/spark/SparkContext.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 8731c56fc9a8..f6684aed3fd1 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1676,8 +1676,8 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli if (addToCurrentThread) { val currentCL = Utils.getContextOrSparkClassLoader currentCL match { - case cl: MutableURLClassLoader => cl.addURL(new URI(key).toURL()) - case _ => logWarning(s"Unsupported cl $currentCL will not update jars for current thread") + case cl: MutableURLClassLoader => cl.addURL(new URI(key).toURL()) + case _ => logWarning(s"Unsupported cl $currentCL will not update jars thread cl") } } } From 8b104b174939421ff92c7457e0bf49e9c0806d42 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Sat, 7 Nov 2015 21:36:29 -0800 Subject: [PATCH 8/9] CR feedback, change param name and update the comments --- core/src/main/scala/org/apache/spark/SparkContext.scala | 9 +++++---- python/pyspark/context.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index f6684aed3fd1..201c9f517c56 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1611,14 +1611,15 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli * Adds a JAR dependency for all tasks to be executed on this SparkContext in the future. * The `path` passed can be either a local file, a file in HDFS (or other Hadoop-supported * filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. - * If addToCurrentThread is true, attempt to add the new class to the current threads class - * loader. + * If addToCurrentClassLoader is true, attempt to add the new class to the current threads' class + * loader. In general adding to the current threads' class loader will impact all other + * application threads unless they have explicitly changed their class loader. */ def addJar(path: String) { addJar(path, false) } - def addJar(path: String, addToCurrentThread: Boolean) { + def addJar(path: String, addToCurrentClassLoader: Boolean) { if (path == null) { logWarning("null specified as parameter to addJar") } else { @@ -1673,7 +1674,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli if (key != null) { addedJars(key) = System.currentTimeMillis logInfo("Added JAR " + path + " at " + key + " with timestamp " + addedJars(key)) - if (addToCurrentThread) { + if (addToCurrentClassLoader) { val currentCL = Utils.getContextOrSparkClassLoader currentCL match { case cl: MutableURLClassLoader => cl.addURL(new URI(key).toURL()) diff --git a/python/pyspark/context.py b/python/pyspark/context.py index 639a81c39543..9e31b20ae3b1 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -806,15 +806,16 @@ def addPyFile(self, path): import importlib importlib.invalidate_caches() - def addJar(self, path, addToCurrentThread=False): + def addJar(self, path, addToCurrentClassLoader=False): """ Adds a JAR dependency for all tasks to be executed on this SparkContext in the future. The `path` passed can be either a local file, a file in HDFS (or other Hadoop-supported filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node. - If addToCurrentThread is true, attempt to add the new class to the current threads class - loader. + If addToCurrentClassLoader is true, attempt to add the new class to the current threads' + class loader. In general adding to the current threads' class loader will impact all other + application threads unless they have explicitly changed their class loader. """ - self._jsc.sc().addJar(path, addToCurrentThread) + self._jsc.sc().addJar(path, addToCurrentClassLoader) def _loadClass(self, className): """ From d9d13758f4e0b4adaeec07bd2fc9b76a9cbbff0c Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Sat, 7 Nov 2015 21:37:25 -0800 Subject: [PATCH 9/9] Revert changes to PythonUtils that are no longer required --- .../main/scala/org/apache/spark/api/python/PythonUtils.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala index bc415280e414..292ac4cfc35b 100644 --- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala +++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala @@ -18,7 +18,6 @@ package org.apache.spark.api.python import java.io.File -import java.net.{URL, URI} import java.util.{List => JList} import scala.collection.JavaConverters._ @@ -26,7 +25,6 @@ import scala.collection.mutable.ArrayBuffer import org.apache.spark.SparkContext import org.apache.spark.api.java.{JavaSparkContext, JavaRDD} -import org.apache.spark.util.{MutableURLClassLoader, Utils} private[spark] object PythonUtils { /** Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from our JAR */