From 87cf15799f85464b33b66411377f7226b53791d7 Mon Sep 17 00:00:00 2001 From: Adam Lewandowski Date: Tue, 13 Oct 2015 21:22:31 -0400 Subject: [PATCH 1/3] Return resources from both the child and parent classloaders in ChildFirstURLClassLoader#getResources() --- .../spark/util/MutableURLClassLoader.scala | 13 +++--- .../util/MutableURLClassLoaderSuite.scala | 41 ++++++++++++++++++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala b/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala index a1c33212cdb2b..945217203be72 100644 --- a/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala +++ b/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala @@ -21,6 +21,8 @@ import java.net.{URLClassLoader, URL} import java.util.Enumeration import java.util.concurrent.ConcurrentHashMap +import scala.collection.JavaConverters._ + /** * URL class loader that exposes the `addURL` and `getURLs` methods in URLClassLoader. */ @@ -82,14 +84,9 @@ private[spark] class ChildFirstURLClassLoader(urls: Array[URL], parent: ClassLoa } override def getResources(name: String): Enumeration[URL] = { - val urls = super.findResources(name) - val res = - if (urls != null && urls.hasMoreElements()) { - urls - } else { - parentClassLoader.getResources(name) - } - res + val childUrls = super.findResources(name).asScala + val parentUrls = parentClassLoader.getResources(name).asScala + (childUrls ++ parentUrls).asJavaEnumeration } override def addURL(url: URL) { diff --git a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala index d3d464e84ffd7..df2a779025a42 100644 --- a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala @@ -19,9 +19,15 @@ package org.apache.spark.util import java.net.URLClassLoader +import scala.collection.JavaConverters._ + import org.apache.spark.{SparkContext, SparkException, SparkFunSuite, TestUtils} -class MutableURLClassLoaderSuite extends SparkFunSuite { +import org.scalatest.Matchers +import org.scalatest.Matchers._ + + +class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers { val urls2 = List(TestUtils.createJarWithClasses( classNames = Seq("FakeClass1", "FakeClass2", "FakeClass3"), @@ -32,6 +38,12 @@ class MutableURLClassLoaderSuite extends SparkFunSuite { toStringValue = "1", classpathUrls = urls2)).toArray + val fileUrlsChild = List(TestUtils.createJarWithFiles(Map( + "resource1" -> "resource1Contents-child", + "resource2" -> "resource2Contents"))).toArray + val fileUrlsParent = List(TestUtils.createJarWithFiles(Map( + "resource1" -> "resource1Contents-parent"))).toArray + test("child first") { val parentLoader = new URLClassLoader(urls2, null) val classLoader = new ChildFirstURLClassLoader(urls, parentLoader) @@ -68,6 +80,33 @@ class MutableURLClassLoaderSuite extends SparkFunSuite { } } + test("default JDK classloader get resources") { + val parentLoader = new URLClassLoader(fileUrlsParent, null) + val classLoader = new URLClassLoader(fileUrlsChild, parentLoader) + assert(classLoader.getResources("resource1").asScala.size == 2) + assert(classLoader.getResources("resource2").asScala.size == 1) + } + + test("parent first get resources") { + val parentLoader = new URLClassLoader(fileUrlsParent, null) + val classLoader = new MutableURLClassLoader(fileUrlsChild, parentLoader) + assert(classLoader.getResources("resource1").asScala.size == 2) + assert(classLoader.getResources("resource2").asScala.size == 1) + } + + test("child first get resources") { + val parentLoader = new URLClassLoader(fileUrlsParent, null) + val classLoader = new ChildFirstURLClassLoader(fileUrlsChild, parentLoader) + + val res1 = classLoader.getResources("resource1").asScala.toList + assert(res1.size == 2) + assert(classLoader.getResources("resource2").asScala.size == 1) + + res1.map(scala.io.Source.fromURL(_).mkString) should contain inOrderOnly + ("resource1Contents-child", "resource1Contents-parent") + } + + test("driver sets context class loader in local mode") { // Test the case where the driver program sets a context classloader and then runs a job // in local mode. This is what happens when ./spark-submit is called with "local" as the From d0aa6aa605914a0d7c2a7e5c953bfdb6eb59a38e Mon Sep 17 00:00:00 2001 From: Adam Lewandowski Date: Wed, 14 Oct 2015 13:07:05 -0400 Subject: [PATCH 2/3] Fix style violations --- .../org/apache/spark/util/MutableURLClassLoaderSuite.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala index df2a779025a42..cc3411b8dd482 100644 --- a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala @@ -21,11 +21,10 @@ import java.net.URLClassLoader import scala.collection.JavaConverters._ -import org.apache.spark.{SparkContext, SparkException, SparkFunSuite, TestUtils} - import org.scalatest.Matchers import org.scalatest.Matchers._ +import org.apache.spark.{SparkContext, SparkException, SparkFunSuite, TestUtils} class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers { From 8f1b6d98e986d576edf36021b4b8a4bc39f884e3 Mon Sep 17 00:00:00 2001 From: Adam Lewandowski Date: Wed, 14 Oct 2015 19:58:22 -0400 Subject: [PATCH 3/3] Fix assertion comparisons --- .../spark/util/MutableURLClassLoaderSuite.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala index cc3411b8dd482..8b53d4f14a6a4 100644 --- a/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/MutableURLClassLoaderSuite.scala @@ -82,15 +82,15 @@ class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers { test("default JDK classloader get resources") { val parentLoader = new URLClassLoader(fileUrlsParent, null) val classLoader = new URLClassLoader(fileUrlsChild, parentLoader) - assert(classLoader.getResources("resource1").asScala.size == 2) - assert(classLoader.getResources("resource2").asScala.size == 1) + assert(classLoader.getResources("resource1").asScala.size === 2) + assert(classLoader.getResources("resource2").asScala.size === 1) } test("parent first get resources") { val parentLoader = new URLClassLoader(fileUrlsParent, null) val classLoader = new MutableURLClassLoader(fileUrlsChild, parentLoader) - assert(classLoader.getResources("resource1").asScala.size == 2) - assert(classLoader.getResources("resource2").asScala.size == 1) + assert(classLoader.getResources("resource1").asScala.size === 2) + assert(classLoader.getResources("resource2").asScala.size === 1) } test("child first get resources") { @@ -98,8 +98,8 @@ class MutableURLClassLoaderSuite extends SparkFunSuite with Matchers { val classLoader = new ChildFirstURLClassLoader(fileUrlsChild, parentLoader) val res1 = classLoader.getResources("resource1").asScala.toList - assert(res1.size == 2) - assert(classLoader.getResources("resource2").asScala.size == 1) + assert(res1.size === 2) + assert(classLoader.getResources("resource2").asScala.size === 1) res1.map(scala.io.Source.fromURL(_).mkString) should contain inOrderOnly ("resource1Contents-child", "resource1Contents-parent")