From db48bf33e867c05b4bb560a3f787153d86c1c393 Mon Sep 17 00:00:00 2001 From: Kalvin Chau Date: Thu, 22 Jun 2017 13:26:41 -0700 Subject: [PATCH 1/2] updated depdency downloaders to exclude org.scala-lang.modules to prevent scala refelection issues TOREE-420 --- .../dependencies/CoursierDependencyDownloader.scala | 2 +- .../toree/dependencies/IvyDependencyDownloader.scala | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala index a75ae1e04..d83ae8916 100644 --- a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala +++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala @@ -84,7 +84,7 @@ class CoursierDependencyDownloader extends DependencyDownloader { // Grab exclusions using base dependencies (always exclude scala lang) val exclusions: Set[(String, String)] = (if (excludeBaseDependencies) { getBaseDependencies.map(_.module).map(m => (m.organization, m.name)) - } else Nil).toSet ++ Set(("org.scala-lang", "*")) + } else Nil).toSet ++ Set(("org.scala-lang", "*"), ("org.scala-lang.modules", "*")) // Mark dependency that we want to download val start = Resolution(Set( diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala index 3ae6de623..85b086250 100644 --- a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala +++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala @@ -127,6 +127,14 @@ class IvyDependencyDownloader( scalaCompilerArtifactId, new RegexpPatternMatcher(), null ) + val scalaLangModuleId = new ModuleId("org.scala-lang.modules", "*") + val scalaLangArtifactId = new ArtifactId( + scalaLangModuleId, "*", "*", "*" + ) + val scalaLangExclusion = new DefaultExcludeRule( + scalaLangArtifactId, new RegexpPatternMatcher(), null + ) + // Create our dependency descriptor val dependencyDescriptor = new DefaultDependencyDescriptor( md, ModuleRevisionId.newInstance(groupId, artifactId, version), @@ -139,6 +147,7 @@ class IvyDependencyDownloader( md.addExcludeRule(sourcesExclusion) md.addExcludeRule(javadocExclusion) md.addExcludeRule(scalaCompilerExclusion) + md.addExcludeRule(scalaLangExclusion) // Exclude our base dependencies if marked to do so if (excludeBaseDependencies) { From 2094e0187543ee111c7ee332d95d613b53a568ef Mon Sep 17 00:00:00 2001 From: Frank Dekervel Date: Mon, 11 Sep 2017 09:58:04 +0200 Subject: [PATCH 2/2] Add --exclude option to AddDeps in order to be able to exclude certain modules The exclude syntax is follows "organisation:name" or "organisation" when everything from a certain organisation is to be excluded --- .../dependencies/CoursierDependencyDownloader.scala | 5 +++-- .../toree/dependencies/DependencyDownloader.scala | 5 ++++- .../toree/dependencies/IvyDependencyDownloader.scala | 12 +++++++++++- .../org/apache/toree/magic/builtin/AddDeps.scala | 12 ++++++++++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala index d83ae8916..89a5e9cce 100644 --- a/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala +++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/CoursierDependencyDownloader.scala @@ -76,7 +76,8 @@ class CoursierDependencyDownloader extends DependencyDownloader { trace: Boolean, configuration: Option[String] = None, artifactType: Option[String] = None, - artifactClassifier: Option[String] = None + artifactClassifier: Option[String] = None, + excludes: Set[(String,String)] = Set.empty ): Seq[URI] = { assert(localDirectory != null) import coursier._ @@ -84,7 +85,7 @@ class CoursierDependencyDownloader extends DependencyDownloader { // Grab exclusions using base dependencies (always exclude scala lang) val exclusions: Set[(String, String)] = (if (excludeBaseDependencies) { getBaseDependencies.map(_.module).map(m => (m.organization, m.name)) - } else Nil).toSet ++ Set(("org.scala-lang", "*"), ("org.scala-lang.modules", "*")) + } else Nil).toSet ++ Set(("org.scala-lang", "*"), ("org.scala-lang.modules", "*")) ++ excludes // Mark dependency that we want to download val start = Resolution(Set( diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala index 2f8396641..36f8b50c0 100644 --- a/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala +++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/DependencyDownloader.scala @@ -23,6 +23,8 @@ import java.nio.file.Files import scala.util.Try import org.apache.toree.utils.FileUtils +case class DependencyExclude(groupId: String, artifactId: String, version: String); + abstract class DependencyDownloader { /** * Retrieves the dependency and all of its dependencies as jars. @@ -57,7 +59,8 @@ abstract class DependencyDownloader { trace: Boolean = false, configuration: Option[String] = None, artifactType: Option[String] = None, - artifactClassifier: Option[String] = None + artifactClassifier: Option[String] = None, + excludes: Set[(String,String)] = Set.empty ): Seq[URI] /** diff --git a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala index 85b086250..e5939afc9 100644 --- a/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala +++ b/kernel-api/src/main/scala/org/apache/toree/dependencies/IvyDependencyDownloader.scala @@ -94,7 +94,8 @@ class IvyDependencyDownloader( trace: Boolean, configuration: Option[String] = None, artifactType: Option[String] = None, - artifactClassifier: Option[String] = None + artifactClassifier: Option[String] = None, + excludes: Set[(String,String)] = Set.empty ): Seq[URI] = { // Start building the ivy.xml file val ivyFile = File.createTempFile("ivy-custom", ".xml") @@ -127,6 +128,7 @@ class IvyDependencyDownloader( scalaCompilerArtifactId, new RegexpPatternMatcher(), null ) + val scalaLangModuleId = new ModuleId("org.scala-lang.modules", "*") val scalaLangArtifactId = new ArtifactId( scalaLangModuleId, "*", "*", "*" @@ -149,6 +151,14 @@ class IvyDependencyDownloader( md.addExcludeRule(scalaCompilerExclusion) md.addExcludeRule(scalaLangExclusion) + + excludes.foreach(x => { + val moduleId = new ModuleId(x._1,x._2); + val artifactId = new ArtifactId(moduleId,"*","*","*"); + val exclusion = new DefaultExcludeRule(artifactId, new RegexpPatternMatcher(), null); + md.addExcludeRule(exclusion); + }) + // Exclude our base dependencies if marked to do so if (excludeBaseDependencies) { getBaseDependencies.foreach(dep => { diff --git a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala index 90f97c2ce..913d64e2c 100644 --- a/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala +++ b/kernel/src/main/scala/org/apache/toree/magic/builtin/AddDeps.scala @@ -52,6 +52,8 @@ class AddDeps extends LineMagic with IncludeInterpreter "abort-on-resolution-errors", "Abort (no downloads) when resolution fails" ) + private val _exclude = parser.accepts("exclude", "exclude dependency").withRequiredArg().ofType(classOf[String]) + private val _repository = parser.accepts( "repository", "Adds an additional repository to available list" ).withRequiredArg().ofType(classOf[String]) @@ -81,6 +83,15 @@ class AddDeps extends LineMagic with IncludeInterpreter val repository = getAll(_repository).getOrElse(Nil) val credentials = getAll(_credentials).getOrElse(Nil) + val excludes = getAll(_exclude).getOrElse(Nil) + + val excludesSet = excludes.map((x: String) => { + if (x.contains(":")) { + (x.split(":")(0), x.split(":")(1)) + } else { + (x, "*") + } + }: (String, String)).toSet val repositoriesWithCreds = dependencyDownloader.resolveRepositoriesAndCredentials(repository, credentials) @@ -95,6 +106,7 @@ class AddDeps extends LineMagic with IncludeInterpreter extraRepositories = repositoriesWithCreds, verbose = _verbose, trace = _trace, + excludes = excludesSet, configuration = get(_configuration), artifactClassifier = get(_classifier) )