Add non-reg tests for Nexus proxies

Ensure everything work fine again with those (things went bad at 1.0.0-RC1, because of the use of directory listings, that may not be exhaustive in proxies - or may be just empty, e.g. currently with nexus 3)
coursier · May 31, 2017 · f5ef7d8 · f5ef7d8
1 parent 1cd57c0
commit f5ef7d8
Show file tree

Hide file tree

Showing 11 changed files with 315 additions and 83 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -5,6 +5,9 @@ os:
   - osx
 script:
   - scripts/travis.sh
+sudo: required
+services:
+  - docker
 # Uncomment once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed
 # after_success:
 #    - bash <(curl -s https://codecov.io/bash)
@@ -13,6 +16,9 @@ matrix:
     - env: SCALA_VERSION=2.12.1 PUBLISH=1
       os: linux
       jdk: oraclejdk8
+      sudo: required
+      services:
+        - docker
     - env: SCALA_VERSION=2.11.11 PUBLISH=1
       os: linux
       jdk: oraclejdk8

diff --git a/build.sbt b/build.sbt
@@ -69,6 +69,19 @@ lazy val tests = crossProject
 lazy val testsJvm = tests.jvm
 lazy val testsJs = tests.js
 
+lazy val `proxy-tests` = project
+  .dependsOn(testsJvm % "test->test")
+  .configs(Integration)
+  .settings(
+    shared,
+    dontPublish,
+    hasITs,
+    coursierPrefix,
+    libs += Deps.scalaAsync.value,
+    utest,
+    sharedTestResources
+  )
+
 lazy val cache = project
   .dependsOn(coreJvm)
   .settings(
@@ -242,6 +255,7 @@ lazy val jvm = project
   .aggregate(
     coreJvm,
     testsJvm,
+    `proxy-tests`,
     cache,
     bootstrap,
     extra,
@@ -297,6 +311,7 @@ lazy val coursier = project
     `fetch-js`,
     testsJvm,
     testsJs,
+    `proxy-tests`,
     cache,
     bootstrap,
     extra,

diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala
@@ -675,16 +675,47 @@ class Helper(
     val task = Task.gatherUnordered(tasks)
 
     val results = task.unsafePerformSync
-    val errors = results.collect{case (artifact, -\/(err)) => artifact -> err }
-    val files0 = results.collect{case (artifact, \/-(f)) => f }
+
+    val (ignoredErrors, errors) = results
+      .collect {
+        case (artifact, -\/(err)) =>
+          artifact -> err
+      }
+      .partition {
+        case (a, err) =>
+          val notFound = err match {
+            case _: FileError.NotFound => true
+            case _ => false
+          }
+          a.isOptional && notFound
+      }
+
+    val files0 = results.collect {
+      case (artifact, \/-(f)) =>
+        f
+    }
 
     logger.foreach(_.stop())
 
+    if (verbosityLevel >= 2)
+      errPrintln(
+        "  Ignoring error(s):\n" +
+        ignoredErrors
+          .map {
+            case (artifact, error) =>
+              s"${artifact.url}: $error"
+          }
+          .mkString("\n")
+      )
+
     exitIf(errors.nonEmpty) {
       s"  Error:\n" +
-      errors.map { case (artifact, error) =>
-        s"${artifact.url}: $error"
-      }.mkString("\n")
+      errors
+        .map {
+          case (artifact, error) =>
+            s"${artifact.url}: $error"
+        }
+        .mkString("\n")
     }
 
     files0

diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala
@@ -198,9 +198,15 @@ final case class Artifact(
 ) {
   def `type`: String = attributes.`type`
   def classifier: String = attributes.classifier
+
+  // TODO make that a proper field after 1.0 (instead of the hack via extra)
+  def isOptional: Boolean = extra.contains(Artifact.optionalKey)
 }
 
 object Artifact {
+
+  private[coursier] val optionalKey = s"$$optional"
+
   trait Source {
     def artifacts(
       dependency: Dependency,

diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala
@@ -19,6 +19,13 @@ final case class MavenSource(
     overrideClassifiers: Option[Seq[String]]
   ): Seq[Artifact] = {
 
+    val packagingTpeMap = project.packagingOpt
+      .filter(_ != Pom.relocatedPackaging)
+      .map { packaging =>
+        (MavenSource.typeDefaultClassifier(packaging), MavenSource.typeExtension(packaging)) -> packaging
+      }
+      .toMap
+
     def artifactOf(publication: Publication) = {
 
       val versioning = project
@@ -60,38 +67,57 @@ final case class MavenSource(
       )
     }
 
-    overrideClassifiers match {
-      case Some(classifiers) =>
+    lazy val defaultPublication = {
 
-        classifiers.map { classifier =>
-          Publication(
-            dependency.module.name,
-            "jar",
-            "jar",
-            classifier
-          )
-        }.map(artifactWithExtra)
+      val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type`
 
-      case None =>
+      val ext = MavenSource.typeExtension(type0)
 
-        val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type`
+      val classifier =
+        if (dependency.attributes.classifier.isEmpty)
+          MavenSource.typeDefaultClassifier(type0)
+        else
+          dependency.attributes.classifier
 
-        val extension = MavenSource.typeExtension(type0)
+      val tpe = packagingTpeMap.getOrElse(
+        (classifier, ext),
+        MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
+      )
 
-        val classifier =
-          if (dependency.attributes.classifier.isEmpty)
-            MavenSource.typeDefaultClassifier(type0)
-          else
-            dependency.attributes.classifier
+      Publication(
+        dependency.module.name,
+        tpe,
+        ext,
+        classifier
+      )
+    }
 
-        Seq(
-          Publication(
-            dependency.module.name,
-            type0,
-            extension,
-            classifier
-          )
-        ).map(artifactWithExtra)
+    overrideClassifiers match {
+      case Some(classifiers) =>
+
+        classifiers
+          .map { classifier =>
+            if (classifier == dependency.attributes.classifier)
+              defaultPublication
+            else {
+              val ext = "jar"
+              val tpe = packagingTpeMap.getOrElse(
+                (classifier, ext),
+                MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
+              )
+
+              Publication(
+                dependency.module.name,
+                tpe,
+                ext,
+                classifier
+              )
+            }
+          }
+          .map(artifactWithExtra)
+
+      case None =>
+        Seq(defaultPublication).map(artifactWithExtra)
     }
   }
 
@@ -107,12 +133,14 @@ final case class MavenSource(
       publication: Publication,
       extra: Map[String, EnrichedPublication]
     ) {
-      def artifact: Artifact = {
+      def artifact: Artifact =
+        artifact(publication.`type`)
+      def artifact(versioningType: String): Artifact = {
 
         val versioning = project
           .snapshotVersioning
           .flatMap(versioning =>
-            mavenVersioning(versioning, publication.classifier, publication.`type`)
+            mavenVersioning(versioning, publication.classifier, versioningType)
           )
 
         val path = dependency.module.organization.split('.').toSeq ++ Seq(
@@ -123,7 +151,7 @@ final case class MavenSource(
 
         val changing0 = changing.getOrElse(project.actualVersion.contains("-SNAPSHOT"))
 
-        val extra0 = extra.mapValues(_.artifact).iterator.toMap
+        val extra0 = extra.mapValues(_.artifact(versioningType)).iterator.toMap
 
         Artifact(
           root + path.mkString("/"),
@@ -211,9 +239,10 @@ final case class MavenSource(
         else if (dependency.attributes.`type`.nonEmpty)
           enrichedPublications.collect {
             case p
-              if p.publication.`type` == dependency.attributes.`type` ||
-                (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow
-            =>
+              if p.publication.classifier.isEmpty && (
+                   p.publication.`type` == dependency.attributes.`type` ||
+                   (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow
+                ) =>
               p.artifact
           }
         else
@@ -226,17 +255,62 @@ final case class MavenSource(
     res.map(withMetadataExtra)
   }
 
+  private val dummyArtifact = Artifact("", Map(), Map(), Attributes("", ""), changing = false, None)
+
   def artifacts(
     dependency: Dependency,
     project: Project,
     overrideClassifiers: Option[Seq[String]]
   ): Seq[Artifact] =
     if (project.packagingOpt.toSeq.contains(Pom.relocatedPackaging))
       Nil
-    else if (project.publications.isEmpty)
-      artifactsUnknownPublications(dependency, project, overrideClassifiers)
-    else
-      artifactsKnownPublications(dependency, project, overrideClassifiers)
+    else {
+
+      def makeOptional(a: Artifact): Artifact =
+        a.copy(
+          extra = a.extra.mapValues(makeOptional).iterator.toMap + (Artifact.optionalKey -> dummyArtifact)
+        )
+
+      def merge(a: Artifact, other: Artifact): Artifact = {
+
+        assert(a.url == other.url, s"Merging artifacts with different URLs (${a.url}, ${other.url})")
+
+        val extra =
+          a.extra.map {
+            case (k, v) =>
+              k -> other.extra.get(k).fold(v)(merge(v, _))
+          } ++
+          other.extra
+            .filterKeys(k => !a.extra.contains(k))
+
+        a.copy(
+          checksumUrls = other.checksumUrls ++ a.checksumUrls,
+          extra = extra
+        )
+      }
+
+      val defaultPublications = artifactsUnknownPublications(dependency, project, overrideClassifiers)
+
+      if (project.publications.isEmpty)
+        defaultPublications
+      else {
+        val listedPublications = artifactsKnownPublications(dependency, project, overrideClassifiers)
+        val listedUrls = listedPublications.map(_.url).toSet
+        val defaultPublications0 = defaultPublications.map(makeOptional)
+        val defaultPublicationsMap = defaultPublications0
+          .map(a => a.url -> a)
+          .toMap
+        val listedPublications0 = listedPublications.map { a =>
+          defaultPublicationsMap
+            .get(a.url)
+            .fold(a)(merge(a, _))
+        }
+        val extraPublications = defaultPublications0
+          .filter(a => !listedUrls(a.url))
+
+        listedPublications0 ++ extraPublications
+      }
+    }
 }
 
 object MavenSource {

diff --git a/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala b/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala
@@ -0,0 +1,9 @@
+package coursier.test
+
+object CentralNexus2ProxyTests extends CentralTests {
+  override def centralBase = "http://localhost:9081/nexus/content/repositories/central"
+}
+
+object CentralNexus3ProxyTests extends CentralTests {
+  override def centralBase = "http://localhost:9082/repository/maven-central"
+}
diff --git a/sbt-coursier/src/main/scala/coursier/Tasks.scala b/sbt-coursier/src/main/scala/coursier/Tasks.scala
@@ -1142,21 +1142,31 @@ object Tasks {
             artifact -> file
         }
 
-        val artifactErrors = artifactFilesOrErrors0.toVector.collect {
-          case (_, -\/(err)) =>
-            err
-        }
+        val (ignoredArtifactErrors, artifactErrors) = artifactFilesOrErrors0
+          .toVector
+          .collect {
+            case (a, -\/(err)) =>
+              a -> err
+          }
+          .partition {
+            case (a, err) =>
+              val notFound = err match {
+                case _: FileError.NotFound => true
+                case _ => false
+              }
+              a.isOptional && notFound
+          }
 
         if (artifactErrors.nonEmpty) {
-          val error = ResolutionError.DownloadErrors(artifactErrors)
+          val error = ResolutionError.DownloadErrors(artifactErrors.map(_._2))
 
           if (ignoreArtifactErrors)
             log.warn(error.description(verbosityLevel >= 1))
           else
             error.throwException()
         }
 
-        // can be non empty only if ignoreArtifactErrors is true
+        // can be non empty only if ignoreArtifactErrors is true or some optional artifacts are not found
         val erroredArtifacts = artifactFilesOrErrors0.collect {
           case (artifact, -\/(_)) =>
             artifact