Merge pull request #243 from alexarchambault/topic/develop

Latest developments

project/travis.sh → .ci/travis.sh

@@ -28,7 +28,16 @@ function isMasterOrDevelop() {
 }
 
 # Required for ~/.ivy2/local repo tests
-~/sbt coreJVM/publish-local
+~/sbt coreJVM/publishLocal simple-web-server/publishLocal
+
+# Required for HTTP authentication tests
+./coursier launch \
+  io.get-coursier:simple-web-server_2.11:1.0.0-SNAPSHOT \
+  -r http://dl.bintray.com/scalaz/releases \
+  -- \
+    -d tests/jvm/src/test/resources/test-repo/http/abc.com \
+    -u user -P pass -r realm \
+    -v &
 
 # TODO Add coverage once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed
 

.travis.yml

@@ -6,7 +6,7 @@ install:
 os:
   - osx
 script:
-  - project/travis.sh "${TRAVIS_SCALA_VERSION:-2.11.8}" "$TRAVIS_PULL_REQUEST" "$TRAVIS_BRANCH" "$PUBLISH"
+  - .ci/travis.sh "${TRAVIS_SCALA_VERSION:-2.11.8}" "$TRAVIS_PULL_REQUEST" "$TRAVIS_BRANCH" "$PUBLISH"
 # Uncomment once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed
 # after_success:
 #    - bash <(curl -s https://codecov.io/bash)

appveyor.yml

@@ -15,10 +15,11 @@ install:
   - cmd: SET SBT_OPTS=-XX:MaxPermSize=2g -Xmx4g
   - cmd: SET COURSIER_NO_TERM=1
 build_script:
-  - sbt ++2.11.8 clean compile coreJVM/publishLocal
+  - sbt ++2.11.8 clean compile coreJVM/publishLocal simple-web-server/publishLocal
   - sbt ++2.10.6 clean compile
   - sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal # to make the scripted tests happy
 test_script:
+  - ps: Start-Job { & java -jar -noverify C:\projects\coursier\coursier launch -r http://dl.bintray.com/scalaz/releases io.get-coursier:simple-web-server_2.11:1.0.0-SNAPSHOT -- -d /C:/projects/coursier/tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm -v }
   - sbt ++2.11.8 testsJVM/test # Would node be around for testsJS/test?
   - sbt ++2.10.6 testsJVM/test plugin/scripted
 cache:

build.sbt

@@ -146,6 +146,21 @@ lazy val core = crossProject
       import com.typesafe.tools.mima.core.ProblemFilters._
 
       Seq(
+        // Since 1.0.0-M12
+        // Extra `authentication` field
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.core.Artifact.apply"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.core.Artifact.copy"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.core.Artifact.this"),
+        ProblemFilters.exclude[MissingTypesProblem]("coursier.ivy.IvyRepository$"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.ivy.IvyRepository.apply"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.ivy.IvyRepository.copy"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.ivy.IvyRepository.this"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.maven.MavenRepository.copy"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.maven.MavenRepository.this"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.maven.MavenSource.apply"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.maven.MavenRepository.apply"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.maven.MavenSource.copy"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.maven.MavenSource.this"),
         // Since 1.0.0-M11
         // Extra parameter with default value added, problem for forward compatibility only
         ProblemFilters.exclude[MissingMethodProblem]("coursier.core.ResolutionProcess.next"),
@@ -226,6 +241,9 @@ lazy val cache = project
       import com.typesafe.tools.mima.core.ProblemFilters._
 
       Seq(
+        // Since 1.0.0-M12
+        // Remove deprecated / unused helper method
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.readFully"),
         // Since 1.0.0-M11
         // Add constructor parameter on FileError - shouldn't be built by users anyway
         ProblemFilters.exclude[MissingMethodProblem]("coursier.FileError.this"),
@@ -488,13 +506,15 @@ lazy val plugin = project
     scriptedBufferLog := false
   )
 
+val http4sVersion = "0.8.6"
+
 lazy val `simple-web-server` = project
   .settings(commonSettings)
   .settings(packAutoSettings)
   .settings(
     libraryDependencies ++= Seq(
-      "org.http4s" %% "http4s-blaze-server" % "0.13.2",
-      "org.http4s" %% "http4s-dsl" % "0.13.2",
+      "org.http4s" %% "http4s-blazeserver" % http4sVersion,
+      "org.http4s" %% "http4s-dsl" % http4sVersion,
       "org.slf4j" % "slf4j-nop" % "1.7.19",
       "com.github.alexarchambault" %% "case-app" % "1.0.0-RC2"
     )

cache/src/main/scala/coursier/Cache.scala

@@ -8,7 +8,9 @@ import java.security.MessageDigest
 import java.util.concurrent.{ ConcurrentHashMap, Executors, ExecutorService }
 import java.util.regex.Pattern
 
+import coursier.core.Authentication
 import coursier.ivy.IvyRepository
+import coursier.util.Base64.Encoder
 
 import scala.annotation.tailrec
 
@@ -18,6 +20,10 @@ import scalaz.concurrent.{ Task, Strategy }
 
 import java.io.{ Serializable => _, _ }
 
+trait AuthenticatedURLConnection extends URLConnection {
+  def authenticate(authentication: Authentication): Unit
+}
+
 object Cache {
 
   // Check SHA-1 if available, else be fine with no checksum
@@ -43,7 +49,7 @@ object Cache {
     }
   }
 
-  private def localFile(url: String, cache: File): File = {
+  private def localFile(url: String, cache: File, user: Option[String]): File = {
     val path =
       if (url.startsWith("file:///"))
         url.stripPrefix("file://")
@@ -62,7 +68,10 @@ object Cache {
               else
                 throw new Exception(s"URL $url doesn't contain an absolute path")
 
-            new File(cache, escape(protocol + "/" + remaining0)) .toString
+            new File(
+              cache,
+              escape(protocol + "/" + user.fold("")(_ + "@") + remaining0.dropWhile(_ == '/'))
+            ).toString
 
           case _ =>
             throw new Exception(s"No protocol found in URL $url")
@@ -204,7 +213,7 @@ object Cache {
         -\/(FileError.ConcurrentDownload(url))
     }
     catch { case e: Exception =>
-      -\/(FileError.DownloadError(s"Caught $e (${e.getMessage})"))
+      -\/(FileError.DownloadError(s"Caught $e${Option(e.getMessage).fold("")(" (" + _ + ")")}"))
     }
 
   private def temporaryFile(file: File): File = {
@@ -232,7 +241,7 @@ object Cache {
 
         def printError(e: Exception): Unit =
           scala.Console.err.println(
-            s"Cannot instantiate $clsName: $e${Option(e.getMessage).map(" ("+_+")")}"
+            s"Cannot instantiate $clsName: $e${Option(e.getMessage).fold("")(" ("+_+")")}"
           )
 
         val handlerOpt = clsOpt.flatMap {
@@ -259,6 +268,17 @@ object Cache {
     }
   }
 
+  private val BasicRealm = (
+    "^" +
+      Pattern.quote("Basic realm=\"") +
+      "([^" + Pattern.quote("\"") + "]*)" +
+      Pattern.quote("\"") +
+    "$"
+  ).r
+
+  private def basicAuthenticationEncode(user: String, password: String): String =
+    (user + ":" + password).getBytes("UTF-8").toBase64
+
   /**
     * Returns a `java.net.URL` for `s`, possibly using the custom protocol handlers found under the
     * `coursier.cache.protocol` namespace.
@@ -289,7 +309,7 @@ object Cache {
     val referenceFileOpt = artifact
       .extra
       .get("metadata")
-      .map(a => localFile(a.url, cache))
+      .map(a => localFile(a.url, cache, a.authentication.map(_.user)))
 
     def referenceFileExists: Boolean = referenceFileOpt.exists(_.exists())
 
@@ -300,6 +320,20 @@ object Cache {
       // (Maven 2 compatibility? - happens for snapshot versioning metadata,
       // this is SO FSCKING CRAZY)
       conn.setRequestProperty("User-Agent", "")
+
+      for (auth <- artifact.authentication)
+        conn match {
+          case authenticated: AuthenticatedURLConnection =>
+            authenticated.authenticate(auth)
+          case conn0: HttpURLConnection =>
+            conn0.setRequestProperty(
+              "Authorization",
+              "Basic " + basicAuthenticationEncode(auth.user, auth.password)
+            )
+          case _ =>
+            // FIXME Authentication is ignored
+        }
+
       conn
     }
 
@@ -384,15 +418,28 @@ object Cache {
       }
     }
 
-    def is404(conn: URLConnection) =
+    def responseCode(conn: URLConnection): Option[Int] =
       conn match {
         case conn0: HttpURLConnection =>
-          conn0.getResponseCode == 404
+          Some(conn0.getResponseCode)
         case _ =>
-          false
+          None
       }
 
-    def remote(file: File, url: String): EitherT[Task, FileError, Unit] =
+    def realm(conn: URLConnection): Option[String] =
+      conn match {
+        case conn0: HttpURLConnection =>
+          Option(conn0.getHeaderField("WWW-Authenticate")).collect {
+            case BasicRealm(realm) => realm
+          }
+        case _ =>
+          None
+      }
+
+    def remote(
+      file: File,
+      url: String
+    ): EitherT[Task, FileError, Unit] =
       EitherT {
         Task {
           withLockFor(cache, file) {
@@ -421,8 +468,10 @@ object Cache {
                 case _ => (false, conn0)
               }
 
-              if (is404(conn))
+              if (responseCode(conn) == Some(404))
                 FileError.NotFound(url, permanent = Some(true)).left
+              else if (responseCode(conn) == Some(401))
+                FileError.Unauthorized(url, realm = realm(conn)).left
               else {
                 for (len0 <- Option(conn.getContentLengthLong) if len0 >= 0L) {
                   val len = len0 + (if (partialDownload) alreadyDownloaded else 0L)
@@ -534,7 +583,7 @@ object Cache {
 
     val tasks =
       for (url <- urls) yield {
-        val file = localFile(url, cache)
+        val file = localFile(url, cache, artifact.authentication.map(_.user))
 
         val res =
           if (url.startsWith("file:/")) {
@@ -618,12 +667,12 @@ object Cache {
 
     implicit val pool0 = pool
 
-    val localFile0 = localFile(artifact.url, cache)
+    val localFile0 = localFile(artifact.url, cache, artifact.authentication.map(_.user))
 
     EitherT {
       artifact.checksumUrls.get(sumType) match {
         case Some(sumUrl) =>
-          val sumFile = localFile(sumUrl, cache)
+          val sumFile = localFile(sumUrl, cache, artifact.authentication.map(_.user))
 
           Task {
             val sumOpt = parseChecksum(
@@ -730,7 +779,7 @@ object Cache {
         checksums = checksums,
         logger = logger,
         pool = pool
-      ).leftMap(_.message).map { f =>
+      ).leftMap(_.describe).map { f =>
         // FIXME Catch error here?
         new String(NioFiles.readAllBytes(f.toPath), "UTF-8")
       }
@@ -811,18 +860,6 @@ object Cache {
     buffer.toByteArray
   }
 
-  def readFully(is: => InputStream) =
-    Task {
-      \/.fromTryCatchNonFatal {
-        val is0 = is
-        val b =
-          try readFullySync(is0)
-          finally is0.close()
-
-        new String(b, "UTF-8")
-      } .leftMap(_.getMessage)
-    }
-
   def withContent(is: InputStream, f: (Array[Byte], Int) => Unit): Unit = {
     val data = Array.ofDim[Byte](16384)
 

cache/src/main/scala/coursier/CacheParse.scala

@@ -2,6 +2,7 @@ package coursier
 
 import java.net.MalformedURLException
 
+import coursier.core.Authentication
 import coursier.ivy.IvyRepository
 import coursier.util.Parse
 
@@ -26,12 +27,48 @@ object CacheParse {
           sys.error(s"Unrecognized repository: $r")
       }
 
-      try {
-        Cache.url(url)
-        repo.success
+      val validatedUrl = try {
+        Cache.url(url).success
       } catch {
         case e: MalformedURLException =>
-          ("Error parsing URL " + url + Option(e.getMessage).map(" (" + _ + ")").mkString).failure
+          ("Error parsing URL " + url + Option(e.getMessage).fold("")(" (" + _ + ")")).failure
+      }
+
+      validatedUrl.flatMap { url =>
+        Option(url.getUserInfo) match {
+          case None =>
+            repo.success
+          case Some(userInfo) =>
+            userInfo.split(":", 2) match {
+              case Array(user, password) =>
+                val baseUrl = new java.net.URL(
+                  url.getProtocol,
+                  url.getHost,
+                  url.getPort,
+                  url.getFile
+                ).toString
+
+                val repo0 = repo match {
+                  case m: MavenRepository =>
+                    m.copy(
+                      root = baseUrl,
+                      authentication = Some(Authentication(user, password))
+                    )
+                  case i: IvyRepository =>
+                    i.copy(
+                      pattern = baseUrl,
+                      authentication = Some(Authentication(user, password))
+                    )
+                  case r =>
+                    sys.error(s"Unrecognized repository: $r")
+                }
+
+                repo0.success
+
+              case _ =>
+                s"No password found in user info of URL $url".failure
+            }
+        }
       }
     }
 

cache/src/main/scala/coursier/FileError.scala

@@ -5,7 +5,9 @@ import java.io.File
 sealed abstract class FileError(
   val `type`: String,
   val message: String
-) extends Product with Serializable
+) extends Product with Serializable {
+  def describe: String = s"${`type`}: $message"
+}
 
 object FileError {
 
@@ -22,6 +24,14 @@ object FileError {
     file
   )
 
+  final case class Unauthorized(
+    file: String,
+    realm: Option[String]
+  ) extends FileError(
+    "unauthorized",
+    file + realm.fold("")(" (" + _ + ")")
+  )
+
   final case class ChecksumNotFound(
     sumType: String,
     file: String

cache/src/main/scala/coursier/Platform.scala

@@ -32,10 +32,10 @@ object Platform {
 
         new String(b, "UTF-8")
       } .leftMap{
-        case e: java.io.FileNotFoundException =>
+        case e: java.io.FileNotFoundException if e.getMessage != null =>
           s"Not found: ${e.getMessage}"
         case e =>
-          s"$e: ${e.getMessage}"
+          s"$e${Option(e.getMessage).fold("")(" (" + _ + ")")}"
       }
     }