Copy local artifact to cache (#831)

bootstrap/src/main/java/coursier/Bootstrap.java

@@ -186,7 +186,8 @@ public Thread newThread(Runnable r) {
             if (protocol.equals("file") || protocol.equals(bootstrapProtocol)) {
                 localURLs.add(url);
             } else {
-                File dest = CachePath.localFile(url.toString(), cacheDir, null);
+                // fourth argument is false because we don't want to store local files when bootstrapping
+                File dest = CachePath.localFile(url.toString(), cacheDir, null, false);
 
                 if (dest.exists()) {
                     localURLs.add(dest.toURI().toURL());
@@ -200,7 +201,8 @@ public Thread newThread(Runnable r) {
             completionService.submit(new Callable<URL>() {
                 @Override
                 public URL call() throws Exception {
-                    final File dest = CachePath.localFile(url.toString(), cacheDir, null);
+                    // fourth argument is false because we don't want to store local files when bootstrapping
+                    final File dest = CachePath.localFile(url.toString(), cacheDir, null, false);
 
                     if (!dest.exists()) {
                         FileOutputStream out = null;

cache/jvm/src/main/scala/coursier/Cache.scala

@@ -37,8 +37,8 @@ object Cache {
   // Check SHA-1 if available, else be fine with no checksum
   val defaultChecksums = Seq(Some("SHA-1"), None)
 
-  def localFile(url: String, cache: File, user: Option[String]): File =
-    CachePath.localFile(url, cache, user.orNull)
+  def localFile(url: String, cache: File, user: Option[String], localArtifactsShouldBeCached: Boolean): File =
+    CachePath.localFile(url, cache, user.orNull, localArtifactsShouldBeCached)
 
   private def readFullyTo(
     in: InputStream,
@@ -363,15 +363,16 @@ object Cache {
     cachePolicy: CachePolicy,
     pool: ExecutorService,
     logger: Option[Logger],
-    ttl: Option[Duration]
+    ttl: Option[Duration],
+    localArtifactsShouldBeCached: Boolean
   )(implicit S: Schedulable[F]): F[Seq[((File, String), Either[FileError, Unit])]] = {
 
     // Reference file - if it exists, and we get not found errors on some URLs, we assume
     // we can keep track of these missing, and not try to get them again later.
     val referenceFileOpt = artifact
       .extra
       .get("metadata")
-      .map(a => localFile(a.url, cache, a.authentication.map(_.user)))
+      .map(a => localFile(a.url, cache, a.authentication.map(_.user), localArtifactsShouldBeCached))
 
     def referenceFileExists: Boolean = referenceFileOpt.exists(_.exists())
 
@@ -779,7 +780,7 @@ object Cache {
       case Some(required) =>
         cachePolicy0 match {
           case CachePolicy.LocalOnly | CachePolicy.LocalUpdateChanging | CachePolicy.LocalUpdate =>
-            val file = localFile(required.url, cache, artifact.authentication.map(_.user))
+            val file = localFile(required.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
             localInfo(file, required.url).flatMap {
               case true =>
                 EitherT(S.point[Either[FileError, Unit]](Right(())))
@@ -793,10 +794,10 @@ object Cache {
 
     val tasks =
       for (url <- urls) yield {
-        val file = localFile(url, cache, artifact.authentication.map(_.user))
+        val file = localFile(url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
 
         def res =
-          if (url.startsWith("file:/")) {
+          if (url.startsWith("file:/") && !localArtifactsShouldBeCached) {
             // for debug purposes, flaky with URL-encoded chars anyway
             // def filtered(s: String) =
             //   s.stripPrefix("file:/").stripPrefix("//").stripSuffix("/")
@@ -879,15 +880,16 @@ object Cache {
     artifact: Artifact,
     sumType: String,
     cache: File,
-    pool: ExecutorService
+    pool: ExecutorService,
+    localArtifactsShouldBeCached: Boolean = false
   )(implicit S: Schedulable[F]): EitherT[F, FileError, Unit] = {
 
-    val localFile0 = localFile(artifact.url, cache, artifact.authentication.map(_.user))
+    val localFile0 = localFile(artifact.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
 
     EitherT {
       artifact.checksumUrls.get(sumType) match {
         case Some(sumUrl) =>
-          val sumFile = localFile(sumUrl, cache, artifact.authentication.map(_.user))
+          val sumFile = localFile(sumUrl, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
 
           S.schedule(pool) {
             val sumOpt = parseRawChecksum(Files.readAllBytes(sumFile.toPath))
@@ -941,7 +943,8 @@ object Cache {
     logger: Option[Logger] = None,
     pool: ExecutorService = defaultPool,
     ttl: Option[Duration] = defaultTtl,
-    retry: Int = 1
+    retry: Int = 1,
+    localArtifactsShouldBeCached: Boolean = false
   )(implicit S: Schedulable[F]): EitherT[F, FileError, File] = {
 
     val checksums0 = if (checksums.isEmpty) Seq(None) else checksums
@@ -954,7 +957,8 @@ object Cache {
         cachePolicy,
         pool,
         logger = logger,
-        ttl = ttl
+        ttl = ttl,
+        localArtifactsShouldBeCached
       )) { results =>
         val checksum = checksums0.find {
           case None => true
@@ -982,7 +986,7 @@ object Cache {
     res.flatMap {
       case (f, None) => EitherT(S.point[Either[FileError, File]](Right(f)))
       case (f, Some(c)) =>
-        validateChecksum(artifact, c, cache, pool).map(_ => f)
+        validateChecksum(artifact, c, cache, pool, localArtifactsShouldBeCached).map(_ => f)
     }.leftFlatMap {
       case err: FileError.WrongChecksum =>
         if (retry <= 0) {
@@ -991,7 +995,7 @@ object Cache {
         else {
           EitherT {
             S.schedule[Either[FileError, Unit]](pool) {
-              val badFile = localFile(artifact.url, cache, artifact.authentication.map(_.user))
+              val badFile = localFile(artifact.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
               badFile.delete()
               logger.foreach(_.removedCorruptFile(artifact.url, badFile, Some(err)))
               Right(())

cli/src/main/scala-2.12/coursier/cli/Helper.scala

@@ -295,7 +295,7 @@ class Helper(
         }
     }.toMap
 
-  val depsWithUrlRepo: FallbackDependenciesRepository = FallbackDependenciesRepository(depsWithUrls)
+  val depsWithUrlRepo: FallbackDependenciesRepository = FallbackDependenciesRepository(depsWithUrls, cacheFileArtifacts)
 
   // Prepend FallbackDependenciesRepository to the repository list
   // so that dependencies with URIs are resolved against this repo
@@ -646,7 +646,8 @@ class Helper(
         logger = logger,
         pool = pool,
         ttl = ttl0,
-        retry = common.retryCount
+        retry = common.retryCount,
+        cacheFileArtifacts
       )
 
       (file(cachePolicies.head) /: cachePolicies.tail)(_ orElse file(_))

cli/src/main/scala-2.12/coursier/cli/options/CommonOptions.scala

@@ -98,7 +98,11 @@ final case class CommonOptions(
     cacheOptions: CacheOptions = CacheOptions(),
 
   @Help("Retry limit for Checksum error when fetching a file")
-    retryCount: Int = 1
+    retryCount: Int = 1,
+
+  @Help("Flag that specifies if a local artifact should be cached.")
+  @Short("cfa")
+    cacheFileArtifacts: Boolean = false
 
 ) {
   val verbosityLevel = Tag.unwrap(verbose) - (if (quiet) 1 else 0)

cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala

@@ -208,7 +208,8 @@ object Assembly {
     extraDependencies: Seq[String],
     options: CommonOptions,
     artifactTypes: Set[String],
-    checksumSeed: Array[Byte] = "v1".getBytes(UTF_8)
+    checksumSeed: Array[Byte] = "v1".getBytes(UTF_8),
+    localArtifactsShouldBeCached: Boolean = false
   ): Either[String, (File, Seq[File])] = {
 
     val helper = sparkJarsHelper(scalaVersion, sparkVersion, yarnVersion, default, extraDependencies, options)
@@ -219,7 +220,7 @@ object Assembly {
     val checksums = artifacts.map { a =>
       val f = a.checksumUrls.get("SHA-1") match {
         case Some(url) =>
-          Cache.localFile(url, helper.cache, a.authentication.map(_.user))
+          Cache.localFile(url, helper.cache, a.authentication.map(_.user), localArtifactsShouldBeCached)
         case None =>
           throw new Exception(s"SHA-1 file not found for ${a.url}")
       }

cli/src/test/scala-2.12/coursier/cli/CliFetchIntegrationTest.scala

@@ -448,43 +448,65 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
 
   /**
    * Result:
-   * |└─ org.apache.commons:commons-compress:1.5
+   * |└─ a:b:c
    */
-  "local dep url" should "have coursier-fetch-test.jar" in withFile() {
+  "local file dep url" should "have coursier-fetch-test.jar and cached for second run" in withFile() {
     (jsonFile, _) => {
       withFile("tada", "coursier-fetch-test", ".jar") {
         (testFile, _) => {
           val path = testFile.getAbsolutePath
           val encodedUrl = encode("file://" + path, "UTF-8")
 
 
-          val commonOpt = CommonOptions(jsonOutputFile = jsonFile.getPath)
+          val commonOpt = CommonOptions(jsonOutputFile = jsonFile.getPath, cacheFileArtifacts = true)
           val fetchOpt = FetchOptions(common = commonOpt)
 
           // fetch with encoded url set to temp jar
           Fetch.run(
             fetchOpt,
             RemainingArgs(
               Seq(
-                "org.apache.commons:commons-compress:1.5,url=" + encodedUrl
+                "a:b:c,url=" + encodedUrl
               ),
               Seq()
             )
           )
 
-          val node: ReportNode = getReportFromJson(jsonFile)
+          val node1: ReportNode = getReportFromJson(jsonFile)
 
-          val depNodes: Seq[DepNode] = node.dependencies
-            .filter(_.coord == "org.apache.commons:commons-compress:1.5")
+          val depNodes1: Seq[DepNode] = node1.dependencies
+            .filter(_.coord == "a:b:c")
             .sortBy(fileNameLength)
-          assert(depNodes.length == 1)
+          assert(depNodes1.length == 1)
 
-          val urlInJsonFile = depNodes.head.file.get
-          assert(urlInJsonFile.contains(path))
+          val urlInJsonFile1 = depNodes1.head.file.get
+          assert(urlInJsonFile1.contains(path))
 
           // open jar and inspect contents
-          val fileContents = Source.fromFile(urlInJsonFile).getLines.mkString
-          assert(fileContents == "tada")
+          val fileContents1 = Source.fromFile(urlInJsonFile1).getLines.mkString
+          assert(fileContents1 == "tada")
+
+          testFile.delete()
+
+          Fetch.run(
+            fetchOpt,
+            RemainingArgs(
+              Seq(
+                "a:b:c,url=" + encodedUrl
+              ),
+              Seq()
+            )
+          )
+
+          val node2: ReportNode = getReportFromJson(jsonFile)
+
+          val depNodes2: Seq[DepNode] = node2.dependencies
+            .filter(_.coord == "a:b:c")
+            .sortBy(fileNameLength)
+          assert(depNodes2.length == 1)
+
+          val urlInJsonFile2 = depNodes2.head.file.get
+          assert(urlInJsonFile2.contains("coursier/cache") && urlInJsonFile2.contains(testFile.toString))
         }
       }
     }
@@ -525,7 +547,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
 
   /**
    * Result:
-   * |└─ a:b:c
+   * |└─ h:i:j
    */
   "external dep url with arbitrary coords" should "fetch junit-4.12.jar" in withFile() {
     (jsonFile, _) => {
@@ -540,7 +562,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
         fetchOpt,
         RemainingArgs(
           Seq(
-            "a:b:c,url=" + externalUrl
+            "h:i:j,url=" + externalUrl
           ),
           Seq()
         )
@@ -549,7 +571,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
       val node: ReportNode = getReportFromJson(jsonFile)
 
       val depNodes: Seq[DepNode] = node.dependencies
-        .filter(_.coord == "a:b:c")
+        .filter(_.coord == "h:i:j")
         .sortBy(fileNameLength)
       assert(depNodes.length == 1)
       depNodes.head.file.map( f => assert(f.contains("junit/junit/4.12/junit-4.12.jar"))).orElse(fail("Not Defined"))

extra/src/main/scala/coursier/FallbackDependenciesRepository.scala

@@ -7,16 +7,22 @@ import coursier.util.{EitherT, Monad}
 
 object FallbackDependenciesRepository {
 
-  def exists(url: URL): Boolean = {
+  def exists(url: URL, localArtifactsShouldBeCached: Boolean): Boolean = {
 
     // Sometimes HEAD attempts fail even though standard GETs are fine.
     // E.g. https://github.com/NetLogo/NetLogo/releases/download/5.3.1/NetLogo.jar
     // returning 403s. Hence the second attempt below.
 
     val protocolSpecificAttemptOpt = {
 
-      def ifFile: Boolean =
-        new File(url.getPath).exists() // FIXME Escaping / de-escaping needed here?
+      def ifFile: Option[Boolean] = {
+        if (localArtifactsShouldBeCached && !new File(url.getPath).exists()) {
+          val cachePath = coursier.Cache.default + "/file"  // Use '/file' here because the protocol becomes part of the cache path
+          Some(new File(cachePath, url.getPath).exists())
+        } else {
+          Some(new File(url.getPath).exists()) // FIXME Escaping / de-escaping needed here?
+        }
+      }
 
       def ifHttp: Option[Boolean] = {
         // HEAD request attempt, adapted from http://stackoverflow.com/questions/22541629/android-how-can-i-make-an-http-head-request/22545275#22545275
@@ -41,7 +47,7 @@ object FallbackDependenciesRepository {
       }
 
       url.getProtocol match {
-        case "file"           => Some(ifFile)
+        case "file"           => ifFile
         case "http" | "https" => ifHttp
         case _                => None
       }
@@ -71,7 +77,8 @@ object FallbackDependenciesRepository {
 }
 
 final case class FallbackDependenciesRepository(
-  fallbacks: Map[(Module, String), (URL, Boolean)]
+  fallbacks: Map[(Module, String), (URL, Boolean)],
+  localArtifactsShouldBeCached: Boolean = false
 ) extends Repository {
 
   private val source: Artifact.Source =
@@ -113,7 +120,7 @@ final case class FallbackDependenciesRepository(
           else {
             val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1)
 
-            if (FallbackDependenciesRepository.exists(url)) {
+            if (FallbackDependenciesRepository.exists(url, localArtifactsShouldBeCached)) {
               val proj = Project(
                 module,
                 version,

paths/src/main/java/coursier/CachePath.java

@@ -36,14 +36,14 @@ private static boolean isUnsafe(char ch) {
         return ch > 128 || " %$&+,:;=?@<>#%".indexOf(ch) >= 0;
     }
 
-    public static File localFile(String url, File cache, String user) throws MalformedURLException {
+    public static File localFile(String url, File cache, String user, boolean localArtifactsShouldBeCached) throws MalformedURLException {
 
         // use the File constructor accepting a URI in case of problem with the two cases below?
 
-        if (url.startsWith("file:///"))
+        if (url.startsWith("file:///") && !localArtifactsShouldBeCached)
             return new File(url.substring("file://".length()));
 
-        if (url.startsWith("file:/"))
+        if (url.startsWith("file:/") && !localArtifactsShouldBeCached)
             return new File(url.substring("file:".length()));
 
         String[] split = url.split(":", 2);