diff --git a/bootstrap/src/main/java/coursier/Bootstrap.java b/bootstrap/src/main/java/coursier/Bootstrap.java index 4eb6236867..e754766041 100644 --- a/bootstrap/src/main/java/coursier/Bootstrap.java +++ b/bootstrap/src/main/java/coursier/Bootstrap.java @@ -186,7 +186,8 @@ public Thread newThread(Runnable r) { if (protocol.equals("file") || protocol.equals(bootstrapProtocol)) { localURLs.add(url); } else { - File dest = CachePath.localFile(url.toString(), cacheDir, null); + // fourth argument is false because we don't want to store local files when bootstrapping + File dest = CachePath.localFile(url.toString(), cacheDir, null, false); if (dest.exists()) { localURLs.add(dest.toURI().toURL()); @@ -200,7 +201,8 @@ public Thread newThread(Runnable r) { completionService.submit(new Callable() { @Override public URL call() throws Exception { - final File dest = CachePath.localFile(url.toString(), cacheDir, null); + // fourth argument is false because we don't want to store local files when bootstrapping + final File dest = CachePath.localFile(url.toString(), cacheDir, null, false); if (!dest.exists()) { FileOutputStream out = null; diff --git a/cache/jvm/src/main/scala/coursier/Cache.scala b/cache/jvm/src/main/scala/coursier/Cache.scala index ffa68312aa..ef6c41c9ae 100644 --- a/cache/jvm/src/main/scala/coursier/Cache.scala +++ b/cache/jvm/src/main/scala/coursier/Cache.scala @@ -37,8 +37,8 @@ object Cache { // Check SHA-1 if available, else be fine with no checksum val defaultChecksums = Seq(Some("SHA-1"), None) - def localFile(url: String, cache: File, user: Option[String]): File = - CachePath.localFile(url, cache, user.orNull) + def localFile(url: String, cache: File, user: Option[String], localArtifactsShouldBeCached: Boolean): File = + CachePath.localFile(url, cache, user.orNull, localArtifactsShouldBeCached) private def readFullyTo( in: InputStream, @@ -363,7 +363,8 @@ object Cache { cachePolicy: CachePolicy, pool: ExecutorService, logger: Option[Logger], - ttl: Option[Duration] + ttl: Option[Duration], + localArtifactsShouldBeCached: Boolean )(implicit S: Schedulable[F]): F[Seq[((File, String), Either[FileError, Unit])]] = { // Reference file - if it exists, and we get not found errors on some URLs, we assume @@ -371,7 +372,7 @@ object Cache { val referenceFileOpt = artifact .extra .get("metadata") - .map(a => localFile(a.url, cache, a.authentication.map(_.user))) + .map(a => localFile(a.url, cache, a.authentication.map(_.user), localArtifactsShouldBeCached)) def referenceFileExists: Boolean = referenceFileOpt.exists(_.exists()) @@ -779,7 +780,7 @@ object Cache { case Some(required) => cachePolicy0 match { case CachePolicy.LocalOnly | CachePolicy.LocalUpdateChanging | CachePolicy.LocalUpdate => - val file = localFile(required.url, cache, artifact.authentication.map(_.user)) + val file = localFile(required.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached) localInfo(file, required.url).flatMap { case true => EitherT(S.point[Either[FileError, Unit]](Right(()))) @@ -793,10 +794,10 @@ object Cache { val tasks = for (url <- urls) yield { - val file = localFile(url, cache, artifact.authentication.map(_.user)) + val file = localFile(url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached) def res = - if (url.startsWith("file:/")) { + if (url.startsWith("file:/") && !localArtifactsShouldBeCached) { // for debug purposes, flaky with URL-encoded chars anyway // def filtered(s: String) = // s.stripPrefix("file:/").stripPrefix("//").stripSuffix("/") @@ -879,15 +880,16 @@ object Cache { artifact: Artifact, sumType: String, cache: File, - pool: ExecutorService + pool: ExecutorService, + localArtifactsShouldBeCached: Boolean = false )(implicit S: Schedulable[F]): EitherT[F, FileError, Unit] = { - val localFile0 = localFile(artifact.url, cache, artifact.authentication.map(_.user)) + val localFile0 = localFile(artifact.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached) EitherT { artifact.checksumUrls.get(sumType) match { case Some(sumUrl) => - val sumFile = localFile(sumUrl, cache, artifact.authentication.map(_.user)) + val sumFile = localFile(sumUrl, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached) S.schedule(pool) { val sumOpt = parseRawChecksum(Files.readAllBytes(sumFile.toPath)) @@ -941,7 +943,8 @@ object Cache { logger: Option[Logger] = None, pool: ExecutorService = defaultPool, ttl: Option[Duration] = defaultTtl, - retry: Int = 1 + retry: Int = 1, + localArtifactsShouldBeCached: Boolean = false )(implicit S: Schedulable[F]): EitherT[F, FileError, File] = { val checksums0 = if (checksums.isEmpty) Seq(None) else checksums @@ -954,7 +957,8 @@ object Cache { cachePolicy, pool, logger = logger, - ttl = ttl + ttl = ttl, + localArtifactsShouldBeCached )) { results => val checksum = checksums0.find { case None => true @@ -982,7 +986,7 @@ object Cache { res.flatMap { case (f, None) => EitherT(S.point[Either[FileError, File]](Right(f))) case (f, Some(c)) => - validateChecksum(artifact, c, cache, pool).map(_ => f) + validateChecksum(artifact, c, cache, pool, localArtifactsShouldBeCached).map(_ => f) }.leftFlatMap { case err: FileError.WrongChecksum => if (retry <= 0) { @@ -991,7 +995,7 @@ object Cache { else { EitherT { S.schedule[Either[FileError, Unit]](pool) { - val badFile = localFile(artifact.url, cache, artifact.authentication.map(_.user)) + val badFile = localFile(artifact.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached) badFile.delete() logger.foreach(_.removedCorruptFile(artifact.url, badFile, Some(err))) Right(()) diff --git a/cli/src/main/scala-2.12/coursier/cli/Helper.scala b/cli/src/main/scala-2.12/coursier/cli/Helper.scala index a8b7baaa58..826434e838 100644 --- a/cli/src/main/scala-2.12/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.12/coursier/cli/Helper.scala @@ -295,7 +295,7 @@ class Helper( } }.toMap - val depsWithUrlRepo: FallbackDependenciesRepository = FallbackDependenciesRepository(depsWithUrls) + val depsWithUrlRepo: FallbackDependenciesRepository = FallbackDependenciesRepository(depsWithUrls, cacheFileArtifacts) // Prepend FallbackDependenciesRepository to the repository list // so that dependencies with URIs are resolved against this repo @@ -646,7 +646,8 @@ class Helper( logger = logger, pool = pool, ttl = ttl0, - retry = common.retryCount + retry = common.retryCount, + cacheFileArtifacts ) (file(cachePolicies.head) /: cachePolicies.tail)(_ orElse file(_)) diff --git a/cli/src/main/scala-2.12/coursier/cli/options/CommonOptions.scala b/cli/src/main/scala-2.12/coursier/cli/options/CommonOptions.scala index 01a1cad1d0..50aecad985 100644 --- a/cli/src/main/scala-2.12/coursier/cli/options/CommonOptions.scala +++ b/cli/src/main/scala-2.12/coursier/cli/options/CommonOptions.scala @@ -98,7 +98,11 @@ final case class CommonOptions( cacheOptions: CacheOptions = CacheOptions(), @Help("Retry limit for Checksum error when fetching a file") - retryCount: Int = 1 + retryCount: Int = 1, + + @Help("Flag that specifies if a local artifact should be cached.") + @Short("cfa") + cacheFileArtifacts: Boolean = false ) { val verbosityLevel = Tag.unwrap(verbose) - (if (quiet) 1 else 0) diff --git a/cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala b/cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala index d94d9ff181..477cbb919a 100644 --- a/cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala +++ b/cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala @@ -208,7 +208,8 @@ object Assembly { extraDependencies: Seq[String], options: CommonOptions, artifactTypes: Set[String], - checksumSeed: Array[Byte] = "v1".getBytes(UTF_8) + checksumSeed: Array[Byte] = "v1".getBytes(UTF_8), + localArtifactsShouldBeCached: Boolean = false ): Either[String, (File, Seq[File])] = { val helper = sparkJarsHelper(scalaVersion, sparkVersion, yarnVersion, default, extraDependencies, options) @@ -219,7 +220,7 @@ object Assembly { val checksums = artifacts.map { a => val f = a.checksumUrls.get("SHA-1") match { case Some(url) => - Cache.localFile(url, helper.cache, a.authentication.map(_.user)) + Cache.localFile(url, helper.cache, a.authentication.map(_.user), localArtifactsShouldBeCached) case None => throw new Exception(s"SHA-1 file not found for ${a.url}") } diff --git a/cli/src/test/scala-2.12/coursier/cli/CliFetchIntegrationTest.scala b/cli/src/test/scala-2.12/coursier/cli/CliFetchIntegrationTest.scala index 279c670e26..34228d3eb2 100644 --- a/cli/src/test/scala-2.12/coursier/cli/CliFetchIntegrationTest.scala +++ b/cli/src/test/scala-2.12/coursier/cli/CliFetchIntegrationTest.scala @@ -448,9 +448,9 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers { /** * Result: - * |└─ org.apache.commons:commons-compress:1.5 + * |└─ a:b:c */ - "local dep url" should "have coursier-fetch-test.jar" in withFile() { + "local file dep url" should "have coursier-fetch-test.jar and cached for second run" in withFile() { (jsonFile, _) => { withFile("tada", "coursier-fetch-test", ".jar") { (testFile, _) => { @@ -458,7 +458,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers { val encodedUrl = encode("file://" + path, "UTF-8") - val commonOpt = CommonOptions(jsonOutputFile = jsonFile.getPath) + val commonOpt = CommonOptions(jsonOutputFile = jsonFile.getPath, cacheFileArtifacts = true) val fetchOpt = FetchOptions(common = commonOpt) // fetch with encoded url set to temp jar @@ -466,25 +466,47 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers { fetchOpt, RemainingArgs( Seq( - "org.apache.commons:commons-compress:1.5,url=" + encodedUrl + "a:b:c,url=" + encodedUrl ), Seq() ) ) - val node: ReportNode = getReportFromJson(jsonFile) + val node1: ReportNode = getReportFromJson(jsonFile) - val depNodes: Seq[DepNode] = node.dependencies - .filter(_.coord == "org.apache.commons:commons-compress:1.5") + val depNodes1: Seq[DepNode] = node1.dependencies + .filter(_.coord == "a:b:c") .sortBy(fileNameLength) - assert(depNodes.length == 1) + assert(depNodes1.length == 1) - val urlInJsonFile = depNodes.head.file.get - assert(urlInJsonFile.contains(path)) + val urlInJsonFile1 = depNodes1.head.file.get + assert(urlInJsonFile1.contains(path)) // open jar and inspect contents - val fileContents = Source.fromFile(urlInJsonFile).getLines.mkString - assert(fileContents == "tada") + val fileContents1 = Source.fromFile(urlInJsonFile1).getLines.mkString + assert(fileContents1 == "tada") + + testFile.delete() + + Fetch.run( + fetchOpt, + RemainingArgs( + Seq( + "a:b:c,url=" + encodedUrl + ), + Seq() + ) + ) + + val node2: ReportNode = getReportFromJson(jsonFile) + + val depNodes2: Seq[DepNode] = node2.dependencies + .filter(_.coord == "a:b:c") + .sortBy(fileNameLength) + assert(depNodes2.length == 1) + + val urlInJsonFile2 = depNodes2.head.file.get + assert(urlInJsonFile2.contains("coursier/cache") && urlInJsonFile2.contains(testFile.toString)) } } } @@ -525,7 +547,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers { /** * Result: - * |└─ a:b:c + * |└─ h:i:j */ "external dep url with arbitrary coords" should "fetch junit-4.12.jar" in withFile() { (jsonFile, _) => { @@ -540,7 +562,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers { fetchOpt, RemainingArgs( Seq( - "a:b:c,url=" + externalUrl + "h:i:j,url=" + externalUrl ), Seq() ) @@ -549,7 +571,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers { val node: ReportNode = getReportFromJson(jsonFile) val depNodes: Seq[DepNode] = node.dependencies - .filter(_.coord == "a:b:c") + .filter(_.coord == "h:i:j") .sortBy(fileNameLength) assert(depNodes.length == 1) depNodes.head.file.map( f => assert(f.contains("junit/junit/4.12/junit-4.12.jar"))).orElse(fail("Not Defined")) diff --git a/extra/src/main/scala/coursier/FallbackDependenciesRepository.scala b/extra/src/main/scala/coursier/FallbackDependenciesRepository.scala index 405ad28a85..d63e14ecb4 100644 --- a/extra/src/main/scala/coursier/FallbackDependenciesRepository.scala +++ b/extra/src/main/scala/coursier/FallbackDependenciesRepository.scala @@ -7,7 +7,7 @@ import coursier.util.{EitherT, Monad} object FallbackDependenciesRepository { - def exists(url: URL): Boolean = { + def exists(url: URL, localArtifactsShouldBeCached: Boolean): Boolean = { // Sometimes HEAD attempts fail even though standard GETs are fine. // E.g. https://github.com/NetLogo/NetLogo/releases/download/5.3.1/NetLogo.jar @@ -15,8 +15,14 @@ object FallbackDependenciesRepository { val protocolSpecificAttemptOpt = { - def ifFile: Boolean = - new File(url.getPath).exists() // FIXME Escaping / de-escaping needed here? + def ifFile: Option[Boolean] = { + if (localArtifactsShouldBeCached && !new File(url.getPath).exists()) { + val cachePath = coursier.Cache.default + "/file" // Use '/file' here because the protocol becomes part of the cache path + Some(new File(cachePath, url.getPath).exists()) + } else { + Some(new File(url.getPath).exists()) // FIXME Escaping / de-escaping needed here? + } + } def ifHttp: Option[Boolean] = { // HEAD request attempt, adapted from http://stackoverflow.com/questions/22541629/android-how-can-i-make-an-http-head-request/22545275#22545275 @@ -41,7 +47,7 @@ object FallbackDependenciesRepository { } url.getProtocol match { - case "file" => Some(ifFile) + case "file" => ifFile case "http" | "https" => ifHttp case _ => None } @@ -71,7 +77,8 @@ object FallbackDependenciesRepository { } final case class FallbackDependenciesRepository( - fallbacks: Map[(Module, String), (URL, Boolean)] + fallbacks: Map[(Module, String), (URL, Boolean)], + localArtifactsShouldBeCached: Boolean = false ) extends Repository { private val source: Artifact.Source = @@ -113,7 +120,7 @@ final case class FallbackDependenciesRepository( else { val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1) - if (FallbackDependenciesRepository.exists(url)) { + if (FallbackDependenciesRepository.exists(url, localArtifactsShouldBeCached)) { val proj = Project( module, version, diff --git a/paths/src/main/java/coursier/CachePath.java b/paths/src/main/java/coursier/CachePath.java index 3d47122b6d..e854ae7e1e 100644 --- a/paths/src/main/java/coursier/CachePath.java +++ b/paths/src/main/java/coursier/CachePath.java @@ -36,14 +36,14 @@ private static boolean isUnsafe(char ch) { return ch > 128 || " %$&+,:;=?@<>#%".indexOf(ch) >= 0; } - public static File localFile(String url, File cache, String user) throws MalformedURLException { + public static File localFile(String url, File cache, String user, boolean localArtifactsShouldBeCached) throws MalformedURLException { // use the File constructor accepting a URI in case of problem with the two cases below? - if (url.startsWith("file:///")) + if (url.startsWith("file:///") && !localArtifactsShouldBeCached) return new File(url.substring("file://".length())); - if (url.startsWith("file:/")) + if (url.startsWith("file:/") && !localArtifactsShouldBeCached) return new File(url.substring("file:".length())); String[] split = url.split(":", 2);