Permalink
Browse files

Add json report to fetch and local exclusion option (#692)

This patch introduces changes for cli with json output #659. Format as follows:
```
{
  "conflict_resolution": {
    "org:name:version" (requested): "org:name:version" (reconciled) 
  },
  "dependencies": [
    {
      "coord": "orgA:nameA:versionA",
      "files": [
        [
          <classifier>,
          <path>
        ]
      ],
      "dependencies": [ // coodinates for its transitive dependencies
        <orgX:nameX:versionX>,
        <orgY:nameY:versionY>,
      ]
    },
    {
      "coord": "orgB:nameB:versionB",
      "files": [
        [
          <classifier>,
          <path>
        ]
      ],
      "dependencies": [ // coodinates for its transitive dependencies
        <orgX:nameX:versionX>,
        <orgZ:nameZ:versionZ>,
      ]
    },
  ]
}
```
For example:
```
fetch -t org.apache.avro:trevni-avro:1.8.2  org.slf4j:slf4j-api:1.7.6 --json-output-file x.out
  Result:
├─ org.apache.avro:trevni-avro:1.8.2
│  ├─ org.apache.avro:trevni-core:1.8.2
│  │  ├─ org.apache.commons:commons-compress:1.8.1
│  │  ├─ org.slf4j:slf4j-api:1.7.7
│  │  └─ org.xerial.snappy:snappy-java:1.1.1.3
│  └─ org.slf4j:slf4j-api:1.7.7
└─ org.slf4j:slf4j-api:1.7.6 -> 1.7.7
```
would produce the following json file:
```
$ jq < x.out 
{
  "conflict_resolution": {
    "org.slf4j:slf4j-api:1.7.6": "org.slf4j:slf4j-api:1.7.7"
  },
  "dependencies": [
    {
      "coord": "org.apache.avro:trevni-core:1.8.2",
      "files": [
        [
          "",
          "/Users/yic/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/apache/avro/trevni-core/1.8.2/trevni-core-1.8.2.jar"
        ]
      ],
      "dependencies": [
        "org.slf4j:slf4j-api:1.7.7",
        "org.xerial.snappy:snappy-java:1.1.1.3",
        "org.apache.commons:commons-compress:1.8.1"
      ]
    },
    {
      "coord": "org.apache.avro:trevni-avro:1.8.2",
      "files": [
        [
          "",
          "/Users/yic/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/apache/avro/trevni-avro/1.8.2/trevni-avro-1.8.2.jar"
        ]
      ],
      "dependencies": [
        "org.apache.avro:trevni-core:1.8.2",
        "org.slf4j:slf4j-api:1.7.7",
        "org.xerial.snappy:snappy-java:1.1.1.3",
        "org.apache.commons:commons-compress:1.8.1"
      ]
    },
    {
      "coord": "org.slf4j:slf4j-api:1.7.7",
      "files": [
        [
          "",
          "/Users/yic/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.7/slf4j-api-1.7.7.jar"
        ]
      ],
      "dependencies": []
    },
    {
      "coord": "org.apache.commons:commons-compress:1.8.1",
      "files": [
        [
          "",
          "/Users/yic/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/apache/commons/commons-compress/1.8.1/commons-compress-1.8.1.jar"
        ]
      ],
      "dependencies": []
    },
    {
      "coord": "org.xerial.snappy:snappy-java:1.1.1.3",
      "files": [
        [
          "",
          "/Users/yic/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/xerial/snappy/snappy-java/1.1.1.3/snappy-java-1.1.1.3.jar"
        ]
      ],
      "dependencies": []
    }
  ]
}

```
  • Loading branch information...
wisechengyi committed Dec 26, 2017
1 parent b21042a commit a4258f48ce7e9a3b432dc5227ab9cd2dfdc2471a
View
@@ -72,3 +72,5 @@ cache:
- $HOME/.ivy2/cache
- $HOME/.sbt
- $HOME/.coursier
# Pants cache
- $HOME/.cache
View
@@ -110,17 +110,6 @@ jar_library(
],
)
jar_library(
name = "jackson-module-scala",
jars = [
scala_jar(
name = "jackson-module-scala",
org = "com.fasterxml.jackson.module",
rev = "2.8.4",
),
],
)
jar_library(
name = "caseapp",
jars = [
@@ -169,3 +158,32 @@ jar_library(
),
],
)
jar_library(
name = "utest",
jars = [
scala_jar(
name = "utest",
org = "com.lihaoyi",
rev = "0.5.4",
),
],
)
jar_library(
name = "async",
jars = [
scala_jar(
name = "scala-async",
org = "org.scala-lang.modules",
rev = "0.9.7",
),
],
)
jar_library(
name = "scalatest",
jars = [
scala_jar("org.scalatest", "scalatest", "3.0.0"),
],
)
View
@@ -153,7 +153,9 @@ lazy val cli = project
if (scalaBinaryVersion.value == "2.11")
Seq(
Deps.caseApp,
Deps.argonautShapeless
Deps.argonautShapeless,
Deps.junit % "test", // to be able to run tests with pants
Deps.scalatest % "test"
)
else
Seq()
@@ -3,8 +3,8 @@ scala_library(
dependencies = [
"3rdparty/jvm:argonaut-shapeless",
"3rdparty/jvm:caseapp",
"core:core",
"cache/src/main/scala:cache",
"core:core",
"extra/src/main/scala/coursier/extra:extra",
"extra/src/main/scala-2.11/coursier/extra:native",
],
@@ -7,7 +7,7 @@ import caseapp._
import scala.language.reflectiveCalls
final case class Fetch(
case class Fetch(
@Recurse
options: FetchOptions
) extends App {
@@ -1,23 +1,22 @@
package coursier
package cli
import java.io.{ OutputStreamWriter, File }
import java.net.{ URL, URLClassLoader }
import java.util.jar.{ Manifest => JManifest }
import java.io.{File, OutputStreamWriter, PrintWriter}
import java.net.{URL, URLClassLoader}
import java.util.concurrent.Executors
import java.util.jar.{Manifest => JManifest}
import coursier.cli.scaladex.Scaladex
import coursier.cli.util.{JsonElem, JsonPrintRequirement, JsonReport}
import coursier.extra.Typelevel
import coursier.ivy.IvyRepository
import coursier.util.{Print, Parse}
import coursier.util.{Parse, Print}
import scala.annotation.tailrec
import scala.concurrent.duration.Duration
import scala.util.Try
import scalaz.concurrent.{Strategy, Task}
import scalaz.{-\/, Failure, Nondeterminism, Success, \/-}
import scalaz.{Failure, Nondeterminism, Success, \/-, -\/}
import scalaz.concurrent.{ Task, Strategy }
import scalaz.std.list._
object Helper {
def fileRepr(f: File) = f.toString
@@ -86,7 +85,6 @@ class Helper(
) {
import common._
import Helper.errPrintln
import Util._
val ttl0 =
@@ -315,18 +313,40 @@ class Helper(
.mkString("\n")
}
val excludes = excludesNoAttr.map { mod =>
val excludes: Set[(String, String)] = excludesNoAttr.map { mod =>
(mod.organization, mod.name)
}.toSet
val localExcludeMap: Map[String, Set[(String, String)]] =
if (localExcludeFile.isEmpty) {
Map()
} else {
val source = scala.io.Source.fromFile(localExcludeFile)
val lines = try source.mkString.split("\n") finally source.close()
lines.map({ str =>
val parent_and_child = str.split("--")
if (parent_and_child.length != 2) {
throw SoftExcludeParsingException(s"Failed to parse $str")
}
val child_org_name = parent_and_child(1).split(":")
if (child_org_name.length != 2) {
throw SoftExcludeParsingException(s"Failed to parse $child_org_name")
}
(parent_and_child(0), (child_org_name(0), child_org_name(1)))
}).groupBy(_._1).mapValues(_.map(_._2).toSet).toMap
}
val baseDependencies = allModuleVersionConfigs.map {
case (module, version, configOpt) =>
Dependency(
module,
version,
attributes = Attributes("", ""),
configuration = configOpt.getOrElse(defaultConfiguration),
exclusions = excludes
exclusions = localExcludeMap.getOrElse(module.orgName, Set()) | excludes
)
}
@@ -611,17 +631,9 @@ class Helper(
val res0 = Option(subset).fold(res)(res.subset)
val artifacts0 =
if (classifier0.nonEmpty || sources || javadoc) {
var classifiers = classifier0
if (sources)
classifiers = classifiers + "sources"
if (javadoc)
classifiers = classifiers + "javadoc"
val depArtTuples: Seq[(Dependency, Artifact)] = getDepArtifactsForClassifier(sources, javadoc, res0)
res0.dependencyClassifiersArtifacts(classifiers.toVector.sorted).map(_._2)
} else
res0.dependencyArtifacts(withOptional = true).map(_._2)
val artifacts0 = depArtTuples.map(_._2)
if (artifactTypes("*"))
artifacts0
@@ -631,6 +643,20 @@ class Helper(
}
}
private def getDepArtifactsForClassifier(sources: Boolean, javadoc: Boolean, res0: Resolution): Seq[(Dependency, Artifact)] = {
if (classifier0.nonEmpty || sources || javadoc) {
var classifiers = classifier0
if (sources)
classifiers = classifiers + "sources"
if (javadoc)
classifiers = classifiers + "javadoc"
//TODO: this function somehow gives duplicated things
res0.dependencyClassifiersArtifacts(classifiers.toVector.sorted)
} else {
res0.dependencyArtifacts(withOptional = true)
}
}
def fetch(
sources: Boolean,
javadoc: Boolean,
@@ -690,8 +716,10 @@ class Helper(
a.isOptional && notFound
}
val artifactToFile: collection.mutable.Map[String, File] = collection.mutable.Map()
val files0 = results.collect {
case (artifact, \/-(f)) =>
case (artifact: Artifact, \/-(f)) =>
artifactToFile.put(artifact.url, f)
f
}
@@ -718,6 +746,37 @@ class Helper(
.mkString("\n")
}
val depToArtifacts: Map[Dependency, Vector[Artifact]] =
getDepArtifactsForClassifier(sources, javadoc, res).groupBy(_._1).mapValues(_.map(_._2).toVector)
if (!jsonOutputFile.isEmpty) {
// TODO(wisechengyi): This is not exactly the root dependencies we are asking for on the command line, but it should be
// a strict super set.
val deps: Seq[Dependency] = Set(getDepArtifactsForClassifier(sources, javadoc, res).map(_._1): _*).toSeq
// A map from requested org:name:version to reconciled org:name:version
val conflictResolutionForRoots: Map[String, String] = dependencies.map({ dep =>
val reconciledVersion: String = res.reconciledVersions
.getOrElse(dep.module, dep.version)
if (reconciledVersion != dep.version) {
Option((s"${dep.module}:${dep.version}", s"${dep.module}:$reconciledVersion"))
}
else {
Option.empty
}
}).filter(_.isDefined).map(_.get).toMap
val artifacts: Seq[(Dependency, Artifact)] = res.dependencyArtifacts
val jsonReq = JsonPrintRequirement(artifactToFile, depToArtifacts, conflictResolutionForRoots)
val roots = deps.toVector.map(JsonElem(_, artifacts, Option(jsonReq), res, printExclusions = verbosityLevel >= 1, excluded = false, colors = false))
val jsonStr = JsonReport(roots, jsonReq.conflictResolutionForRoots)(_.children, _.reconciledVersionStr, _.requestedVersionStr, _.downloadedFiles)
val pw = new PrintWriter(new File(jsonOutputFile))
pw.write(jsonStr)
pw.close()
}
files0
}
@@ -849,3 +908,7 @@ class Helper(
mainClass
}
}
case class SoftExcludeParsingException(private val message: String = "",
private val cause: Throwable = None.orNull)
extends Exception(message, cause)
@@ -49,7 +49,16 @@ final case class CommonOptions(
@Help("Exclude module")
@Value("organization:name")
@Short("E")
@Help("Global level exclude")
exclude: List[String] = Nil,
@Short("x")
@Help("Path to the local exclusion file. " +
"Syntax: <org:name>--<org:name>. `--` means minus. Example file content:\n\t" +
"\tcom.twitter.penguin:korean-text--com.twitter:util-tunable-internal_2.11\n\t" +
"\torg.apache.commons:commons-math--com.twitter.search:core-query-nodes\n\t" +
"Behavior: If root module A excludes module X, but root module B requires X, module X will still be fetched.")
localExcludeFile: String = "",
@Help("Default scala version")
@Short("e")
scalaVersion: String = scala.util.Properties.versionNumberString,
@@ -83,6 +92,11 @@ final case class CommonOptions(
@Value("profile")
@Short("F")
profile: List[String] = Nil,
@Help("Specify path for json output")
@Short("j")
jsonOutputFile: String = "",
@Help("Swap the mainline Scala JARs by Typelevel ones")
typelevel: Boolean = false,
@Recurse
@@ -169,7 +183,7 @@ final case class IsolatedLoaderOptions(
}
object ArtifactOptions {
def defaultArtifactTypes = Set("jar", "bundle")
def defaultArtifactTypes = Set("jar", "bundle", "test-jar")
}
final case class ArtifactOptions(
Oops, something went wrong.

0 comments on commit a4258f4

Please sign in to comment.