Enable cross-repo navigation.

build.sbt

@@ -15,6 +15,7 @@ lazy val V =
     def scala213 = "2.13.4"
     def scala212 = "2.12.12"
     def scalameta = "4.4.8"
+    def requests = "0.6.5"
   }
 
 inThisBuild(
@@ -102,7 +103,7 @@ lazy val plugin = project
     fatjarPackageSettings,
     javaOnlySettings,
     moduleName := "semanticdb-javac",
-    javaToolchainVersion := "1.8",
+    javaToolchainVersion := "8",
     assemblyShadeRules.in(assembly) :=
       Seq(
         ShadeRule
@@ -120,7 +121,7 @@ lazy val lsif = project
   .in(file("lsif-semanticdb"))
   .settings(
     moduleName := "lsif-semanticdb",
-    javaToolchainVersion := "1.8",
+    javaToolchainVersion := "8",
     javaOnlySettings,
     libraryDependencies +=
       "com.google.protobuf" % "protobuf-java-util" % V.protobuf,
@@ -146,6 +147,9 @@ lazy val cli = project
     buildInfoPackage := "com.sourcegraph.lsif_java",
     libraryDependencies ++=
       List(
+        "io.get-coursier" %% "coursier" % V.coursier,
+        "org.scala-lang.modules" %% "scala-xml" % "2.0.0-RC1",
+        "com.lihaoyi" %% "requests" % V.requests,
         "org.scalameta" %% "moped" % V.moped,
         "org.scalameta" %% "ascii-graphs" % "0.1.2"
       ),
@@ -299,7 +303,6 @@ lazy val bench = project
   .in(file("tests/benchmarks"))
   .settings(
     moduleName := "lsif-java-bench",
-    javaToolchainVersion := "1.8",
     fork.in(run) := true,
     skip.in(publish) := true
   )

lsif-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala

@@ -9,6 +9,15 @@ import java.nio.file.attribute.BasicFileAttributes
 
 class DeleteVisitor(deleteFile: Path => Boolean = _ => true)
     extends SimpleFileVisitor[Path] {
+  override def preVisitDirectory(
+      dir: Path,
+      attrs: BasicFileAttributes
+  ): FileVisitResult = {
+    if (!deleteFile(dir))
+      FileVisitResult.SKIP_SUBTREE
+    else
+      super.preVisitDirectory(dir, attrs)
+  }
   override def visitFile(
       file: Path,
       attrs: BasicFileAttributes

tests/unit/src/main/scala/tests/Dependencies.scala → lsif-java/src/main/scala/com/sourcegraph/lsif_java/Dependencies.scala

@@ -1,14 +1,14 @@
-package tests
+package com.sourcegraph.lsif_java
 
 import java.io.File
-import java.nio.file.FileSystems
+import java.nio.file.Path
 
 import scala.concurrent.duration.Duration
 import scala.xml.XML
 
-import scala.meta.io.AbsolutePath
-
+import com.sourcegraph.lsif_java.BuildInfo
 import coursier.Fetch
+import coursier.Repositories
 import coursier.Resolve
 import coursier.cache.Cache
 import coursier.cache.CachePolicy
@@ -18,9 +18,12 @@ import coursier.parse.DependencyParser
 import coursier.util.Task
 
 case class Dependencies(
-    sources: Seq[AbsolutePath],
-    classpath: Seq[AbsolutePath]
+    dependencies: List[Dependency],
+    sourcesResult: Fetch.Result,
+    classpathResult: Fetch.Result
 ) {
+  val sources: Seq[Path] = sourcesResult.files.map(_.toPath())
+  val classpath: Seq[Path] = classpathResult.files.map(_.toPath())
   def classpathSyntax: String = classpath.mkString(File.pathSeparator)
 }
 
@@ -30,34 +33,46 @@ object Dependencies {
     .withCachePolicies(List(CachePolicy.LocalOnly, CachePolicy.Update))
     .withTtl(Duration.Inf)
     .withChecksums(Nil)
-  private val jarPattern = FileSystems.getDefault.getPathMatcher("glob:**.jar")
+  private val defaultExtraRepositories = List[Repository](
+    Repositories.google,
+    Repositories.clojars,
+    Repositories.jitpack,
+    Repositories.centralGcs
+  )
 
   def resolveDependencies(
       dependencies: List[String],
-      repos: List[Repository]
+      transitive: Boolean = true
   ): Dependencies = {
     val deps = dependencies.map(parseDependency)
-    val provided = deps.flatMap(d => resolveProvidedDeps(d, repos))
+    val provided = deps.flatMap(d => resolveProvidedDeps(d))
+    def nonTransitiveDeps = deps.map(_.withTransitive(false))
     val fetch = Fetch[Task](Cache.default)
       .addDependencies(deps: _*)
       .addDependencies(provided: _*)
-      .addRepositories(repos: _*)
+      .addRepositories(defaultExtraRepositories: _*)
 
-    val classpath = fetch.run()
-    val sources = fetch.withClassifiers(Set(Classifier.sources)).run()
+    val classpath = fetch.runResult()
+    val sources = fetch
+      .withDependencies(
+        if (transitive)
+          fetch.dependencies
+        else
+          nonTransitiveDeps
+      )
+      .withClassifiers(Set(Classifier.sources))
+      .runResult()
     Dependencies(
-      sources = sources.map(AbsolutePath(_)),
-      classpath = classpath.map(AbsolutePath(_))
+      dependencies = deps,
+      sourcesResult = sources,
+      classpathResult = classpath
     )
   }
 
-  private def resolveProvidedDeps(
-      dep: Dependency,
-      repos: List[Repository]
-  ): Seq[Dependency] = {
+  def resolveProvidedDeps(dep: Dependency): Seq[Dependency] = {
     val artifacts = Resolve[Task](Cache.default)
       .addDependencies(dep)
-      .addRepositories(repos: _*)
+      .addRepositories(defaultExtraRepositories: _*)
       .run()
       .artifacts()
     for {
@@ -83,10 +98,13 @@ object Dependencies {
     )
   }.toList
 
-  private def parseDependency(lib: String): Dependency = {
-    val dep = DependencyParser
+  def parseDependencyEither(lib: String): Either[String, Dependency] = {
+    DependencyParser
       .dependency(lib, defaultScalaVersion = BuildInfo.scalaVersion)
-    dep match {
+  }
+
+  private def parseDependency(lib: String): Dependency = {
+    parseDependencyEither(lib) match {
       case Left(error) =>
         throw new IllegalArgumentException(error)
       case Right(value) =>

lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala

@@ -1,29 +1,42 @@
 package com.sourcegraph.lsif_java
+
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.StandardCopyOption
 
+import moped.reporters.Reporter
+import os.CommandResult
+
 object Embedded {
 
   def semanticdbJar(tmpDir: Path): Path =
     copyFile(tmpDir, "semanticdb-plugin.jar")
 
   def agentJar(tmpDir: Path): Path = copyFile(tmpDir, "semanticdb-agent.jar")
+  def coursier(tmpDir: Path): Path = {
+    val result = copyFile(tmpDir, "lsif-java/coursier")
+    result.toFile().setExecutable(true)
+    result
+  }
+
+  private def javacErrorpath(tmp: Path) = tmp.resolve("errorpath.txt")
 
   def customJavac(sourceroot: Path, targetroot: Path, tmp: Path): Path = {
     val javac = tmp.resolve("javac")
     val pluginpath = Embedded.semanticdbJar(tmp)
-    val oldArguments = tmp.resolve("javac_oldarguments")
-    val newArguments = tmp.resolve("javac_newarguments")
-    val launcherArgs = tmp.resolve("javac_launcherarguments")
+    val errorpath = javacErrorpath(tmp)
+    val javacopts = targetroot.resolve("javacopts.txt")
+    Files.createDirectories(targetroot)
+    val newJavacopts = tmp.resolve("javac_newarguments")
     val injectSemanticdbArguments = List[String](
       "java",
+      s"-Dsemanticdb.errorpath=$errorpath",
       s"-Dsemanticdb.pluginpath=$pluginpath",
       s"-Dsemanticdb.sourceroot=$sourceroot",
       s"-Dsemanticdb.targetroot=$targetroot",
-      s"-Dsemanticdb.output=$newArguments",
-      s"-Dsemanticdb.old-output=$oldArguments",
+      s"-Dsemanticdb.output=$$NEW_JAVAC_OPTS",
+      s"-Dsemanticdb.old-output=$javacopts",
       s"-classpath $pluginpath",
       "com.sourcegraph.semanticdb_javac.InjectSemanticdbOptions",
       """"$@""""
@@ -32,27 +45,53 @@ object Embedded {
       s"""#!/usr/bin/env bash
          |set -eu
          |LAUNCHER_ARGS=()
-         |echo $$@ >> $launcherArgs
+         |NEW_JAVAC_OPTS="$newJavacopts-$$RANDOM"
          |for arg in "$$@"; do
          |  if [[ $$arg == -J* ]]; then
          |    LAUNCHER_ARGS+=("$$arg")
          |  fi
          |done
          |$injectSemanticdbArguments
          |if [ $${#LAUNCHER_ARGS[@]} -eq 0 ]; then
-         |  javac "@$newArguments"
+         |  javac "@$$NEW_JAVAC_OPTS"
          |else
-         |  javac "@$newArguments" "$${LAUNCHER_ARGS[@]}"
+         |  javac "@$$NEW_JAVAC_OPTS" "$${LAUNCHER_ARGS[@]}"
          |fi
          |""".stripMargin
     Files.write(javac, script.getBytes(StandardCharsets.UTF_8))
     javac.toFile.setExecutable(true)
     javac
   }
 
+  /**
+   * The custom javac wrapper reports errors to a specific file if unexpected
+   * errors happen. The javac wrapper gets invoked by builds tools like
+   * Gradle/Maven, which hide the actual errors from the script because they
+   * assume the standard output is from javac. This file is used a side-channel
+   * to avoid relying on the error reporting from Gradle/Maven.
+   */
+  def reportUnexpectedJavacErrors(
+      reporter: Reporter,
+      tmp: Path
+  ): Option[CommandResult] = {
+    val errorpath = javacErrorpath(tmp)
+    if (Files.isRegularFile(errorpath)) {
+      reporter.error("unexpected javac compile errors")
+      Files
+        .readAllLines(errorpath)
+        .forEach { line =>
+          reporter.error(line)
+        }
+      Some(CommandResult(1, Nil))
+    } else {
+      None
+    }
+  }
+
   private def copyFile(tmpDir: Path, filename: String): Path = {
     val in = this.getClass.getResourceAsStream(s"/$filename")
     val out = tmpDir.resolve(filename)
+    Files.createDirectories(out.getParent())
     try Files.copy(in, out, StandardCopyOption.REPLACE_EXISTING)
     finally in.close()
     out

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala

@@ -12,6 +12,10 @@ abstract class BuildTool(val name: String, index: IndexCommand) {
 
   protected def defaultTargetroot: Path
 
+  def isHidden: Boolean = false
+
+  def indexJdk(): Boolean = true
+
   final def targetroot: Path =
     AbsolutePath
       .of(index.targetroot.getOrElse(defaultTargetroot), index.workingDirectory)
@@ -24,5 +28,11 @@ abstract class BuildTool(val name: String, index: IndexCommand) {
 
 object BuildTool {
   def all(index: IndexCommand): List[BuildTool] =
-    List(new GradleBuildTool(index), new MavenBuildTool(index))
+    List(
+      new GradleBuildTool(index),
+      new MavenBuildTool(index),
+      new LsifBuildTool(index)
+    )
+  def allNames: String =
+    all(IndexCommand()).filterNot(_.isHidden).map(_.name).mkString(", ")
 }