Add build tool support for sbt.

Previously, the `lsif-java index` command only worked with Gradle and
Maven builds. This commits adds support for sbt as well. The Scala
community mostly uses sbt so this new feature is a significant milestone
towards supporting Scala as well.

Author: olafurpg

build.sbt

@@ -17,7 +17,7 @@ lazy val V =
     def scala211 = "2.11.12"
     def scala3 = "3.0.1"
     def metals = "0.10.6-M1"
-    def scalameta = "4.4.25"
+    def scalameta = "4.4.26"
     def testcontainers = "0.39.3"
     def requests = "0.6.5"
   }
@@ -145,8 +145,12 @@ lazy val cli = project
     buildInfoKeys :=
       Seq[BuildInfoKey](
         version,
+        sbtVersion,
         scalaVersion,
-        "mtags" -> V.metals,
+        "sbtSourcegraphVersion" ->
+          com.sourcegraph.sbtsourcegraph.BuildInfo.version,
+        "semanticdbVersion" -> V.scalameta,
+        "mtagsVersion" -> V.metals,
         "scala211" -> V.scala211,
         "scala212" -> V.scala212,
         "scala213" -> V.scala213,
@@ -268,6 +272,7 @@ lazy val minimizedSettings = List[Def.Setting[_]](
       s"-Arandomtimestamp=${System.nanoTime()}",
       List(
         s"-Xplugin:semanticdb",
+        s"-build-tool:sbt",
         s"-text:on",
         s"-verbose",
         s"-sourceroot:${(ThisBuild / baseDirectory).value}",
@@ -290,7 +295,11 @@ lazy val minimized8 = project
 
 lazy val minimized15 = project
   .in(file("tests/minimized/.j15"))
-  .settings(minimizedSettings, javaToolchainVersion := "15")
+  .settings(
+    minimizedSettings,
+    javaToolchainVersion := "15",
+    Compile / javaHome := None
+  )
   .dependsOn(agent, plugin)
   .disablePlugins(JavaFormatterPlugin)
 

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala

@@ -1,38 +1,53 @@
 package com.sourcegraph.lsif_java.buildtools
 
+import java.nio.file.Files
 import java.nio.file.Path
 
-import com.sourcegraph.io.AbsolutePath
 import com.sourcegraph.lsif_java.commands.IndexCommand
+import com.sourcegraph.lsif_java.commands.IndexSemanticdbCommand
+import os.CommandResult
 
 /**
  * A build tool such as Gradle, Maven or Bazel.
  */
 abstract class BuildTool(val name: String, index: IndexCommand) {
-
-  protected def defaultTargetroot: Path
-
   def isHidden: Boolean = false
-  def buildKind: String = ""
-
   final def sourceroot: Path = index.workingDirectory
-  final def targetroot: Path =
-    AbsolutePath
-      .of(index.targetroot.getOrElse(defaultTargetroot), index.workingDirectory)
-
   def usedInCurrentDirectory(): Boolean
-
-  def generateSemanticdb(): os.CommandResult
-
+  def generateLsif(): Int
 }
 
 object BuildTool {
   def all(index: IndexCommand): List[BuildTool] =
     List(
       new GradleBuildTool(index),
       new MavenBuildTool(index),
-      new LsifBuildTool(index)
+      new LsifBuildTool(index),
+      new SbtBuildTool(index)
     )
   def allNames: String =
     all(IndexCommand()).filterNot(_.isHidden).map(_.name).mkString(", ")
+
+  def generateLsifFromTargetroot(
+      generateSemanticdbResult: CommandResult,
+      targetroot: Path,
+      index: IndexCommand,
+      buildKind: String = ""
+  ): Int = {
+    if (!Files.isDirectory(targetroot)) {
+      generateSemanticdbResult.exitCode
+    } else if (index.app.reporter.hasErrors()) {
+      index.app.reporter.exitCode()
+    } else if (generateSemanticdbResult.exitCode != 0) {
+      generateSemanticdbResult.exitCode
+    } else {
+      IndexSemanticdbCommand(
+        output = index.finalOutput,
+        targetroot = List(targetroot),
+        packagehub = index.packagehub,
+        buildKind = buildKind,
+        app = index.app
+      ).run()
+    }
+  }
 }

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala

@@ -12,17 +12,23 @@ import os.CommandResult
 
 class GradleBuildTool(index: IndexCommand) extends BuildTool("Gradle", index) {
 
-  override def defaultTargetroot: Path =
-    Paths.get("build", "semanticdb-targetroot")
-
   override def usedInCurrentDirectory(): Boolean = {
     Files.isRegularFile(index.workingDirectory.resolve("settings.gradle")) ||
     Files.isRegularFile(index.workingDirectory.resolve("gradlew")) ||
     Files.isRegularFile(index.workingDirectory.resolve("build.gradle")) ||
     Files.isRegularFile(index.workingDirectory.resolve("build.gradle.kts"))
   }
 
-  override def generateSemanticdb(): CommandResult = {
+  override def generateLsif(): Int = {
+    BuildTool
+      .generateLsifFromTargetroot(generateSemanticdb(), targetroot, index)
+  }
+
+  def targetroot: Path = index.finalTargetroot(defaultTargetroot)
+
+  private def defaultTargetroot: Path =
+    Paths.get("build", "semanticdb-targetroot")
+  private def generateSemanticdb(): CommandResult = {
     val gradleWrapper: Path = index
       .workingDirectory
       .resolve(

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala

@@ -76,14 +76,25 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
     .getDefault
     .getPathMatcher("glob:**.{java,scala}")
   private val moduleInfo = Paths.get("module-info.java")
-  protected def defaultTargetroot: Path = Paths.get("target")
+
+  override def usedInCurrentDirectory(): Boolean =
+    Files.isRegularFile(configFile)
+  override def isHidden: Boolean = true
+  override def generateLsif(): Int = {
+    BuildTool.generateLsifFromTargetroot(
+      generateSemanticdb(),
+      index.finalTargetroot(defaultTargetroot),
+      index,
+      buildKind
+    )
+  }
+
+  private def targetroot: Path = index.finalTargetroot(defaultTargetroot)
+  private def defaultTargetroot: Path = Paths.get("target")
   private def configFile =
     index.workingDirectory.resolve(LsifBuildTool.ConfigFileName)
-  def usedInCurrentDirectory(): Boolean = Files.isRegularFile(configFile)
-  override def isHidden: Boolean = true
-  override def buildKind: String =
-    parsedConfig.fold(_.kind, _ => super.buildKind)
-  def generateSemanticdb(): CommandResult = {
+  private def buildKind: String = parsedConfig.fold(_.kind, _ => "")
+  private def generateSemanticdb(): CommandResult = {
     parsedConfig match {
       case ValueResult(value) =>
         clean()
@@ -224,7 +235,7 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
         )
       }
     val mtags = Dependencies.resolveDependencies(
-      List(s"org.scalameta:mtags_${scalaVersion}:${BuildInfo.mtags}")
+      List(s"org.scalameta:mtags_${scalaVersion}:${BuildInfo.mtagsVersion}")
     )
     val scalaLibrary = mtags
       .classpath

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala

@@ -11,12 +11,22 @@ import com.sourcegraph.lsif_java.commands.IndexCommand
 import os.CommandResult
 
 class MavenBuildTool(index: IndexCommand) extends BuildTool("Maven", index) {
-  override def defaultTargetroot: Path =
-    Paths.get("target", "semanticdb-targetroot")
+
   override def usedInCurrentDirectory(): Boolean =
     Files.isRegularFile(index.workingDirectory.resolve("pom.xml"))
 
-  override def generateSemanticdb(): CommandResult = {
+  override def generateLsif(): Int = {
+    BuildTool.generateLsifFromTargetroot(
+      generateSemanticdb(),
+      index.finalTargetroot(defaultTargetroot),
+      index
+    )
+  }
+
+  private def defaultTargetroot: Path =
+    Paths.get("target", "semanticdb-targetroot")
+
+  private def generateSemanticdb(): CommandResult = {
     TemporaryFiles.withDirectory(index) { tmp =>
       val mvnw = index.workingDirectory.resolve("mvnw")
       val mavenScript =

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/SbtBuildTool.scala

@@ -0,0 +1,93 @@
+package com.sourcegraph.lsif_java.buildtools
+
+import java.nio.file.Files
+import java.nio.file.StandardCopyOption
+import java.util.Properties
+
+import scala.util.Using
+
+import com.sourcegraph.lsif_java.BuildInfo
+import com.sourcegraph.lsif_java.commands.IndexCommand
+
+class SbtBuildTool(index: IndexCommand) extends BuildTool("sbt", index) {
+  override def usedInCurrentDirectory(): Boolean = {
+    Files.isRegularFile(index.workingDirectory.resolve("build.sbt")) ||
+    sbtVersion().isDefined
+  }
+
+  override def generateLsif(): Int = {
+    sbtVersion() match {
+      case Some(version) =>
+        if (isSupportedSbtVersion(version)) {
+          unconditionallyGenerateLsif()
+        } else {
+          failFast(
+            s"Unsupported sbt version '$version'. " +
+              s"To fix this problem, upgrade to sbt ${BuildInfo.sbtVersion} and try again."
+          )
+        }
+      case None =>
+        failFast(
+          s"No sbt version detected. " +
+            s"To fix this problem, run the following command and try again: " +
+            s"echo 'sbt.version=${BuildInfo.sbtVersion}' >> project/build.properties"
+        )
+    }
+  }
+
+  private def failFast(message: String): Int = {
+    index.app.error(message)
+    1
+  }
+
+  private def unconditionallyGenerateLsif(): Int =
+    Using.resource(sourcegraphSbtPluginFile()) { _ =>
+      val sourcegraphLsif = index
+        .process(List("sbt", "sourcegraphEnable", "sourcegraphLsif"))
+      val inputDump = index
+        .workingDirectory
+        .resolve("target")
+        .resolve("sbt-sourcegraph")
+        .resolve("dump.lsif")
+      if (sourcegraphLsif.exitCode == 0 && Files.isRegularFile(inputDump)) {
+        val outputDump = index.workingDirectory.resolve("dump.lsif")
+        Files.copy(inputDump, outputDump, StandardCopyOption.REPLACE_EXISTING)
+        index.app.info(outputDump.toString)
+      }
+      sourcegraphLsif.exitCode
+    }
+
+  private def isSupportedSbtVersion(version: String): Boolean = {
+    (!version.startsWith("0.13") || version.startsWith("0.13.17")) &&
+    !version.startsWith("1.0") && !version.startsWith("1.1")
+  }
+
+  private def sbtVersion(): Option[String] = {
+    val buildProperties = index
+      .workingDirectory
+      .resolve("project")
+      .resolve("build.properties")
+    if (Files.isRegularFile(buildProperties)) {
+      val props = new Properties()
+      val in = Files.newInputStream(buildProperties)
+      try props.load(in)
+      finally in.close()
+      Option(props.getProperty("sbt.version"))
+    } else {
+      None
+    }
+  }
+
+  private def sourcegraphSbtPluginFile(): AutoDeletedFile = {
+    val addSbtPluginFile = index
+      .workingDirectory
+      .resolve("project")
+      .resolve("sourcegraph_generated.sbt")
+    val version = BuildInfo.sbtSourcegraphVersion
+    AutoDeletedFile.fromPath(
+      addSbtPluginFile,
+      s"""addSbtPlugin("com.sourcegraph" % "sbt-sourcegraph" % "$version")
+         |""".stripMargin
+    )
+  }
+}

lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala

@@ -66,7 +66,16 @@ case class IndexCommand(
       shellable: Shellable,
       env: Map[String, String] = Map.empty
   ): CommandResult = {
-    app.out.println(Color.DarkGray(shellable.value.mkString("$ ", " ", "")))
+    val commandSyntax = shellable
+      .value
+      .map { line =>
+        if (line.contains(" "))
+          s"""'$line'"""
+        else
+          line
+      }
+      .mkString("$ ", " ", "")
+    app.out.println(Color.DarkGray(commandSyntax))
     app
       .process(shellable)
       .call(
@@ -149,22 +158,7 @@ case class IndexCommand(
         }
         1
       case tool :: Nil =>
-        val generateSemanticdbResult = tool.generateSemanticdb()
-        if (!Files.isDirectory(tool.targetroot)) {
-          generateSemanticdbResult.exitCode
-        } else if (app.reporter.hasErrors()) {
-          app.reporter.exitCode()
-        } else if (generateSemanticdbResult.exitCode != 0) {
-          generateSemanticdbResult.exitCode
-        } else {
-          IndexSemanticdbCommand(
-            output = finalOutput,
-            targetroot = List(tool.targetroot),
-            packagehub = packagehub,
-            buildKind = tool.buildKind,
-            app = app
-          ).run()
-        }
+        tool.generateLsif()
       case many =>
         val names = many.map(_.name).mkString(", ")
         app.error(

lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java

@@ -1,5 +1,6 @@
 package com.sourcegraph.lsif_semanticdb;
 
+import com.google.protobuf.CodedInputStream;
 import com.sourcegraph.lsif_protocol.MarkupKind;
 import com.sourcegraph.semanticdb_javac.Semanticdb;
 import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolInformation;
@@ -188,11 +189,13 @@ private Integer processDocumentUnsafe(
 
   private Stream<LsifTextDocument> parseTextDocument(Path semanticdbPath) {
     try {
-      return Semanticdb.TextDocuments.parseFrom(Files.readAllBytes(semanticdbPath))
-          .getDocumentsList().stream()
+      CodedInputStream in = CodedInputStream.newInstance(Files.readAllBytes(semanticdbPath));
+      in.setRecursionLimit(1000);
+      return Semanticdb.TextDocuments.parseFrom(in).getDocumentsList().stream()
           .filter(sdb -> !sdb.getOccurrencesList().isEmpty())
           .map(sdb -> new LsifTextDocument(semanticdbPath, sdb, options.sourceroot));
     } catch (IOException e) {
+      options.reporter.error("invalid protobuf: " + semanticdbPath);
       options.reporter.error(e);
       return Stream.empty();
     }