Add support for Kotlin files in lsif-java.json builds

build.sbt

@@ -18,6 +18,7 @@ lazy val V =
     def scala3 = "3.0.1"
     def metals = "0.10.6-M1"
     def scalameta = "4.4.26"
+    def semanticdbKotlinc = "0.0.2"
     def testcontainers = "0.39.3"
     def requests = "0.6.5"
   }
@@ -156,6 +157,7 @@ lazy val cli = project
         "sbtSourcegraphVersion" ->
           com.sourcegraph.sbtsourcegraph.BuildInfo.version,
         "semanticdbVersion" -> V.scalameta,
+        "semanticdbKotlincVersion" -> V.semanticdbKotlinc,
         "mtagsVersion" -> V.metals,
         "scala211" -> V.scala211,
         "scala212" -> V.scala212,
@@ -172,7 +174,8 @@ lazy val cli = project
         "org.scala-lang.modules" %% "scala-xml" % "1.3.0",
         "com.lihaoyi" %% "requests" % V.requests,
         "org.scalameta" %% "moped" % V.moped,
-        "org.scalameta" %% "ascii-graphs" % "0.1.2"
+        "org.scalameta" %% "ascii-graphs" % "0.1.2",
+        "org.jetbrains.kotlin" % "kotlin-compiler-embeddable" % "1.5.21"
       ),
     (Compile / resourceGenerators) +=
       Def

lsif-java/src/main/scala/com/sourcegraph/lsif_java/Dependencies.scala

@@ -4,9 +4,9 @@ import java.io.File
 import java.nio.file.Path
 
 import scala.concurrent.duration.Duration
+import scala.util.Try
 import scala.xml.XML
 
-import com.sourcegraph.lsif_java.BuildInfo
 import coursier.Fetch
 import coursier.Repositories
 import coursier.Resolve
@@ -41,6 +41,37 @@ object Dependencies {
     Repositories.centralGcs
   )
 
+  /**
+   * Attempts to find the "common definitions" JAR for a potentially
+   * MultiPlatform Project. We only support JVM for now, native and JS are not
+   * supported. If it ends with '-jvm', we search for a JAR with the classifier
+   * truncated. If it does not end with -jvm, we search for a JAR with the
+   * -common classifier. This is non-exhaustive, and the classifiers are
+   * completely arbitrary.
+   */
+  def kotlinMPPCommon(
+      group: String,
+      artifact: String,
+      version: String
+  ): Option[Path] =
+    Try {
+      val task = Fetch[Task](Cache.default)
+        .withClassifiers(Set(Classifier.sources))
+        .addRepositories(defaultExtraRepositories: _*)
+
+      if (artifact.endsWith("-jvm")) {
+        val dependency = Dependencies
+          .parseDependency(s"$group:${artifact.stripSuffix("-jvm")}:$version")
+        val result = task.addDependencies(dependency).runResult()
+        return Some(result.files.head.toPath)
+      }
+
+      val dependency = Dependencies
+        .parseDependency(s"$group:$artifact-common:$version")
+      val result = task.addDependencies(dependency).runResult()
+      result.files.head.toPath
+    }.toOption
+
   def resolveDependencies(
       dependencies: List[String],
       transitive: Boolean = true

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala

@@ -1,6 +1,7 @@
 package com.sourcegraph.lsif_java.buildtools
 
 import java.io.File
+import java.io.FileOutputStream
 import java.io.IOException
 import java.net.URLClassLoader
 import java.nio.file.FileSystems
@@ -17,9 +18,11 @@ import java.util.Collections
 import java.util.Optional
 import java.util.ServiceLoader
 import java.util.concurrent.TimeUnit
+import java.util.jar.JarFile
 
 import scala.collection.mutable.ListBuffer
 import scala.jdk.CollectionConverters._
+import scala.language.postfixOps
 import scala.util.Failure
 import scala.util.Success
 import scala.util.Try
@@ -46,6 +49,14 @@ import moped.macros.ClassShape
 import moped.parsers.JsonParser
 import moped.reporters.Diagnostic
 import moped.reporters.Input
+import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
+import org.jetbrains.kotlin.cli.common.arguments.ParseCommandLineArgumentsKt.parseCommandLineArguments
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSourceLocation
+import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
+import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
+import org.jetbrains.kotlin.config.Services
 import os.CommandResult
 import os.ProcessOutput.Readlines
 import os.SubprocessException
@@ -72,9 +83,12 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
   private val scalaPattern = FileSystems
     .getDefault
     .getPathMatcher("glob:**.scala")
+  private val kotlinPattern = FileSystems
+    .getDefault
+    .getPathMatcher("glob:**.kt")
   private val allPatterns = FileSystems
     .getDefault
-    .getPathMatcher("glob:**.{java,scala}")
+    .getPathMatcher("glob:**.{java,scala,kt}")
   private val moduleInfo = Paths.get("module-info.java")
 
   override def usedInCurrentDirectory(): Boolean =
@@ -139,20 +153,26 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
       throw new NoSuchFileException(sourceroot.toString)
     }
     val allSourceFiles = collectAllSourceFiles(sourceroot)
-    val javaFiles = allSourceFiles.filter(path => javaPattern.matches(path))
-    val scalaFiles = allSourceFiles.filter(path => scalaPattern.matches(path))
-    if (javaFiles.isEmpty && scalaFiles.isEmpty) {
+    val javaFiles = allSourceFiles.filter(javaPattern.matches)
+    val scalaFiles = allSourceFiles.filter(scalaPattern.matches)
+    val kotlinFiles = allSourceFiles.filter(kotlinPattern.matches)
+    if (javaFiles.isEmpty && scalaFiles.isEmpty && kotlinFiles.isEmpty) {
       index
         .app
         .warning(
-          s"doing nothing, no files matching pattern '$sourceroot/**.{java,scala}'"
+          s"doing nothing, no files matching pattern '$sourceroot/**.{java,scala,kt}'"
         )
       return CommandResult(0, Nil)
     }
-    val errors = ListBuffer.empty[Try[Unit]]
-    compileJavaFiles(tmp, deps, config, javaFiles)
-      .recover(e => errors.addOne(Failure(e)))
-    compileScalaFiles(deps, scalaFiles).recover(e => errors.addOne(Failure(e)))
+
+    val compileAttemtps = ListBuffer.empty[Try[Unit]]
+    compileAttemtps += compileJavaFiles(tmp, deps, config, javaFiles)
+    compileAttemtps += compileScalaFiles(deps, scalaFiles)
+    compileAttemtps += compileKotlinFiles(deps, config, kotlinFiles)
+    val errors = compileAttemtps.collect { case Failure(exception) =>
+      exception
+    }
+
     if (index.cleanup) {
       Files.walkFileTree(tmp, new DeleteVisitor)
     }
@@ -168,14 +188,141 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
           .info(
             "Some SemanticDB files got generated even if there were compile errors. " +
               "In most cases, this means that lsif-java managed to index everything " +
-              "except the locations that had compile errors and you can ignore the compile errors." +
-              errors.mkString("\n")
+              "except the locations that had compile errors and you can ignore the compile errors."
           )
+        errors.foreach { error =>
+          index.app.reporter.info(error.getMessage())
+        }
       }
       CommandResult(0, Nil)
     }
   }
 
+  private def compileKotlinFiles(
+      deps: Dependencies,
+      config: Config,
+      allKotlinFiles: List[Path]
+  ): Try[Unit] = {
+    if (allKotlinFiles.isEmpty)
+      return Success()
+    val filesPaths = allKotlinFiles.map(_.toString)
+
+    val plugin =
+      Dependencies
+        .resolveDependencies(
+          List(
+            s"com.sourcegraph:semanticdb-kotlinc:${BuildInfo.semanticdbKotlincVersion}"
+          ),
+          transitive = false
+        )
+        .classpath
+        .head
+
+    val self = config.dependencies.head
+    val commonKotlinFiles: List[Path] =
+      Dependencies
+        .kotlinMPPCommon(self.groupId, self.artifactId, self.version) match {
+        case Some(common) =>
+          val outdir = Files.createTempDirectory("lsifjava-kotlin")
+          val file = common.toFile
+          val basename = file
+            .getName
+            .substring(0, file.getName.lastIndexOf("."))
+          val newFiles = ListBuffer[Path]()
+          val jar = new JarFile(file)
+          val enu = jar.entries
+          while (enu.hasMoreElements) {
+            val entry = enu.nextElement
+            val entryPath =
+              if (entry.getName.startsWith(basename))
+                entry.getName.substring(basename.length)
+              else
+                entry.getName
+
+            if (entry.isDirectory) {
+              new File(outdir.toString, entryPath).mkdirs
+            } else if (entry.getName.endsWith(".kt")) {
+              val newFile = new File(outdir.toString, entryPath)
+              newFiles.addOne(newFile.toPath)
+              val istream = jar.getInputStream(entry)
+              val ostream = new FileOutputStream(newFile)
+              Iterator
+                .continually(istream.read)
+                .takeWhile(-1 !=)
+                .foreach(ostream.write)
+              ostream.close()
+              istream.close()
+            }
+          }
+          newFiles.toList
+        case None =>
+          List[Path]()
+      }
+
+    val kargs: K2JVMCompilerArguments = new K2JVMCompilerArguments()
+    val args = ListBuffer[String](
+      "-nowarn",
+      "-no-reflect",
+      "-no-stdlib",
+      "-Xmulti-platform",
+      "-Xno-check-actual",
+      "-Xopt-in=kotlin.RequiresOptIn",
+      "-Xopt-in=kotlin.ExperimentalUnsignedTypes",
+      "-Xopt-in=kotlin.ExperimentalStdlibApi",
+      "-Xopt-in=kotlin.ExperimentalMultiplatform",
+      "-Xopt-in=kotlin.contracts.ExperimentalContracts",
+      "-Xallow-kotlin-package",
+      s"-Xplugin=$plugin",
+      "-P",
+      s"plugin:semanticdb-kotlinc:sourceroot=$sourceroot",
+      "-P",
+      s"plugin:semanticdb-kotlinc:targetroot=$targetroot",
+      "-classpath",
+      deps.classpathSyntax
+    )
+
+    if (commonKotlinFiles.nonEmpty) {
+      args +=
+        s"-Xcommon-sources=${commonKotlinFiles.map(_.toAbsolutePath.toString).mkString(",")}"
+    }
+
+    args ++= filesPaths ++ commonKotlinFiles.map(_.toAbsolutePath.toString)
+
+    parseCommandLineArguments(args.asJava, kargs)
+
+    val exit = new K2JVMCompiler().exec(
+      new MessageCollector {
+        private val errors = new util.LinkedList[String]
+        override def clear(): Unit = errors.clear()
+
+        override def hasErrors: Boolean = !errors.isEmpty
+
+        override def report(
+            compilerMessageSeverity: CompilerMessageSeverity,
+            s: String,
+            compilerMessageSourceLocation: CompilerMessageSourceLocation
+        ): Unit = {
+          if (
+            s.endsWith("without a body must be abstract") ||
+            s.endsWith("must have a body")
+          )
+            return // we get these when indexing the stdlib, no other solution found yet
+          val msg = MessageRenderer
+            .PLAIN_FULL_PATHS
+            .render(compilerMessageSeverity, s, compilerMessageSourceLocation)
+          index.app.reporter.debug(msg)
+          errors.push(msg)
+        }
+      },
+      Services.EMPTY,
+      kargs
+    )
+    if (exit.getCode == 0)
+      Success(())
+    else
+      Failure(new Exception(exit.toString))
+  }
+
   private def compileScalaFiles(
       deps: Dependencies,
       allScalaFiles: List[Path]