From e8245db41b86084d3d89d353fa67eff413673962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93lafur=20P=C3=A1ll=20Geirsson?= Date: Mon, 8 Mar 2021 16:06:38 +0100 Subject: [PATCH 1/2] Implement LSIF emitter in Java. Previously, we shelled out to `lsif-semanticdb` to convert SemanticDB files into LSIF. This commit ports the original Go implemementation to Java so that we can call a normal method instead of shelling out to a separate process. The biggest problem with the previous solution is that it complicated installation for users. As we evolve lsif-java, the conversion logic will also evolve and we want to avoid the situation where users have an outdated version of lsif-semanticdb installed. With this commit, users only need to install `lsif-java` and everything should work correctly. This commit also adds a new `snapshot-lsif` command to pretty-print an LSIF index, similar to how we pretty-print SemanticDB payloads. This command is mostly useful for testing purposes, but could be helpful to troubleshoot user issues. --- .github/workflows/native.yml | 48 -- build.sbt | 110 +++-- .../lsif_java/IndexSemanticdbCommand.scala | 55 --- docs/contributing.md | 9 +- .../com/sourcegraph/io}/AbsolutePath.scala | 2 +- .../com/sourcegraph/io}/DeleteVisitor.scala | 2 +- .../com/sourcegraph/lsif_java/Embedded.scala | 0 .../com/sourcegraph/lsif_java/LsifJava.scala | 7 +- .../lsif_java/SemanticdbPrinters.scala | 2 +- .../lsif_java/buildtools/BuildTool.scala | 4 +- .../buildtools/GradleBuildTool.scala | 2 +- .../buildtools/GradleJavaCompiler.scala | 2 +- .../buildtools/GradleJavaToolchains.scala | 2 +- .../lsif_java/buildtools/MavenBuildTool.scala | 2 +- .../lsif_java/buildtools/TemporaryFiles.scala | 2 +- .../lsif_java/commands}/IndexCommand.scala | 28 +- .../commands/IndexSemanticdbCommand.scala | 76 ++++ .../lsif_java/commands}/SnapshotCommand.scala | 29 +- .../commands/SnapshotLsifCommand.scala | 423 ++++++++++++++++++ .../ConsoleLsifSemanticdbReporter.scala | 44 ++ .../LsifSemanticdbProgressRenderer.scala | 28 ++ .../lsif_semanticdb/LsifByteOutputStream.java | 15 + .../lsif_semanticdb/LsifOutputFormat.java | 11 + .../lsif_semanticdb/LsifOutputStream.java | 79 ++++ .../lsif_semanticdb/LsifSemanticdb.java | 132 ++++++ .../LsifSemanticdbOptions.java | 37 ++ .../LsifSemanticdbReporter.java | 21 + .../lsif_semanticdb/LsifTextDocument.java | 53 +++ .../lsif_semanticdb/LsifWriter.java | 160 +++++++ .../lsif_semanticdb/RangeComparator.java | 29 ++ .../lsif_semanticdb/ResultIds.java | 18 + .../lsif_semanticdb/ResultSets.java | 50 +++ .../lsif_semanticdb/SemanticdbWalker.java | 47 ++ lsif-semanticdb/src/main/protobuf/lsif.proto | 45 ++ .../semanticdb_javac/SemanticdbSymbols.java | 0 .../src/main/protobuf/semanticdb.proto | 0 .../semanticdb_javac/GlobalSymbolsCache.java | 4 +- ...tions.java => SemanticdbJavacOptions.java} | 8 +- .../semanticdb_javac/SemanticdbPlugin.java | 2 +- .../SemanticdbTaskListener.java | 6 +- .../semanticdb_javac/SemanticdbVisitor.java | 4 +- .../main/scala/benchmarks/CompileBench.scala | 52 ++- .../benchmarks/LsifSemanticdbBench.scala | 77 ++++ .../main/generated/index-semanticdb/locals | 56 +++ .../main/generated/index-semanticdb/multifile | 65 +++ .../main/generated/index-semanticdb/reference | 57 +++ .../src/main/generated/snapshot-lsif.diff | 0 .../tests/LibrarySnapshotGenerator.scala | 2 +- .../tests/LsifGraphSnapshotGenerator.scala | 150 +++++++ .../MinimizedLsifSnapshotGenerator.scala | 86 ++++ .../scala/tests/MinimizedSnapshotSuite.scala | 5 - .../scala/tests/SaveSnapshotHandler.scala | 2 +- .../SemanticdbJavacSnapshotGenerator.scala | 7 +- .../src/test/scala/tests/SnapshotSuite.scala | 6 + .../main/scala/tests/TempDirectories.scala | 2 +- .../src/main/scala/tests/TestCompiler.scala | 26 +- .../scala/tests/SemanticdbPrintersSuite.scala | 87 ++++ .../src/test/scala/tests/TargetedSuite.scala | 2 +- 58 files changed, 2048 insertions(+), 232 deletions(-) delete mode 100644 .github/workflows/native.yml delete mode 100644 cli/src/main/scala/com/sourcegraph/lsif_java/IndexSemanticdbCommand.scala rename {cli/src/main/scala/com/sourcegraph/lsif_java => lsif-java/src/main/scala/com/sourcegraph/io}/AbsolutePath.scala (92%) rename {cli/src/main/scala/com/sourcegraph/lsif_java => lsif-java/src/main/scala/com/sourcegraph/io}/DeleteVisitor.scala (96%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala (100%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/LsifJava.scala (73%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/SemanticdbPrinters.scala (97%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala (86%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala (98%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaCompiler.scala (98%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala (98%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala (97%) rename {cli => lsif-java}/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala (88%) rename {cli/src/main/scala/com/sourcegraph/lsif_java => lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands}/IndexCommand.scala (90%) create mode 100644 lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala rename {cli/src/main/scala/com/sourcegraph/lsif_java => lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands}/SnapshotCommand.scala (77%) create mode 100644 lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala create mode 100644 lsif-java/src/main/scala/com/sourcegraph/lsif_semanticdb/ConsoleLsifSemanticdbReporter.scala create mode 100644 lsif-java/src/main/scala/com/sourcegraph/lsif_semanticdb/LsifSemanticdbProgressRenderer.scala create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifByteOutputStream.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputFormat.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputStream.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbOptions.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbReporter.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifTextDocument.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifWriter.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/RangeComparator.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/ResultIds.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/ResultSets.java create mode 100644 lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbWalker.java create mode 100644 lsif-semanticdb/src/main/protobuf/lsif.proto rename {semanticdb-javac => semanticdb-java}/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbSymbols.java (100%) rename {semanticdb-javac => semanticdb-java}/src/main/protobuf/semanticdb.proto (100%) rename semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/{SemanticdbOptions.java => SemanticdbJavacOptions.java} (90%) create mode 100644 tests/benchmarks/src/main/scala/benchmarks/LsifSemanticdbBench.scala create mode 100644 tests/snapshots/src/main/generated/index-semanticdb/locals create mode 100644 tests/snapshots/src/main/generated/index-semanticdb/multifile create mode 100644 tests/snapshots/src/main/generated/index-semanticdb/reference create mode 100644 tests/snapshots/src/main/generated/snapshot-lsif.diff create mode 100644 tests/snapshots/src/main/scala/tests/LsifGraphSnapshotGenerator.scala create mode 100644 tests/snapshots/src/main/scala/tests/MinimizedLsifSnapshotGenerator.scala delete mode 100644 tests/snapshots/src/main/scala/tests/MinimizedSnapshotSuite.scala create mode 100644 tests/unit/src/test/scala/tests/SemanticdbPrintersSuite.scala diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml deleted file mode 100644 index b555309e..00000000 --- a/.github/workflows/native.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Native Image -on: - push: - branches: - - master - release: - types: [published] -jobs: - unix: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [macOS-latest, ubuntu-latest] - include: - - os: macOS-latest - uploaded_filename: lsif-java-x86_64-apple-darwin - local_path: cli/target/native-image/lsif-java - - os: ubuntu-latest - uploaded_filename: lsif-java-x86_64-pc-linux - local_path: cli/target/native-image/lsif-java - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: olafurpg/setup-scala@v10 - - uses: actions/setup-go@v2 - with: - go-version: "^1.13.1" - - run: go get github.com/sourcegraph/lsif-semanticdb/cmd/lsif-semanticdb - - name: sbt nativeImage - shell: bash - run: | - sbt cli/nativeImage "cli/nativeImageRun index --cwd tests/gradle-example" - - uses: actions/upload-artifact@master - with: - path: ${{ matrix.local_path }} - name: ${{ matrix.uploaded_filename }} - - name: Upload release - if: github.event_name == 'release' - uses: actions/upload-release-asset@v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: ${{ matrix.local_path }} - asset_name: ${{ matrix.uploaded_filename }} - asset_content_type: application/zip diff --git a/build.sbt b/build.sbt index ea5ec8cb..c1a35a11 100644 --- a/build.sbt +++ b/build.sbt @@ -1,11 +1,13 @@ import scala.xml.{Node => XmlNode, NodeSeq => XmlNodeSeq, _} import scala.xml.transform.{RewriteRule, RuleTransformer} import java.io.File +import java.nio.file.Files import java.util.Properties import scala.collection.mutable.ListBuffer lazy val V = new { + val protobuf = "3.15.6" val coursier = "2.0.8" val bloop = "1.4.7" val bsp = "2.0.0-M13" @@ -64,11 +66,20 @@ commands += "javafmtCheckAll" :: "publishLocal" :: "docs/docusaurusCreateSite" :: s } +lazy val semanticdb = project + .in(file("semanticdb-java")) + .settings( + moduleName := "semanticdb-java", + javaOnlySettings, + PB.targets.in(Compile) := + Seq(PB.gens.java(V.protobuf) -> (Compile / sourceManaged).value) + ) + lazy val agent = project .in(file("semanticdb-agent")) .settings( fatjarPackageSettings, - autoScalaLibrary := false, + javaOnlySettings, moduleName := "semanticdb-agent", libraryDependencies ++= List( @@ -76,10 +87,6 @@ lazy val agent = project "net.bytebuddy" % "byte-buddy" % "1.10.20", "net.bytebuddy" % "byte-buddy-agent" % "1.10.20" ), - incOptions ~= { old => - old.withEnabled(false) - }, - crossPaths := false, Compile / packageBin / packageOptions += Package.ManifestAttributes( "Agent-Class" -> "com.sourcegraph.semanticdb_javac.SemanticdbAgent", @@ -92,13 +99,10 @@ lazy val agent = project lazy val plugin = project .in(file("semanticdb-javac")) .settings( + fatjarPackageSettings, + javaOnlySettings, moduleName := "semanticdb-javac", javaToolchainVersion := "1.8", - autoScalaLibrary := false, - incOptions ~= { old => - old.withEnabled(false) - }, - fatjarPackageSettings, assemblyShadeRules.in(assembly) := Seq( ShadeRule @@ -108,14 +112,26 @@ lazy val plugin = project "org.relaxng.**" -> "com.sourcegraph.shaded.relaxng.@1" ) .inAll - ), - crossPaths := false, + ) + ) + .dependsOn(semanticdb) + +lazy val lsif = project + .in(file("lsif-semanticdb")) + .settings( + moduleName := "lsif-semanticdb", + javaToolchainVersion := "1.8", + javaOnlySettings, + libraryDependencies += + "com.google.protobuf" % "protobuf-java-util" % V.protobuf, PB.targets.in(Compile) := - Seq(PB.gens.java -> (Compile / sourceManaged).value) + Seq(PB.gens.java(V.protobuf) -> (Compile / sourceManaged).value), + Compile / PB.protocOptions := Seq("--experimental_allow_proto3_optional") ) + .dependsOn(semanticdb) lazy val cli = project - .in(file("cli")) + .in(file("lsif-java")) .settings( moduleName := "lsif-java", mainClass.in(Compile) := Some("com.sourcegraph.lsif_java.LsifJava"), @@ -128,7 +144,11 @@ lazy val cli = project "bspVersion" -> V.bsp ), buildInfoPackage := "com.sourcegraph.lsif_java", - libraryDependencies ++= List("org.scalameta" %% "moped" % V.moped), + libraryDependencies ++= + List( + "org.scalameta" %% "moped" % V.moped, + "org.scalameta" %% "ascii-graphs" % "0.1.2" + ), resourceGenerators.in(Compile) += Def .task[Seq[File]] { @@ -161,11 +181,25 @@ lazy val cli = project propsFile :: copiedJars.toList } .taskValue, - nativeImageOptions ++= List("-H:IncludeResources=^semanticdb-.*jar$"), + nativeImageOptions ++= + List( + "-H:IncludeResources=^semanticdb-.*jar$", + s"-H:ReflectionConfigurationFiles=${target.value / "native-image-configs" / "reflect-config.json"}" + ), nativeImageOutput := target.in(NativeImage).value / "lsif-java" ) .enablePlugins(NativeImagePlugin, BuildInfoPlugin) - .dependsOn(plugin) + .dependsOn(lsif) + +commands += + Command.command("nativeImageProfiled") { s => + val targetroot = + file("tests/minimized/.j11/target/scala-2.13/meta").absolutePath + val output = Files.createTempFile("lsif-java", "dump.lsif") + "minimized/compile" :: + s"""nativeImageRunAgent " index-semanticdb --output=$output $targetroot"""" :: + "nativeImage" :: s + } def minimizedSourceDirectory = file("tests/minimized/src/main/java").getAbsoluteFile @@ -264,12 +298,37 @@ lazy val bench = project .in(file("tests/benchmarks")) .settings( moduleName := "lsif-java-bench", + javaToolchainVersion := "1.8", fork.in(run) := true, skip.in(publish) := true ) .dependsOn(unit) .enablePlugins(JmhPlugin) +lazy val docs = project + .in(file("lsif-java-docs")) + .settings( + mdocOut := + baseDirectory.in(ThisBuild).value / "website" / "target" / "docs", + fork := false, + mdocVariables := + Map[String, String]( + "VERSION" -> version.value, + "SCALA_VERSION" -> scalaVersion.value, + "STABLE_VERSION" -> version.value.replaceFirst("\\-.*", "") + ) + ) + .dependsOn(unit) + .enablePlugins(DocusaurusPlugin) + +lazy val javaOnlySettings = List[Def.Setting[_]]( + autoScalaLibrary := false, + incOptions ~= { old => + old.withEnabled(false) + }, + crossPaths := false +) + lazy val testSettings = List( skip.in(publish) := true, autoScalaLibrary := true, @@ -279,7 +338,6 @@ lazy val testSettings = List( "org.scalameta" %% "munit" % "0.7.10", "org.scalameta" %% "moped-testkit" % V.moped, "org.scalameta" %% "scalameta" % V.scalameta, - "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.0", "io.get-coursier" %% "coursier" % V.coursier, "com.lihaoyi" %% "pprint" % "0.6.1" ) @@ -334,19 +392,3 @@ lazy val fatjarPackageSettings = List[Def.Setting[_]]( ).transform(node).head } ) - -lazy val docs = project - .in(file("lsif-java-docs")) - .settings( - mdocOut := - baseDirectory.in(ThisBuild).value / "website" / "target" / "docs", - fork := false, - mdocVariables := - Map[String, String]( - "VERSION" -> version.value, - "SCALA_VERSION" -> scalaVersion.value, - "STABLE_VERSION" -> version.value.replaceFirst("\\-.*", "") - ) - ) - .dependsOn(unit) - .enablePlugins(DocusaurusPlugin) diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/IndexSemanticdbCommand.scala b/cli/src/main/scala/com/sourcegraph/lsif_java/IndexSemanticdbCommand.scala deleted file mode 100644 index 55449b99..00000000 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/IndexSemanticdbCommand.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.sourcegraph.lsif_java - -import java.nio.file.Path - -import scala.collection.mutable.ListBuffer - -import moped.annotations.CommandName -import moped.annotations.Description -import moped.annotations.ExampleUsage -import moped.annotations.Inline -import moped.annotations.PositionalArguments -import moped.annotations.Usage -import moped.cli.Application -import moped.cli.Command -import moped.cli.CommandParser -import os.Inherit -import os.Shellable - -@Description("Converts SemanticDB files into a single LSIF index file.") -@Usage("lsif-java index-semanticdb [OPTIONS ...] [POSITIONAL ARGUMENTS ...]") -@ExampleUsage( - "lsif-java index-semanticdb --out=myindex.lsif my/targetroot1 my/targetroot2" -) -@CommandName("index-semanticdb") -final case class IndexSemanticdbCommand( - @Description( - "The name of the output file. Defaults to 'dump.lsif'" - ) out: Option[Path] = None, - @Description( - "SemanticDB file paths or directories that contain SemanticDB files." - ) - @PositionalArguments() directories: List[Path] = Nil, - @Inline() app: Application = Application.default -) extends Command { - def run(): Int = { - val arguments = ListBuffer.empty[String] - arguments += "lsif-semanticdb" - out.foreach { dir => - arguments += s"--out=$dir" - } - directories.foreach { dir => - arguments += s"--semanticdbDir=$dir" - } - app.info(arguments.mkString(" ")) - app - .process(Shellable(arguments.toList)) - .call(check = false, stderr = Inherit, stdout = Inherit) - .exitCode - } -} - -object IndexSemanticdbCommand { - val default = IndexSemanticdbCommand() - implicit val parser = CommandParser.derive(default) -} diff --git a/docs/contributing.md b/docs/contributing.md index e7789c0a..18ea7f88 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -63,7 +63,14 @@ First, install the community edition is [open source](https://github.com/JetBrains/intellij-community) and free to use. -Next, install the IntelliJ Scala plugin. +Next, install the following IntelliJ plugins: + +- IntelliJ Scala plugin. +- Google Java Format + +Next, follow +[these instructions](https://github.com/HPI-Information-Systems/Metanome/wiki/Installing-the-google-styleguide-settings-in-intellij-and-eclipse) +here to configure the Google Java formatter. Finally, run "File > Project From Existing Sources" to import the sbt build into IntelliJ. Select the "sbt" option if it asks you to choose between diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/AbsolutePath.scala b/lsif-java/src/main/scala/com/sourcegraph/io/AbsolutePath.scala similarity index 92% rename from cli/src/main/scala/com/sourcegraph/lsif_java/AbsolutePath.scala rename to lsif-java/src/main/scala/com/sourcegraph/io/AbsolutePath.scala index 7bbfbd19..80e77174 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/AbsolutePath.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/io/AbsolutePath.scala @@ -1,4 +1,4 @@ -package com.sourcegraph.lsif_java +package com.sourcegraph.io import java.nio.file.Path import java.nio.file.Paths diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/DeleteVisitor.scala b/lsif-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala similarity index 96% rename from cli/src/main/scala/com/sourcegraph/lsif_java/DeleteVisitor.scala rename to lsif-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala index 22ea0358..4a1a9aa9 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/DeleteVisitor.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/io/DeleteVisitor.scala @@ -1,4 +1,4 @@ -package com.sourcegraph.lsif_java +package com.sourcegraph.io import java.io.IOException import java.nio.file.FileVisitResult diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala similarity index 100% rename from cli/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/LsifJava.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/LsifJava.scala similarity index 73% rename from cli/src/main/scala/com/sourcegraph/lsif_java/LsifJava.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/LsifJava.scala index 1cf3ee3f..bf7b5e46 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/LsifJava.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/LsifJava.scala @@ -2,6 +2,10 @@ package com.sourcegraph.lsif_java import java.io.PrintStream +import com.sourcegraph.lsif_java.commands.IndexCommand +import com.sourcegraph.lsif_java.commands.IndexSemanticdbCommand +import com.sourcegraph.lsif_java.commands.SnapshotCommand +import com.sourcegraph.lsif_java.commands.SnapshotLsifCommand import moped.cli.Application import moped.cli.CommandParser import moped.commands.HelpCommand @@ -17,7 +21,8 @@ object LsifJava { CommandParser[VersionCommand], CommandParser[IndexCommand], CommandParser[IndexSemanticdbCommand], - CommandParser[SnapshotCommand] + CommandParser[SnapshotCommand], + CommandParser[SnapshotLsifCommand] ) ) def main(args: Array[String]): Unit = { diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/SemanticdbPrinters.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/SemanticdbPrinters.scala similarity index 97% rename from cli/src/main/scala/com/sourcegraph/lsif_java/SemanticdbPrinters.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/SemanticdbPrinters.scala index 6ce3d1ce..12782647 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/SemanticdbPrinters.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/SemanticdbPrinters.scala @@ -17,7 +17,7 @@ object SemanticdbPrinters { .linesWithSeparators .zipWithIndex .foreach { case (line, i) => - out.append(line) + out.append(line.replace("\t", "→")) val occurences = occurrencesByLine .getOrElse(i, Nil) .sortBy(o => diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala similarity index 86% rename from cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala index f4b11f39..34ac4e7c 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/BuildTool.scala @@ -2,8 +2,8 @@ package com.sourcegraph.lsif_java.buildtools import java.nio.file.Path -import com.sourcegraph.lsif_java.AbsolutePath -import com.sourcegraph.lsif_java.IndexCommand +import com.sourcegraph.io.AbsolutePath +import com.sourcegraph.lsif_java.commands.IndexCommand /** * A build tool such as Gradle, Maven or Bazel. diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala similarity index 98% rename from cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala index 836adc05..ec9d5b0c 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala @@ -6,7 +6,7 @@ import java.nio.file._ import scala.collection.mutable.ListBuffer import scala.util.Properties -import com.sourcegraph.lsif_java.IndexCommand +import com.sourcegraph.lsif_java.commands.IndexCommand import os.CommandResult class GradleBuildTool(index: IndexCommand) extends BuildTool("Gradle", index) { diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaCompiler.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaCompiler.scala similarity index 98% rename from cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaCompiler.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaCompiler.scala index b0a703b9..2af92432 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaCompiler.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaCompiler.scala @@ -9,7 +9,7 @@ import java.nio.file.StandardCopyOption import scala.collection.mutable.ListBuffer import com.sourcegraph.lsif_java.Embedded -import com.sourcegraph.lsif_java.IndexCommand +import com.sourcegraph.lsif_java.commands.IndexCommand /** * Metadata about the Java compiler that is used by a Gradle build. diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala similarity index 98% rename from cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala index e0f46584..abd1161a 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala @@ -7,7 +7,7 @@ import java.nio.file.Path import scala.jdk.CollectionConverters._ import com.sourcegraph.lsif_java.Embedded -import com.sourcegraph.lsif_java.IndexCommand +import com.sourcegraph.lsif_java.commands.IndexCommand case class GradleJavaToolchains( toolchains: List[GradleJavaCompiler], diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala similarity index 97% rename from cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala index ae028d22..e6b5a61d 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala @@ -7,7 +7,7 @@ import java.nio.file.Paths import scala.collection.mutable.ListBuffer import com.sourcegraph.lsif_java.Embedded -import com.sourcegraph.lsif_java.IndexCommand +import com.sourcegraph.lsif_java.commands.IndexCommand import os.CommandResult class MavenBuildTool(index: IndexCommand) extends BuildTool("Maven", index) { diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala similarity index 88% rename from cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala index 55292678..0e994c34 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala @@ -3,7 +3,7 @@ package com.sourcegraph.lsif_java.buildtools import java.nio.file.Files import java.nio.file.Path -import com.sourcegraph.lsif_java.DeleteVisitor +import com.sourcegraph.io.DeleteVisitor object TemporaryFiles { def withDirectory[T](cleanup: Boolean)(fn: Path => T): T = { diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/IndexCommand.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala similarity index 90% rename from cli/src/main/scala/com/sourcegraph/lsif_java/IndexCommand.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala index 4e8e4cab..29c8f89b 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/IndexCommand.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala @@ -1,16 +1,12 @@ -package com.sourcegraph.lsif_java +package com.sourcegraph.lsif_java.commands import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths +import com.sourcegraph.io.AbsolutePath import com.sourcegraph.lsif_java.buildtools.BuildTool -import moped.annotations.Description -import moped.annotations.ExampleUsage -import moped.annotations.ExampleValue -import moped.annotations.Inline -import moped.annotations.TrailingArguments -import moped.annotations.Usage +import moped.annotations._ import moped.cli.Application import moped.cli.Command import moped.cli.CommandParser @@ -148,17 +144,15 @@ case class IndexCommand( if (!Files.isDirectory(tool.targetroot)) { generateSemanticdbResult.exitCode } else { - val generateLsifResult = process( - "lsif-semanticdb", - s"--out=${finalOutput}", - s"--semanticdbDir=${tool.targetroot}" - ) - if ( - generateLsifResult.exitCode == 0 && Files.isRegularFile(finalOutput) - ) { - app.info(finalOutput.toAbsolutePath().toString()) + val generateLsifResult = IndexSemanticdbCommand( + output = finalOutput, + targetroot = List(tool.targetroot), + app = app + ).run() + if (generateLsifResult == 0 && Files.isRegularFile(finalOutput)) { + app.info(finalOutput.toAbsolutePath.toString()) } - generateSemanticdbResult.exitCode + generateLsifResult.exitCode + generateSemanticdbResult.exitCode + generateLsifResult } case many => val names = many.map(_.name).mkString(", ") diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala new file mode 100644 index 00000000..c542a3c5 --- /dev/null +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala @@ -0,0 +1,76 @@ +package com.sourcegraph.lsif_java.commands + +import java.nio.file.Path +import java.nio.file.Paths + +import scala.jdk.CollectionConverters._ + +import com.sourcegraph.io.AbsolutePath +import com.sourcegraph.lsif_java.BuildInfo +import com.sourcegraph.lsif_protocol.LsifToolInfo +import com.sourcegraph.lsif_semanticdb.ConsoleLsifSemanticdbReporter +import com.sourcegraph.lsif_semanticdb.LsifOutputFormat +import com.sourcegraph.lsif_semanticdb.LsifSemanticdb +import com.sourcegraph.lsif_semanticdb.LsifSemanticdbOptions +import moped.annotations._ +import moped.cli.Application +import moped.cli.Command +import moped.cli.CommandParser + +@Description("Converts SemanticDB files into a single LSIF index file.") +@Usage("lsif-java index-semanticdb [OPTIONS ...] [POSITIONAL ARGUMENTS ...]") +@ExampleUsage( + "lsif-java index-semanticdb --out=myindex.lsif my/targetroot1 my/targetroot2" +) +@CommandName("index-semanticdb") +final case class IndexSemanticdbCommand( + @Description("The name of the output file.") output: Path = Paths + .get("dump.lsif"), + @Description( + "Whether to process the SemanticDB files in parallel" + ) parallel: Boolean = true, + @Description("Directories that contain SemanticDB files.") + @PositionalArguments() targetroot: List[Path] = Nil, + @Inline() app: Application = Application.default +) extends Command { + def sourceroot: Path = AbsolutePath.of(app.env.workingDirectory) + def isProtobufFormat: Boolean = + IndexSemanticdbCommand.isProtobufFormat(output) + def run(): Int = { + val reporter = new ConsoleLsifSemanticdbReporter(app) + val format = + if (isProtobufFormat) + LsifOutputFormat.PROTOBUF + else + LsifOutputFormat.JSON + val options = + new LsifSemanticdbOptions( + targetroot.map(ts => AbsolutePath.of(ts, sourceroot)).asJava, + AbsolutePath.of(output, sourceroot), + sourceroot, + reporter, + LsifToolInfo + .newBuilder() + .setName("lsif-java") + .setVersion(BuildInfo.version) + .build(), + "java", + format, + parallel + ) + LsifSemanticdb.run(options) + if (!app.reporter.hasErrors()) { + app.info(options.output.toString) + } + app.reporter.exitCode() + } +} + +object IndexSemanticdbCommand { + def isJsonFormat(path: Path): Boolean = + path.getFileName.toString.endsWith(".lsif") + def isProtobufFormat(path: Path): Boolean = + path.getFileName.toString.endsWith(".lsif-protobuf") + val default = IndexSemanticdbCommand() + implicit val parser = CommandParser.derive(default) +} diff --git a/cli/src/main/scala/com/sourcegraph/lsif_java/SnapshotCommand.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotCommand.scala similarity index 77% rename from cli/src/main/scala/com/sourcegraph/lsif_java/SnapshotCommand.scala rename to lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotCommand.scala index f97e1eeb..3cddd2e6 100644 --- a/cli/src/main/scala/com/sourcegraph/lsif_java/SnapshotCommand.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotCommand.scala @@ -1,25 +1,17 @@ -package com.sourcegraph.lsif_java +package com.sourcegraph.lsif_java.commands import java.nio.charset.StandardCharsets -import java.nio.file.FileSystems -import java.nio.file.FileVisitResult -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import java.nio.file.SimpleFileVisitor +import java.nio.file._ import java.nio.file.attribute.BasicFileAttributes import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters._ +import com.sourcegraph.io.DeleteVisitor +import com.sourcegraph.lsif_java.SemanticdbPrinters import com.sourcegraph.semanticdb_javac.Semanticdb.TextDocument import com.sourcegraph.semanticdb_javac.Semanticdb.TextDocuments -import moped.annotations.CommandName -import moped.annotations.Description -import moped.annotations.ExampleUsage -import moped.annotations.Inline -import moped.annotations.PositionalArguments -import moped.annotations.Usage +import moped.annotations._ import moped.cli.Application import moped.cli.Command import moped.cli.CommandParser @@ -82,10 +74,7 @@ case class SnapshotCommand( } semanticdbFiles.foreach { doc => - val document = SemanticdbPrinters.printTextDocument(doc) - val snapshotOutput = output.resolve(doc.getUri) - Files.createDirectories(snapshotOutput.getParent) - Files.write(snapshotOutput, document.getBytes(StandardCharsets.UTF_8)) + SnapshotCommand.writeSnapshot(doc, output) } 0 @@ -93,5 +82,11 @@ case class SnapshotCommand( } object SnapshotCommand { + def writeSnapshot(doc: TextDocument, outputDirectory: Path): Unit = { + val document = SemanticdbPrinters.printTextDocument(doc) + val snapshotOutput = outputDirectory.resolve(doc.getUri) + Files.createDirectories(snapshotOutput.getParent) + Files.write(snapshotOutput, document.getBytes(StandardCharsets.UTF_8)) + } implicit val parser = CommandParser.derive(SnapshotCommand()) } diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala new file mode 100644 index 00000000..e50fbaa3 --- /dev/null +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala @@ -0,0 +1,423 @@ +package com.sourcegraph.lsif_java.commands + +import java.net.URI +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.atomic.AtomicInteger +import java.util.stream.Collectors +import java.util.stream.Stream + +import scala.collection.immutable.SortedSet +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer +import scala.collection.mutable.ListBuffer +import scala.jdk.CollectionConverters._ + +import com.google.common.graph.EndpointPair +import com.google.common.graph.Graph +import com.google.common.graph.GraphBuilder +import com.google.common.graph.MutableNetwork +import com.google.common.graph.NetworkBuilder +import com.google.protobuf.InvalidProtocolBufferException +import com.google.protobuf.util.JsonFormat +import com.sourcegraph.io.AbsolutePath +import com.sourcegraph.io.DeleteVisitor +import com.sourcegraph.lsif_protocol.LsifHover +import com.sourcegraph.lsif_protocol.LsifObject +import com.sourcegraph.lsif_protocol.LsifPosition +import com.sourcegraph.semanticdb_javac.Semanticdb +import com.sourcegraph.semanticdb_javac.Semanticdb.Language +import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence +import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence.Role +import com.sourcegraph.semanticdb_javac.Semanticdb.TextDocument +import moped.annotations.CommandName +import moped.annotations.Inline +import moped.annotations.PositionalArguments +import moped.cli.Application +import moped.cli.Command +import moped.cli.CommandParser +import moped.reporters.Input +import moped.reporters.Position +import org.scalameta.ascii.layout.prefs.LayoutPrefsImpl + +@CommandName("snapshot-lsif") +case class SnapshotLsifCommand( + @Inline() app: Application = Application.default, + output: Path = Paths.get("generated"), + @PositionalArguments() input: List[Path] = List(Paths.get("dump.lsif")) +) extends Command { + + private val finalOutput = AbsolutePath.of(output, sourceroot) + def sourceroot: Path = AbsolutePath.of(app.env.workingDirectory) + + def run(): Int = { + Files.walkFileTree(finalOutput, new DeleteVisitor()) + for { + inputPath <- input + in = AbsolutePath.of(inputPath, sourceroot) + doc <- SnapshotLsifCommand.parseTextDocument(in, sourceroot) + } { + SnapshotCommand.writeSnapshot(doc, finalOutput) + } + 0 + } + +} + +object SnapshotLsifCommand { + private val jsonParser = JsonFormat.parser().ignoringUnknownFields() + def parseTextDocument(input: Path, sourceroot: Path): List[TextDocument] = { + parseSemanticdb(input, parseInput(input), sourceroot) + } + + def parseSemanticdb( + input: Path, + objects: mutable.Buffer[LsifObject], + sourceroot: Path + ): List[TextDocument] = { + val lsif = new IndexedLsif(input, objects, sourceroot) + lsif + .ranges + .foreach { o => + val docId = lsif.contains(o.getId) + val doc = lsif.textDocument(docId) + val isDefinition = lsif + .G + .predecessors(o) + .asScala + .exists(_.getLabel == "definitionResult") + val role = + if (isDefinition) + Role.DEFINITION + else + Role.REFERENCE + + val symbol = + ( + for { + resultSetId <- lsif.next.get(o.getId) + monikerId <- lsif.moniker.get(resultSetId) + moniker <- lsif.monikerIdentifier.get(monikerId) + } yield moniker + ).getOrElse { + val id = lsif.next.getOrElse(o.getId, o.getId) + s"local$id" + } + val occ = SymbolOccurrence + .newBuilder() + .setRange( + Semanticdb + .Range + .newBuilder() + .setStartLine(o.getStart.getLine) + .setStartCharacter(o.getStart.getCharacter) + .setEndLine(o.getEnd.getLine) + .setEndCharacter(o.getEnd.getCharacter) + ) + .setRole(role) + .setSymbol(symbol) + .build() + doc.addOccurrences(occ) + } + lsif.documents.values.map(_.build()).toList + } + + class IndexedLsif( + val path: Path, + val objects: mutable.Buffer[LsifObject], + val sourceroot: Path + ) { + val documents = mutable.Map.empty[Int, TextDocument.Builder] + val next = mutable.Map.empty[Int, Int] + val monikerIdentifier = mutable.Map.empty[Int, String] + val moniker = mutable.Map.empty[Int, Int] + val monikerInverse = mutable.Map.empty[Int, Int] + val item = mutable.Map.empty[Int, mutable.Buffer[Integer]] + val contains = mutable.Map.empty[Int, Int] + val hoverVertexes = mutable.Map.empty[Int, LsifHover] + val hoverEdges = mutable.Map.empty[Int, Int] + val ranges: ArrayBuffer[LsifObject] = mutable.ArrayBuffer.empty[LsifObject] + val isDefinitionResult = mutable.Set.empty[Integer] + + def textDocument(id: Int): TextDocument.Builder = { + documents.getOrElseUpdate(id, TextDocument.newBuilder()) + } + + val monikers: Map[String, LsifObject] = + objects + .iterator + .filter(_.getType == "vertex") + .filter(_.getLabel == "moniker") + .map(m => m.getIdentifier -> m) + .toMap + val hovers: Map[String, String] = monikers.map { case (sym, obj) => + val hoverMessage = + for { + resultSet <- monikerInverse.get(obj.getId).toList + hoverEdge <- hoverEdges.get(resultSet).toList + hoverResult <- hoverVertexes.get(hoverEdge).toList + contents <- hoverResult.getContentsList.asScala + } yield contents.getValue.trim + sym -> hoverMessage.mkString("\n\n") + } + + def visualizeGraph(): String = { + import scala.sys.process._ + s"lsif-visualize $path".!! + } + def asciiGraph(symbol: String): String = { + import org.scalameta.ascii.layout._ + import org.scalameta.ascii.graph.Graph + val List(moniker) = + byLabel("moniker") + .filter(o => o.getType() == "vertex" && o.getIdentifier() == symbol) + .toList + val S = subgraph(moniker) + val vertices = SortedSet.newBuilder[String] + val edges = ListBuffer.empty[(String, String)] + val inputs = mutable.Map.empty[String, Input] + def input(range: LsifObject): Input = { + val uri = G + .predecessors(range) + .asScala + .iterator + .filter(_.getLabel() == "document") + .next() + .getUri() + inputs.getOrElseUpdate(uri, Input.path(Paths.get(URI.create(uri)))) + } + def renderPos(pos: LsifPosition): String = { + s"${pos.getLine()}:${pos.getCharacter()}" + } + val addedVertexes = mutable.Set.empty[String] + def render(node: LsifObject): String = { + (node.getType(), node.getLabel()) match { + case ("vertex", "document") => + val filename = sourceroot + .relativize(Paths.get(URI.create(node.getUri()))) + .iterator() + .asScala + .mkString("/") + s"document ${filename}" + case ("vertex", "hoverResult") => + val contents = node + .getResult() + .getContentsList() + .asScala + .map(_.getValue()) + .mkString("\n") + .replace("\n", "\\n") + .trim() + s"hoverResult(${node.getId()}) ${contents}" + case ("vertex", "moniker") => + s"moniker ${node.getIdentifier()}" + case ("vertex", "range") => + val i = input(node) + val p = Position.range( + i, + node.getStart().getLine(), + node.getStart().getCharacter(), + node.getEnd().getLine(), + node.getEnd().getCharacter() + ) + s"range(${node.getId()}) ${renderPos(node.getStart())} '${p.text}'" + case ("vertex", label) => + s"${label}(${node.getId()})" + case _ => + s"${node.getType}/${node.getLabel} (${node.getId()})" + } + } + def isAdded(vertex: LsifObject) = addedVertexes.contains(render(vertex)) + val addedEdges = mutable.Set.empty[(String, String)] + S.nodes() + .forEach { node => + val r = render(node) + addedVertexes += r + vertices += r + } + S.edges() + .forEach { pair => + val edge = render(pair.target()) -> render(pair.source()) + if ( + addedEdges.add(edge) && isAdded(pair.source()) && + isAdded(pair.source()) + ) { + edges += edge + } + } + val output = GraphLayout.renderGraph( + Graph[String](vertices.result(), edges.toList.sorted), + layoutPrefs = LayoutPrefsImpl + .DEFAULT + .copy(rounded = true, explicitAsciiBends = true) + ) + output + .linesIterator + .map { line => + line.replaceAll(" +$", "") + } + .mkString("\n") + } + + /** + * Runs `lsif-visualize` against this LSIF dump and opens the generated SVG + * file in your browser. + */ + def visualizeOpenBrowser(): Unit = { + import scala.sys.process._ + val svg = Files.createTempFile("lsif-java", "dump.svg") + (s"lsif-visualize $path" #| s"dot -Tsvg -o $svg").! + s"open $svg".! + } + + val isSuccessorRelevantLabel = Set("resultSet") + def subgraph(node: LsifObject): Graph[LsifObject] = { + val S = GraphBuilder.directed().immutable[LsifObject]() + val visited = mutable.Set.empty[Int] + def loop(l: LsifObject): Unit = { + if (visited.add(l.getId())) { + S.addNode(l) + G.predecessors(l) + .forEach { n => + S.putEdge(EndpointPair.ordered(l, n)) + loop(n) + } + if (isSuccessorRelevantLabel(l.getLabel())) { + G.successors(l) + .forEach { n => + S.putEdge(EndpointPair.ordered(n, l)) + loop(n) + } + } + } + } + loop(node) + S.build() + } + + val byId = objects.iterator.map(o => o.getId -> o).toMap + val byType = objects.groupBy(_.getType) + val byLabel = objects.groupBy(_.getLabel) + + val G: MutableNetwork[LsifObject, LsifObject] = { + val B: MutableNetwork[LsifObject, LsifObject] = NetworkBuilder + .directed() + .allowsParallelEdges(true) + .allowsSelfLoops(true) + .expectedEdgeCount(byType("edge").size) + .expectedNodeCount(byType("vertex").size) + .build() + objects.foreach { o => + o.getType match { + case "vertex" => + B.addNode(o) + case "edge" => + byId + .get(o.getOutV) + .foreach { outV => + for { + inId <- + Iterator(o.getInV: Integer) ++ + o.getInVsList.asScala.iterator + inV <- byId.get(inId) + } { + B.addEdge( + EndpointPair.ordered(outV, inV), + o.toBuilder().clearInVs().setInV(inId).build() + ) + } + } + case _ => + } + } + B + } + + objects.foreach { o => + o.getType match { + case "edge" => + o.getLabel match { + case "item" => + o.getInVsList + .forEach { inV => + item.getOrElseUpdate(inV, ListBuffer.empty) += o.getOutV() + } + case "contains" => + o.getInVsList.forEach(inV => contains(inV) = o.getOutV) + case "next" => + next(o.getOutV) = o.getInV + case "moniker" => + moniker(o.getOutV) = o.getInV + monikerInverse(o.getInV) = o.getOutV + case "textDocument/hover" => + hoverEdges(o.getOutV) = o.getInV + case _ => + } + case "vertex" => + o.getLabel match { + case "moniker" => + monikerIdentifier(o.getId) = o.getIdentifier + case "document" => + val relativeFile = Paths.get(URI.create(o.getUri)) + val absoluteFile = sourceroot.resolve(relativeFile) + val text = + new String( + Files.readAllBytes(absoluteFile), + StandardCharsets.UTF_8 + ) + val relativeUri = sourceroot + .relativize(absoluteFile) + .iterator() + .asScala + .mkString("/") + val language = Language + .values() + .find(_.name().compareToIgnoreCase(o.getLanguage) == 0) + .getOrElse(Language.UNKNOWN_LANGUAGE) + textDocument(o.getId) + .setUri(relativeUri) + .setLanguage(language) + .setText(text) + case "definitionResult" => + isDefinitionResult += o.getId() + case "hoverResult" => + hoverVertexes(o.getId) = o.getResult + case "range" => + ranges += o + case _ => + } + case _ => + } + } + } + + def parseInput(input: Path): mutable.Buffer[LsifObject] = { + if (IndexSemanticdbCommand.isJsonFormat(input)) { + val lines = Files.lines(input) + try { + lines.parallel().flatMap(parseLine).collect(Collectors.toList()).asScala + } finally { + lines.close() + } + } else { + throw new UnsupportedOperationException(input.toString) + } + } + def parseLine(line: String): Stream[LsifObject] = { + val builder = LsifObject.newBuilder() + try { + jsonParser.merge(line, builder) + Stream.of(builder.build()) + } catch { + case e: InvalidProtocolBufferException + if e.getMessage.startsWith("Expect message object but got") => + // Ignore: usually it's a failure to decode the hover message + // and we don't use hovers anyways. + Stream.empty[LsifObject]() + } + } + val default = SnapshotLsifCommand() + implicit val parser = CommandParser.derive(default) +} diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_semanticdb/ConsoleLsifSemanticdbReporter.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_semanticdb/ConsoleLsifSemanticdbReporter.scala new file mode 100644 index 00000000..92156b48 --- /dev/null +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_semanticdb/ConsoleLsifSemanticdbReporter.scala @@ -0,0 +1,44 @@ +package com.sourcegraph.lsif_semanticdb + +import java.io.PrintWriter +import java.nio.file.NoSuchFileException + +import moped.cli.Application +import moped.progressbars.InteractiveProgressBar +import moped.reporters.Diagnostic + +/** + * Console reporter for index-semanticdb command. + */ +class ConsoleLsifSemanticdbReporter(app: Application) + extends LsifSemanticdbReporter { + + val renderer = new LsifSemanticdbProgressRenderer + val progressbar = + new InteractiveProgressBar( + new PrintWriter(app.env.standardError), + renderer, + isDynamicPartEnabled = app.env.isProgressBarEnabled + ) + override def error(e: Throwable): Unit = { + e match { + case _: NoSuchFileException => + app.reporter.error(s"no such file: ${e.getMessage}") + case _ => + app.reporter.log(Diagnostic.exception(e)) + } + } + + override def hasErrors: Boolean = app.reporter.hasErrors() + override def startProcessing(taskSize: Int): Unit = { + renderer.totalSize = taskSize + progressbar.start() + } + override def processedOneItem(): Unit = { + renderer.currentSize.incrementAndGet() + } + override def endProcessing(): Unit = { + progressbar.stop() + } + +} diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_semanticdb/LsifSemanticdbProgressRenderer.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_semanticdb/LsifSemanticdbProgressRenderer.scala new file mode 100644 index 00000000..3c6ceb16 --- /dev/null +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_semanticdb/LsifSemanticdbProgressRenderer.scala @@ -0,0 +1,28 @@ +package com.sourcegraph.lsif_semanticdb + +import java.util.concurrent.atomic.AtomicInteger + +import moped.progressbars.ProgressRenderer +import moped.progressbars.ProgressStep +import org.typelevel.paiges.Doc + +/** + * Progress bar for the lsif-semanticdb command. + */ +class LsifSemanticdbProgressRenderer() extends ProgressRenderer { + var totalSize = 0 + val currentSize = new AtomicInteger() + override def renderStep(): ProgressStep = { + if (totalSize < 100) + return ProgressStep.empty + + val current = currentSize.get() + val ratio = current.toDouble / totalSize + val progress: Int = (ratio * 10).toInt + val percentage: String = s"${(ratio * 100).toInt}%".padTo(4, ' ') + val bars = ("#" * progress).padTo(10, ' ') + val render = + f"Generating LSIF... [$bars] $percentage $current%,.0f files processed" + ProgressStep(static = Doc.empty, dynamic = Doc.text(render)) + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifByteOutputStream.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifByteOutputStream.java new file mode 100644 index 00000000..d340c335 --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifByteOutputStream.java @@ -0,0 +1,15 @@ +package com.sourcegraph.lsif_semanticdb; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; + +/** Wrapper around a ByteArrayOutputStream and OutputStreamWriter. */ +public class LsifByteOutputStream { + public final ByteArrayOutputStream output; + public final OutputStreamWriter writer; + + public LsifByteOutputStream() { + this.output = new ByteArrayOutputStream(); + this.writer = new OutputStreamWriter(output); + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputFormat.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputFormat.java new file mode 100644 index 00000000..13306372 --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputFormat.java @@ -0,0 +1,11 @@ +package com.sourcegraph.lsif_semanticdb; + +/** + * Whether to generate dump.lsif (JSON) or dump.lsif-protobuf (Protobuf). + * + *

The Protobuf format is experimental and currently only exists as a proof-of-concept. + */ +public enum LsifOutputFormat { + JSON, + PROTOBUF +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputStream.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputStream.java new file mode 100644 index 00000000..eee4f6dc --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifOutputStream.java @@ -0,0 +1,79 @@ +package com.sourcegraph.lsif_semanticdb; + +import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.util.JsonFormat; +import com.google.protobuf.util.JsonFormat.Printer; +import com.sourcegraph.lsif_protocol.LsifObject; +import com.sourcegraph.lsif_protocol.LsifPosition; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Low-level methods to write raw LSIF objects into the output stream. */ +public class LsifOutputStream { + private final LsifSemanticdbOptions options; + private final OutputStream out; + private final ConcurrentLinkedDeque buffer; + private final AtomicBoolean isFlushing; + private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8); + private final ThreadLocal baos = + ThreadLocal.withInitial(LsifByteOutputStream::new); + private final Printer jsonPrinter; + + public LsifOutputStream(LsifSemanticdbOptions options, OutputStream out) { + this.options = options; + this.out = out; + buffer = new ConcurrentLinkedDeque<>(); + isFlushing = new AtomicBoolean(false); + + Set fieldsToAlwaysInclude = + new HashSet<>(LsifPosition.getDescriptor().getFields()); + fieldsToAlwaysInclude.add(LsifObject.getDescriptor().findFieldByName("id")); + jsonPrinter = + JsonFormat.printer() + .includingDefaultValueFields(fieldsToAlwaysInclude) + .omittingInsignificantWhitespace(); + } + + public void write(byte[] bytes) { + if (bytes.length == 0) return; + buffer.add(bytes); + } + + public void writeLsifObject(LsifObject.Builder object) { + LsifByteOutputStream b = baos.get(); + b.output.reset(); + try { + switch (options.format) { + case PROTOBUF: + object.buildPartial().writeTo(b.output); + break; + case JSON: + default: + jsonPrinter.appendTo(object, b.writer); + b.writer.flush(); + break; + } + } catch (IOException e) { + options.reporter.error(e); + } + write(b.output.toByteArray()); + } + + public void flush() throws IOException { + if (isFlushing.compareAndSet(false, true)) { + byte[] bytes = buffer.poll(); + while (bytes != null) { + out.write(bytes); + out.write(NEWLINE); + bytes = buffer.poll(); + } + out.flush(); + isFlushing.set(false); + } + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java new file mode 100644 index 00000000..bf775fc7 --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java @@ -0,0 +1,132 @@ +package com.sourcegraph.lsif_semanticdb; + +import com.sourcegraph.semanticdb_javac.Semanticdb; +import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolInformation; +import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence; +import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence.Role; +import com.sourcegraph.semanticdb_javac.SemanticdbSymbols; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** The core logic that converts SemanticDB into LSIF. */ +public class LsifSemanticdb { + private final LsifWriter writer; + private final LsifSemanticdbOptions options; + private final Map globals; + + public LsifSemanticdb(LsifWriter writer, LsifSemanticdbOptions options) { + this.writer = writer; + this.options = options; + this.globals = new ConcurrentHashMap<>(); + } + + public static void run(LsifSemanticdbOptions options) throws IOException { + LsifWriter writer = new LsifWriter(options); + new LsifSemanticdb(writer, options).run(); + } + + private void run() throws IOException { + writer.emitMetaData(); + int projectId = writer.emitProject(options.language); + + List files = SemanticdbWalker.findSemanticdbFiles(options); + if (options.reporter.hasErrors()) return; + options.reporter.startProcessing(files.size()); + + Set isExportedSymbol = exportSymbols(files); + List documentIds = + filesStream(files) + .flatMap(d -> processPath(d, isExportedSymbol)) + .collect(Collectors.toList()); + + writer.emitContains(projectId, documentIds); + + writer.build(); + options.reporter.endProcessing(); + } + + private Stream filesStream(List files) { + return options.parallel ? files.parallelStream() : files.stream(); + } + + private Set exportSymbols(List files) { + return filesStream(files) + .flatMap(this::parseTextDocument) + .flatMap( + d -> + d.semanticdb.getOccurrencesList().stream() + .filter(occ -> occ.getRole() == Role.DEFINITION) + .map(SymbolOccurrence::getSymbol) + .filter(SemanticdbSymbols::isGlobal)) + .collect(Collectors.toSet()); + } + + private Stream processPath(Path path, Set isExportedSymbol) { + return parseTextDocument(path).map(d -> processDocument(d, isExportedSymbol)); + } + + private Integer processDocument(LsifTextDocument doc, Set isExportedSymbol) { + int documentId = writer.emitDocument(doc); + Set localDefinitions = + doc.semanticdb.getOccurrencesList().stream() + .filter( + occ -> + occ.getRole() == Role.DEFINITION && SemanticdbSymbols.isLocal(occ.getSymbol())) + .map(SymbolOccurrence::getSymbol) + .collect(Collectors.toSet()); + doc.id = documentId; + ResultSets results = new ResultSets(writer, globals, isExportedSymbol, localDefinitions); + Set rangeIds = new LinkedHashSet<>(); + + for (SymbolOccurrence occ : doc.sortedSymbolOccurrences()) { + SymbolInformation symbolInformation = + doc.symbols.getOrDefault(occ.getSymbol(), SymbolInformation.getDefaultInstance()); + ResultIds ids = results.getOrInsertResultSet(occ.getSymbol()); + int rangeId = writer.emitRange(occ.getRange()); + rangeIds.add(rangeId); + + // Range + writer.emitNext(rangeId, ids.resultSet); + + // Reference + writer.emitItem(ids.referenceResult, rangeId, doc.id); + + // Definition + if (occ.getRole() == SymbolOccurrence.Role.DEFINITION && ids.isDefinitionDefined()) { + writer.emitItem(ids.definitionResult, rangeId, doc.id); + + // Hover + String documentation = symbolInformation.getDocumentation().getMessage(); + if (!documentation.isEmpty()) { + int hoverId = writer.emitHoverResult(doc.semanticdb.getLanguage(), documentation); + writer.emitHoverEdge(ids.resultSet, hoverId); + } + } + } + writer.emitContains(doc.id, new ArrayList<>(rangeIds)); + writer.flush(); + options.reporter.processedOneItem(); + return documentId; + } + + private Stream parseTextDocument(Path semanticdbPath) { + try { + return Semanticdb.TextDocuments.parseFrom(Files.readAllBytes(semanticdbPath)) + .getDocumentsList().stream() + .filter(sdb -> !sdb.getOccurrencesList().isEmpty()) + .map(sdb -> new LsifTextDocument(semanticdbPath, sdb, options.sourceroot)); + } catch (IOException e) { + options.reporter.error(e); + return Stream.empty(); + } + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbOptions.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbOptions.java new file mode 100644 index 00000000..c0d6f54e --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbOptions.java @@ -0,0 +1,37 @@ +package com.sourcegraph.lsif_semanticdb; + +import com.sourcegraph.lsif_protocol.LsifToolInfo; +import java.nio.file.Path; +import java.util.List; + +/** Configuration options to tweak the lsif-semanticdb command. */ +public class LsifSemanticdbOptions { + + public final List targetroots; + public final Path output; + public final Path sourceroot; + public final LsifSemanticdbReporter reporter; + public final LsifToolInfo toolInfo; + public final String language; + public final LsifOutputFormat format; + public final boolean parallel; + + public LsifSemanticdbOptions( + List targetroots, + Path output, + Path sourceroot, + LsifSemanticdbReporter reporter, + LsifToolInfo toolInfo, + String language, + LsifOutputFormat format, + boolean parallel) { + this.targetroots = targetroots; + this.output = output; + this.sourceroot = sourceroot; + this.reporter = reporter; + this.toolInfo = toolInfo; + this.language = language; + this.format = format; + this.parallel = parallel; + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbReporter.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbReporter.java new file mode 100644 index 00000000..bbf56725 --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdbReporter.java @@ -0,0 +1,21 @@ +package com.sourcegraph.lsif_semanticdb; + +/** + * API to hook into the event stream of the lsif-semanticdb command. + * + *

The lsif-semanticdb command doesn't fail fast on the first error. Clients are expected to + * handle errors through the error method. + */ +public abstract class LsifSemanticdbReporter { + public void error(Throwable e) {} + + public void startProcessing(int taskSize) {} + + public void processedOneItem() {} + + public void endProcessing() {} + + public boolean hasErrors() { + return false; + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifTextDocument.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifTextDocument.java new file mode 100644 index 00000000..7127fe5a --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifTextDocument.java @@ -0,0 +1,53 @@ +package com.sourcegraph.lsif_semanticdb; + +import com.sourcegraph.semanticdb_javac.Semanticdb; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Wrapper around a SemanticDB TextDocument with LSIF-related utilities. */ +public class LsifTextDocument { + public final Path semanticdbPath; + public Semanticdb.TextDocument semanticdb; + public int id; + public final Map symbols; + public final Map localSymbols; + + public LsifTextDocument( + Path semanticdbPath, Semanticdb.TextDocument semanticdb, Path sourceroot) { + this.semanticdbPath = semanticdbPath; + this.symbols = new HashMap<>(); + this.localSymbols = new HashMap<>(); + String uri = sourceroot.resolve(semanticdb.getUri()).toUri().toString(); + setSemanticdb(Semanticdb.TextDocument.newBuilder(semanticdb).setUri(uri).build()); + } + + @Override + public String toString() { + return "LsifDocument{" + + "path=" + + semanticdbPath + + ", semanticdb=" + + semanticdb + + ", id=" + + id + + '}'; + } + + public List sortedSymbolOccurrences() { + ArrayList result = + new ArrayList<>(semanticdb.getOccurrencesList().size()); + result.addAll(semanticdb.getOccurrencesList()); + result.sort((o1, o2) -> new RangeComparator().compare(o1.getRange(), o2.getRange())); + return result; + } + + private void setSemanticdb(Semanticdb.TextDocument semanticdb) { + this.semanticdb = semanticdb; + for (Semanticdb.SymbolInformation info : semanticdb.getSymbolsList()) { + symbols.put(info.getSymbol(), info); + } + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifWriter.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifWriter.java new file mode 100644 index 00000000..2fd65b54 --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifWriter.java @@ -0,0 +1,160 @@ +package com.sourcegraph.lsif_semanticdb; + +import com.sourcegraph.lsif_protocol.LsifHover; +import com.sourcegraph.lsif_protocol.LsifHover.Content; +import com.sourcegraph.lsif_protocol.LsifObject; +import com.sourcegraph.lsif_protocol.LsifPosition; +import com.sourcegraph.semanticdb_javac.Semanticdb; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; + +/** High-level utility methods to write LSIF vertex/edge objects into the LSIF output stream. */ +public class LsifWriter implements AutoCloseable { + + private final Path tmp; + private final LsifOutputStream output; + private final LsifSemanticdbOptions options; + private final AtomicInteger id = new AtomicInteger(); + + public LsifWriter(LsifSemanticdbOptions options) throws IOException { + this.tmp = Files.createTempFile("lsif-semanticdb", "dump.lsif"); + this.output = + new LsifOutputStream(options, new BufferedOutputStream(Files.newOutputStream(tmp))); + this.options = options; + } + + public void emitMetaData() { + emitObject( + lsifVertex("metaData") + .setVersion("0.4.3") + .setProjectRoot(options.sourceroot.toUri().toString()) + .setPositionEncoding("utf-16") + .setToolInfo(options.toolInfo)); + } + + public int emitProject(String language) { + return emitObject(lsifVertex("project").setKind(language)); + } + + public int emitDocument(LsifTextDocument doc) { + return emitObject( + lsifVertex("document") + .setUri(doc.semanticdb.getUri()) + .setLanguage(doc.semanticdb.getLanguage().toString().toLowerCase())); + } + + public void emitContains(int outV, List inVs) { + emitObject(lsifEdge("contains").setOutV(outV).addAllInVs(inVs)); + } + + public int emitRange(Semanticdb.Range range) { + return emitObject( + lsifVertex("range") + .setStart( + LsifPosition.newBuilder() + .setLine(range.getStartLine()) + .setCharacter(range.getStartCharacter())) + .setEnd( + LsifPosition.newBuilder() + .setLine(range.getEndLine()) + .setCharacter(range.getEndCharacter()))); + } + + public int emitResultSet() { + return emitObject(lsifVertex("resultSet")); + } + + public void emitNext(int outV, int inV) { + emitObject(lsifEdge("next").setOutV(outV).setInV(inV)); + } + + public void emitEdge(String label, int outV, int inV) { + emitObject(lsifEdge(label).setOutV(outV).setInV(inV)); + } + + public int emitReferenceResult(int resultSet) { + int referenceResult = emitObject(lsifVertex("referenceResult")); + emitEdge("textDocument/reference", resultSet, referenceResult); + return referenceResult; + } + + public int emitDefinitionResult(int resultSet) { + int definitionResult = emitObject(lsifVertex("definitionResult")); + emitEdge("textDocument/definition", resultSet, definitionResult); + return definitionResult; + } + + public int emitHoverResult(Semanticdb.Language language, String value) { + return emitObject( + lsifVertex("hoverResult") + .setResult( + LsifHover.newBuilder() + .addContents( + Content.newBuilder() + .setLanguage(language.toString().toLowerCase(Locale.ROOT)) + .setValue(value)))); + } + + public void emitHoverEdge(int outV, int inV) { + emitObject(lsifEdge("textDocument/hover").setOutV(outV).setInV(inV)); + } + + public void emitMonikerEdge(int outV, int inV) { + emitObject(lsifEdge("moniker").setOutV(outV).setInV(inV)); + } + + public int emitMonikerVertex(String symbol, boolean isExportedSymbol) { + String kind = isExportedSymbol ? "export" : "import"; + return emitObject( + lsifVertex("moniker").setKind(kind).setScheme("semanticdb").setIdentifier(symbol)); + } + + public void emitItem(int outV, int inV, int document) { + emitObject(lsifEdge("item").setOutV(outV).addInVs(inV).setDocument(document)); + } + + public void build() throws IOException { + close(); + Files.move(tmp, options.output, StandardCopyOption.REPLACE_EXISTING); + } + + @Override + public void close() throws IOException { + output.flush(); + } + + public void flush() { + try { + output.flush(); + } catch (IOException e) { + options.reporter.error(e); + } + } + + public int emitObject(LsifObject.Builder object) { + output.writeLsifObject(object); + return object.getId(); + } + + private LsifObject.Builder lsifObject() { + return LsifObject.newBuilder().setId(nextId()); + } + + private LsifObject.Builder lsifEdge(String label) { + return lsifObject().setType("edge").setLabel(label); + } + + private LsifObject.Builder lsifVertex(String label) { + return lsifObject().setType("vertex").setLabel(label); + } + + private int nextId() { + return id.incrementAndGet(); + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/RangeComparator.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/RangeComparator.java new file mode 100644 index 00000000..6a368d4f --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/RangeComparator.java @@ -0,0 +1,29 @@ +package com.sourcegraph.lsif_semanticdb; + +import com.sourcegraph.semanticdb_javac.Semanticdb.Range; + +/** + * Comparator that sorts SemanticDB ranges by appearance in the document. + * + *

We can't guarantee ordering of SymbolOccurrence in SemanticDB payloads so it's good to sort + * them before processing. + */ +public class RangeComparator implements java.util.Comparator { + + @Override + public int compare(Range r1, Range r2) { + int byStartLine = Integer.compare(r1.getStartLine(), r2.getStartLine()); + if (byStartLine != 0) { + return byStartLine; + } + int byStartCharacter = Integer.compare(r1.getStartCharacter(), r2.getStartCharacter()); + if (byStartCharacter != 0) { + return byStartCharacter; + } + int byEndLine = Integer.compare(r1.getEndLine(), r2.getEndLine()); + if (byEndLine != 0) { + return byEndLine; + } + return Integer.compare(r1.getEndCharacter(), r2.getEndCharacter()); + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/ResultIds.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/ResultIds.java new file mode 100644 index 00000000..557ec296 --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/ResultIds.java @@ -0,0 +1,18 @@ +package com.sourcegraph.lsif_semanticdb; + +/** Utility to deal with a group of IDs for a result set, definition result and reference result. */ +public class ResultIds { + public final int resultSet; + public final int definitionResult; + public final int referenceResult; + + public ResultIds(int resultSet, int definitionResult, int referenceResult) { + this.resultSet = resultSet; + this.definitionResult = definitionResult; + this.referenceResult = referenceResult; + } + + public boolean isDefinitionDefined() { + return definitionResult > 0; + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/ResultSets.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/ResultSets.java new file mode 100644 index 00000000..9b6dc955 --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/ResultSets.java @@ -0,0 +1,50 @@ +package com.sourcegraph.lsif_semanticdb; + +import com.sourcegraph.semanticdb_javac.SemanticdbSymbols; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +/** Utility to deal with the creation of result sets. */ +public class ResultSets implements Function { + + private final LsifWriter writer; + private final Map globals; + private final HashMap locals; + private final Set exportedSymbols; + private final Set localDefinitions; + + public ResultSets( + LsifWriter writer, + Map globals, + Set exportedSymbols, + Set localDefinitions) { + this.writer = writer; + this.globals = globals; + this.exportedSymbols = exportedSymbols; + this.localDefinitions = localDefinitions; + locals = new HashMap<>(); + } + + public ResultIds getOrInsertResultSet(String symbol) { + boolean isLocal = SemanticdbSymbols.isLocal(symbol); + Map cache = isLocal ? locals : globals; + return cache.computeIfAbsent(symbol, this); + } + + @Override + public ResultIds apply(String symbol) { + boolean isExportedSymbol = exportedSymbols.contains(symbol); + boolean hasDefinitionResult = isExportedSymbol || localDefinitions.contains(symbol); + int resultSet = writer.emitResultSet(); + + // Moniker + int monikerId = writer.emitMonikerVertex(symbol, isExportedSymbol); + writer.emitMonikerEdge(resultSet, monikerId); + + int definitionId = hasDefinitionResult ? writer.emitDefinitionResult(resultSet) : -1; + + return new ResultIds(resultSet, definitionId, writer.emitReferenceResult(resultSet)); + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbWalker.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbWalker.java new file mode 100644 index 00000000..2157a26b --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbWalker.java @@ -0,0 +1,47 @@ +package com.sourcegraph.lsif_semanticdb; + +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +/** A file visitor that recursively collects all SemanticDB files in a given directory. */ +public class SemanticdbWalker extends SimpleFileVisitor { + private final ArrayList result; + private final LsifSemanticdbOptions options; + private final PathMatcher semanticdbPattern; + + public SemanticdbWalker(LsifSemanticdbOptions options) { + this.options = options; + result = new ArrayList<>(); + semanticdbPattern = FileSystems.getDefault().getPathMatcher("glob:**.semanticdb"); + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (semanticdbPattern.matches(file)) { + result.add(file); + } + return super.visitFile(file, attrs); + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + options.reporter.error(exc); + return FileVisitResult.CONTINUE; + } + + public static List findSemanticdbFiles(LsifSemanticdbOptions options) throws IOException { + SemanticdbWalker walker = new SemanticdbWalker(options); + for (Path root : options.targetroots) { + Files.walkFileTree(root, walker); + } + return walker.result; + } +} diff --git a/lsif-semanticdb/src/main/protobuf/lsif.proto b/lsif-semanticdb/src/main/protobuf/lsif.proto new file mode 100644 index 00000000..c3518486 --- /dev/null +++ b/lsif-semanticdb/src/main/protobuf/lsif.proto @@ -0,0 +1,45 @@ +syntax = "proto3"; + +option java_multiple_files = true; +package com.sourcegraph.lsif_protocol; + +message LsifObject { + int32 id = 1; + string type = 2; + string label = 3; + string version = 4; + string projectRoot = 5; + string positionEncoding = 6; + LsifToolInfo toolInfo = 7; + string kind = 8; + string uri = 9; + string language = 10; + int32 outV = 12; + int32 inV = 13; + repeated int32 inVs = 14; + int32 document = 15; + LsifHover result = 16; + string scheme = 17; + string identifier = 18; + LsifPosition start = 19; + LsifPosition end = 20; +} + +message LsifToolInfo { + string name = 1; + string version = 2; + repeated string args = 3; +} + +message LsifPosition { + int32 line = 1; + int32 character = 2; +} + +message LsifHover { + repeated Content contents = 1; + message Content { + string language = 1; + string value = 2; + } +} diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbSymbols.java b/semanticdb-java/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbSymbols.java similarity index 100% rename from semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbSymbols.java rename to semanticdb-java/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbSymbols.java diff --git a/semanticdb-javac/src/main/protobuf/semanticdb.proto b/semanticdb-java/src/main/protobuf/semanticdb.proto similarity index 100% rename from semanticdb-javac/src/main/protobuf/semanticdb.proto rename to semanticdb-java/src/main/protobuf/semanticdb.proto diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/GlobalSymbolsCache.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/GlobalSymbolsCache.java index 1d2105b9..2af332e6 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/GlobalSymbolsCache.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/GlobalSymbolsCache.java @@ -12,9 +12,9 @@ public final class GlobalSymbolsCache { private final IdentityHashMap globals = new IdentityHashMap<>(); - private final SemanticdbOptions options; + private final SemanticdbJavacOptions options; - public GlobalSymbolsCache(SemanticdbOptions options) { + public GlobalSymbolsCache(SemanticdbJavacOptions options) { this.options = options; } diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbOptions.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbJavacOptions.java similarity index 90% rename from semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbOptions.java rename to semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbJavacOptions.java index 89d3d168..5d20bd35 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbOptions.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbJavacOptions.java @@ -5,7 +5,7 @@ import java.util.ArrayList; /** Settings that can be configured alongside the -Xplugin compiler option. */ -public class SemanticdbOptions { +public class SemanticdbJavacOptions { /** The directory to place */ public Path targetroot; @@ -15,7 +15,7 @@ public class SemanticdbOptions { public boolean verboseEnabled = false; public final ArrayList errors; - public SemanticdbOptions() { + public SemanticdbJavacOptions() { errors = new ArrayList<>(); } @@ -26,8 +26,8 @@ public static String missingRequiredDirectoryOption(String option) { option, option); } - public static SemanticdbOptions parse(String[] args) { - SemanticdbOptions result = new SemanticdbOptions(); + public static SemanticdbJavacOptions parse(String[] args) { + SemanticdbJavacOptions result = new SemanticdbJavacOptions(); for (String arg : args) { if (arg.startsWith("-targetroot:")) { result.targetroot = Paths.get(arg.substring("-targetroot:".length())); diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbPlugin.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbPlugin.java index c8e5ff9c..83881c69 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbPlugin.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbPlugin.java @@ -13,7 +13,7 @@ public String getName() { @Override public void init(JavacTask task, String... args) { SemanticdbReporter reporter = new SemanticdbReporter(); - SemanticdbOptions options = SemanticdbOptions.parse(args); + SemanticdbJavacOptions options = SemanticdbJavacOptions.parse(args); GlobalSymbolsCache globals = new GlobalSymbolsCache(options); if (!options.errors.isEmpty()) { for (String error : options.errors) { diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbTaskListener.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbTaskListener.java index 88eb7f00..ce7d8a92 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbTaskListener.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbTaskListener.java @@ -15,13 +15,13 @@ * source file. */ public final class SemanticdbTaskListener implements TaskListener { - private final SemanticdbOptions options; + private final SemanticdbJavacOptions options; private final JavacTask task; private final GlobalSymbolsCache globals; private final SemanticdbReporter reporter; public SemanticdbTaskListener( - SemanticdbOptions options, + SemanticdbJavacOptions options, JavacTask task, GlobalSymbolsCache globals, SemanticdbReporter reporter) { @@ -69,7 +69,7 @@ private void writeSemanticdb(Path output, Semanticdb.TextDocument textDocument) } } - private Result semanticdbOutputPath(SemanticdbOptions options, TaskEvent e) { + private Result semanticdbOutputPath(SemanticdbJavacOptions options, TaskEvent e) { Path absolutePath = Paths.get(e.getSourceFile().toUri()).normalize(); if (absolutePath.startsWith(options.sourceroot)) { Path relativePath = options.sourceroot.relativize(absolutePath); diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbVisitor.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbVisitor.java index c7b763cf..901950ae 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbVisitor.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbVisitor.java @@ -28,15 +28,15 @@ public class SemanticdbVisitor extends TreePathScanner { private final LocalSymbolsCache locals; private final JavacTask task; private final TaskEvent event; - private final SemanticdbOptions options; private final Trees trees; + private final SemanticdbJavacOptions options; private final EndPosTable endPosTable; private final ArrayList occurrences; private final ArrayList symbolInfos; private String source; public SemanticdbVisitor( - JavacTask task, GlobalSymbolsCache globals, TaskEvent event, SemanticdbOptions options) { + JavacTask task, GlobalSymbolsCache globals, TaskEvent event, SemanticdbJavacOptions options) { this.task = task; this.globals = globals; // Reused cache between compilation units. this.locals = new LocalSymbolsCache(); // Fresh cache per compilation unit. diff --git a/tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala b/tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala index 3d9b1ed4..1e7f9402 100644 --- a/tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala +++ b/tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala @@ -4,13 +4,12 @@ import java.nio.charset.StandardCharsets import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path -import java.nio.file.PathMatcher import java.util.concurrent.TimeUnit import scala.meta.inputs.Input import scala.meta.internal.io.FileIO -import com.sourcegraph.lsif_java.DeleteVisitor +import com.sourcegraph.io.DeleteVisitor import org.openjdk.jmh.annotations._ import tests.Dependencies import tests.TestCompiler @@ -21,7 +20,7 @@ class CompileBench { var deps: Dependencies = _ var tmp: Path = _ var compiler: TestCompiler = _ - var javaPattern: PathMatcher = _ + @Param(Array("bytebuddy", "guava")) var lib: String = _ @@ -32,7 +31,6 @@ class CompileBench { @Setup() def setup(): Unit = { - javaPattern = FileSystems.getDefault.getPathMatcher("glob:**.java") tmp = Files.createTempDirectory("benchmarks") deps = Dependencies.resolveDependencies(List(libs(lib)), Nil) compiler = new TestCompiler(deps.classpathSyntax, List.empty[String], tmp) @@ -43,7 +41,32 @@ class CompileBench { Files.walkFileTree(tmp, new DeleteVisitor) } - def foreachSource(fn: Seq[Input.VirtualFile] => Int): Long = { + @Benchmark + @BenchmarkMode(Array(Mode.SingleShotTime)) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + def compile(): Long = { + CompileBench.foreachSource(deps) { inputs => + compiler.compile(inputs).byteCode.length + } + } + + @Benchmark + @BenchmarkMode(Array(Mode.SingleShotTime)) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + def compileSemanticdb(): Long = { + CompileBench.foreachSource(deps) { inputs => + compiler.compileSemanticdb(inputs).textDocument.getOccurrencesCount + } + } + +} +object CompileBench { + private val javaPattern = FileSystems + .getDefault + .getPathMatcher("glob:**.java") + def foreachSource( + deps: Dependencies + )(fn: Seq[Input.VirtualFile] => Int): Long = { var sum = 0L deps .sources @@ -65,23 +88,4 @@ class CompileBench { } sum } - - @Benchmark - @BenchmarkMode(Array(Mode.SingleShotTime)) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - def compile(): Long = { - foreachSource { inputs => - compiler.compile(inputs).byteCode.length - } - } - - @Benchmark - @BenchmarkMode(Array(Mode.SingleShotTime)) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - def compileSemanticdb(): Long = { - foreachSource { inputs => - compiler.compileSemanticdb(inputs).textDocument.getOccurrencesCount - } - } - } diff --git a/tests/benchmarks/src/main/scala/benchmarks/LsifSemanticdbBench.scala b/tests/benchmarks/src/main/scala/benchmarks/LsifSemanticdbBench.scala new file mode 100644 index 00000000..89e1a594 --- /dev/null +++ b/tests/benchmarks/src/main/scala/benchmarks/LsifSemanticdbBench.scala @@ -0,0 +1,77 @@ +package benchmarks + +import java.nio.file.Files +import java.nio.file.Path +import java.util.concurrent.TimeUnit + +import com.sourcegraph.io.DeleteVisitor +import com.sourcegraph.lsif_java.LsifJava +import org.openjdk.jmh.annotations._ +import tests.Dependencies +import tests.TestCompiler + +@State(Scope.Benchmark) +class LsifSemanticdbBench { + + var targetroot: Path = _ + var deps: Dependencies = _ + + @Setup + def setup(): Unit = { + targetroot = Files.createTempDirectory("lsif-java") + deps = Dependencies + .resolveDependencies(List("com.google.guava:guava:30.1-jre"), Nil) + val compiler = + new TestCompiler(deps.classpathSyntax, List.empty[String], targetroot) + CompileBench.foreachSource(deps) { inputs => + compiler.compileSemanticdb(inputs).byteCode.length + } + } + + @TearDown + def teardown(): Unit = { + Files.walkFileTree(targetroot, new DeleteVisitor()) + } + + @Benchmark + @BenchmarkMode(Array(Mode.SingleShotTime)) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + def jsonParallel(): Unit = run("dump.lsif", parallel = true) + + @Benchmark + @BenchmarkMode(Array(Mode.SingleShotTime)) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + def json(): Unit = run("dump.lsif", parallel = false) + + @Benchmark + @BenchmarkMode(Array(Mode.SingleShotTime)) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + def protobufParallel(): Unit = run("dump.lsif-protobuf", parallel = true) + + @Benchmark + @BenchmarkMode(Array(Mode.SingleShotTime)) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + def protobuf(): Unit = run("dump.lsif-protobuf", parallel = false) + + private def run(filename: String, parallel: Boolean): Unit = { + val output = Files.createTempFile("lsif-java", filename) + val parallelFlag = + if (parallel) + "--parallel" + else + "--no-parallel" + val exit = LsifJava + .app + .run( + List( + "index-semanticdb", + s"--output=$output", + parallelFlag, + targetroot.toString + ) + ) + + require(exit == 0, exit) + } + +} diff --git a/tests/snapshots/src/main/generated/index-semanticdb/locals b/tests/snapshots/src/main/generated/index-semanticdb/locals new file mode 100644 index 00000000..628e1e22 --- /dev/null +++ b/tests/snapshots/src/main/generated/index-semanticdb/locals @@ -0,0 +1,56 @@ +─────────────── +│ Source code │ +─────────────── +/example/Example.java +package example; + +public class Example { +// ^^^^^^^ definition example/Example# +// ^^^^^^^ definition example/Example#``(). + public static int increment(int n) { return n + 1; } +// ^^^^^^^^^ definition example/Example#increment(). +// ^ definition local0 +// ^ reference local0 +} + + +────────────────────────────────── +│ LSIF Graph for symbol 'local0' │ +────────────────────────────────── + ╭──────────╮ + │project(2)│ + ╰─────┬────╯ + │ + v + ╭─────────────────────────────╮ ╭───────────────────╮ + │document example/Example.java│ │referenceResult(42)│ + ╰──┬─────────────┬────────────╯ ╰────┬──────┬───────╯ + │ │ ^ │ │ + │ │ │ │ ╭──╯ + │ │ ╭───────────┼─╯ │ + │ │ │ │ │ + │ v v │ │ + │ ╭──────────────────╮ │ │ + │ │range(48) 3:46 'n'│ │ │ + │ ╰──────────┬───────╯ │ │ + │ │ ╭────────╯ │ + │ v │ │ + │ ╭─────────┴───╮ │ + │ │resultSet(37)│ │ + │ ╰──┬───┬──────╯ │ + │ │ │ ^ │ + │ │ │ ╰──────────╮ │ + │ │ ╰──────────╮ │ │ + │ v │ │ │ + │ ╭────────────────────╮ │ │ │ + │ │definitionResult(40)│ │ │ │ + │ ╰──────────┬─────────╯ │ │ │ + │ │ │ │ │ + │ ╰─────────────╮│ │ │ + ╰─────────────────────────╮ ││ │ │ + ╭────────────────┼───┼╯ │ │ + │ │ │ │ │ + v v v │ v + ╭──────────────╮ ╭───────────┴──────╮ + │moniker local0│ │range(44) 3:34 'n'│ + ╰──────────────╯ ╰──────────────────╯ diff --git a/tests/snapshots/src/main/generated/index-semanticdb/multifile b/tests/snapshots/src/main/generated/index-semanticdb/multifile new file mode 100644 index 00000000..116b5579 --- /dev/null +++ b/tests/snapshots/src/main/generated/index-semanticdb/multifile @@ -0,0 +1,65 @@ +─────────────── +│ Source code │ +─────────────── +/example/Example1.java +package example; + + /** Example1 docstring */ +public class Example1 {} +// ^^^^^^^^ definition example/Example1# +// ^^^^^^^^ definition example/Example1#``(). + +/example/Example2.java +package example; + +public class Example2 { +// ^^^^^^^^ definition example/Example2# +// ^^^^^^^^ definition example/Example2#``(). + public Example1 example; +// ^^^^^^^^ reference example/Example1# +// ^^^^^^^ definition example/Example2#example. +} + + +───────────────────────────────────────────── +│ LSIF Graph for symbol 'example/Example1#' │ +───────────────────────────────────────────── + ╭──────────╮ + │project(2)│ + ╰───┬─────┬╯ + │ │ + ╭──────────────────╯ │ + │ │ + v │ + ╭──────────────────────────────╮ │ ╭──────────────────╮ + │document example/Example2.java│ │ │referenceResult(9)│ + ╰───────────┬──────────────────╯ │ ╰────┬───┬─────────╯ + │ ╰─────────╮ ^ │ │ + │ │ │ │ ╰─────────────────╮ + │ ╭──────────────────────────────┼──┼─╯ │ + │ │ │ │ │ + v v │ │ │ + ╭────────────────────────╮ │ │ │ + │range(52) 3:9 'Example1'│ │ │ │ + ╰──────────┬─────────────╯ │ │ │ + │ ╭────────────────────────────────┼──╯ │ + v │ │ │ + ╭────────┴───╮ │ │ + │resultSet(4)│ │ │ + ╰───┬───┬──┬─╯ │ │ + │ │ ^│ │ │ + v │ ││ v │ + ╭───────────────────╮ │ ││ ╭──────────────────────────────╮ │ + │definitionResult(7)│ │ ││ │document example/Example1.java│ │ + ╰─────────┬─────────╯ │ ││ ╰───────────────┬──────────────╯ │ + │ │ ││ │ │ + │ │ ││ ╰─────────────────────╮ │ + │ │ ╰┼───────────────────────────────────────────────╮ │ │ + ╰───────────┼──┼──────────────────────────────────────────╮ │ │ │ + ╰──┼─────────────────╮ │ │ │ │ + ╭─────────────╯ │ │ │ │ │ + │ │ │ │ │ │ + v v v │ v v + ╭─────────────────────────╮ ╭──────────────────────────────────╮ ╭──────────┴──────────────╮ + │moniker example/Example1#│ │hoverResult(15) Example1 docstring│ │range(11) 3:13 'Example1'│ + ╰─────────────────────────╯ ╰──────────────────────────────────╯ ╰─────────────────────────╯ diff --git a/tests/snapshots/src/main/generated/index-semanticdb/reference b/tests/snapshots/src/main/generated/index-semanticdb/reference new file mode 100644 index 00000000..a2fb097d --- /dev/null +++ b/tests/snapshots/src/main/generated/index-semanticdb/reference @@ -0,0 +1,57 @@ +─────────────── +│ Source code │ +─────────────── +/example/Example.java +package example; + /** Docstring for Example */ +public class Example { +// ^^^^^^^ definition example/Example# +// ^^^^^^^ definition example/Example#``(). + Example next; +//^^^^^^^ reference example/Example# +// ^^^^ definition example/Example#next. +} + + +──────────────────────────────────────────── +│ LSIF Graph for symbol 'example/Example#' │ +──────────────────────────────────────────── + ╭──────────╮ + │project(2)│ + ╰─────┬────╯ + │ + ╭─────────────────╯ + │ + v + ╭─────────────────────────────╮ ╭──────────────────╮ + │document example/Example.java│ │referenceResult(9)│ + ╰────┬─────────────┬──────────╯ ╰────┬───────┬─────╯ + │ │ ^ │ │ + ╭─────────╯ │ │ │ │ + │ │ ╭────────────────────┼─╯ │ + │ │ │ │ │ + │ v v │ │ + │ ╭───────────────────────╮ │ │ + │ │range(28) 3:2 'Example'│ │ │ + │ ╰──────────────────┬────╯ │ │ + │ │ ╭─────────────╯ │ + │ v │ │ + │ ╭────────┴───╮ │ + │ │resultSet(4)│ │ + │ ╰───┬───┬──┬─╯ │ + │ │ │ ^│ │ + │ ╭─────────────╯ │╭┼┼─────────────────────╯ + │ │ ││││ + │ v ││││ + │ ╭───────────────────╮ ││││ + │ │definitionResult(7)│ ││││ + │ ╰──────────────────┬╯ ││││ + │ │ ││││ + │ │ ╰┼┼┼───────────────────────╮ + ╰───────────────────────╮ │ │││ │ + ╭──────────────────┼────┼─────────┼┼╯ │ + │ │ │ ╭────┼╯ │ + v v v │ v v + ╭────────────────────────╮ ╭──────────────┴─────────╮ ╭─────────────────────────────────────╮ + │moniker example/Example#│ │range(11) 2:13 'Example'│ │hoverResult(15) Docstring for Example│ + ╰────────────────────────╯ ╰────────────────────────╯ ╰─────────────────────────────────────╯ diff --git a/tests/snapshots/src/main/generated/snapshot-lsif.diff b/tests/snapshots/src/main/generated/snapshot-lsif.diff new file mode 100644 index 00000000..e69de29b diff --git a/tests/snapshots/src/main/scala/tests/LibrarySnapshotGenerator.scala b/tests/snapshots/src/main/scala/tests/LibrarySnapshotGenerator.scala index 6abfc8bd..795a4d9d 100644 --- a/tests/snapshots/src/main/scala/tests/LibrarySnapshotGenerator.scala +++ b/tests/snapshots/src/main/scala/tests/LibrarySnapshotGenerator.scala @@ -9,7 +9,7 @@ import scala.meta.inputs.Input import scala.meta.internal.io.FileIO import scala.meta.io.AbsolutePath -import com.sourcegraph.lsif_java.DeleteVisitor +import com.sourcegraph.io.DeleteVisitor import com.sourcegraph.lsif_java.SemanticdbPrinters import coursier.core.Repository import coursier.maven.MavenRepository diff --git a/tests/snapshots/src/main/scala/tests/LsifGraphSnapshotGenerator.scala b/tests/snapshots/src/main/scala/tests/LsifGraphSnapshotGenerator.scala new file mode 100644 index 00000000..ef2c90a1 --- /dev/null +++ b/tests/snapshots/src/main/scala/tests/LsifGraphSnapshotGenerator.scala @@ -0,0 +1,150 @@ +package tests + +import java.nio.file.Files +import java.nio.file.Path + +import com.sourcegraph.io.DeleteVisitor +import com.sourcegraph.lsif_java.LsifJava +import com.sourcegraph.lsif_java.commands.SnapshotLsifCommand +import com.sourcegraph.lsif_java.commands.SnapshotLsifCommand.IndexedLsif +import moped.testkit.FileLayout + +class LsifGraphSnapshotGenerator extends SnapshotGenerator { + + override def run(context: SnapshotContext, handler: SnapshotHandler): Unit = { + val gen = new Gen(context, handler) + gen.indexSemanticdb( + "reference", + "example/Example#", + """/example/Example.java + |package example; + | /** Docstring for Example */ + |public class Example { + | Example next; + |} + |""".stripMargin + ) + + gen.indexSemanticdb( + "multifile", + "example/Example1#", + """|/example/Example1.java + |package example; + | + | /** Example1 docstring */ + |public class Example1 {} + |/example/Example2.java + |package example; + | + |public class Example2 { + | public Example1 example; + |} + |""".stripMargin + ) + + gen.indexSemanticdb( + "locals", + "local0", + """|/example/Example.java + |package example; + | + |public class Example { + | public static int increment(int n) { return n + 1; } + |} + |""".stripMargin + ) + + handler.onFinished(context) + + } + + class Gen(context: SnapshotContext, handler: SnapshotHandler) { + def runSuccessfully(args: List[String], sourceroot: Path): Unit = { + val exit = LsifJava + .app + .withEnv(LsifJava.app.env.withWorkingDirectory(sourceroot)) + .run(args) + require(exit == 0) + } + def indexSemanticdb( + name: String, + symbol: String, + original: String + ): Unit = { + val expectFile = context + .expectDirectory + .resolve("index-semanticdb") + .resolve(name) + handler.onSnapshotTest( + context, + expectFile, + () => { + val tmp = Files.createTempDirectory("lsif-java") + val targetroot = tmp.resolve("targetroot") + val outputLsif = tmp.resolve("dump.lsif") + val outputSnapshot = tmp.resolve("snapshot") + val sourceroot = tmp.resolve("workingDirectory") + FileLayout.fromString(original, sourceroot) + val compiler = + new TestCompiler( + TestCompiler.PROCESSOR_PATH, + Nil, + targetroot, + sourceroot + ) + val result = compiler.compileSemanticdbDirectory(sourceroot) + require(result.isSuccess, result) + runSuccessfully( + List( + "index-semanticdb", + "--no-parallel", + "--output", + outputLsif.toString, + targetroot.toString + ), + sourceroot + ) + val objects = SnapshotLsifCommand.parseInput(outputLsif) + val lsif = new IndexedLsif(outputLsif, objects, sourceroot) + runSuccessfully( + List( + "snapshot-lsif", + "--input", + outputLsif.toString(), + "--output", + outputSnapshot.toString() + ), + sourceroot + ) + val snapshotLsif = FileLayout.asString(outputSnapshot) + val obtained = new StringBuilder() + .header("Source code") + .append(snapshotLsif) + .append("\n") + .header(s"LSIF Graph for symbol '$symbol'") + .append(lsif.asciiGraph(symbol)) + .append("\n") + .toString() + Files.walkFileTree(tmp, new DeleteVisitor()) + obtained + } + ) + } + } + + implicit class XtensionStringBuilder(s: StringBuilder) { + def header(msg: String): StringBuilder = { + val len = msg.length() + 4 + val banner = "─" * len + s.append(banner) + .append("\n") + .append("│ ") + .append(msg) + .append(" │") + .append("\n") + .append(banner) + .append("\n") + } + } + +} diff --git a/tests/snapshots/src/main/scala/tests/MinimizedLsifSnapshotGenerator.scala b/tests/snapshots/src/main/scala/tests/MinimizedLsifSnapshotGenerator.scala new file mode 100644 index 00000000..f5d7edab --- /dev/null +++ b/tests/snapshots/src/main/scala/tests/MinimizedLsifSnapshotGenerator.scala @@ -0,0 +1,86 @@ +package tests + +import java.nio.charset.StandardCharsets +import java.nio.file.Files + +import scala.collection.mutable.ListBuffer + +import scala.meta.internal.io.FileIO +import scala.meta.io.AbsolutePath + +import com.sourcegraph.io.DeleteVisitor +import com.sourcegraph.lsif_java.LsifJava +import munit.internal.console.AnsiColors +import munit.internal.difflib.Diffs + +class MinimizedLsifSnapshotGenerator extends SnapshotGenerator { + def run(args: List[String]): Unit = { + val exit = LsifJava.app.run(args) + require(exit == 0) + } + def onFinished(context: SnapshotContext): Unit = () + override def run(context: SnapshotContext, handler: SnapshotHandler): Unit = { + val sourceroot = AbsolutePath(BuildInfo.sourceroot) + val lsifOutput = Files.createTempDirectory("lsif-java").resolve("dump.lsif") + val snapshotOutput = AbsolutePath(Files.createTempDirectory("lsif-java")) + try { + val javaTargetroot = AbsolutePath(BuildInfo.minimizedJavaTargetroot) + val javaSourceDirectory = AbsolutePath( + BuildInfo.minimizedJavaSourceDirectory + ) + run( + List( + "index-semanticdb", + "--cwd", + sourceroot.toString(), + "--output", + lsifOutput.toString, + "--targetroot", + javaTargetroot.toString() + ) + ) + run( + List( + "snapshot-lsif", + "--cwd", + sourceroot.toString(), + "--input", + lsifOutput.toString, + "--output", + snapshotOutput.toString + ) + ) + val diffs = ListBuffer.empty[String] + FileIO + .listAllFilesRecursively(snapshotOutput) + .foreach { file => + val relativeToSourceroot = file.toRelative(snapshotOutput) + val absoluteSource = sourceroot.resolve(relativeToSourceroot) + val relativeToSourceDirectory = absoluteSource + .toRelative(javaSourceDirectory) + val expectOutput = AbsolutePath( + context.expectDirectory.resolve(relativeToSourceDirectory.toNIO) + ) + val obtainedOutput = FileIO.slurp(file, StandardCharsets.UTF_8) + val expectedOutput = FileIO + .slurp(expectOutput, StandardCharsets.UTF_8) + val diff = Diffs.create(obtainedOutput, expectedOutput).unifiedDiff + if (diff.nonEmpty) { + val filename = relativeToSourceDirectory.toURI(false).toString + val header = "=" * (filename.length + 2) + diffs += header + diffs += s"= $filename" + diffs += header + diffs += "++ expected" + diffs += "-- obtained" + diffs += AnsiColors.filterAnsi(diff) + } + } + val expectFile = context.expectDirectory.resolve("snapshot-lsif.diff") + handler.onSnapshotTest(context, expectFile, () => diffs.mkString("\n")) + } finally { + Files.walkFileTree(lsifOutput, new DeleteVisitor()) + Files.walkFileTree(snapshotOutput.toNIO, new DeleteVisitor()) + } + } +} diff --git a/tests/snapshots/src/main/scala/tests/MinimizedSnapshotSuite.scala b/tests/snapshots/src/main/scala/tests/MinimizedSnapshotSuite.scala deleted file mode 100644 index 9d11735e..00000000 --- a/tests/snapshots/src/main/scala/tests/MinimizedSnapshotSuite.scala +++ /dev/null @@ -1,5 +0,0 @@ -package tests - -import munit.FunSuite - -class MinimizedSnapshotSuite extends FunSuite with AssertSnapshotHandlers {} diff --git a/tests/snapshots/src/main/scala/tests/SaveSnapshotHandler.scala b/tests/snapshots/src/main/scala/tests/SaveSnapshotHandler.scala index 9ab266d3..5283d622 100644 --- a/tests/snapshots/src/main/scala/tests/SaveSnapshotHandler.scala +++ b/tests/snapshots/src/main/scala/tests/SaveSnapshotHandler.scala @@ -7,7 +7,7 @@ import java.util.concurrent.ConcurrentLinkedDeque import scala.jdk.CollectionConverters._ -import com.sourcegraph.lsif_java.DeleteVisitor +import com.sourcegraph.io.DeleteVisitor class SaveSnapshotHandler extends SnapshotHandler { private val writtenTests = new ConcurrentLinkedDeque[Path]() diff --git a/tests/snapshots/src/main/scala/tests/SemanticdbJavacSnapshotGenerator.scala b/tests/snapshots/src/main/scala/tests/SemanticdbJavacSnapshotGenerator.scala index 323f0f26..3872d9e6 100644 --- a/tests/snapshots/src/main/scala/tests/SemanticdbJavacSnapshotGenerator.scala +++ b/tests/snapshots/src/main/scala/tests/SemanticdbJavacSnapshotGenerator.scala @@ -2,5 +2,10 @@ package tests object SemanticdbJavacSnapshotGenerator extends AggregateSnapshotGenerator( - List(new LibrarySnapshotGenerator(), new MinimizedSnapshotGenerator()) + List( + new LibrarySnapshotGenerator(), + new MinimizedSnapshotGenerator(), + new MinimizedLsifSnapshotGenerator(), + new LsifGraphSnapshotGenerator() + ) ) diff --git a/tests/snapshots/src/test/scala/tests/SnapshotSuite.scala b/tests/snapshots/src/test/scala/tests/SnapshotSuite.scala index 2fb3463b..1560a1f8 100644 --- a/tests/snapshots/src/test/scala/tests/SnapshotSuite.scala +++ b/tests/snapshots/src/test/scala/tests/SnapshotSuite.scala @@ -15,3 +15,9 @@ class LibrarySnapshotSuite extends SnapshotSuite(new LibrarySnapshotGenerator) class MinimizedSnapshotSuite extends SnapshotSuite(new MinimizedSnapshotGenerator) + +class MinimizedLsifSnapshotSuite + extends SnapshotSuite(new MinimizedLsifSnapshotGenerator) + +class LsifGraphSnapshotSuite + extends SnapshotSuite(new LsifGraphSnapshotGenerator) diff --git a/tests/unit/src/main/scala/tests/TempDirectories.scala b/tests/unit/src/main/scala/tests/TempDirectories.scala index 62193c2e..86e6a571 100644 --- a/tests/unit/src/main/scala/tests/TempDirectories.scala +++ b/tests/unit/src/main/scala/tests/TempDirectories.scala @@ -3,7 +3,7 @@ package tests import java.nio.file.Files import java.nio.file.Path -import com.sourcegraph.lsif_java.DeleteVisitor +import com.sourcegraph.io.DeleteVisitor import munit.FunSuite trait TempDirectories { diff --git a/tests/unit/src/main/scala/tests/TestCompiler.scala b/tests/unit/src/main/scala/tests/TestCompiler.scala index e578d575..6bd80077 100644 --- a/tests/unit/src/main/scala/tests/TestCompiler.scala +++ b/tests/unit/src/main/scala/tests/TestCompiler.scala @@ -1,6 +1,7 @@ package tests import java.io.StringWriter +import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Path import javax.tools.ToolProvider @@ -9,20 +10,22 @@ import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters._ import scala.meta.Input +import scala.meta.internal.io.FileIO +import scala.meta.io.AbsolutePath import com.sourcegraph.semanticdb_javac.Semanticdb object TestCompiler { - private val PROCESSOR_PATH = System.getProperty("java.class.path") + val PROCESSOR_PATH = System.getProperty("java.class.path") } class TestCompiler( val classpath: String, val options: List[String], - val targetroot: Path + val targetroot: Path, + val sourceroot: Path = Files.createTempDirectory("semanticdb-javac") ) { - private val sourceroot = Files.createTempDirectory("semanticdb-javac") private val compiler = ToolProvider.getSystemJavaCompiler private val fileManager = new SimpleFileManager(compiler.getStandardFileManager(null, null, null)) @@ -31,6 +34,10 @@ class TestCompiler( this(TestCompiler.PROCESSOR_PATH, Nil, targetroot) } + def compileSemanticdbDirectory(dir: Path): CompileResult = { + compileSemanticdb(inputsFromDirectory(dir)) + } + def compileSemanticdb(inputs: Seq[Input.VirtualFile]): CompileResult = { compile( inputs, @@ -90,4 +97,17 @@ class TestCompiler( val stdout = output.toString CompileResult(bytecode, stdout, textDocuments.build(), isSuccess) } + + private def inputsFromDirectory(dir: Path): Seq[Input.VirtualFile] = { + val root = AbsolutePath(dir) + FileIO + .listAllFilesRecursively(root) + .filter(_.toNIO.getFileName.toString.endsWith(".java")) + .map(p => + Input.VirtualFile( + p.toRelative(root).toString(), + FileIO.slurp(p, StandardCharsets.UTF_8) + ) + ) + } } diff --git a/tests/unit/src/test/scala/tests/SemanticdbPrintersSuite.scala b/tests/unit/src/test/scala/tests/SemanticdbPrintersSuite.scala new file mode 100644 index 00000000..aabcb429 --- /dev/null +++ b/tests/unit/src/test/scala/tests/SemanticdbPrintersSuite.scala @@ -0,0 +1,87 @@ +package tests + +import com.sourcegraph.lsif_java.SemanticdbPrinters +import com.sourcegraph.semanticdb_javac.Semanticdb.Range +import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence +import com.sourcegraph.semanticdb_javac.Semanticdb.SymbolOccurrence.Role +import com.sourcegraph.semanticdb_javac.Semanticdb.TextDocument +import munit.FunSuite + +class SemanticdbPrintersSuite extends FunSuite { + test("tabs") { + val doc = TextDocument + .newBuilder() + .setText("type Indexer interface {\n\tFoo()\n\t\tBar()\n\t\t\tQux()\n}\n") + .addOccurrences( + SymbolOccurrence + .newBuilder() + .setRole(Role.DEFINITION) + .setSymbol("foo/Indexer#") + .setRange( + Range + .newBuilder() + .setStartLine(0) + .setStartCharacter(5) + .setEndLine(0) + .setEndCharacter(5 + "Indexer".length) + ) + ) + .addOccurrences( + SymbolOccurrence + .newBuilder() + .setRole(Role.DEFINITION) + .setSymbol("foo/Indexer#Foo().") + .setRange( + Range + .newBuilder() + .setStartLine(1) + .setStartCharacter(1) + .setEndLine(1) + .setEndCharacter(4) + ) + ) + .addOccurrences( + SymbolOccurrence + .newBuilder() + .setRole(Role.DEFINITION) + .setSymbol("foo/Indexer#Bar().") + .setRange( + Range + .newBuilder() + .setStartLine(2) + .setStartCharacter(2) + .setEndLine(2) + .setEndCharacter(5) + ) + ) + .addOccurrences( + SymbolOccurrence + .newBuilder() + .setRole(Role.DEFINITION) + .setSymbol("foo/Indexer#Qux().") + .setRange( + Range + .newBuilder() + .setStartLine(3) + .setStartCharacter(3) + .setEndLine(3) + .setEndCharacter(6) + ) + ) + val obtained = SemanticdbPrinters.printTextDocument(doc.build()) + assertNoDiff( + obtained, + """|type Indexer interface { + |// ^^^^^^^ definition foo/Indexer# + |→Foo() + |//^^ definition foo/Indexer#Foo(). + |→→Bar() + |//^^^ definition foo/Indexer#Bar(). + |→→→Qux() + |// ^^^ definition foo/Indexer#Qux(). + |} + |""".stripMargin + ) + } + +} diff --git a/tests/unit/src/test/scala/tests/TargetedSuite.scala b/tests/unit/src/test/scala/tests/TargetedSuite.scala index 499bedf0..eb19edf7 100644 --- a/tests/unit/src/test/scala/tests/TargetedSuite.scala +++ b/tests/unit/src/test/scala/tests/TargetedSuite.scala @@ -79,7 +79,7 @@ class TargetedSuite extends FunSuite with TempDirectories { } checkDoc( - "overloading", + "issue-24", """package example; |/** Docstring for class. */ |class Test { From b4a249dc66d58f4cea0f4a210cdb642d8e05f992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93lafur=20P=C3=A1ll=20Geirsson?= Date: Wed, 24 Mar 2021 15:40:17 +0100 Subject: [PATCH 2/2] Fix CI and address comment --- .../lsif_java/commands/SnapshotLsifCommand.scala | 1 - .../sourcegraph/lsif_semanticdb/LsifSemanticdb.java | 11 +++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala index e50fbaa3..68d2fa5a 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/SnapshotLsifCommand.scala @@ -5,7 +5,6 @@ import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import java.util.concurrent.atomic.AtomicInteger import java.util.stream.Collectors import java.util.stream.Stream diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java index bf775fc7..6308a165 100644 --- a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java @@ -12,6 +12,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -101,8 +102,14 @@ private Integer processDocument(LsifTextDocument doc, Set isExportedSymb writer.emitItem(ids.referenceResult, rangeId, doc.id); // Definition - if (occ.getRole() == SymbolOccurrence.Role.DEFINITION && ids.isDefinitionDefined()) { - writer.emitItem(ids.definitionResult, rangeId, doc.id); + if (occ.getRole() == SymbolOccurrence.Role.DEFINITION) { + if (ids.isDefinitionDefined()) { + writer.emitItem(ids.definitionResult, rangeId, doc.id); + } else { + options.reporter.error( + new NoSuchElementException( + String.format("no definition ID for symbol '%s'", occ.getSymbol()))); + } // Hover String documentation = symbolInformation.getDocumentation().getMessage();