diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 00000000..e5261b9c --- /dev/null +++ b/.bazelignore @@ -0,0 +1,2 @@ +examples/ + diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 00000000..ac14c3df --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +5.1.1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1c642ef..4f0fc1c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,20 @@ jobs: with: go-version: "^1.13.1" - run: sbt test + bazel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: yarn global add @bazel/bazelisk + - run: bazel build //... --//semanticdb-javac:enabled=true + - run: bazel run lsif-semanticdb:bazel -- --sourceroot "$PWD" + - run: du -h dump.lsif-typed + - run: bazel build //... --@lsif_java//semanticdb-javac:enabled=true + working-directory: examples/bazel-example + - run: bazel run @lsif_java//lsif-semanticdb:bazel -- --sourceroot "$PWD" + working-directory: examples/bazel-example + - run: du -h dump.lsif-typed + working-directory: examples/bazel-example check: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 058511cb..3bd2cd23 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,7 @@ dump.lsif ./generated /sources +bazel-bin +bazel-lsif-java +bazel-out +bazel-testlogs diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000..68e582a9 --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +golang 1.17.5 diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 00000000..2b2d0418 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,59 @@ +workspace(name = "lsif_java_tests") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +############## +# Bazel stdlib +############## +# To update this version, copy-paste instructions from https://github.com/bazelbuild/bazel-skylib/releases +http_archive( + name = "bazel_skylib", + sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + ], +) + +########## +# Protobuf +########## +# To update this version, copy-paste instructions from https://github.com/bazelbuild/rules_proto/releases +http_archive( + name = "rules_proto", + sha256 = "e017528fd1c91c5a33f15493e3a398181a9e821a804eb7ff5acdd1d2d6c2b18d", + strip_prefix = "rules_proto-4.0.0-3.20.0", + urls = [ + "https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0-3.20.0.tar.gz", + ], +) +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") +rules_proto_dependencies() +rules_proto_toolchains() + +############## +# JVM External +############## +# To update this version, copy-paste instructions from https://github.com/bazelbuild/rules_jvm_external/releases +RULES_JVM_EXTERNAL_TAG = "4.2" +RULES_JVM_EXTERNAL_SHA = "cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca" +http_archive( + name = "rules_jvm_external", + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + sha256 = RULES_JVM_EXTERNAL_SHA, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) +load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") +rules_jvm_external_deps() +load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") +rules_jvm_external_setup() +load("@rules_jvm_external//:defs.bzl", "maven_install") +maven_install( + artifacts = [ + "com.google.protobuf:protobuf-java:3.15.6", + "com.google.protobuf:protobuf-java-util:3.15.6", + "org.projectlombok:lombok:1.18.22", + ], + repositories = [ + "https://repo1.maven.org/maven2", + ], +) diff --git a/examples/bazel-example/WORKSPACE b/examples/bazel-example/WORKSPACE new file mode 100644 index 00000000..60542b42 --- /dev/null +++ b/examples/bazel-example/WORKSPACE @@ -0,0 +1,70 @@ + +# This is an end-to-end example of how to consume lsif-java from an external repository. +workspace(name = "lsif_java_example") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +############## +# Bazel stdlib +############## +# To update this version, copy-paste instructions from https://github.com/bazelbuild/bazel-skylib/releases +http_archive( + name = "bazel_skylib", + sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + ], +) + +############## +# lsif-java +############## +local_repository( + name = "lsif_java", + path = "../.." +) +# TODO: add commented out `http_archive` example once this workspace file is available on GitHub. + +########## +# Protobuf +########## +# To update this version, copy-paste instructions from https://github.com/bazelbuild/rules_proto/releases +http_archive( + name = "rules_proto", + sha256 = "e017528fd1c91c5a33f15493e3a398181a9e821a804eb7ff5acdd1d2d6c2b18d", + strip_prefix = "rules_proto-4.0.0-3.20.0", + urls = [ + "https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0-3.20.0.tar.gz", + ], +) +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") +rules_proto_dependencies() +rules_proto_toolchains() + +############## +# JVM External +############## +# To update this version, copy-paste instructions from https://github.com/bazelbuild/rules_jvm_external/releases +RULES_JVM_EXTERNAL_TAG = "4.2" +RULES_JVM_EXTERNAL_SHA = "cd1a77b7b02e8e008439ca76fd34f5b07aecb8c752961f9640dea15e9e5ba1ca" +http_archive( + name = "rules_jvm_external", + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + sha256 = RULES_JVM_EXTERNAL_SHA, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) +load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") +rules_jvm_external_deps() +load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") +rules_jvm_external_setup() +load("@rules_jvm_external//:defs.bzl", "maven_install") +maven_install( + artifacts = [ + "com.google.protobuf:protobuf-java:3.15.6", # Required dependency by lsif-java. + "com.google.protobuf:protobuf-java-util:3.15.6", # Required dependency by lsif-java. + "com.google.guava:guava:31.0-jre", # Not required dependency, only used by tests. + ], + repositories = [ + "https://repo1.maven.org/maven2", + ], +) diff --git a/examples/bazel-example/src/main/java/example/BUILD b/examples/bazel-example/src/main/java/example/BUILD new file mode 100644 index 00000000..c6c3898e --- /dev/null +++ b/examples/bazel-example/src/main/java/example/BUILD @@ -0,0 +1,19 @@ +# We import the custom `java_library` implementation that automatically adds the +# SemanticDB compiler plugin based on the presence of the flag +# `--@lsif_java//semanticdb-javac:enabled=true`. By default, this java_library +# rule works just like the official java_library rule. Feel free to copy-paste +# the `semanticdb:defs.bzl` file and adapt to your needs. This example is only +# for demonstration purposes. +load("@lsif_java//semanticdb-javac:defs.bzl", "java_library") + +package( + default_visibility = ["//visibility:public"], +) + +java_library( + name = "example", + srcs = ["Example.java"], + deps = [ + "@maven//:com_google_guava_guava", + ], +) diff --git a/examples/bazel-example/src/main/java/example/Example.java b/examples/bazel-example/src/main/java/example/Example.java new file mode 100644 index 00000000..cf9b7d18 --- /dev/null +++ b/examples/bazel-example/src/main/java/example/Example.java @@ -0,0 +1,9 @@ +package example; +import com.google.common.util.concurrent.Futures; + +public class Example { + public static void hello() { + System.out.println(Futures.immediateCancelledFuture()); + } + +} diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala index a36375f6..aa899bef 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexSemanticdbCommand.scala @@ -33,6 +33,9 @@ final case class IndexSemanticdbCommand( @Description( "Whether to process the SemanticDB files in parallel" ) parallel: Boolean = true, + @Description( + "Whether to infer the location of SemanticDB files based as produced by Bazel" + ) bazel: Boolean = true, @Description("URL to a PackageHub instance") @Hidden packagehub: Option[String] = None, diff --git a/lsif-semanticdb/BUILD b/lsif-semanticdb/BUILD new file mode 100644 index 00000000..5017374d --- /dev/null +++ b/lsif-semanticdb/BUILD @@ -0,0 +1,36 @@ +load("@rules_java//java:defs.bzl", "java_binary", "java_library", "java_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +package( + default_visibility = ["//visibility:public"], +) + +java_binary( + name = "bazel", + main_class = "com.sourcegraph.lsif_semanticdb.BazelBuildTool", + runtime_deps = [ + ":lsif-semanticdb", + ], +) + +java_library( + name = "lsif-semanticdb", + srcs = glob(["src/main/java/**/*.java"]), + deps = [ + ":all_java_proto", + "//semanticdb-java", + "//semanticdb-java/src/main/protobuf:semanticdb_java_proto", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + ], +) + +java_proto_library( + name = "all_java_proto", + deps = [":all_proto"], +) + +proto_library( + name = "all_proto", + srcs = glob(["src/main/protobuf/*.proto"]), +) diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/BazelBuildTool.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/BazelBuildTool.java new file mode 100644 index 00000000..a545013c --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/BazelBuildTool.java @@ -0,0 +1,130 @@ +package com.sourcegraph.lsif_semanticdb; + +import com.sourcegraph.lsif_java.Bazelbuild; +import com.sourcegraph.lsif_protocol.LsifToolInfo; + +import java.io.*; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.sql.Array; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class BazelBuildTool { + + public static int runAndReturnExitCode(String[] args) throws IOException { + Optional maybeOptions = BazelOptions.parse(args); + if (!maybeOptions.isPresent()) { + return 1; + } + BazelOptions options = maybeOptions.get(); + if (options.isHelp) { + BazelOptions.printHelp(); + return 0; + } + if (options.isVersion) { + System.out.println( + "This command-line tool has no version because it is supposed to be compiled from source."); + return 0; + } + + List mavenPackages = mavenPackages(options); + LsifSemanticdbOptions lsifOptions = + new LsifSemanticdbOptions( + options.targetroots, + options.output, + options.sourceroot, + new LsifSemanticdbReporter() { + @Override + public void error(Throwable e) { + e.printStackTrace(System.err); + } + }, + LsifToolInfo.newBuilder().setName("lsif-java").setVersion("HEAD").build(), + "java", + LsifOutputFormat.TYPED_PROTOBUF, + options.parallel, + mavenPackages, + ""); + LsifSemanticdb.run(lsifOptions); + if (!lsifOptions.reporter.hasErrors()) { + System.out.println("done: " + lsifOptions.output); + } + return 0; + } + + public static List mavenPackages(BazelOptions options) throws IOException { + ArrayList result = new ArrayList<>(); + if (!options.isQueryMavenImports) { + return result; + } + Bazelbuild.QueryResult jvmImports = runBazelQuery(options, "kind('.*_import', @maven//...)"); + Path baseDirectory = + options.sourceroot.resolve("bazel-bin").resolve("external").resolve("maven"); + PathMatcher jarPattern = FileSystems.getDefault().getPathMatcher("glob:**.jar"); + for (Bazelbuild.Target target : jvmImports.getTargetList()) { + if (target.getType() != Bazelbuild.Target.Discriminator.RULE) { + continue; + } + Bazelbuild.Rule rule = target.getRule(); + MavenPackage basePackage = null; + List jarDirectories = new ArrayList<>(); + for (Bazelbuild.Attribute attribute : rule.getAttributeList()) { + if (attribute.getName().equals("jars")) { + for (String tag : attribute.getStringListValueList()) { + if (tag.startsWith("@maven//:")) { + jarDirectories.add( + baseDirectory.resolve(tag.substring("@maven//:".length())).getParent()); + } + } + } + if (attribute.getName().equals("tags")) { + for (String tag : attribute.getStringListValueList()) { + if (tag.startsWith("maven_coordinates=")) { + String[] parts = tag.substring("maven_coordinates=".length()).split(":"); + if (parts.length == 3) { + // The jar part is populated via `withJar()` below. + basePackage = new MavenPackage(/* jar = */ null, parts[0], parts[1], parts[2]); + } + } + } + } + } + if (basePackage == null) { + continue; + } + for (Path directory : jarDirectories) { + try (Stream list = Files.list(directory)) { + List jars = list.filter(jarPattern::matches).collect(Collectors.toList()); + for (Path jar : jars) { + result.add(basePackage.withJar(jar)); + } + } + } + } + return result; + } + + public static Bazelbuild.QueryResult runBazelQuery(BazelOptions options, String query) + throws IOException { + List command = Arrays.asList(options.bazelBinary, "query", query, "--output=proto"); + System.out.println("running: " + String.join(" ", command)); + Process bazelQuery = new ProcessBuilder(command).directory(options.sourceroot.toFile()).start(); + byte[] bytes = InputStreamBytes.readAll(bazelQuery.getInputStream()); + if (bazelQuery.isAlive()) { + throw new RuntimeException(new String(InputStreamBytes.readAll(bazelQuery.getErrorStream()))); + } + int exitValue = bazelQuery.exitValue(); + if (exitValue != 0) { + throw new RuntimeException("bazel command failed\n" + new String(bytes)); + } + return Bazelbuild.QueryResult.parseFrom(bytes); + } + + public static void main(String[] args) throws IOException { + System.exit(runAndReturnExitCode(args)); + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/BazelOptions.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/BazelOptions.java new file mode 100644 index 00000000..5a454550 --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/BazelOptions.java @@ -0,0 +1,180 @@ +package com.sourcegraph.lsif_semanticdb; + +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; + +public class BazelOptions { + public final List targetroots = new ArrayList<>(); + public Path sourceroot; + public Path output; + public String bazelBinary = "bazel"; + public boolean isQueryMavenImports = true; + public boolean isHelp = false; + public boolean isVersion = false; + public boolean parallel = false; + + public static void printHelp() { + System.out.println("bazel run @lsif_java//lsif-semanticdb:bazel -- [..options]"); + System.out.println(); + System.out.println("Command-line tool to generate LSIF for Java targets in a Bazel build."); + System.out.println( + "The idiomatic way to run this tool is to compile it from source via Bazel\n" + + "and invoke it through `bazel run ...`."); + System.out.println(); + System.out.println("OPTIONS:"); + System.out.println( + " --sourceroot the absolute path to the root directory of the Bazel codebase"); + System.out.println( + " TIP: use --sourceroot \"$PWD\" to pass the current working directory"); + System.out.println(" --output the absolute path to the file that should be generated"); + System.out.println(" --parallel whether to process files in parallel"); + System.out.println( + " --no-query-maven-imports whether to skip the `bazel query @maven//...` phase."); + } + + public static Optional parse(String[] args) throws IOException { + + ArrayList errors = new ArrayList<>(); + BazelOptions options = new BazelOptions(); + + if (args.length == 1 + && (args[0].equals("--help") + || args[0].equals("help") + || args[0].equals("-h") + || args[0].equals("-help"))) { + + options.isHelp = true; + return Optional.of(options); + } + if (args.length == 1 + && (args[0].equals("--version") + || args[0].equals("version") + || args[0].equals("-v") + || args[0].equals("-version"))) { + options.isVersion = true; + return Optional.of(options); + } + + Iterator it = Arrays.asList(args).iterator(); + while (it.hasNext()) { + String argument = it.next(); + if (argument.equals("--sourceroot") && it.hasNext()) { + options.sourceroot = Paths.get(it.next()); + } + + if (argument.equals("--output")) { + if (it.hasNext()) { + options.output = Paths.get(it.next()); + } else { + errors.add("missing argument for flag --output "); + } + } + + if (argument.equals("--parallel")) { + options.parallel = true; + } + + if (argument.equals("--query-maven-imports")) { + options.isQueryMavenImports = true; + } + if (argument.equals("--no-query-maven-imports")) { + options.isQueryMavenImports = false; + } + + if (argument.equals("--bazel-binary")) { + if (it.hasNext()) { + options.bazelBinary = it.next(); + } else { + errors.add("missing argument for flag --bazel-binary "); + } + } + } + + if (options.sourceroot == null) { + if (args.length == 0) { + errors.add( + "missing required flag --sourceroot . To fix this problem, pass in the `--sourceroot` flag like this: bazel run @lsif_java//lsif-semanticdb:bazel -- --sourceroot \"$PWD\""); + } else { + errors.add("missing required flag --sourceroot "); + } + } else if (!options.sourceroot.isAbsolute()) { + // result.sourceroot must be an absolute path because `System.getProperty("user.dir")` is a + // temporary directory that's generated by Bazel. + errors.add( + String.format( + "relative path --sourceroot '%s'. To fix this problem, pass in an an absolute path.", + options.sourceroot)); + } else if (!Files.isDirectory(options.sourceroot)) { + errors.add(String.format("not a directory --sourceroot '%s'", options.sourceroot)); + } + + if (options.output == null) { + options.output = Paths.get("dump.lsif-typed"); + } + + if (!errors.isEmpty()) { + if (errors.size() == 1) { + System.err.printf("error: %s\n", errors.get(0)); + } else { + for (int i = 0; i < errors.size(); i++) { + System.err.printf("[%d] error: %s\n", i, errors.get(i)); + } + } + return Optional.empty(); + } + + inferTargetrootsAndPackages(options); + + options.output = absolutePath(options.sourceroot, options.output); + + return Optional.of(options); + } + + public static void inferTargetrootsAndPackages(BazelOptions options) throws IOException { + PathMatcher paramsPattern = FileSystems.getDefault().getPathMatcher("glob:**-0.params"); + Path bazelBin = options.sourceroot.resolve("bazel-bin"); + if (!Files.exists(bazelBin)) { + return; + } + Files.walkFileTree( + bazelBin, + Collections.singleton(FileVisitOption.FOLLOW_LINKS), + 1000, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + exc.printStackTrace(System.err); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + if (paramsPattern.matches(file)) { + visitParamsFile(file, options); + } + return super.visitFile(file, attrs); + } + }); + } + + private static Path absolutePath(Path sourceroot, Path path) { + return path.isAbsolute() ? path : sourceroot.resolve(path); + } + + public static void visitParamsFile(Path paramsFile, BazelOptions options) throws IOException { + List lines = Files.readAllLines(paramsFile); + Optional semanticdbPlugin = + lines.stream().filter(line -> line.startsWith("-Xplugin:semanticdb")).findFirst(); + if (semanticdbPlugin.isPresent()) { + String jarFileName = paramsFile.getFileName().toString(); + jarFileName = jarFileName.substring(0, jarFileName.length() - "-0.param".length() - 1); + Path jarFile = paramsFile.resolveSibling(jarFileName); + if (Files.isRegularFile(jarFile)) { + options.targetroots.add(jarFile); + } + } + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/InputStreamBytes.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/InputStreamBytes.java new file mode 100644 index 00000000..82de93ac --- /dev/null +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/InputStreamBytes.java @@ -0,0 +1,22 @@ +package com.sourcegraph.lsif_semanticdb; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +public class InputStreamBytes { + public static byte[] readAll(InputStream in) throws IOException { + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int nread; + do { + nread = in.read(buffer, 0, buffer.length); + if (nread != -1) baos.write(buffer, 0, nread); + } while (nread != -1); + return baos.toByteArray(); + } finally { + in.close(); + } + } +} diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java index 716a01ff..5fa76ef2 100644 --- a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/LsifSemanticdb.java @@ -10,12 +10,13 @@ import lib.codeintel.lsif_typed.LsifTyped; import java.io.IOException; +import java.io.InputStream; import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -322,8 +323,31 @@ private Stream parseTextDocument(Path semanticdbPath) { } } + private static PathMatcher jarPattern = FileSystems.getDefault().getPathMatcher("glob:**.jar"); + private Semanticdb.TextDocuments textDocumentsParseFrom(Path semanticdbPath) throws IOException { - byte[] bytes = Files.readAllBytes(semanticdbPath); + if (jarPattern.matches(semanticdbPath)) { + return textDocumentsParseJarFile(semanticdbPath); + } + return textDocumentsParseFromBytes(Files.readAllBytes(semanticdbPath)); + } + + private Semanticdb.TextDocuments textDocumentsParseJarFile(Path jarFile) throws IOException { + Semanticdb.TextDocuments.Builder result = Semanticdb.TextDocuments.newBuilder(); + try (JarFile file = new JarFile(jarFile.toFile())) { + Enumeration entries = file.entries(); + while (entries.hasMoreElements()) { + JarEntry element = entries.nextElement(); + if (element.getName().endsWith(".semanticdb")) { + byte[] bytes = InputStreamBytes.readAll(file.getInputStream(element)); + result.addAllDocuments(textDocumentsParseFromBytes(bytes).getDocumentsList()); + } + } + } + return result.build(); + } + + private Semanticdb.TextDocuments textDocumentsParseFromBytes(byte[] bytes) throws IOException { try { CodedInputStream in = CodedInputStream.newInstance(bytes); in.setRecursionLimit(1000); diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/MavenPackage.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/MavenPackage.java index 92afe0a5..2ae26388 100644 --- a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/MavenPackage.java +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/MavenPackage.java @@ -15,6 +15,10 @@ public MavenPackage(Path jar, String groupId, String artifactId, String version) this.version = version; } + public MavenPackage withJar(Path newJar) { + return new MavenPackage(newJar, this.groupId, this.artifactId, this.version); + } + @Override public String repoName() { return String.format("maven/%s/%s", groupId, artifactId); diff --git a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbWalker.java b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbWalker.java index 2157a26b..b092973d 100644 --- a/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbWalker.java +++ b/lsif-semanticdb/src/main/java/com/sourcegraph/lsif_semanticdb/SemanticdbWalker.java @@ -15,12 +15,12 @@ public class SemanticdbWalker extends SimpleFileVisitor { private final ArrayList result; private final LsifSemanticdbOptions options; - private final PathMatcher semanticdbPattern; + private final PathMatcher semanticdbPattern = + FileSystems.getDefault().getPathMatcher("glob:**.semanticdb"); public SemanticdbWalker(LsifSemanticdbOptions options) { this.options = options; result = new ArrayList<>(); - semanticdbPattern = FileSystems.getDefault().getPathMatcher("glob:**.semanticdb"); } @Override @@ -39,8 +39,13 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) { public static List findSemanticdbFiles(LsifSemanticdbOptions options) throws IOException { SemanticdbWalker walker = new SemanticdbWalker(options); + PathMatcher jarPattern = FileSystems.getDefault().getPathMatcher("glob:**.jar"); for (Path root : options.targetroots) { - Files.walkFileTree(root, walker); + if (jarPattern.matches(root)) { + walker.result.add(root); + } else { + Files.walkFileTree(root, walker); + } } return walker.result; } diff --git a/lsif-semanticdb/src/main/protobuf/bazelbuild.proto b/lsif-semanticdb/src/main/protobuf/bazelbuild.proto new file mode 100644 index 00000000..97334f51 --- /dev/null +++ b/lsif-semanticdb/src/main/protobuf/bazelbuild.proto @@ -0,0 +1,531 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file contains the protocol buffer representation of a build +// file or 'blaze query --output=proto' call. + +syntax = "proto2"; + +package bazel_query; + +// option cc_api_version = 2; +// option java_api_version = 1; +option java_package = "com.sourcegraph.lsif_java"; + +message License { + repeated string license_type = 1; + repeated string exception = 2; +} + +message StringDictEntry { + required string key = 1; + required string value = 2; +} + +message LabelDictUnaryEntry { + required string key = 1; + required string value = 2; +} + +message LabelListDictEntry { + required string key = 1; + repeated string value = 2; +} + +message LabelKeyedStringDictEntry { + required string key = 1; + required string value = 2; +} + +message StringListDictEntry { + required string key = 1; + repeated string value = 2; +} + +// Represents an entry attribute of a Fileset rule in a build file. +message FilesetEntry { + // Indicates what to do when a source file is actually a symlink. + enum SymlinkBehavior { + COPY = 1; + DEREFERENCE = 2; + } + + // The label pointing to the source target where files are copied from. + required string source = 1; + + // The relative path within the fileset rule where files will be mapped. + required string destination_directory = 2; + + // Whether the files= attribute was specified. This is necessary because + // no files= attribute and files=[] mean different things. + optional bool files_present = 7; + + // A list of file labels to include from the source directory. + repeated string file = 3; + + // If this is a fileset entry representing files within the rule + // package, this lists relative paths to files that should be excluded from + // the set. This cannot contain values if 'file' also has values. + repeated string exclude = 4; + + // This field is optional because there will be some time when the new + // PB is used by tools depending on blaze query, but the new blaze version + // is not yet released. + // released that outputs this field. + optional SymlinkBehavior symlink_behavior = 5 [default = COPY]; + + // The prefix to strip from the path of the files in this FilesetEntry. Note + // that no value and the empty string as the value mean different things here. + optional string strip_prefix = 6; +} + +// A rule attribute. Each attribute must have a type and one of the various +// value fields populated - for the most part. +// +// Attributes of BOOLEAN and TRISTATE type may set all of the int, bool, and +// string values for backwards compatibility with clients that expect them to +// be set. +// +// Attributes of INTEGER, STRING, LABEL, LICENSE, BOOLEAN, and TRISTATE type +// may set *none* of the values. This can happen if the Attribute message is +// prepared for a client that doesn't support SELECTOR_LIST, but the rule has +// a selector list value for the attribute. (Selector lists for attributes of +// other types--the collection types--are handled differently when prepared +// for such a client. The possible collection values are gathered together +// and flattened.) +// +// By checking the type, the appropriate value can be extracted - see the +// comments on each type for the associated value. The order of lists comes +// from the blaze parsing. If an attribute is of a list type, the associated +// list should never be empty. +message Attribute { + reserved 12, 16; + + // Indicates the type of attribute. + enum Discriminator { + INTEGER = 1; // int_value + STRING = 2; // string_value + LABEL = 3; // string_value + OUTPUT = 4; // string_value + STRING_LIST = 5; // string_list_value + LABEL_LIST = 6; // string_list_value + OUTPUT_LIST = 7; // string_list_value + DISTRIBUTION_SET = 8; // string_list_value - order is unimportant + LICENSE = 9; // license + STRING_DICT = 10; // string_dict_value + FILESET_ENTRY_LIST = 11; // fileset_list_value + LABEL_LIST_DICT = 12; // label_list_dict_value + STRING_LIST_DICT = 13; // string_list_dict_value + BOOLEAN = 14; // int, bool and string value + TRISTATE = 15; // tristate, int and string value + INTEGER_LIST = 16; // int_list_value + UNKNOWN = 18; // unknown type, use only for build extensions + LABEL_DICT_UNARY = 19; // label_dict_unary_value + SELECTOR_LIST = 20; // selector_list + LABEL_KEYED_STRING_DICT = 21; // label_keyed_string_dict + + DEPRECATED_STRING_DICT_UNARY = 17; + } + + // Values for the TriState field type. + enum Tristate { + NO = 0; + YES = 1; + AUTO = 2; + } + + message SelectorEntry { + reserved 12; + + // The key of the selector entry. At this time, this is the label of a + // config_setting rule, or the pseudo-label "//conditions:default". + optional string label = 1; + + // True if the entry's value is the default value for the type as a + // result of the condition value being specified as None (ie: + // {"//condition": None}). + optional bool is_default_value = 16; + + // Exactly one of the following fields (except for glob_criteria) must be + // populated - note that the BOOLEAN and TRISTATE caveat in Attribute's + // comment does not apply here. The type field in the SelectorList + // containing this entry indicates which of these fields is populated, + // in accordance with the comments on Discriminator enum values above. + // (To be explicit: BOOLEAN populates the boolean_value field and TRISTATE + // populates the tristate_value field.) + optional int32 int_value = 2; + optional string string_value = 3; + optional bool boolean_value = 4; + optional Tristate tristate_value = 5; + repeated string string_list_value = 6; + optional License license = 7; + repeated StringDictEntry string_dict_value = 8; + repeated FilesetEntry fileset_list_value = 9; + repeated LabelListDictEntry label_list_dict_value = 10; + repeated StringListDictEntry string_list_dict_value = 11; + repeated int32 int_list_value = 13; + repeated LabelDictUnaryEntry label_dict_unary_value = 15; + repeated LabelKeyedStringDictEntry label_keyed_string_dict_value = 17; + + repeated bytes DEPRECATED_string_dict_unary_value = 14; + } + + message Selector { + // The list of (label, value) pairs in the map that defines the selector. + // At this time, this cannot be empty, i.e. a selector has at least one + // entry. + repeated SelectorEntry entries = 1; + + // Whether or not this has any default values. + optional bool has_default_value = 2; + + // The error message when no condition matches. + optional string no_match_error = 3; + } + + message SelectorList { + // The type that this selector list evaluates to, and the type that each + // selector in the list evaluates to. At this time, this cannot be + // SELECTOR_LIST, i.e. selector lists do not nest. + optional Discriminator type = 1; + + // The list of selector elements in this selector list. At this time, this + // cannot be empty, i.e. a selector list is never empty. + repeated Selector elements = 2; + } + + // The name of the attribute + required string name = 1; + + // Whether the attribute was explicitly specified + optional bool explicitly_specified = 13; + + // If this attribute has a string value or a string list value, then this + // may be set to indicate that the value may be treated as a label that + // isn't a dependency of this attribute's rule. + optional bool nodep = 20; + + // The type of attribute. This message is used for all of the different + // attribute types so the discriminator helps for figuring out what is + // stored in the message. + required Discriminator type = 2; + + // If this attribute has an integer value this will be populated. + // Boolean and TriState also use this field as [0,1] and [-1,0,1] + // for [false, true] and [auto, no, yes] respectively. + optional int32 int_value = 3; + + // If the attribute has a string value this will be populated. Label and + // path attributes use this field as the value even though the type may + // be LABEL or something else other than STRING. + optional string string_value = 5; + + // If the attribute has a boolean value this will be populated. + optional bool boolean_value = 14; + + // If the attribute is a Tristate value, this will be populated. + optional Tristate tristate_value = 15; + + // The value of the attribute has a list of string values (label and path + // note from STRING applies here as well). + repeated string string_list_value = 6; + + // If this is a license attribute, the license information is stored here. + optional License license = 7; + + // If this is a string dict, each entry will be stored here. + repeated StringDictEntry string_dict_value = 8; + + // If the attribute is part of a Fileset, the fileset entries are stored in + // this field. + repeated FilesetEntry fileset_list_value = 9; + + // If this is a label list dict, each entry will be stored here. + repeated LabelListDictEntry label_list_dict_value = 10; + + // If this is a string list dict, each entry will be stored here. + repeated StringListDictEntry string_list_dict_value = 11; + + // The value of the attribute has a list of int32 values + repeated int32 int_list_value = 17; + + // If this is a label dict unary, each entry will be stored here. + repeated LabelDictUnaryEntry label_dict_unary_value = 19; + + // If this is a label-keyed string dict, each entry will be stored here. + repeated LabelKeyedStringDictEntry label_keyed_string_dict_value = 22; + + // If this attribute's value is an expression containing one or more select + // expressions, then its type is SELECTOR_LIST and a SelectorList will be + // stored here. + optional SelectorList selector_list = 21; + + repeated bytes DEPRECATED_string_dict_unary_value = 18; +} + +// A rule instance (e.g., cc_library foo, java_binary bar). +message Rule { + reserved 8, 11; + + // The name of the rule (formatted as an absolute label, e.g. //foo/bar:baz). + required string name = 1; + + // The rule class (e.g., java_library) + required string rule_class = 2; + + // The BUILD file and line number of the location (formatted as + // ::) in the rule's package's + // BUILD file where the rule instance was instantiated. The line number will + // be that of a rule invocation or macro call (that in turn invoked a + // rule). See + // https://docs.bazel.build/versions/master/skylark/macros.html#macro-creation + optional string location = 3; + + // All of the attributes that describe the rule. + repeated Attribute attribute = 4; + + // All of the inputs to the rule (formatted as absolute labels). These are + // predecessors in the dependency graph. + repeated string rule_input = 5; + + // All of the outputs of the rule (formatted as absolute labels). These are + // successors in the dependency graph. + repeated string rule_output = 6; + + // The set of all "features" inherited from the rule's package declaration. + repeated string default_setting = 7; + + // The rule's class's public by default value. + optional bool DEPRECATED_public_by_default = 9; + + optional bool DEPRECATED_is_skylark = 10; + + // Hash encapsulating the behavior of this Starlark rule. Any change to this + // rule's definition that could change its behavior will be reflected here. + optional string skylark_environment_hash_code = 12; + + // The Starlark call stack at the moment the rule was instantiated. + // Each entry has the form "file:line:col: function". + // The outermost stack frame ("", the BUILD file) appears first; + // the frame for the rule function itself is omitted. + // The file name may be relative to package's source root directory. + // + // Requires --proto:instantiation_stack=true. + repeated string instantiation_stack = 13; + + // The Starlark call stack for the definition of the rule class of this + // particular rule instance. If empty, either populating the field was not + // enabled on the command line with the --proto:definition_stack flag or the + // rule is a native one. + repeated string definition_stack = 14; +} + +// Summary of all transitive dependencies of 'rule,' where each dependent +// rule is included only once in the 'dependency' field. Gives complete +// information to analyze the single build target labeled rule.name, +// including optional location of target in BUILD file. +message RuleSummary { + required Rule rule = 1; + repeated Rule dependency = 2; + optional string location = 3; +} + +// A package group. Aside from the name, it contains the list of packages +// present in the group (as specified in the BUILD file). +message PackageGroup { + reserved 4; + + // The name of the package group + required string name = 1; + + // The list of packages as specified in the BUILD file. Currently this is + // only a list of packages, but some time in the future, there might be + // some type of wildcard mechanism. + repeated string contained_package = 2; + + // The list of sub package groups included in this one. + repeated string included_package_group = 3; +} + +// An environment group. +message EnvironmentGroup { + // The name of the environment group. + required string name = 1; + + // The environments that belong to this group (as labels). + repeated string environment = 2; + + // The member environments that rules implicitly support if not otherwise + // specified. + repeated string default = 3; +} + +// A file that is an input into the build system. +// Next-Id: 10 +message SourceFile { + reserved 7; + + // The name of the source file (a label). + required string name = 1; + + // The location of the source file. This is a path with a line number and a + // column number not a label in the build system. + optional string location = 2; + + // Labels of .bzl (Starlark) files that are transitively loaded in this BUILD + // file. This is present only when the SourceFile represents a BUILD file that + // loaded .bzl files. + repeated string subinclude = 3; + + // Labels of package groups that are mentioned in the visibility declaration + // for this source file. + repeated string package_group = 4; + + // Labels mentioned in the visibility declaration (including :__pkg__ and + // //visibility: ones) + repeated string visibility_label = 5; + + // The package-level features enabled for this package. Only present if the + // SourceFile represents a BUILD file. + repeated string feature = 6; + + // License attribute for the file. + optional License license = 8; + + // True if the package contains an error. Only present if the SourceFile + // represents a BUILD file. + optional bool package_contains_errors = 9; +} + +// A file that is the output of a build rule. +message GeneratedFile { + // The name of the generated file (a label). + required string name = 1; + + // The label of the target that generates the file. + required string generating_rule = 2; + + // The path, line number, and column number of the output file (not a label). + optional string location = 3; +} + +// A target from a blaze query execution. Similar to the Attribute message, +// the Discriminator is used to determine which field contains information. +// For any given type, only one of these can be populated in a single Target. +message Target { + enum Discriminator { + RULE = 1; + SOURCE_FILE = 2; + GENERATED_FILE = 3; + PACKAGE_GROUP = 4; + ENVIRONMENT_GROUP = 5; + } + + // The type of target contained in the message. + required Discriminator type = 1; + + // If this target represents a rule, the rule is stored here. + optional Rule rule = 2; + + // A file that is not generated by the build system (version controlled + // or created by the test harness). + optional SourceFile source_file = 3; + + // A generated file that is the output of a rule. + optional GeneratedFile generated_file = 4; + + // A package group. + optional PackageGroup package_group = 5; + + // An environment group. + optional EnvironmentGroup environment_group = 6; +} + +// Container for all of the blaze query results. +message QueryResult { + // All of the targets returned by the blaze query. + repeated Target target = 1; +} + +//////////////////////////////////////////////////////////////////////////// +// Messages dealing with querying the BUILD language itself. For now, this is +// quite simplistic: Blaze can only tell the names of the rule classes, their +// attributes with their type. + +// Information about allowed rule classes for a specific attribute of a rule. +message AllowedRuleClassInfo { + enum AllowedRuleClasses { + ANY = 1; // Any rule is allowed to be in this attribute + SPECIFIED = 2; // Only the explicitly listed rules are allowed + } + + required AllowedRuleClasses policy = 1; + + // Rule class names of rules allowed in this attribute, e.g "cc_library", + // "py_binary". Only present if the allowed_rule_classes field is set to + // SPECIFIED. + repeated string allowed_rule_class = 2; +} + +// This message represents a single attribute of a single rule. +// See docs.bazel.build/versions/master/skylark/lib/attr.html. +message AttributeDefinition { + required string name = 1; // e.g. "name", "srcs" + required Attribute.Discriminator type = 2; + optional bool mandatory = 3; + optional AllowedRuleClassInfo allowed_rule_classes = 4; // type=label* + optional string documentation = 5; + optional bool allow_empty = 6; // type=*_list|*_dict + optional bool allow_single_file = 7; // type=label + optional AttributeValue default = + 9; // simple (not computed/late-bound) values only + optional bool executable = 10; // type=label + optional bool configurable = 11; + optional bool nodep = + 12; // label-valued edge does not establish a dependency + optional bool cfg_is_host = + 13; // edge entails a transition to "host" configuration +} + +// An AttributeValue represents the value of an attribute. +// A single field, determined by the attribute type, is populated. +// +// It is used only for AttributeDefinition.default. Attribute and +// SelectorEntry do their own thing for unfortunate historical reasons. +message AttributeValue { + optional int32 int = 1; // type=int|tristate + optional string string = 2; // type=string|label|output + optional bool bool = 3; // type=bool + repeated AttributeValue list = 4; // type=*_list|distrib + repeated DictEntry dict = 5; // type=*_dict + + message DictEntry { + required string key = 1; + required AttributeValue value = 2; + } +} + +message RuleDefinition { + required string name = 1; + // Only contains documented attributes + repeated AttributeDefinition attribute = 2; + optional string documentation = 3; + // Only for build extensions: label to file that defines the extension + optional string label = 4; +} + +message BuildLanguage { + // Only contains documented rule definitions + repeated RuleDefinition rule = 1; +} \ No newline at end of file diff --git a/lsif-semanticdb/src/main/protobuf/lsif.proto b/lsif-semanticdb/src/main/protobuf/lsif.proto index 9c05520f..22440081 100644 --- a/lsif-semanticdb/src/main/protobuf/lsif.proto +++ b/lsif-semanticdb/src/main/protobuf/lsif.proto @@ -51,3 +51,5 @@ enum MarkupKind { PLAINTEXT = 0; MARKDOWN = 1; } + + diff --git a/semanticdb-java/BUILD b/semanticdb-java/BUILD new file mode 100644 index 00000000..96f8844f --- /dev/null +++ b/semanticdb-java/BUILD @@ -0,0 +1,24 @@ +load("@rules_java//java:defs.bzl", "java_library", "java_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +package( + default_visibility = ["//visibility:public"], +) + +java_proto_library( + name = "semanticdb_java_proto", + deps = [":semanticdb_proto"], +) + +proto_library( + name = "semanticdb_proto", + srcs = glob(["src/main/protobuf/*.proto"]), +) + +java_library( + name = "semanticdb-java", + srcs = glob(["src/main/java/**/*.java"]), + deps = [ + "//semanticdb-java/src/main/protobuf:semanticdb_java_proto", + ], +) diff --git a/semanticdb-java/src/main/protobuf/BUILD b/semanticdb-java/src/main/protobuf/BUILD new file mode 100644 index 00000000..88669718 --- /dev/null +++ b/semanticdb-java/src/main/protobuf/BUILD @@ -0,0 +1,16 @@ +load("@rules_java//java:defs.bzl", "java_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +package( + default_visibility = ["//visibility:public"], +) + +java_proto_library( + name = "semanticdb_java_proto", + deps = [":semanticdb_proto"], +) + +proto_library( + name = "semanticdb_proto", + srcs = ["semanticdb.proto"] +) diff --git a/semanticdb-javac/BUILD b/semanticdb-javac/BUILD new file mode 100644 index 00000000..37a6e13f --- /dev/null +++ b/semanticdb-javac/BUILD @@ -0,0 +1,41 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load("@rules_java//java:defs.bzl", "java_import", "java_library", "java_plugin") + +package( + default_visibility = ["//visibility:public"], +) + +config_setting( + name = "is_enabled", + flag_values = {":enabled": "true"}, +) + +string_flag( + name = "enabled", + values = ["true", "false"], + build_setting_default = "false", +) + +java_import( + name = "javac-import", + jars = ["@bazel_tools//third_party/java/jdk/langtools:javac_jar"], +) + +java_plugin( + name = "plugin", + deps = [ + ":semanticdb-javac", + ], +) + + +java_library( + name = "semanticdb-javac", + srcs = glob(["src/main/java/**/*.java"]), + resources = ["src/main/resources/META-INF/services/com.sun.source.util.Plugin"], + deps = [ + "//semanticdb-java/src/main/protobuf:semanticdb_java_proto", + "//semanticdb-java", + ":javac-import", + ], +) diff --git a/semanticdb-javac/defs.bzl b/semanticdb-javac/defs.bzl new file mode 100644 index 00000000..05310045 --- /dev/null +++ b/semanticdb-javac/defs.bzl @@ -0,0 +1,27 @@ +"""Java rules that automatically register the SemanticDB compiler plugin based on the presence of a string flag.""" +load("@rules_java//java:defs.bzl", native_java_library="java_library", native_java_binary="java_binary") + +def java_library(javacopts=[], plugins=[],**kwargs): + native_java_library( + javacopts=_actual_javacopts(javacopts), + plugins=_actual_plugins(plugins), + **kwargs) + + +def java_binary(javacopts=[], plugins=[],**kwargs): + native_java_binary( + javacopts=_actual_javacopts(javacopts), + plugins=_actual_plugins(plugins), + **kwargs) + +def _actual_javacopts(javacopts): + return select({ + "@lsif_java//semanticdb-javac:is_enabled": ["'-Xplugin:semanticdb -build-tool:bazel'"] + javacopts, + "//conditions:default": javacopts, + }) + +def _actual_plugins(plugins): + return select({ + "@lsif_java//semanticdb-javac:is_enabled": ["@lsif_java//semanticdb-javac:plugin"] + plugins, + "//conditions:default": plugins, + }) diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbJavacOptions.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbJavacOptions.java index 4867f721..b98cc58c 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbJavacOptions.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbJavacOptions.java @@ -23,6 +23,7 @@ public class SemanticdbJavacOptions { public boolean includeText = false; public boolean verboseEnabled = false; public final ArrayList errors; + public boolean alreadyReportedErrors = false; public UriScheme uriScheme = UriScheme.DEFAULT; public static String stubClassName = "META-INF-semanticdb-stub"; @@ -56,6 +57,10 @@ public static SemanticdbJavacOptions parse(String[] args, Context ctx) { result.sourceroot = Paths.get(arg.substring("-sourceroot:".length())).normalize(); } else if (arg.equals("-build-tool:sbt")) { result.uriScheme = UriScheme.SBT; + } else if (arg.equals("-build-tool:bazel")) { + result.uriScheme = UriScheme.BAZEL; + useJavacClassesDir = true; + result.targetroot = getJavacClassesDir(result, ctx); } else if (arg.equals("-text:on")) { result.includeText = true; } else if (arg.equals("-text:off")) { @@ -73,12 +78,23 @@ public static SemanticdbJavacOptions parse(String[] args, Context ctx) { if (result.targetroot == null && !useJavacClassesDir) { result.errors.add(missingRequiredDirectoryOption("targetroot")); } - if (result.sourceroot == null) { + if (!isSourcerootDefined(result)) { + // When using -build-tool:bazel, the sourceroot is automatically inferred from the first + // compilation unit. + // See `SemanticdbTaskListener.inferBazelSourceroot()` for the method that infers the + // sourceroot. result.errors.add(missingRequiredDirectoryOption("sourceroot")); } return result; } + private static boolean isSourcerootDefined(SemanticdbJavacOptions options) { + if (options.uriScheme == UriScheme.BAZEL) { + return true; // The sourceroot is automatically inferred for Bazel. + } + return options.sourceroot != null; + } + private static Path getJavacClassesDir(SemanticdbJavacOptions result, Context ctx) { // I'm not aware of a better way to get the class output directory from javac Path outputDir = null; diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbPlugin.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbPlugin.java index 48a68afa..69408d61 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbPlugin.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbPlugin.java @@ -1,10 +1,10 @@ package com.sourcegraph.semanticdb_javac; -import com.sun.source.util.JavacTask; -import com.sun.source.util.Plugin; +import com.sun.source.util.*; import com.sun.tools.javac.api.BasicJavacTask; import com.sun.tools.javac.model.JavacTypes; import com.sun.tools.javac.util.Context; +import com.sun.tools.javac.util.Options; /** Entrypoint of the semanticdb-javac compiler plugin. */ public class SemanticdbPlugin implements Plugin { @@ -18,17 +18,10 @@ public String getName() { public void init(JavacTask task, String... args) { Context ctx = ((BasicJavacTask) task).getContext(); - SemanticdbReporter reporter = new SemanticdbReporter(); + SemanticdbReporter reporter = new SemanticdbReporter(Trees.instance(task)); SemanticdbJavacOptions options = SemanticdbJavacOptions.parse(args, ctx); GlobalSymbolsCache globals = new GlobalSymbolsCache(options); JavacTypes javacTypes = JavacTypes.instance(ctx); - if (!options.errors.isEmpty()) { - for (String error : options.errors) { - reporter.error(error); - } - } else { - task.addTaskListener( - new SemanticdbTaskListener(options, task, globals, reporter, javacTypes)); - } + task.addTaskListener(new SemanticdbTaskListener(options, task, globals, reporter, javacTypes)); } } diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbReporter.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbReporter.java index 3bcb5d7e..3ca71fdb 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbReporter.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbReporter.java @@ -1,6 +1,12 @@ package com.sourcegraph.semanticdb_javac; -import java.io.PrintStream; +import com.sun.source.tree.CompilationUnitTree; +import com.sun.source.tree.Tree; +import com.sun.source.util.Trees; + +import javax.tools.Diagnostic; +import java.io.ByteArrayOutputStream; +import java.io.PrintWriter; /** * Utilities to report error messages. @@ -9,22 +15,26 @@ * This class can be removed if the Java compiler has APIs to report info/warning/error messages. */ public class SemanticdbReporter { - private final PrintStream out; + private final Trees trees; - public SemanticdbReporter() { - this.out = System.err; + public SemanticdbReporter(Trees trees) { + this.trees = trees; } - public void exception(Throwable e) { - e.printStackTrace(out); - out.println( + public void exception(Throwable e, Tree tree, CompilationUnitTree root) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(baos); + e.printStackTrace(writer); + writer.println( "Please report a bug to https://github.com/sourcegraph/semanticdb-java with the stack trace above."); + trees.printMessage(Diagnostic.Kind.ERROR, baos.toString(), tree, root); } - public void error(String message) { + public void error(String message, Tree tree, CompilationUnitTree root) { // NOTE(olafur): ideally, this message should be reported as a compiler diagnostic, but I dind't // find // the reporter API so the message goes to stderr instead for now. - out.printf("semanticdb-javac: %s\n", message); + trees.printMessage( + Diagnostic.Kind.ERROR, String.format("semanticdb-javac: %s", message), tree, root); } } diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbTaskListener.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbTaskListener.java index d464e68c..61c59965 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbTaskListener.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbTaskListener.java @@ -3,9 +3,11 @@ import com.sun.source.util.JavacTask; import com.sun.source.util.TaskEvent; import com.sun.source.util.TaskListener; -import com.sourcegraph.semanticdb_javac.Semanticdb; +import com.sun.source.util.Trees; import com.sun.tools.javac.model.JavacTypes; +import javax.tools.Diagnostic; +import javax.tools.JavaFileObject; import java.io.IOException; import java.net.URI; import java.nio.file.Files; @@ -22,6 +24,7 @@ public final class SemanticdbTaskListener implements TaskListener { private final GlobalSymbolsCache globals; private final SemanticdbReporter reporter; private final JavacTypes javacTypes; + private final Trees trees; public SemanticdbTaskListener( SemanticdbJavacOptions options, @@ -34,6 +37,7 @@ public SemanticdbTaskListener( this.globals = globals; this.reporter = reporter; this.javacTypes = javacTypes; + this.trees = Trees.instance(task); } @Override @@ -42,6 +46,20 @@ public void started(TaskEvent e) {} @Override public void finished(TaskEvent e) { if (e.getKind() != TaskEvent.Kind.ANALYZE) return; + if (!options.errors.isEmpty()) { + if (!options.alreadyReportedErrors) { + options.alreadyReportedErrors = true; + for (String error : options.errors) { + trees.printMessage( + Diagnostic.Kind.ERROR, + "semanticdb-javac: " + error, + e.getCompilationUnit(), + e.getCompilationUnit()); + } + } + return; + } + inferBazelSourceroot(e.getSourceFile()); try { onFinishedAnalyze(e); } catch (Throwable ex) { @@ -54,7 +72,8 @@ public void finished(TaskEvent e) { new CompilationUnitException( String.valueOf(e.getSourceFile().toUri().toString()), throwable); } - reporter.exception(throwable); + reporter.exception(throwable, e.getCompilationUnit(), e.getCompilationUnit()); + throw new RuntimeException("boom", throwable); } } @@ -64,24 +83,25 @@ private void onFinishedAnalyze(TaskEvent e) { Semanticdb.TextDocument textDocument = new SemanticdbVisitor(task, globals, e, options, javacTypes) .buildTextDocument(e.getCompilationUnit()); - writeSemanticdb(path.getOrThrow(), textDocument); + writeSemanticdb(e, path.getOrThrow(), textDocument); } else { - reporter.error(path.getErrorOrThrow()); + reporter.error(path.getErrorOrThrow(), e.getCompilationUnit(), e.getCompilationUnit()); } } - private void writeSemanticdb(Path output, Semanticdb.TextDocument textDocument) { + private void writeSemanticdb(TaskEvent event, Path output, Semanticdb.TextDocument textDocument) { try { byte[] bytes = Semanticdb.TextDocuments.newBuilder().addDocuments(textDocument).build().toByteArray(); Files.createDirectories(output.getParent()); Files.write(output, bytes); } catch (IOException e) { - reporter.exception(e); + reporter.exception(e, event.getCompilationUnit(), event.getCompilationUnit()); } } - public static Path absolutePathFromUri(SemanticdbJavacOptions options, URI uri) { + public static Path absolutePathFromUri(SemanticdbJavacOptions options, JavaFileObject file) { + URI uri = file.toUri(); if (options.uriScheme == UriScheme.SBT && uri.getScheme().equals("vf") && uri.toString().startsWith("vf://tmp/")) { @@ -91,13 +111,63 @@ public static Path absolutePathFromUri(SemanticdbJavacOptions options, URI uri) } else { throw new IllegalArgumentException("unsupported URI: " + uri); } + } else if (options.uriScheme == UriScheme.BAZEL) { + String toString = file.toString(); + // This solution is hacky, and it would be very nice to use a dedicated API instead. + // The Bazel Java compiler constructs `SimpleFileObject` with a "user-friendly" name that + // points to the original source file and an underlying/actual file path in a temporary + // directory. We're constrained by having to use only public APIs of the Java compiler + // and `toString()` seems to be the only way to access the user-friendly path. + if (toString.startsWith("SimpleFileObject[") && toString.endsWith("]")) { + return Paths.get(toString.substring("SimpleFileObject[".length(), toString.length() - 1)); + } else { + throw new IllegalArgumentException("unsupported source file: " + toString); + } } else { return Paths.get(uri); } } + // Infers the `-sourceroot:` flag from the provided file. + // FIXME: add unit tests https://github.com/sourcegraph/lsif-java/issues/444 + private void inferBazelSourceroot(JavaFileObject file) { + if (options.uriScheme != UriScheme.BAZEL || options.sourceroot != null) { + return; + } + Path absolutePath = absolutePathFromUri(options, file); + Path uriPath = Paths.get(file.toUri()); + // absolutePath is the "human-readable" original path, for example + // /home/repo/com/example/Hello.java + // uriPath is the sandbox/temporary file path, for example + // /private/var/tmp/com/example/Hello.java + // + // We infer sourceroot by iterating the names of both files in reverse order + // and stop at the first entry where the two paths are different. For the + // example above, we compare "Hello.java", then "example", then "com", and + // when we reach "repo" != "tmp" then we guess that "/home/repo" is the + // sourceroot. This logic is brittle and it would be nice to use more + // dedicated APIs, but Bazel actively makes an effort to sandbox + // compilation and hide access to the original workspace, which is why we + // resort to solutions like this. + int relativePathDepth = 0; + int uriPathDepth = uriPath.getNameCount(); + int absolutePathDepth = absolutePath.getNameCount(); + while (relativePathDepth < uriPathDepth && relativePathDepth < absolutePathDepth) { + String uriName = uriPath.getName(uriPathDepth - relativePathDepth - 1).toString(); + String pathName = absolutePath.getName(absolutePathDepth - relativePathDepth - 1).toString(); + if (!uriName.equals(pathName)) { + break; + } + relativePathDepth++; + } + options.sourceroot = + absolutePath + .getRoot() + .resolve(absolutePath.subpath(0, absolutePathDepth - relativePathDepth)); + } + private Result semanticdbOutputPath(SemanticdbJavacOptions options, TaskEvent e) { - Path absolutePath = absolutePathFromUri(options, e.getSourceFile().toUri()); + Path absolutePath = absolutePathFromUri(options, e.getSourceFile()); if (absolutePath.startsWith(options.sourceroot)) { Path relativePath = options.sourceroot.relativize(absolutePath); String filename = relativePath.getFileName().toString() + ".semanticdb"; diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbVisitor.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbVisitor.java index f88287c7..9bb622bb 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbVisitor.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbVisitor.java @@ -435,8 +435,7 @@ private Semanticdb.Access semanticdbAccess(Symbol sym) { } private String semanticdbUri() { - Path absolutePath = - SemanticdbTaskListener.absolutePathFromUri(options, event.getSourceFile().toUri()); + Path absolutePath = SemanticdbTaskListener.absolutePathFromUri(options, event.getSourceFile()); Path relativePath = options.sourceroot.relativize(absolutePath); StringBuilder out = new StringBuilder(); Iterator it = relativePath.iterator(); diff --git a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/UriScheme.java b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/UriScheme.java index d633209c..b953c4b9 100644 --- a/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/UriScheme.java +++ b/semanticdb-javac/src/main/java/com/sourcegraph/semanticdb_javac/UriScheme.java @@ -2,5 +2,6 @@ public enum UriScheme { DEFAULT, - SBT + SBT, + BAZEL } diff --git a/tests/minimized/BUILD b/tests/minimized/BUILD new file mode 100644 index 00000000..0d647b12 --- /dev/null +++ b/tests/minimized/BUILD @@ -0,0 +1,15 @@ +load("@rules_java//java:defs.bzl", "java_library") + +java_library( + name = "minimized", + srcs = glob(["src/main/java/minimized/*.java"]), + deps = [ + "@maven//:org_projectlombok_lombok", + ], + plugins = [ + "//semanticdb-javac:plugin", + ], + javacopts = [ + "'-Xplugin:semanticdb -build-tool:bazel'", + ], +) diff --git a/tests/minimized/src/main/java/minimized/Enums.java b/tests/minimized/src/main/java/minimized/Enums.java index 2ed0755f..a367dc05 100644 --- a/tests/minimized/src/main/java/minimized/Enums.java +++ b/tests/minimized/src/main/java/minimized/Enums.java @@ -1,6 +1,7 @@ package minimized; import java.util.Arrays; +import java.util.stream.Collectors; enum Enums { A("A", 420), @@ -13,7 +14,7 @@ enum Enums { } public static String app() { - String all = Arrays.stream(values()).map(e -> e.value).map(Enums::valueOf).toString(); + String all = Arrays.stream(values()).map(e -> e.value).map(Enums::valueOf).collect(Collectors.toList()).toString(); return all + A.value + B.value + C.value; } } diff --git a/tests/minimized/src/main/java/minimized/InnerClasses.java b/tests/minimized/src/main/java/minimized/InnerClasses.java index fe111e37..e3cb47da 100644 --- a/tests/minimized/src/main/java/minimized/InnerClasses.java +++ b/tests/minimized/src/main/java/minimized/InnerClasses.java @@ -56,7 +56,7 @@ private static B runInnerInterface(InnerInterface fn, A a) { } public static void testEnum(InnerEnum magicEnum) { - if (System.nanoTime() > System.nanoTime()) { + if (System.nanoTime() > System.currentTimeMillis()) { magicEnum = InnerEnum.B; } switch (magicEnum) { diff --git a/tests/minimized/src/main/java/minimized/TabIndented.java b/tests/minimized/src/main/java/minimized/TabIndented.java index e53e569c..4686e1fe 100644 --- a/tests/minimized/src/main/java/minimized/TabIndented.java +++ b/tests/minimized/src/main/java/minimized/TabIndented.java @@ -8,6 +8,11 @@ public boolean equals(Object other) { return false; } + @Override + public int hashCode() { + return System.identityHashCode(this); + } + @Override public String toString() { return ""; diff --git a/tests/snapshots/src/main/generated/minimized/Enums.java b/tests/snapshots/src/main/generated/minimized/Enums.java index 02f494da..d90d6a13 100644 --- a/tests/snapshots/src/main/generated/minimized/Enums.java +++ b/tests/snapshots/src/main/generated/minimized/Enums.java @@ -5,6 +5,11 @@ // ^^^^ reference java/ // ^^^^ reference java/util/ // ^^^^^^ reference java/util/Arrays# +import java.util.stream.Collectors; +// ^^^^ reference java/ +// ^^^^ reference java/util/ +// ^^^^^^ reference java/util/stream/ +// ^^^^^^^^^^ reference java/util/stream/Collectors# enum Enums { // ^^^^^ definition minimized/Enums# enum Enums @@ -34,7 +39,7 @@ enum Enums { public static String app() { // ^^^^^^ reference java/lang/String# // ^^^ definition minimized/Enums#app(). public static String app() - String all = Arrays.stream(values()).map(e -> e.value).map(Enums::valueOf).toString(); + String all = Arrays.stream(values()).map(e -> e.value).map(Enums::valueOf).collect(Collectors.toList()).toString(); // ^^^^^^ reference java/lang/String# // ^^^ definition local2 String all // ^^^^^^ reference java/util/Arrays# @@ -47,7 +52,10 @@ public static String app() { // ^^^ reference java/util/stream/Stream#map(). // ^^^^^ reference minimized/Enums# // ^^^^^^^ reference minimized/Enums#valueOf(). -// ^^^^^^^^ reference java/lang/Object#toString(). +// ^^^^^^^ reference java/util/stream/Stream#collect(+1). +// ^^^^^^^^^^ reference java/util/stream/Collectors# +// ^^^^^^ reference java/util/stream/Collectors#toList(). +// ^^^^^^^^ reference java/lang/Object#toString(). return all + A.value + B.value + C.value; // ^^^ reference local2 // ^ reference minimized/Enums#A. diff --git a/tests/snapshots/src/main/generated/minimized/InnerClasses.java b/tests/snapshots/src/main/generated/minimized/InnerClasses.java index 66e83c12..b92c54f2 100644 --- a/tests/snapshots/src/main/generated/minimized/InnerClasses.java +++ b/tests/snapshots/src/main/generated/minimized/InnerClasses.java @@ -129,11 +129,11 @@ public static void testEnum(InnerEnum magicEnum) { // ^^^^^^^^^ reference minimized/InnerClasses#InnerEnum#``(). // ^^^^^^^^^ reference minimized/InnerClasses#InnerEnum# // ^^^^^^^^^ definition local6 InnerEnum magicEnum - if (System.nanoTime() > System.nanoTime()) { + if (System.nanoTime() > System.currentTimeMillis()) { // ^^^^^^ reference java/lang/System# // ^^^^^^^^ reference java/lang/System#nanoTime(). // ^^^^^^ reference java/lang/System# -// ^^^^^^^^ reference java/lang/System#nanoTime(). +// ^^^^^^^^^^^^^^^^^ reference java/lang/System#currentTimeMillis(). magicEnum = InnerEnum.B; // ^^^^^^^^^ reference local6 // ^^^^^^^^^ reference minimized/InnerClasses#InnerEnum# diff --git a/tests/snapshots/src/main/generated/minimized/TabIndented.java b/tests/snapshots/src/main/generated/minimized/TabIndented.java index 447c4ae1..74622974 100644 --- a/tests/snapshots/src/main/generated/minimized/TabIndented.java +++ b/tests/snapshots/src/main/generated/minimized/TabIndented.java @@ -14,15 +14,24 @@ public class TabIndented { →→→public boolean equals(Object other) { // ^^^^^^ definition local3 @Override public boolean equals(Object other) // ^^^^^^ reference java/lang/Object# -// ^^^^^ definition local5 Object other +// ^^^^^ definition local6 Object other →→→→return false; →→→} +→→→@Override +// ^^^^^^^^ reference java/lang/Override# +→→→public int hashCode() { +// ^^^^^^^^ definition local4 @Override public int hashCode() +→→→→return System.identityHashCode(this); +// ^^^^^^ reference java/lang/System# +// ^^^^^^^^^^^^^^^^ reference java/lang/System#identityHashCode(). +→→→} + →→→@Override // ^^^^^^^^ reference java/lang/Override# →→→public String toString() { // ^^^^^^ reference java/lang/String# -// ^^^^^^^^ definition local4 @Override public String toString() +// ^^^^^^^^ definition local5 @Override public String toString() →→→→return ""; →→→} →→};