From ab736c4add403de629de9c6aa0ae7ae840590fdc Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Wed, 21 Oct 2020 16:03:33 +0100 Subject: [PATCH 1/9] Complete migration to kotlin for best of both worlds --- src/main/java/lsifjava/ArgumentParser.java | 79 ------ src/main/java/lsifjava/Arguments.java | 13 - src/main/java/lsifjava/DefinitionMeta.java | 17 -- src/main/java/lsifjava/FileCollector.java | 57 ---- src/main/java/lsifjava/LanguageConstruct.java | 93 ------- src/main/java/lsifjava/LanguageUtils.java | 262 ------------------ src/main/java/lsifjava/Main.java | 47 ---- .../java/lsifjava/PositionCalculator.java | 185 ------------- src/main/java/lsifjava/ProjectIndexer.java | 75 ----- src/main/java/lsifjava/SourceFileManager.java | 137 --------- src/main/java/lsifjava/SourceFileObject.java | 113 -------- src/main/kotlin/lsifjava/ArgumentParser.kt | 70 +++++ src/main/kotlin/lsifjava/DocumentIndexer.kt | 7 +- src/main/kotlin/lsifjava/FileCollector.kt | 44 +++ src/main/kotlin/lsifjava/LanguageUtils.kt | 20 ++ src/main/kotlin/lsifjava/Main.kt | 38 +++ src/main/kotlin/lsifjava/ProjectIndexer.kt | 66 +++++ src/main/kotlin/lsifjava/SourceFileManager.kt | 139 ++++++++++ src/main/kotlin/lsifjava/SourceFileObject.kt | 89 ++++++ 19 files changed, 472 insertions(+), 1079 deletions(-) delete mode 100644 src/main/java/lsifjava/ArgumentParser.java delete mode 100644 src/main/java/lsifjava/Arguments.java delete mode 100644 src/main/java/lsifjava/DefinitionMeta.java delete mode 100644 src/main/java/lsifjava/FileCollector.java delete mode 100644 src/main/java/lsifjava/LanguageConstruct.java delete mode 100644 src/main/java/lsifjava/LanguageUtils.java delete mode 100644 src/main/java/lsifjava/Main.java delete mode 100644 src/main/java/lsifjava/PositionCalculator.java delete mode 100644 src/main/java/lsifjava/ProjectIndexer.java delete mode 100644 src/main/java/lsifjava/SourceFileManager.java delete mode 100644 src/main/java/lsifjava/SourceFileObject.java create mode 100644 src/main/kotlin/lsifjava/ArgumentParser.kt create mode 100644 src/main/kotlin/lsifjava/FileCollector.kt create mode 100644 src/main/kotlin/lsifjava/LanguageUtils.kt create mode 100644 src/main/kotlin/lsifjava/Main.kt create mode 100644 src/main/kotlin/lsifjava/ProjectIndexer.kt create mode 100644 src/main/kotlin/lsifjava/SourceFileManager.kt create mode 100644 src/main/kotlin/lsifjava/SourceFileObject.kt diff --git a/src/main/java/lsifjava/ArgumentParser.java b/src/main/java/lsifjava/ArgumentParser.java deleted file mode 100644 index 60803745..00000000 --- a/src/main/java/lsifjava/ArgumentParser.java +++ /dev/null @@ -1,79 +0,0 @@ -package lsifjava; - -import java.io.IOException; -import java.nio.file.Paths; - -import org.apache.commons.cli.*; - -public class ArgumentParser { - public static final String VERSION = "0.1.0"; - public static final String PROTOCOL_VERSION = "0.4.0"; - - public static Arguments parse(String[] args) { - var options = createOptions(); - var parser = new DefaultParser(); - var formatter = new HelpFormatter(); - - CommandLine cmd; - try { - cmd = parser.parse(options, args); - } catch (ParseException e) { - System.out.println(e.getMessage()); - formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, ""); - System.exit(1); - return null; - } - - if (cmd.hasOption("help")) { - formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, ""); - System.exit(0); - } - - if (cmd.hasOption("version")) { - System.out.printf("%s, protocol version %s\n", VERSION, PROTOCOL_VERSION); - System.exit(0); - } - - String projectRoot; - try { - projectRoot = Paths.get(cmd.getOptionValue("projectRoot", ".")).toFile().getCanonicalPath(); - } catch(IOException e) { - throw new RuntimeException(String.format("Directory %s could not be found", cmd.getOptionValue("projectRoot", "."))); - } - var outFile = cmd.getOptionValue("out", projectRoot + "/dump.lsif"); - var verbosity = cmd.hasOption("verbose"); - - return new Arguments(projectRoot, outFile, verbosity); - } - - private static Options createOptions() { - var options = new Options(); - - options.addOption(new Option( - "help", false, - "Show help." - )); - - options.addOption(new Option( - "version", false, - "Show version." - )); - - options.addOption(new Option( - "verbose", false, - "Display verbose information." - )); - - options.addOption(new Option( - "projectRoot", true, - "Specifies the project root. Defaults to the current working directory." - )); - - options.addOption(new Option( - "out", true, - "The output file the dump is save to." - )); - - return options; - } -} diff --git a/src/main/java/lsifjava/Arguments.java b/src/main/java/lsifjava/Arguments.java deleted file mode 100644 index 6209dc84..00000000 --- a/src/main/java/lsifjava/Arguments.java +++ /dev/null @@ -1,13 +0,0 @@ -package lsifjava; - -public class Arguments { - public final String projectRoot; - public final String outFile; - public final boolean verbose; - - public Arguments(String projectRoot, String outFile, boolean verbose) { - this.projectRoot = projectRoot; - this.outFile = outFile; - this.verbose = verbose; - } -} diff --git a/src/main/java/lsifjava/DefinitionMeta.java b/src/main/java/lsifjava/DefinitionMeta.java deleted file mode 100644 index 221be1f3..00000000 --- a/src/main/java/lsifjava/DefinitionMeta.java +++ /dev/null @@ -1,17 +0,0 @@ -package lsifjava; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -public class DefinitionMeta { - public String rangeId; - public String resultSetId; - public String definitionResultId; - public Map> referenceRangeIds = new HashMap<>(); - - public DefinitionMeta(String rangeId, String resultSetId) { - this.rangeId = rangeId; - this.resultSetId = resultSetId; - } -} diff --git a/src/main/java/lsifjava/FileCollector.java b/src/main/java/lsifjava/FileCollector.java deleted file mode 100644 index c1bf2b69..00000000 --- a/src/main/java/lsifjava/FileCollector.java +++ /dev/null @@ -1,57 +0,0 @@ -package lsifjava; - -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Map; - -public class FileCollector extends SimpleFileVisitor { - - private final Map indexers; - private final Emitter emitter; - private final Arguments args; - private final String projectId; - - public FileCollector(String projectId, Arguments args, Emitter emitter, Map indexers) { - this.emitter = emitter; - this.projectId = projectId; - this.args = args; - this.indexers = indexers; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - if (attrs.isSymbolicLink()) { - return FileVisitResult.SKIP_SUBTREE; - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes _attrs) { - if (isJavaFile(file)) { - indexers.put(file, new DocumentIndexer( - args.projectRoot, - args.verbose, - file, - projectId, - emitter, - indexers) - ); - } - return FileVisitResult.CONTINUE; - } - - static boolean isJavaFile(Path file) { - var name = file.getFileName().toString(); - // We hide module-info.java from javac, because when javac sees module-info.java - // it goes into "module mode" and starts looking for classes on the module class path. - // This becomes evident when javac starts recompiling *way too much* on each task, - // because it doesn't realize there are already up-to-date .class files. - // The better solution would be for java-language server to detect the presence of module-info.java, - // and go into its own "module mode" where it infers a module source path and a module class path. - return name.endsWith(".java") && !Files.isDirectory(file) && !name.equals("module-info.java"); - } -} diff --git a/src/main/java/lsifjava/LanguageConstruct.java b/src/main/java/lsifjava/LanguageConstruct.java deleted file mode 100644 index 80109c12..00000000 --- a/src/main/java/lsifjava/LanguageConstruct.java +++ /dev/null @@ -1,93 +0,0 @@ -package lsifjava; - -import org.eclipse.lsp4j.Hover; -import org.eclipse.lsp4j.Location; -import org.eclipse.lsp4j.MarkedString; - -import javax.lang.model.element.Element; -import javax.lang.model.element.ElementKind; -import javax.lang.model.type.TypeMirror; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -public class LanguageConstruct { - - private final Element element; - - private String docComment; - - private Location location; - - public LanguageConstruct(String fileName, Element element, TypeMirror typeMirror) { - this.element = element; - } - - public Element getElement() { - return element; - } - - public void setDocComment(String docComment) { - this.docComment = docComment; - } - - public void setLocation(Location location) { - this.location = location; - } - - public static class Signature { - public ElementKind elementKind; - public String simpleName; - public String qualifiedName; - public String outermostContainerName; - public String packageName; - - public static Signature of(ElementKind elementKind, String simpleName, String qualifiedName, String outermostContainerName, String packageName) { - return new Signature(elementKind, simpleName, qualifiedName, outermostContainerName, packageName); - } - - private Signature(ElementKind elementKind, String simpleName, String qualifiedName, String outermostContainerName, String packageName) { - this.elementKind = elementKind; - this.simpleName = simpleName; - this.qualifiedName = qualifiedName; - this.outermostContainerName = outermostContainerName; - this.packageName = packageName; - } - - @Override - public String toString() { - return String.join( - ":", - elementKind.toString(), - qualifiedName - ); - } - - @Override - public boolean equals(Object object) { - if (this == object) return true; - if (object == null || getClass() != object.getClass()) return false; - - Signature signature = (Signature) object; - - if (elementKind != signature.elementKind) return false; - if (!Objects.equals(simpleName, signature.simpleName)) - return false; - if (!Objects.equals(qualifiedName, signature.qualifiedName)) - return false; - if (!Objects.equals(packageName, signature.packageName)) - return false; - return Objects.equals(outermostContainerName, signature.outermostContainerName); - } - - @Override - public int hashCode() { - int result = elementKind != null ? elementKind.hashCode() : 0; - result = 31 * result + (simpleName != null ? simpleName.hashCode() : 0); - result = 31 * result + (qualifiedName != null ? qualifiedName.hashCode() : 0); - result = 31 * result + (packageName != null ? packageName.hashCode() : 0); - result = 31 * result + (outermostContainerName != null ? outermostContainerName.hashCode() : 0); - return result; - } - } -} diff --git a/src/main/java/lsifjava/LanguageUtils.java b/src/main/java/lsifjava/LanguageUtils.java deleted file mode 100644 index 084e82a9..00000000 --- a/src/main/java/lsifjava/LanguageUtils.java +++ /dev/null @@ -1,262 +0,0 @@ -package lsifjava; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.lsp4j.SymbolKind; - -import javax.lang.model.element.*; -import javax.lang.model.type.ExecutableType; -import javax.lang.model.type.TypeKind; -import javax.lang.model.type.TypeMirror; -import java.io.UnsupportedEncodingException; -import java.net.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.stream.Collectors; - -public class LanguageUtils { - public static Element getTopLevelClass(Element element) { - Element highestClass = null; - for (; element != null; element = element.getEnclosingElement()) { - ElementKind kind = element.getKind(); - if (isTopLevel(kind)) { - highestClass = element; - } - } - return highestClass; - } - - public static boolean isTopLevel(ElementKind kind) { - return kind.isClass() || kind.isInterface(); - } - - public static Path uriToPath(String uri) { - if (uri.startsWith("file://")) { - String hostAndPath = uri.substring("file://".length()); - String[] components = StringUtils.split(hostAndPath, '/'); - for (int i = 0; i < components.length; i++) { - try { - components[i] = URLDecoder.decode(components[i], StandardCharsets.UTF_8.name()); - } catch (UnsupportedEncodingException e) { - // does not happen - } - } - uri = '/' + StringUtils.join(components, '/'); - return Paths.get(uri); - } - return Paths.get(URI.create(uri)); - } - - public static String getPackageName(Element element) { - for (; element != null; element = element.getEnclosingElement()) { - if (element instanceof PackageElement) { - return ((PackageElement) element).getQualifiedName().toString(); - } - } - return null; - } - - public static String getQualifiedName(Element element) { - return getQualifiedName(element, false); - } - - public static String getCrossRepoQualifiedName(Element element) { - return getQualifiedName(element, true); - } - - private static String getQualifiedName(Element element, boolean crossRepo) { - List names = new ArrayList<>(); - for (; element != null; element = element.getEnclosingElement()) { - if (element instanceof QualifiedNameable) { - names.add(((QualifiedNameable) element).getQualifiedName().toString()); - break; - } - // construct a simpler signature for cross-repo method lookups -- see comment for getCrossRepoMethodName - if (crossRepo && element instanceof ExecutableElement) { - names.add(getCrossRepoMethodName((ExecutableElement) element)); - } else { - names.add(element.toString()); - } - } - Collections.reverse(names); - return String.join(".", names); - } - - /** - * The signatures returned for workspace/symbol and textDocument/xDefinition don't quite match because the - * former are constructed from raw parse trees, while the latter are constructed after a full type-check. This - * will cause cross-repo lookups to fail, so we need xDefinition to return simplified method signatures that - * match what workspace/symbol returns, in the case where an externally-defined method is requested. - * - * @return simplified method signature matching what would be returned by SymbolVisitor::getMethodString - */ - private static String getCrossRepoMethodName(ExecutableElement element) { - - StringBuilder methodSig = new StringBuilder(); - - String typeParams = element.getTypeParameters().stream() - .map(TypeParameterElement::getSimpleName) - .map(Name::toString) - .reduce((l, r) -> l + "," + r) - .orElse(""); - if (!typeParams.isEmpty()) { - methodSig.append('<').append(typeParams).append('>'); - } - - String name = element.getSimpleName().toString(); - if ("".equals(name)) { - name = element.getEnclosingElement().getSimpleName().toString(); - } - methodSig.append(name); - - String params = element.getParameters().stream() - .map(VariableElement::asType) - .map(TypeMirror::toString) - .map(param -> param.contains(".") ? StringUtils.substringAfterLast(param, ".") : param) - .reduce((l, r) -> l + "," + r) - .orElse(""); - methodSig.append('(').append(params).append(')'); - - // KLUDGE -- turning a type into a string sometimes puts spaces after commas, sometimes not -- this seems - // to depend mostly on whether we're serializing Trees or TypeMirrors. Just normalize it here so that it'll - // match what workspace/symbol returns via SymbolVisitor::getMethodString - return methodSig.toString().replace(", ", ","); - } - - public static String getElementSignature(Element element) { - ElementKind kind = element.getKind(); - if (kind == ElementKind.PACKAGE) { - return "package " + element.toString(); - } else if (kind == ElementKind.ANNOTATION_TYPE) { - return getModifiersPrefix(element) + "@interface " + element.asType(); - } else if (kind == ElementKind.CLASS) { - return getModifiersPrefix(element) + "class " + element.asType(); - } else if (kind == ElementKind.INTERFACE) { - return getModifiersPrefix(element) + "interface " + element.asType(); - } else if (kind == ElementKind.CONSTRUCTOR) { - return getMethodSignature(element); - } else if (kind == ElementKind.ENUM) { - return getModifiersPrefix(element) + "enum " + element.asType(); - } else if (kind == ElementKind.ENUM_CONSTANT || - kind == ElementKind.EXCEPTION_PARAMETER || - kind == ElementKind.FIELD || - kind == ElementKind.LOCAL_VARIABLE || - kind == ElementKind.PARAMETER) { - return getModifiersPrefix(element) + element.asType().toString() + ' ' + element.toString(); - - } else if (kind == ElementKind.METHOD) { - return getMethodSignature(element); - } - return element.toString(); - } - - public static String getMethodSignature(Element element) { - - String name = element.getSimpleName().toString(); - boolean ctor = name.equals(""); - StringBuilder sig = new StringBuilder(getModifiersPrefix(element)); - TypeMirror typeMirror = element.asType(); - ExecutableType methodType = (ExecutableType) typeMirror; - ExecutableElement executableElement = (ExecutableElement) element; - if (!ctor) { - sig.append(getTypeSignature(methodType.getReturnType())); - } - ArrayList typeVarSigs = executableElement.getTypeParameters().stream() - .map(LanguageUtils::getElementSignature) - .map(String::trim) - .collect(Collectors.toCollection(ArrayList::new)); - if (!typeVarSigs.isEmpty()) { - sig.append(" <"); - sig.append(String.join(", ", typeVarSigs)); - sig.append(">"); - } - if (!ctor) { - sig.append(' ').append(name); - } else { - sig.append(element.getEnclosingElement().getSimpleName()); - } - sig.append('('); - ArrayList paramSigs = executableElement.getParameters().stream() - .map(LanguageUtils::getElementSignature) - .map(String::trim) - .collect(Collectors.toCollection(ArrayList::new)); - sig.append(String.join(", ", paramSigs)); - sig.append(")"); - ArrayList thrownSigs = executableElement.getThrownTypes().stream() - .map(LanguageUtils::getTypeSignature) - .collect(Collectors.toCollection(ArrayList::new)); - if (!thrownSigs.isEmpty()) { - sig.append(" throws "); - sig.append(String.join(", ", thrownSigs)); - } - return sig.toString(); - } - - public static String getTypeSignature(TypeMirror typeMirror) { - - if (typeMirror == null) return ""; - - StringBuilder sig = new StringBuilder(); - // TODO: treat generic classes differently? We might need to use the TypeElement instead. - switch (typeMirror.getKind()) { - case DECLARED -> sig.append(typeMirror.toString()); - case EXECUTABLE -> { - ExecutableType methodType = (ExecutableType) typeMirror; - ArrayList typeVarSigs = methodType.getTypeVariables().stream() - .map(LanguageUtils::getTypeSignature) - .collect(Collectors.toCollection(ArrayList::new)); - if (!typeVarSigs.isEmpty()) { - sig.append("<"); - sig.append(String.join(", ", typeVarSigs)); - sig.append("> "); - } - TypeMirror receiverType = methodType.getReceiverType(); - if (receiverType != null && receiverType.getKind() != TypeKind.NONE) { - sig.append(getTypeSignature(receiverType)); - sig.append("::"); - } - sig.append("("); - ArrayList paramSigs = methodType.getParameterTypes().stream() - .map(LanguageUtils::getTypeSignature) - .collect(Collectors.toCollection(ArrayList::new)); - sig.append(String.join(", ", paramSigs)); - sig.append(") -> "); - sig.append(getTypeSignature(methodType.getReturnType())); - ArrayList thrownSigs = methodType.getThrownTypes().stream() - .map(LanguageUtils::getTypeSignature) - .collect(Collectors.toCollection(ArrayList::new)); - if (!thrownSigs.isEmpty()) { - sig.append(" throws "); - sig.append(String.join(", ", thrownSigs)); - } - } - default -> sig.append(typeMirror.toString()); - } - return sig.toString(); - } - - public static SymbolKind toSymbolKind(ElementKind elementKind) { - if (elementKind == null) return null; - return switch (elementKind) { - case INTERFACE -> SymbolKind.Interface; - case CLASS -> SymbolKind.Class; - case PACKAGE -> SymbolKind.Package; - case METHOD -> SymbolKind.Method; - case CONSTRUCTOR -> SymbolKind.Constructor; - case FIELD -> SymbolKind.Field; - case ENUM -> SymbolKind.Enum; - default -> null; - }; - } - - private static String getModifiersPrefix(Element element) { - Collection modifiers = element.getModifiers(); - if (CollectionUtils.isEmpty(modifiers)) { - return StringUtils.EMPTY; - } - return StringUtils.join(modifiers, " ") + ' '; - } -} - diff --git a/src/main/java/lsifjava/Main.java b/src/main/java/lsifjava/Main.java deleted file mode 100644 index a92a58b9..00000000 --- a/src/main/java/lsifjava/Main.java +++ /dev/null @@ -1,47 +0,0 @@ -package lsifjava; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; - -public class Main { - public static void main(String[] args) throws IOException { - System.out.println("Running JVM " + System.getProperty("java.version")); - - var arguments = ArgumentParser.parse(args); - var writer = createWriter(arguments); - var emitter = new Emitter(writer); - var indexer = new ProjectIndexer(arguments, emitter); - - long start = System.nanoTime(); - - try { - indexer.index(); - } finally { - writer.flush(); - writer.close(); - } - - displayStats(indexer, emitter, start); - } - - private static PrintWriter createWriter(Arguments arguments) { - try { - return new PrintWriter(new File(arguments.outFile)); - } catch (IOException ex) { - throw new RuntimeException(String.format("Failed to open file %s", arguments.outFile)); - } - } - - private static void displayStats(ProjectIndexer indexer, Emitter emitter, long start) { - System.out.printf( - "%d file(s), %d def(s), %d element(s)\n", - indexer.numFiles, - indexer.numDefinitions, - emitter.numElements() - ); - - System.out.printf("Processed in %.0fms", (System.nanoTime() - start) / 1e6); - } - -} diff --git a/src/main/java/lsifjava/PositionCalculator.java b/src/main/java/lsifjava/PositionCalculator.java deleted file mode 100644 index de975432..00000000 --- a/src/main/java/lsifjava/PositionCalculator.java +++ /dev/null @@ -1,185 +0,0 @@ -package lsifjava; - -import com.sun.source.tree.*; -import com.sun.source.util.SourcePositions; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.eclipse.lsp4j.Location; -import org.eclipse.lsp4j.Position; -import org.eclipse.lsp4j.Range; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.tools.Diagnostic; -import javax.tools.JavaFileObject; -import java.io.IOException; -import java.util.HashMap; - -public class PositionCalculator { - - private static final Logger log = LoggerFactory.getLogger(PositionCalculator.class); - - private final SourcePositions sourcePositions; - - private final CompilationUnitTree compilationUnit; - - private final String fileUri; - - private final String content; - - private final HashMap positions; - - public PositionCalculator(SourcePositions sourcePositions, CompilationUnitTree compilationUnit) { - this.sourcePositions = sourcePositions; - this.compilationUnit = compilationUnit; - JavaFileObject file = compilationUnit.getSourceFile(); - this.fileUri = file.getName(); - try { - this.content = file.getCharContent(true).toString(); - } catch (IOException exception) { - log.error("Unable to extract content of {}", fileUri); - throw new RuntimeException(exception); - } - this.positions = new HashMap<>(); - } - - public Range getRange(Tree tree) { - int startOffset = getStartOffset(tree); - int endOffset = getEndOffset(tree); - if (endOffset < 0) endOffset = startOffset; - return new Range( - positions.computeIfAbsent(startOffset, this::calculatePosition), - positions.computeIfAbsent(endOffset, this::calculatePosition) - ); - } - - public Location getLocation(MethodTree tree, String name) { - return new Location(fileUri, getRange(getBoundingBox(tree, name))); - } - - public Location getLocation(Tree tree) { - - Range range; - - if (tree instanceof ClassTree) { - range = getRange(getBoundingBox((ClassTree) tree)); - } else if (tree instanceof MethodTree) { - range = getRange(getBoundingBox((MethodTree) tree)); - } else if (tree instanceof VariableTree) { - range = getRange(getBoundingBox((VariableTree) tree)); - } else if (tree instanceof MemberSelectTree) { - range = getRange(getBoundingBox((MemberSelectTree) tree)); - } else { - range = getRange(tree); - } - - return new Location(fileUri, range); - } - - public Pair getBoundingBox(ClassTree tree) { - String name = tree.getSimpleName().toString(); - // the modifiers might contain the classname itself (e.g., if a class is self-annotated), so exclude the - // modifiers when searching for the classname - int offset = -1; - if (tree.getModifiers() != null) { - offset = getEndOffset(tree.getModifiers()); - } - if (offset == -1) { - // end offset of tree modifiers may be null too, see - // https://github.com/sourcegraph/java-langserver/issues/148 - offset = getStartOffset(tree); - } - int startOffset = content.indexOf(name, offset); - return Pair.of(startOffset, startOffset + name.length()); - } - - public Pair getBoundingBox(MethodTree tree) { - return getBoundingBox(tree, tree.getName().toString()); - } - - public Pair getBoundingBox(MemberSelectTree tree) { - String name = tree.getIdentifier().toString(); - int startOffset = content.indexOf(name, getEndOffset(tree.getExpression(), tree)); - return Pair.of(startOffset, startOffset + name.length()); - } - - public Pair getBoundingBox(VariableTree tree) { - String name = tree.getName().toString(); - int startOffset = content.indexOf(name, getEndOffset(tree.getType(), tree)); - return Pair.of(startOffset, startOffset + name.length()); - } - - public Pair getBoundingBox(MethodTree tree, String name) { - int startOffset; - if (tree.getReturnType() != null) { - startOffset = content.indexOf(name, getEndOffset(tree.getReturnType(), tree)); - } else if (tree.getModifiers() != null) { - startOffset = content.indexOf(name, getEndOffset(tree.getModifiers(), tree)); - } else { - startOffset = content.indexOf(name, getStartOffset(tree)); - } - return Pair.of(startOffset, startOffset + name.length()); - } - - public int getJavaSourceRange(Tree tree) { - int startOffset = getStartOffset(tree); - int endOffset = getEndOffset(tree); - if (endOffset < 0) endOffset = startOffset; - return endOffset - startOffset; - } - - private Position calculatePosition(int rawOffset) { - if (rawOffset < 0 || rawOffset > content.length()) { - return new Position(-1, -1); - } - String precedingText = content.substring(0, rawOffset); - int linesPreceding = StringUtils.countMatches(precedingText, '\n'); - int charsPreceding = StringUtils.substringAfterLast(precedingText, "\n").length(); - return new Position(linesPreceding, charsPreceding); - } - - private int getStartOffset(Tree tree) { - // would prefer not to cast, but all the string manipulation methods take ints - return (int) sourcePositions.getStartPosition(compilationUnit, tree); - } - - private int getEndOffset(Tree tree) { - // would prefer not to cast, but all the string manipulation methods take ints - return (int) sourcePositions.getEndPosition(compilationUnit, tree); - } - - /** - * @param tree tree - * @param parent paremt tree - * @return end offset of a given tree or start offset of a parent tree if the former is NOPOS - */ - private int getEndOffset(Tree tree, Tree parent) { - int ret = getEndOffset(tree); - return ret == Diagnostic.NOPOS ? getStartOffset(parent) : ret; - } - - public static int positionToOffset(CharSequence content, Position position) { - - int currentLine = 0; - int offset = position.getCharacter(); - - if (currentLine == position.getLine()) return offset; - - for (int i = 0; i < content.length(); ++i) { - if (content.charAt(i) == '\n') { - ++currentLine; - if (currentLine == position.getLine()) return i + 1 + offset; - } - } - - return -1; - } - - private Range getRange(Pair offsets) { - return new Range( - calculatePosition(offsets.getLeft()), - calculatePosition(offsets.getRight()) - ); - } -} - diff --git a/src/main/java/lsifjava/ProjectIndexer.java b/src/main/java/lsifjava/ProjectIndexer.java deleted file mode 100644 index 23a5e876..00000000 --- a/src/main/java/lsifjava/ProjectIndexer.java +++ /dev/null @@ -1,75 +0,0 @@ -package lsifjava; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; - -public class ProjectIndexer { - public int numFiles; - public int numDefinitions; - - private final Arguments arguments; - private final Emitter emitter; - - public ProjectIndexer(Arguments arguments, Emitter emitter) { - this.arguments = arguments; - this.emitter = emitter; - } - - public void index() throws IOException { - emitter.emitVertex("metaData", Map.of( - "version", "0.4.0", - "positionEncoding", "utf-16", - "projectRoot", String.format("file://%s", Paths.get(arguments.projectRoot).toFile().getCanonicalPath()), - "toolInfo", Map.of("name", "lsif-java", "version", "0.1.0") - )); - - var projectId = emitter.emitVertex("project", Map.of( - "kind", "java" - )); - - var indexers = new HashMap(); - var collector = new FileCollector(projectId, arguments, emitter, indexers); - - try(GradleInterface gradleInterface = new GradleInterface(arguments.projectRoot)) { - for (Path it : gradleInterface.sourceDirectories()) { - Files.walkFileTree(it, collector); - } - } - - var fileManager = new SourceFileManager(indexers.keySet()); - - for(var indexer : indexers.values()) { - indexer.preIndex(fileManager); - } - - for(var indexer : indexers.values()) { - indexer.index(); - } - - /* if(arguments.verbose) - System.out.println("\nEMITTING DEFINITIONS\n-------------------------------------------------"); - - for(var indexer : indexers.values()) { - indexer.visitDefinitions(); - } - - if(arguments.verbose) - System.out.println("\nEMITTING REFERENCES\n-------------------------------------------------"); - - for(var indexer : indexers.values()) { - indexer.visitReferences(); - } - */ - for(var indexer : indexers.values()) { - indexer.postIndex(); - } - - for(var indexer : indexers.values()) { - numFiles++; - numDefinitions += indexer.numDefinitions(); - } - } -} diff --git a/src/main/java/lsifjava/SourceFileManager.java b/src/main/java/lsifjava/SourceFileManager.java deleted file mode 100644 index 038efa1b..00000000 --- a/src/main/java/lsifjava/SourceFileManager.java +++ /dev/null @@ -1,137 +0,0 @@ -package lsifjava; - -import com.sun.tools.javac.file.JavacFileManager; -import com.sun.tools.javac.util.Context; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Path; -import java.util.*; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Stream; -import javax.tools.*; - -public class SourceFileManager extends JavacFileManager { - private final Set paths; - SourceFileManager(Set paths) { - super(new Context(), false, Charset.defaultCharset()); - this.paths = paths; - } - - @Override - public Iterable list( - Location location, String packageName, Set kinds, boolean recurse) throws IOException { - if (location == StandardLocation.SOURCE_PATH) { - - final Stream sourceFileObjectStream = list(packageName); - return sourceFileObjectStream::iterator; - } else { - return super.list(location, packageName, kinds, recurse); - } - } - - private Stream list(String packageName) { - return paths.stream().map(path -> { - try { - return new SourceFileObject(path); - } catch(IOException e) { - return null; - } - }).filter(obj -> packageName(obj).equals(packageName)).map(obj -> obj); - } - - static String packageName(SourceFileObject file) { - var packagePattern = Pattern.compile("^package +(.*);"); - var startOfClass = Pattern.compile("^[\\w ]*class +\\w+"); - var lines = file.contents.lines().iterator(); - for (var line = lines.next(); line != null; line = lines.next()) { - if (startOfClass.matcher(line).find()) return ""; - var matchPackage = packagePattern.matcher(line); - if (matchPackage.matches()) { - return matchPackage.group(1); - } - } - // TODO fall back on parsing file - return ""; - } - - private JavaFileObject asJavaFileObject(Path file) throws IOException { - // TODO erase method bodies of files that are not open - return new SourceFileObject(file); - } - - @Override - public String inferBinaryName(Location location, JavaFileObject file) { - if (location == StandardLocation.SOURCE_PATH) { - var source = (SourceFileObject) file; - var packageName = packageName(source); - var className = removeExtension(source.path.getFileName().toString()); - if (!packageName.isEmpty()) className = packageName + "." + className; - return className; - } else { - return super.inferBinaryName(location, file); - } - } - - private String removeExtension(String fileName) { - var lastDot = fileName.lastIndexOf("."); - return (lastDot == -1 ? fileName : fileName.substring(0, lastDot)); - } - - @Override - public boolean hasLocation(Location location) { - return location == StandardLocation.SOURCE_PATH || super.hasLocation(location); - } - - @Override - public JavaFileObject getJavaFileForInput(Location location, String className, JavaFileObject.Kind kind) throws IOException { - // FileStore shadows disk - if (location == StandardLocation.SOURCE_PATH) { - if (className.equals("module-info")) return null; - var packageName = mostName(className); - var simpleClassName = lastName(className); - Iterator iterator = list(packageName).map(obj -> (SourceFileObject)obj).iterator(); - for (var f = iterator.next(); iterator.hasNext(); f = iterator.next()) { - if (f.path.getFileName().toString().equals(simpleClassName + kind.extension)) { - return f; - } - } - // Fall through to disk in case we have .jar or .zip files on the source path - } - return super.getJavaFileForInput(location, className, kind); - } - - // TODO this doesn't work for inner classes, eliminate - static String mostName(String name) { - var lastDot = name.lastIndexOf('.'); - return lastDot == -1 ? "" : name.substring(0, lastDot); - } - - // TODO this doesn't work for inner classes, eliminate - static String lastName(String name) { - int i = name.lastIndexOf('.'); - if (i == -1) return name; - else return name.substring(i + 1); - } - - @Override - public FileObject getFileForInput(Location location, String packageName, String relativeName) throws IOException { - if (location == StandardLocation.SOURCE_PATH) { - return null; - } - return super.getFileForInput(location, packageName, relativeName); - } - - @Override - public boolean contains(Location location, FileObject file) throws IOException { - if (location == StandardLocation.SOURCE_PATH) { - var source = (SourceFileObject) file; - return paths.contains(source.path); - } else { - return super.contains(location, file); - } - } - - private static final Logger LOG = Logger.getLogger("main"); -} diff --git a/src/main/java/lsifjava/SourceFileObject.java b/src/main/java/lsifjava/SourceFileObject.java deleted file mode 100644 index 5f7d2c4d..00000000 --- a/src/main/java/lsifjava/SourceFileObject.java +++ /dev/null @@ -1,113 +0,0 @@ -package lsifjava; - -import java.io.*; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.time.Instant; -import javax.lang.model.element.Modifier; -import javax.lang.model.element.NestingKind; -import javax.tools.JavaFileObject; - -import com.google.common.io.Files; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.input.CharSequenceReader; - -public class SourceFileObject implements JavaFileObject { - /** path is the absolute path to this file on disk */ - final Path path; - /** contents is the text in this file, or null if we should use the text in FileStore */ - final String contents; - - public SourceFileObject(Path path) throws IOException { - this(path, Files.asCharSource(path.toFile(), StandardCharsets.UTF_8).read()); - } - - public SourceFileObject(Path path, String contents) { - this.path = path; - this.contents = contents; - } - - @Override - public boolean equals(Object other) { - if (other.getClass() != SourceFileObject.class) return false; - var that = (SourceFileObject) other; - return this.path.equals(that.path); - } - - @Override - public int hashCode() { - return this.path.hashCode(); - } - - @Override - public Kind getKind() { - return Kind.SOURCE; - } - - @Override - public boolean isNameCompatible(String simpleName, Kind kind) { - return path.getFileName().toString().equals(simpleName + kind.extension); - } - - @Override - public NestingKind getNestingKind() { - return null; - } - - @Override - public Modifier getAccessLevel() { - return null; - } - - @Override - public URI toUri() { - return path.toUri(); - } - - @Override - public String getName() { - return path.toString(); - } - - @Override - public InputStream openInputStream() { - return IOUtils.toInputStream(contents, StandardCharsets.UTF_8); - } - - @Override - public OutputStream openOutputStream() { - throw new UnsupportedOperationException(); - } - - @Override - public Reader openReader(boolean ignoreEncodingErrors) { - return new CharSequenceReader(contents); - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return contents; - } - - @Override - public Writer openWriter() { - throw new UnsupportedOperationException(); - } - - @Override - public long getLastModified() { - return Instant.EPOCH.toEpochMilli(); - } - - @Override - public boolean delete() { - throw new UnsupportedOperationException(); - } - - @Override - public String toString() { - return path.toString(); - } -} diff --git a/src/main/kotlin/lsifjava/ArgumentParser.kt b/src/main/kotlin/lsifjava/ArgumentParser.kt new file mode 100644 index 00000000..31f3e96b --- /dev/null +++ b/src/main/kotlin/lsifjava/ArgumentParser.kt @@ -0,0 +1,70 @@ +package lsifjava + +import org.apache.commons.cli.* +import java.io.IOException +import java.nio.file.Paths +import kotlin.system.exitProcess + +const val VERSION = "0.1.0" +const val PROTOCOL_VERSION = "0.4.0" + +data class Arguments( + public val projectRoot: String, + public val outFile: String, + public val verbose: Boolean) + +fun parse(args: Array): Arguments { + val options = createOptions() + val parser = DefaultParser() + val formatter = HelpFormatter() + val cmd: CommandLine + cmd = try { + parser.parse(options, args) + } catch (e: ParseException) { + println(e.message) + formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, "") + exitProcess(1) + } + if (cmd.hasOption("help")) { + formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, "") + exitProcess(0) + } + if (cmd.hasOption("version")) { + println("${VERSION}, protocol version $PROTOCOL_VERSION") + exitProcess(0) + } + val projectRoot: String + projectRoot = try { + Paths.get(cmd.getOptionValue("projectRoot", ".")).toFile().canonicalPath + } catch (e: IOException) { + throw RuntimeException(String.format("Directory %s could not be found", cmd.getOptionValue("projectRoot", "."))) + } + val outFile = cmd.getOptionValue("out", "$projectRoot/dump.lsif") + val verbosity = cmd.hasOption("verbose") + return Arguments(projectRoot, outFile, verbosity) +} + +private fun createOptions(): Options { + val options = Options() + options.addOption(Option( + "help", false, + "Show help." + )) + options.addOption(Option( + "version", false, + "Show version." + )) + options.addOption(Option( + "verbose", false, + "Display verbose information." + )) + options.addOption(Option( + "projectRoot", true, + "Specifies the project root. Defaults to the current working directory." + )) + options.addOption(Option( + "out", true, + "The output file the dump is save to." + )) + return options +} diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index 8bcd4a02..9e6ce3bc 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -25,6 +25,11 @@ class DocumentIndexer( companion object { private val systemProvider = JavacTool.create() } + + data class DefinitionMeta(val rangeId: String, val resultSetId: String) { + var definitionResultId: String? = null + val referenceRangeIds = HashMap>() + } private lateinit var documentId: String private var indexed = false @@ -175,7 +180,7 @@ class DocumentIndexer( } emitter.emitEdge("item", mapOf( - "outV" to meta.definitionResultId, + "outV" to meta.definitionResultId!!, "inVs" to arrayOf(meta.rangeId), "document" to indexer.documentId )) diff --git a/src/main/kotlin/lsifjava/FileCollector.kt b/src/main/kotlin/lsifjava/FileCollector.kt new file mode 100644 index 00000000..7076279e --- /dev/null +++ b/src/main/kotlin/lsifjava/FileCollector.kt @@ -0,0 +1,44 @@ +package lsifjava + +import java.nio.file.FileVisitResult +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes + + +class FileCollector(private val projectId: String, private val args: Arguments, private val emitter: Emitter, private val indexers: MutableMap) : SimpleFileVisitor() { + var classpath: Classpath? = null + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { + return if (attrs.isSymbolicLink) { + FileVisitResult.SKIP_SUBTREE + } else FileVisitResult.CONTINUE + } + + override fun visitFile(file: Path, _attrs: BasicFileAttributes): FileVisitResult { + if (isJavaFile(file)) { + indexers[file] = DocumentIndexer( + args.projectRoot, + args.verbose, + file, + projectId, + emitter, + indexers, + classpath!!) + } + return FileVisitResult.CONTINUE + } + + companion object { + fun isJavaFile(file: Path): Boolean { + val name = file.fileName.toString() + // We hide module-info.java from javac, because when javac sees module-info.java + // it goes into "module mode" and starts looking for classes on the module class path. + // This becomes evident when javac starts recompiling *way too much* on each task, + // because it doesn't realize there are already up-to-date .class files. + // The better solution would be for java-language server to detect the presence of module-info.java, + // and go into its own "module mode" where it infers a module source path and a module class path. + return name.endsWith(".java") && !Files.isDirectory(file) && name != "module-info.java" + } + } +} diff --git a/src/main/kotlin/lsifjava/LanguageUtils.kt b/src/main/kotlin/lsifjava/LanguageUtils.kt new file mode 100644 index 00000000..dce0e3f0 --- /dev/null +++ b/src/main/kotlin/lsifjava/LanguageUtils.kt @@ -0,0 +1,20 @@ +package lsifjava + +import javax.lang.model.element.* + +fun getTopLevelClass(element: Element?): Element? { + var element = element + var highestClass: Element? = null + while (element != null) { + val kind = element.kind + if (isTopLevel(kind)) { + highestClass = element + } + element = element.enclosingElement + } + return highestClass +} + +fun isTopLevel(kind: ElementKind): Boolean { + return kind.isClass || kind.isInterface +} diff --git a/src/main/kotlin/lsifjava/Main.kt b/src/main/kotlin/lsifjava/Main.kt new file mode 100644 index 00000000..66b522ed --- /dev/null +++ b/src/main/kotlin/lsifjava/Main.kt @@ -0,0 +1,38 @@ +package lsifjava + +import java.io.File +import java.io.PrintWriter + +fun main(args: Array) { + println("Running JVM ${System.getProperty("java.version")}") + + val arguments = parse(args) + val writer = createWriter(arguments) + val emitter = Emitter(writer) + val indexer = ProjectIndexer(arguments, emitter) + + val start = System.nanoTime() + + try { + indexer.index() + } finally { + writer.flush() + writer.close() + } + + displayStats(indexer, emitter, start) +} + +private fun createWriter(args: Arguments): PrintWriter { + return PrintWriter(File(args.outFile)) +} + +private fun displayStats(indexer: ProjectIndexer, emitter: Emitter, start: Long) { + System.out.printf( + "%d file(s), %d def(s), %d element(s)\n", + indexer.numFiles, + indexer.numDefinitions, + emitter.numElements() + ) + System.out.printf("Processed in %.0fms", (System.nanoTime() - start) / 1e6) +} \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/ProjectIndexer.kt b/src/main/kotlin/lsifjava/ProjectIndexer.kt new file mode 100644 index 00000000..24537af7 --- /dev/null +++ b/src/main/kotlin/lsifjava/ProjectIndexer.kt @@ -0,0 +1,66 @@ +package lsifjava + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +class ProjectIndexer(private val arguments: Arguments, private val emitter: Emitter) { + var numFiles = 0 + var numDefinitions = 0 + fun index() { + emitter.emitVertex("metaData", mapOf( + "version" to "0.4.0", + "positionEncoding" to "utf-16", + "projectRoot" to String.format("file://%s", Paths.get(arguments.projectRoot).toFile().canonicalPath), + "toolInfo" to mapOf("name" to "lsif-java", "version" to "0.1.0") + )) + + val projectId = emitter.emitVertex("project", mapOf( + "kind" to "java" + )) + + val indexers = HashMap() + val collector = FileCollector(projectId, arguments, emitter, indexers) + GradleInterface(arguments.projectRoot).use { gradleInterface -> + val classpaths = gradleInterface.getClasspaths() + gradleInterface.getSourceDirectories().forEachIndexed { i, paths -> + collector.classpath = classpaths[i] + paths.forEach { Files.walkFileTree(it, collector) } + } + } + + val fileManager = SourceFileManager(indexers.keys) + + for (indexer in indexers.values) { + indexer.preIndex(fileManager) + } + for (indexer in indexers.values) { + indexer.index() + } + + /* if(arguments.verbose) + System.out.println("\nEMITTING DEFINITIONS\n-------------------------------------------------"); + + for(var indexer : indexers.values()) { + indexer.visitDefinitions(); + } + + if(arguments.verbose) + System.out.println("\nEMITTING REFERENCES\n-------------------------------------------------"); + + for(var indexer : indexers.values()) { + indexer.visitReferences(); + }*/ + + for (indexer in indexers.values) { + indexer.postIndex() + } + + for (indexer in indexers.values) { + numFiles++ + numDefinitions += indexer.numDefinitions() + } + } +} + + diff --git a/src/main/kotlin/lsifjava/SourceFileManager.kt b/src/main/kotlin/lsifjava/SourceFileManager.kt new file mode 100644 index 00000000..ff6252c4 --- /dev/null +++ b/src/main/kotlin/lsifjava/SourceFileManager.kt @@ -0,0 +1,139 @@ +@file:Suppress("JAVA_MODULE_DOES_NOT_EXPORT_PACKAGE") +package lsifjava + +import com.sun.tools.javac.file.JavacFileManager +import com.sun.tools.javac.util.Context +import java.io.IOException +import java.nio.charset.Charset +import java.nio.file.Path +import java.util.logging.Logger +import java.util.regex.Pattern +import java.util.stream.Stream +import javax.tools.FileObject +import javax.tools.JavaFileManager +import javax.tools.JavaFileObject +import javax.tools.StandardLocation + + +class SourceFileManager internal constructor(private val paths: Set) : JavacFileManager(Context(), false, Charset.defaultCharset()) { + @Throws(IOException::class) + override fun list( + location: JavaFileManager.Location, packageName: String, kinds: Set, recurse: Boolean): Iterable { + return if (location === StandardLocation.SOURCE_PATH) { + val sourceFileObjectStream = list(packageName) + Iterable { sourceFileObjectStream.iterator() } + } else { + super.list(location, packageName, kinds, recurse) + } + } + + private fun list(packageName: String): Stream { + return paths.stream().map { path: Path? -> + try { + return@map SourceFileObject(path!!) + } catch (e: IOException) { + return@map null + } + }.filter { obj: SourceFileObject? -> packageName(obj!!) == packageName } + .map { obj: SourceFileObject? -> obj } + } + + @Throws(IOException::class) + private fun asJavaFileObject(file: Path): JavaFileObject { + // TODO erase method bodies of files that are not open + return SourceFileObject(file) + } + + override fun inferBinaryName(location: JavaFileManager.Location, file: JavaFileObject): String { + return if (location === StandardLocation.SOURCE_PATH) { + val source = file as SourceFileObject + val packageName = packageName(source) + var className = removeExtension(source.path.fileName.toString()) + if (packageName.isNotEmpty()) className = "$packageName.$className" + className + } else { + super.inferBinaryName(location, file) + } + } + + private fun removeExtension(fileName: String): String { + val lastDot = fileName.lastIndexOf(".") + return if (lastDot == -1) fileName else fileName.substring(0, lastDot) + } + + override fun hasLocation(location: JavaFileManager.Location): Boolean { + return location === StandardLocation.SOURCE_PATH || super.hasLocation(location) + } + + @Throws(IOException::class) + override fun getJavaFileForInput(location: JavaFileManager.Location, className: String, kind: JavaFileObject.Kind): JavaFileObject? { + // FileStore shadows disk + if (location === StandardLocation.SOURCE_PATH) { + if (className == "module-info") return null + val packageName = mostName(className) + val simpleClassName = lastName(className) + val iterator: Iterator = list(packageName).map { obj: JavaFileObject -> + obj as SourceFileObject + }.iterator() + var f = iterator.next() + while (iterator.hasNext()) { + if (f.path.fileName.toString() == simpleClassName + kind.extension) { + return f + } + f = iterator.next() + } + // Fall through to disk in case we have .jar or .zip files on the source path + } + return super.getJavaFileForInput(location, className, kind) + } + + @Throws(IOException::class) + override fun getFileForInput(location: JavaFileManager.Location, packageName: String, relativeName: String): FileObject? { + return if (location === StandardLocation.SOURCE_PATH) { + null + } else super.getFileForInput(location, packageName, relativeName) + } + + @Throws(IOException::class) + override fun contains(location: JavaFileManager.Location, file: FileObject): Boolean { + return if (location === StandardLocation.SOURCE_PATH) { + val source = file as SourceFileObject + paths.contains(source.path) + } else { + super.contains(location, file) + } + } + + companion object { + fun packageName(file: SourceFileObject): String { + val packagePattern = Pattern.compile("^package +(.*);") + val startOfClass = Pattern.compile("^[\\w ]*class +\\w+") + val lines = file.contents.lines().iterator() + var line = lines.next() + while (line != null) { + if (startOfClass.matcher(line).find()) return "" + val matchPackage = packagePattern.matcher(line) + if (matchPackage.matches()) { + return matchPackage.group(1) + } + line = lines.next() + } + // TODO fall back on parsing file + return "" + } + + // TODO this doesn't work for inner classes, eliminate + fun mostName(name: String): String { + val lastDot = name.lastIndexOf('.') + return if (lastDot == -1) "" else name.substring(0, lastDot) + } + + // TODO this doesn't work for inner classes, eliminate + fun lastName(name: String): String { + val i = name.lastIndexOf('.') + return if (i == -1) name else name.substring(i + 1) + } + + private val LOG = Logger.getLogger("main") + } +} diff --git a/src/main/kotlin/lsifjava/SourceFileObject.kt b/src/main/kotlin/lsifjava/SourceFileObject.kt new file mode 100644 index 00000000..1effdf2a --- /dev/null +++ b/src/main/kotlin/lsifjava/SourceFileObject.kt @@ -0,0 +1,89 @@ +package lsifjava + +import com.google.common.io.Files +import org.apache.commons.io.IOUtils +import org.apache.commons.io.input.CharSequenceReader +import java.io.InputStream +import java.io.OutputStream +import java.io.Reader +import java.io.Writer +import java.net.URI +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.time.Instant +import javax.lang.model.element.Modifier +import javax.lang.model.element.NestingKind +import javax.tools.JavaFileObject + + +class SourceFileObject @JvmOverloads constructor( + /** path is the absolute path to this file on disk */ + val path: Path, + /** contents is the text in this file, or null if we should use the text in FileStore */ + val contents: String = Files.asCharSource(path.toFile(), StandardCharsets.UTF_8).read()) : JavaFileObject { + override fun equals(other: Any?): Boolean { + if (other!!.javaClass != SourceFileObject::class.java) return false + val that = other as SourceFileObject? + return path == that!!.path + } + + override fun hashCode(): Int { + return path.hashCode() + } + + override fun getKind(): JavaFileObject.Kind { + return JavaFileObject.Kind.SOURCE + } + + override fun isNameCompatible(simpleName: String, kind: JavaFileObject.Kind): Boolean { + return path.fileName.toString() == simpleName + kind.extension + } + + override fun getNestingKind(): NestingKind? { + return null + } + + override fun getAccessLevel(): Modifier? { + return null + } + + override fun toUri(): URI { + return path.toUri() + } + + override fun getName(): String { + return path.toString() + } + + override fun openInputStream(): InputStream { + return IOUtils.toInputStream(contents, StandardCharsets.UTF_8) + } + + override fun openOutputStream(): OutputStream { + throw UnsupportedOperationException() + } + + override fun openReader(ignoreEncodingErrors: Boolean): Reader { + return CharSequenceReader(contents) + } + + override fun getCharContent(ignoreEncodingErrors: Boolean): CharSequence { + return contents + } + + override fun openWriter(): Writer { + throw UnsupportedOperationException() + } + + override fun getLastModified(): Long { + return Instant.EPOCH.toEpochMilli() + } + + override fun delete(): Boolean { + throw UnsupportedOperationException() + } + + override fun toString(): String { + return path.toString() + } +} From 7902d4a6f97f1f838c88355d90d4d63cba94e29b Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 23 Oct 2020 15:01:29 +0100 Subject: [PATCH 2/9] Fixed mkDoc function when signature contains newlines, throwing off the syntax indent remover --- src/main/kotlin/lsifjava/DocumentIndexer.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index 9e6ce3bc..c1a322c2 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -193,10 +193,7 @@ class DocumentIndexer( } private fun mkDoc(signature: String, docComment: String?): String { - return """ - ```java - $signature - ```""".trimIndent() + + return "```java\n$signature\n```" + if (docComment == null || docComment == "") "" else "\n---\n${docComment.trim()}" } From ad73053c57dfcb1329be6c84901e3938b00014e6 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 23 Oct 2020 15:03:32 +0100 Subject: [PATCH 3/9] Kotlin-ified some code, lazy-sequence basic package name finder rather than eager line splitting --- src/main/kotlin/lsifjava/SourceFileManager.kt | 66 ++++++++----------- src/main/kotlin/lsifjava/SourceFileObject.kt | 61 +++++------------ 2 files changed, 43 insertions(+), 84 deletions(-) diff --git a/src/main/kotlin/lsifjava/SourceFileManager.kt b/src/main/kotlin/lsifjava/SourceFileManager.kt index ff6252c4..91eb9b51 100644 --- a/src/main/kotlin/lsifjava/SourceFileManager.kt +++ b/src/main/kotlin/lsifjava/SourceFileManager.kt @@ -6,7 +6,6 @@ import com.sun.tools.javac.util.Context import java.io.IOException import java.nio.charset.Charset import java.nio.file.Path -import java.util.logging.Logger import java.util.regex.Pattern import java.util.stream.Stream import javax.tools.FileObject @@ -38,12 +37,6 @@ class SourceFileManager internal constructor(private val paths: Set) : Jav .map { obj: SourceFileObject? -> obj } } - @Throws(IOException::class) - private fun asJavaFileObject(file: Path): JavaFileObject { - // TODO erase method bodies of files that are not open - return SourceFileObject(file) - } - override fun inferBinaryName(location: JavaFileManager.Location, file: JavaFileObject): String { return if (location === StandardLocation.SOURCE_PATH) { val source = file as SourceFileObject @@ -56,9 +49,8 @@ class SourceFileManager internal constructor(private val paths: Set) : Jav } } - private fun removeExtension(fileName: String): String { - val lastDot = fileName.lastIndexOf(".") - return if (lastDot == -1) fileName else fileName.substring(0, lastDot) + private fun removeExtension(fileName: String): String = fileName.lastIndexOf(".").let { + if(it == -1) fileName else fileName.substring(0, it) } override fun hasLocation(location: JavaFileManager.Location): Boolean { @@ -103,37 +95,33 @@ class SourceFileManager internal constructor(private val paths: Set) : Jav super.contains(location, file) } } +} - companion object { - fun packageName(file: SourceFileObject): String { - val packagePattern = Pattern.compile("^package +(.*);") - val startOfClass = Pattern.compile("^[\\w ]*class +\\w+") - val lines = file.contents.lines().iterator() - var line = lines.next() - while (line != null) { - if (startOfClass.matcher(line).find()) return "" - val matchPackage = packagePattern.matcher(line) - if (matchPackage.matches()) { - return matchPackage.group(1) - } - line = lines.next() - } - // TODO fall back on parsing file - return "" - } - - // TODO this doesn't work for inner classes, eliminate - fun mostName(name: String): String { - val lastDot = name.lastIndexOf('.') - return if (lastDot == -1) "" else name.substring(0, lastDot) +private fun packageName(file: SourceFileObject): String { + val packagePattern = Pattern.compile("^package +(.*);") + val startOfClass = Pattern.compile("^[\\w ]*class +\\w+") + val lines = file.contents.lineSequence().iterator() + var line = lines.next() + do { + if (startOfClass.matcher(line).find()) return "" + val matchPackage = packagePattern.matcher(line) + if (matchPackage.matches()) { + return matchPackage.group(1) } + line = lines.next() + } while(lines.hasNext()) + // TODO fall back on parsing file + return "" +} - // TODO this doesn't work for inner classes, eliminate - fun lastName(name: String): String { - val i = name.lastIndexOf('.') - return if (i == -1) name else name.substring(i + 1) - } +// TODO this doesn't work for inner classes, eliminate +private fun mostName(name: String): String { + val lastDot = name.lastIndexOf('.') + return if (lastDot == -1) "" else name.substring(0, lastDot) +} - private val LOG = Logger.getLogger("main") - } +// TODO this doesn't work for inner classes, eliminate +private fun lastName(name: String): String { + val i = name.lastIndexOf('.') + return if (i == -1) name else name.substring(i + 1) } diff --git a/src/main/kotlin/lsifjava/SourceFileObject.kt b/src/main/kotlin/lsifjava/SourceFileObject.kt index 1effdf2a..944f4ac7 100644 --- a/src/main/kotlin/lsifjava/SourceFileObject.kt +++ b/src/main/kotlin/lsifjava/SourceFileObject.kt @@ -27,63 +27,34 @@ class SourceFileObject @JvmOverloads constructor( return path == that!!.path } - override fun hashCode(): Int { - return path.hashCode() - } + override fun hashCode() = path.hashCode() - override fun getKind(): JavaFileObject.Kind { - return JavaFileObject.Kind.SOURCE - } + override fun getKind() = JavaFileObject.Kind.SOURCE - override fun isNameCompatible(simpleName: String, kind: JavaFileObject.Kind): Boolean { - return path.fileName.toString() == simpleName + kind.extension - } + override fun isNameCompatible(simpleName: String, kind: JavaFileObject.Kind) = + path.fileName.toString() == simpleName + kind.extension - override fun getNestingKind(): NestingKind? { - return null - } + override fun getNestingKind(): NestingKind? = null - override fun getAccessLevel(): Modifier? { - return null - } + override fun getAccessLevel(): Modifier? = null - override fun toUri(): URI { - return path.toUri() - } + override fun toUri(): URI = path.toUri() - override fun getName(): String { - return path.toString() - } + override fun getName() = path.toString() - override fun openInputStream(): InputStream { - return IOUtils.toInputStream(contents, StandardCharsets.UTF_8) - } + override fun openInputStream(): InputStream = IOUtils.toInputStream(contents, StandardCharsets.UTF_8) - override fun openOutputStream(): OutputStream { - throw UnsupportedOperationException() - } + override fun openOutputStream(): OutputStream = throw UnsupportedOperationException() - override fun openReader(ignoreEncodingErrors: Boolean): Reader { - return CharSequenceReader(contents) - } + override fun openReader(ignoreEncodingErrors: Boolean) = CharSequenceReader(contents) - override fun getCharContent(ignoreEncodingErrors: Boolean): CharSequence { - return contents - } + override fun getCharContent(ignoreEncodingErrors: Boolean): CharSequence = contents - override fun openWriter(): Writer { - throw UnsupportedOperationException() - } + override fun openWriter(): Writer = throw UnsupportedOperationException() - override fun getLastModified(): Long { - return Instant.EPOCH.toEpochMilli() - } + override fun getLastModified() = Instant.EPOCH.toEpochMilli() - override fun delete(): Boolean { - throw UnsupportedOperationException() - } + override fun delete(): Boolean = throw UnsupportedOperationException() - override fun toString(): String { - return path.toString() - } + override fun toString() = path.toString() } From f6055ad719910d268277f17ce5f56a268e8f85f0 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 23 Oct 2020 19:29:26 +0100 Subject: [PATCH 4/9] Collection of classpaths per sourcedir group and collection of sourcedir groups from Gradle --- src/main/kotlin/lsifjava/GradleInterface.kt | 25 ++++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/lsifjava/GradleInterface.kt b/src/main/kotlin/lsifjava/GradleInterface.kt index f622b8b5..b2fff3f5 100644 --- a/src/main/kotlin/lsifjava/GradleInterface.kt +++ b/src/main/kotlin/lsifjava/GradleInterface.kt @@ -1,9 +1,7 @@ package lsifjava import org.gradle.tooling.GradleConnector -import org.gradle.tooling.model.GradleProject import org.gradle.tooling.model.eclipse.EclipseProject -import org.gradle.tooling.model.idea.IdeaProject import java.nio.file.Path import java.nio.file.Paths @@ -11,17 +9,32 @@ import java.nio.file.Paths class GradleInterface(private val projectDir: String): AutoCloseable { private val projectConnection by lazy { // TODO(nsc) version override, 6.0 < x < 6.3 seems to be an issue - GradleConnector.newConnector().forProjectDirectory(Paths.get(projectDir).toFile()).connect() + GradleConnector.newConnector() + .forProjectDirectory(Paths.get(projectDir).toFile()) + //.useGradleVersion("6.3") + .connect() } private val eclipseModel by lazy { projectConnection.getModel(EclipseProject::class.java) } - fun classpath() = Classpath(eclipseModel.classpath.map { it.file.canonicalPath }) + fun getClasspaths() = classpath(eclipseModel) + + private fun classpath(project: EclipseProject): List { + val classPaths = arrayListOf() + classPaths += Classpath(project.classpath.map { it.file.canonicalPath }) + project.children.forEach { classpath(it).toCollection(classPaths) } + return classPaths + } + + fun getSourceDirectories() = sourceDirectories(eclipseModel) - fun sourceDirectories() = eclipseModel.sourceDirectories.map { - Paths.get(eclipseModel.projectDirectory.path, it.path) + private fun sourceDirectories(project: EclipseProject): List> { + val sourceDirs = arrayListOf>() + sourceDirs += project.sourceDirectories.map { Paths.get(project.projectDirectory.path, it.path) } + project.children.forEach { sourceDirectories(it).toCollection(sourceDirs) } + return sourceDirs } override fun close() = projectConnection.close() From 065f8ed0eaca22f4d457f572ca40f848a9fb2075 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 23 Oct 2020 19:37:35 +0100 Subject: [PATCH 5/9] Potential null-symbol handling if symbols arent resolved (gotta keep indexing even on partial failures) --- src/main/kotlin/lsifjava/IndexingVisitor.kt | 67 ++++++++++++--------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/src/main/kotlin/lsifjava/IndexingVisitor.kt b/src/main/kotlin/lsifjava/IndexingVisitor.kt index 45ff0a75..eb4c2119 100644 --- a/src/main/kotlin/lsifjava/IndexingVisitor.kt +++ b/src/main/kotlin/lsifjava/IndexingVisitor.kt @@ -5,7 +5,7 @@ import com.sun.source.tree.* import com.sun.source.util.* import com.sun.tools.javac.code.Flags import com.sun.tools.javac.code.Symbol -import com.sun.tools.javac.tree.JCTree +import com.sun.tools.javac.tree.JCTree.* import org.eclipse.lsp4j.Location import org.eclipse.lsp4j.Position import org.eclipse.lsp4j.Range @@ -40,7 +40,7 @@ class IndexingVisitor( val packagePrefix = compUnit.packageName?.toString()?.plus(".") ?: "" // TODO(nsc): records - val classOrEnum = if ((node as JCTree.JCClassDecl).sym.flags() and Flags.ENUM.toLong() != 0L) "enum " else "class " + val classOrEnum = if ((node as JCClassDecl).sym.flags() and Flags.ENUM.toLong() != 0L) "enum " else "class " val range = findLocation(currentPath, node.simpleName.toString())?.range ?: return super.visitClass(node, p) @@ -54,7 +54,7 @@ class IndexingVisitor( } override fun visitMethod(node: MethodTree, p: Unit?): Unit? { - if((node as JCTree.JCMethodDecl).mods.flags and Flags.GENERATEDCONSTR != 0L) return null + if((node as JCMethodDecl).mods.flags and Flags.GENERATEDCONSTR != 0L) return null val methodName = node.name.toString().let { if(it == "") node.sym.owner.name.toString() else it @@ -77,23 +77,34 @@ class IndexingVisitor( } override fun visitNewClass(node: NewClassTree, p: Unit?): Unit? { + (node as JCNewClass).constructor ?: return super.visitNewClass(node, p) // ignore auto-genned constructors (for now) - if((node as JCTree.JCNewClass).constructor.flags() and Flags.GENERATEDCONSTR != 0L) return null - - val ident = when(node.identifier) { - is JCTree.JCIdent -> node.identifier - is JCTree.JCTypeApply -> (node.identifier as JCTree.JCTypeApply).clazz - else -> { - println("identifier was neither JCIdent nor JCTypeApply, but ${node.identifier.javaClass}") + if(node.constructor.flags() and Flags.GENERATEDCONSTR != 0L) return null + + // refactor to extension methods? + val identifier = node.identifier + val identName = when(identifier) { + is JCIdent -> identifier.name + is JCFieldAccess -> identifier.name + is JCTypeApply -> when(identifier.clazz) { + is JCIdent -> (identifier.clazz as JCIdent).name + is JCFieldAccess -> (identifier.clazz as JCFieldAccess).name + else -> { + println("clazz was not JCIdent|JCFieldAccess|JCTypeApply, but ${identifier.clazz.javaClass}:\n${identifier.clazz}") + return super.visitNewClass(node, p) + } + } + else -> { + println("identifier was not JCIdent|JCTypeApply|JCFieldAccess, but ${identifier.javaClass}:\n${identifier}") return super.visitNewClass(node, p) } - } as JCTree.JCIdent + }.toString() val identPath = TreePath(currentPath, node.identifier) val identElement = node.constructor - val defContainer = LanguageUtils.getTopLevelClass(identElement) as Symbol.ClassSymbol? ?: return super.visitNewClass(node, p) - val (useRange, refRange, defPath) = findReference(identElement, ident.name.toString(), defContainer, path = identPath) ?: return super.visitNewClass(node, p) + val defContainer = getTopLevelClass(identElement) as Symbol.ClassSymbol? ?: return super.visitNewClass(node, p) + val (useRange, refRange, defPath) = findReference(identElement, identName, defContainer, path = identPath) ?: return super.visitNewClass(node, p) indexer.emitUse(useRange, refRange, defPath) @@ -101,9 +112,9 @@ class IndexingVisitor( } override fun visitMethodInvocation(node: MethodInvocationTree?, p: Unit?): Unit? { - val symbol = when((node as JCTree.JCMethodInvocation).meth) { - is JCTree.JCFieldAccess -> (node.meth as JCTree.JCFieldAccess).sym - is JCTree.JCIdent -> (node.meth as JCTree.JCIdent).sym + val symbol = when((node as JCMethodInvocation).meth) { + is JCFieldAccess -> (node.meth as JCFieldAccess).sym + is JCIdent -> (node.meth as JCIdent).sym else -> { println("method receiver tree was neither JCFieldAccess nor JCIdent but ${node.meth.javaClass}") return super.visitMethodInvocation(node, p) @@ -116,10 +127,10 @@ class IndexingVisitor( if(it == "") "this" else it } ?: return super.visitMethodInvocation(node, p) val methodPath = TreePath(currentPath, node.meth) - val defContainer = LanguageUtils.getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMethodInvocation(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMethodInvocation(node, p) // this gives us the start pos of the name of the method, so override defStartPos - val overrideStartOffset = (trees.getPath(symbol)?.leaf as JCTree.JCMethodDecl?)?.pos ?: return super.visitMethodInvocation(node, p) + val overrideStartOffset = (trees.getPath(symbol)?.leaf as JCMethodDecl?)?.pos ?: return super.visitMethodInvocation(node, p) val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, path = methodPath, defStartPos = overrideStartOffset) ?: return super.visitMethodInvocation(node, p) indexer.emitUse(useRange, refRange, defPath) @@ -129,13 +140,13 @@ class IndexingVisitor( // does not match `var` or constructor calls override fun visitIdentifier(node: IdentifierTree?, p: Unit?): Unit? { - val symbol = (node as JCTree.JCIdent).sym ?: return super.visitIdentifier(node, p) + val symbol = (node as JCIdent).sym ?: return super.visitIdentifier(node, p) if(symbol is Symbol.PackageSymbol) return super.visitIdentifier(node, p) val name = symbol.name.toString() if(name == "") return super.visitIdentifier(node, p) // handled by visitMethodInvocation - val defContainer = LanguageUtils.getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitIdentifier(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitIdentifier(node, p) val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) ?: return super.visitIdentifier(node, p) @@ -145,23 +156,20 @@ class IndexingVisitor( } override fun visitTypeParameter(node: TypeParameterTree?, p: Unit?): Unit? { - val range = findLocation(currentPath, node!!.name.toString(), (node as JCTree.JCTypeParameter).pos)?.range ?: return super.visitTypeParameter(node, p) + val range = findLocation(currentPath, node!!.name.toString(), (node as JCTypeParameter).pos)?.range ?: return super.visitTypeParameter(node, p) - indexer.emitDefinition( - range, - "type-parameter $node" - ) + indexer.emitDefinition(range, "type-parameter $node") return super.visitTypeParameter(node, p) } // function references eg test::myfunc or banana::new override fun visitMemberReference(node: MemberReferenceTree?, p: Unit?): Unit? { - val symbol = (node as JCTree.JCMemberReference).sym ?: return super.visitMemberReference(node, p) + val symbol = (node as JCMemberReference).sym ?: return super.visitMemberReference(node, p) val name = symbol.name.toString() - val defContainer = LanguageUtils.getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMemberReference(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMemberReference(node, p) val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) ?: return super.visitMemberReference(node, p) @@ -171,7 +179,7 @@ class IndexingVisitor( } override fun visitMemberSelect(node: MemberSelectTree?, p: Unit?): Unit? { - if((node as JCTree.JCFieldAccess).sym == null) return super.visitMemberSelect(node, p) + if((node as JCFieldAccess).sym == null) return super.visitMemberSelect(node, p) val symbol = when(node.sym) { is Symbol.ClassSymbol -> node.sym @@ -181,7 +189,7 @@ class IndexingVisitor( val name = symbol.name.toString() - val defContainer = LanguageUtils.getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMemberSelect(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMemberSelect(node, p) // need to narrow down start pos of the field access here, so override refStartPos val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, refStartPos = node.pos) ?: return super.visitMemberSelect(node, p) @@ -232,6 +240,7 @@ class IndexingVisitor( start = findNameIn(path.compilationUnit, name, start, end) end = start + name.length } + if (start == -1) return null val startLine = lines.getLineNumber(start.toLong()).toInt() val startColumn = lines.getColumnNumber(start.toLong()).toInt() val endLine = lines.getLineNumber(end.toLong()).toInt() From 9e5191e396dd76d079047fec5aca70a8759b7637 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 23 Oct 2020 19:38:43 +0100 Subject: [PATCH 6/9] Sourcedir groups handling in projectindexer and if sourcedir doesnt exist --- src/main/kotlin/lsifjava/ProjectIndexer.kt | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/lsifjava/ProjectIndexer.kt b/src/main/kotlin/lsifjava/ProjectIndexer.kt index 24537af7..5df7581a 100644 --- a/src/main/kotlin/lsifjava/ProjectIndexer.kt +++ b/src/main/kotlin/lsifjava/ProjectIndexer.kt @@ -25,7 +25,10 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit val classpaths = gradleInterface.getClasspaths() gradleInterface.getSourceDirectories().forEachIndexed { i, paths -> collector.classpath = classpaths[i] - paths.forEach { Files.walkFileTree(it, collector) } + paths.forEach { + if (Files.notExists(it)) return@forEach + Files.walkFileTree(it, collector) + } } } @@ -38,20 +41,6 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit indexer.index() } - /* if(arguments.verbose) - System.out.println("\nEMITTING DEFINITIONS\n-------------------------------------------------"); - - for(var indexer : indexers.values()) { - indexer.visitDefinitions(); - } - - if(arguments.verbose) - System.out.println("\nEMITTING REFERENCES\n-------------------------------------------------"); - - for(var indexer : indexers.values()) { - indexer.visitReferences(); - }*/ - for (indexer in indexers.values) { indexer.postIndex() } From 9e15410be30bec4c88a9b5d8fd2aa87ba310b33a Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 23 Oct 2020 19:40:39 +0100 Subject: [PATCH 7/9] Fixed SimpleContext using JavaCompiler.compilerKey is error in java8 as is protected. Adapted to use passed in classpath object --- src/main/kotlin/lsifjava/DocumentIndexer.kt | 31 ++++++++------------- src/main/kotlin/lsifjava/FileCollector.kt | 25 +++++++++++++++-- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index c1a322c2..b882584d 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -8,7 +8,7 @@ import com.sun.tools.javac.main.JavaCompiler import com.sun.tools.javac.util.Context import org.eclipse.lsp4j.Position import org.eclipse.lsp4j.Range -import java.io.OutputStreamWriter +import java.io.Writer import java.nio.file.Path import java.util.* import kotlin.collections.HashMap @@ -20,7 +20,9 @@ class DocumentIndexer( private val path: Path, private val projectId: String, private val emitter: Emitter, - private val indexers: Map + private val indexers: Map, + private val classpath: Classpath, + private val javacOutput: Writer ) { companion object { private val systemProvider = JavacTool.create() @@ -60,28 +62,16 @@ class DocumentIndexer( val (task, compUnit) = analyzeFile() IndexingVisitor(task, compUnit, this, indexers).scan(compUnit, null) referencesBacklog.forEach { it() } + + println("finished analyzing and visiting $path") } private fun analyzeFile(): Pair { val context = SimpleContext() val task = systemProvider.getTask( - OutputStreamWriter.nullWriter(), fileManager, null, - listOf("--enable-preview", "-source", "15", "--add-exports", - "jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "--add-exports", - "jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "--add-exports", - "jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED", - "--add-exports", - "jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", - "--add-exports", - "jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "--add-exports", - "jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "--add-exports", - "jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", - "--add-exports", - "jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED"), + javacOutput, fileManager, null, + listOf(/*"--enable-preview", */"-source", "8", "-proc:none", + "-classpath", classpath.toString()), listOf(), listOf(SourceFileObject(path)), context ) val compUnit = task.parse().iterator().next() @@ -92,13 +82,14 @@ class DocumentIndexer( internal class SimpleContext: Context() { init { - put(JavaCompiler.compilerKey, SimpleCompiler.factory) + put(SimpleCompiler.getContextKey(), SimpleCompiler.factory) } } internal class SimpleCompiler(context: Context?): JavaCompiler(context) { companion object { var factory = Context.Factory { context: Context? -> SimpleCompiler(context) } + fun getContextKey(): Context.Key = compilerKey } init { diff --git a/src/main/kotlin/lsifjava/FileCollector.kt b/src/main/kotlin/lsifjava/FileCollector.kt index 7076279e..515694e8 100644 --- a/src/main/kotlin/lsifjava/FileCollector.kt +++ b/src/main/kotlin/lsifjava/FileCollector.kt @@ -1,5 +1,7 @@ package lsifjava +import org.apache.commons.io.output.NullOutputStream +import java.io.* import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.Path @@ -8,13 +10,16 @@ import java.nio.file.attribute.BasicFileAttributes class FileCollector(private val projectId: String, private val args: Arguments, private val emitter: Emitter, private val indexers: MutableMap) : SimpleFileVisitor() { - var classpath: Classpath? = null + lateinit var classpath: Classpath + private val javacOutStream by lazy { createJavacOutWriter(args) } + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { return if (attrs.isSymbolicLink) { FileVisitResult.SKIP_SUBTREE } else FileVisitResult.CONTINUE } + // TODO(nsc) uncouple FileCollector + DocumentIndexer override fun visitFile(file: Path, _attrs: BasicFileAttributes): FileVisitResult { if (isJavaFile(file)) { indexers[file] = DocumentIndexer( @@ -24,12 +29,28 @@ class FileCollector(private val projectId: String, private val args: Arguments, projectId, emitter, indexers, - classpath!!) + classpath, + javacOutStream + ) } return FileVisitResult.CONTINUE } companion object { + fun createJavacOutWriter(args: Arguments): Writer { + return when(args.javacOutWriter) { + "stdout" -> System.out.writer() + "stderr" -> System.err.writer() + //"none" -> OutputStreamWriter.nullWriter() CANT USE IN JAVA 8 :( + "none" -> object : Writer() { + override fun close() {} + override fun flush() {} + override fun write(p0: CharArray, p1: Int, p2: Int) {} + } + else -> PrintStream(File(args.javacOutWriter)).writer() + } + } + fun isJavaFile(file: Path): Boolean { val name = file.fileName.toString() // We hide module-info.java from javac, because when javac sees module-info.java From 4afd46045d198b858d5e5ba8a0c76018b56af9f3 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 23 Oct 2020 19:41:01 +0100 Subject: [PATCH 8/9] Taking in javac output destination as arg --- src/main/kotlin/lsifjava/ArgumentParser.kt | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/lsifjava/ArgumentParser.kt b/src/main/kotlin/lsifjava/ArgumentParser.kt index 31f3e96b..41dd3de8 100644 --- a/src/main/kotlin/lsifjava/ArgumentParser.kt +++ b/src/main/kotlin/lsifjava/ArgumentParser.kt @@ -9,9 +9,11 @@ const val VERSION = "0.1.0" const val PROTOCOL_VERSION = "0.4.0" data class Arguments( - public val projectRoot: String, - public val outFile: String, - public val verbose: Boolean) + val projectRoot: String, + val outFile: String, + val verbose: Boolean, + val javacOutWriter: String +) fun parse(args: Array): Arguments { val options = createOptions() @@ -33,15 +35,17 @@ fun parse(args: Array): Arguments { println("${VERSION}, protocol version $PROTOCOL_VERSION") exitProcess(0) } - val projectRoot: String - projectRoot = try { + + val projectRoot = try { Paths.get(cmd.getOptionValue("projectRoot", ".")).toFile().canonicalPath } catch (e: IOException) { throw RuntimeException(String.format("Directory %s could not be found", cmd.getOptionValue("projectRoot", "."))) } val outFile = cmd.getOptionValue("out", "$projectRoot/dump.lsif") val verbosity = cmd.hasOption("verbose") - return Arguments(projectRoot, outFile, verbosity) + val javacOutWriter = cmd.getOptionValue("javacOut", "none") + + return Arguments(projectRoot, outFile, verbosity, javacOutWriter) } private fun createOptions(): Options { @@ -66,5 +70,9 @@ private fun createOptions(): Options { "out", true, "The output file the dump is save to." )) + options.addOption(Option( + "javacOut", true, + "The output location for output from javac. Options include stdout, stderr or filepath." + )) return options } From 049429879261cafd0281a16e3501fdf203972f9f Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 23 Oct 2020 22:45:19 +0100 Subject: [PATCH 9/9] Reporting of total javac errors, passing of java source version, bump of lsif-java version --- src/main/kotlin/lsifjava/DocumentIndexer.kt | 19 ++++++++++++++----- src/main/kotlin/lsifjava/FileCollector.kt | 7 ++++--- src/main/kotlin/lsifjava/GradleInterface.kt | 9 +++++++++ src/main/kotlin/lsifjava/ProjectIndexer.kt | 7 ++++++- src/main/kotlin/lsifjava/SourceFileManager.kt | 6 +----- 5 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index b882584d..bb18a10a 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -11,18 +11,20 @@ import org.eclipse.lsp4j.Range import java.io.Writer import java.nio.file.Path import java.util.* +import javax.tools.Diagnostic +import javax.tools.DiagnosticListener import kotlin.collections.HashMap import kotlin.collections.HashSet class DocumentIndexer( - private val projectRoot: String, private val verbose: Boolean, private val path: Path, private val projectId: String, private val emitter: Emitter, private val indexers: Map, private val classpath: Classpath, - private val javacOutput: Writer + private val javaSourceVersion: String, + private val javacOutput: Writer? ) { companion object { private val systemProvider = JavacTool.create() @@ -40,6 +42,10 @@ class DocumentIndexer( private lateinit var fileManager: SourceFileManager private val referencesBacklog: LinkedList<() -> Unit> = LinkedList() + + var javacDiagnostics = LinkedList>() + private set + private val diagnosticsListener = DiagnosticListener { javacDiagnostics.add(it) } fun numDefinitions(): Int { return definitions.size @@ -63,15 +69,18 @@ class DocumentIndexer( IndexingVisitor(task, compUnit, this, indexers).scan(compUnit, null) referencesBacklog.forEach { it() } + //javacDiagnostics.forEach(::println) + println("finished analyzing and visiting $path") } private fun analyzeFile(): Pair { val context = SimpleContext() + + // TODO(nsc) diagnosticsListener not being null seems to interfere with javacOutput val task = systemProvider.getTask( - javacOutput, fileManager, null, - listOf(/*"--enable-preview", */"-source", "8", "-proc:none", - "-classpath", classpath.toString()), + javacOutput, fileManager, diagnosticsListener, + listOf("-source", "8", "-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString()), listOf(), listOf(SourceFileObject(path)), context ) val compUnit = task.parse().iterator().next() diff --git a/src/main/kotlin/lsifjava/FileCollector.kt b/src/main/kotlin/lsifjava/FileCollector.kt index 515694e8..3c8c5c59 100644 --- a/src/main/kotlin/lsifjava/FileCollector.kt +++ b/src/main/kotlin/lsifjava/FileCollector.kt @@ -11,6 +11,7 @@ import java.nio.file.attribute.BasicFileAttributes class FileCollector(private val projectId: String, private val args: Arguments, private val emitter: Emitter, private val indexers: MutableMap) : SimpleFileVisitor() { lateinit var classpath: Classpath + var javaSourceVersion: String? = null private val javacOutStream by lazy { createJavacOutWriter(args) } override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { @@ -23,13 +24,13 @@ class FileCollector(private val projectId: String, private val args: Arguments, override fun visitFile(file: Path, _attrs: BasicFileAttributes): FileVisitResult { if (isJavaFile(file)) { indexers[file] = DocumentIndexer( - args.projectRoot, args.verbose, file, projectId, emitter, indexers, classpath, + javaSourceVersion!!, javacOutStream ) } @@ -37,9 +38,9 @@ class FileCollector(private val projectId: String, private val args: Arguments, } companion object { - fun createJavacOutWriter(args: Arguments): Writer { + fun createJavacOutWriter(args: Arguments): Writer? { return when(args.javacOutWriter) { - "stdout" -> System.out.writer() + "stdout" -> null // getTask interprets this as stdout "stderr" -> System.err.writer() //"none" -> OutputStreamWriter.nullWriter() CANT USE IN JAVA 8 :( "none" -> object : Writer() { diff --git a/src/main/kotlin/lsifjava/GradleInterface.kt b/src/main/kotlin/lsifjava/GradleInterface.kt index b2fff3f5..bf43832c 100644 --- a/src/main/kotlin/lsifjava/GradleInterface.kt +++ b/src/main/kotlin/lsifjava/GradleInterface.kt @@ -36,6 +36,15 @@ class GradleInterface(private val projectDir: String): AutoCloseable { project.children.forEach { sourceDirectories(it).toCollection(sourceDirs) } return sourceDirs } + + fun javaSourceVersions() = javaSourceVersion(eclipseModel) + + private fun javaSourceVersion(project: EclipseProject): List { + val javaVersions = arrayListOf() + javaVersions.add(project.javaSourceSettings?.sourceLanguageLevel?.toString()) + project.children.forEach { javaSourceVersion(it).toCollection(javaVersions) } + return javaVersions + } override fun close() = projectConnection.close() } diff --git a/src/main/kotlin/lsifjava/ProjectIndexer.kt b/src/main/kotlin/lsifjava/ProjectIndexer.kt index 5df7581a..b836b834 100644 --- a/src/main/kotlin/lsifjava/ProjectIndexer.kt +++ b/src/main/kotlin/lsifjava/ProjectIndexer.kt @@ -7,12 +7,14 @@ import java.nio.file.Paths class ProjectIndexer(private val arguments: Arguments, private val emitter: Emitter) { var numFiles = 0 var numDefinitions = 0 + var numJavacErrors = 0 + fun index() { emitter.emitVertex("metaData", mapOf( "version" to "0.4.0", "positionEncoding" to "utf-16", "projectRoot" to String.format("file://%s", Paths.get(arguments.projectRoot).toFile().canonicalPath), - "toolInfo" to mapOf("name" to "lsif-java", "version" to "0.1.0") + "toolInfo" to mapOf("name" to "lsif-java", "version" to "0.7.0") )) val projectId = emitter.emitVertex("project", mapOf( @@ -23,8 +25,10 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit val collector = FileCollector(projectId, arguments, emitter, indexers) GradleInterface(arguments.projectRoot).use { gradleInterface -> val classpaths = gradleInterface.getClasspaths() + val javaSourceVersions = gradleInterface.javaSourceVersions() gradleInterface.getSourceDirectories().forEachIndexed { i, paths -> collector.classpath = classpaths[i] + collector.javaSourceVersion = javaSourceVersions[i] paths.forEach { if (Files.notExists(it)) return@forEach Files.walkFileTree(it, collector) @@ -48,6 +52,7 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit for (indexer in indexers.values) { numFiles++ numDefinitions += indexer.numDefinitions() + numJavacErrors += indexer.javacDiagnostics.size } } } diff --git a/src/main/kotlin/lsifjava/SourceFileManager.kt b/src/main/kotlin/lsifjava/SourceFileManager.kt index 91eb9b51..9eb32872 100644 --- a/src/main/kotlin/lsifjava/SourceFileManager.kt +++ b/src/main/kotlin/lsifjava/SourceFileManager.kt @@ -13,9 +13,8 @@ import javax.tools.JavaFileManager import javax.tools.JavaFileObject import javax.tools.StandardLocation +class SourceFileManager internal constructor(private val paths: Set): JavacFileManager(Context(), false, Charset.defaultCharset()) { -class SourceFileManager internal constructor(private val paths: Set) : JavacFileManager(Context(), false, Charset.defaultCharset()) { - @Throws(IOException::class) override fun list( location: JavaFileManager.Location, packageName: String, kinds: Set, recurse: Boolean): Iterable { return if (location === StandardLocation.SOURCE_PATH) { @@ -57,7 +56,6 @@ class SourceFileManager internal constructor(private val paths: Set) : Jav return location === StandardLocation.SOURCE_PATH || super.hasLocation(location) } - @Throws(IOException::class) override fun getJavaFileForInput(location: JavaFileManager.Location, className: String, kind: JavaFileObject.Kind): JavaFileObject? { // FileStore shadows disk if (location === StandardLocation.SOURCE_PATH) { @@ -79,14 +77,12 @@ class SourceFileManager internal constructor(private val paths: Set) : Jav return super.getJavaFileForInput(location, className, kind) } - @Throws(IOException::class) override fun getFileForInput(location: JavaFileManager.Location, packageName: String, relativeName: String): FileObject? { return if (location === StandardLocation.SOURCE_PATH) { null } else super.getFileForInput(location, packageName, relativeName) } - @Throws(IOException::class) override fun contains(location: JavaFileManager.Location, file: FileObject): Boolean { return if (location === StandardLocation.SOURCE_PATH) { val source = file as SourceFileObject