From 5a1ee9d853c0bf17c343adcd0dad1a059a511bc4 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Wed, 28 Oct 2020 16:10:37 +0000 Subject: [PATCH 1/6] Introduced CountingDiagnosticListener and CanonicalPath types, Decoupled FileCollector from DocumentIndexer map and made it async Decoupled the filetree visitor type from creating the documentindexer map by making the process async and sending discovered files down a channel CanonicalPath simplifies some code that derives a canonical path from a basic Path type CountingDiagnosticListener combines both an output writer and a counter for diagnostics. By implementing the DiagnosticListener, we can accurately count individual diagnostics rather than counting individual writes to a java.io.Writer, which is innacurate as a single diagnostic generally results in multiple write calls --- build.gradle | 16 ++- src/main/kotlin/lsifjava/ArgumentParser.kt | 44 ++++--- ...adleInterface.kt => BuildToolInterface.kt} | 23 ++-- .../lsifjava/CountingDiagnosticWriter.kt | 47 ++++++++ src/main/kotlin/lsifjava/DocumentIndexer.kt | 46 ++++---- src/main/kotlin/lsifjava/FileCollector.kt | 107 ++++++++++++------ src/main/kotlin/lsifjava/IndexingVisitor.kt | 15 +++ src/main/kotlin/lsifjava/LanguageUtils.kt | 20 ---- src/main/kotlin/lsifjava/ProjectIndexer.kt | 29 ++--- 9 files changed, 206 insertions(+), 141 deletions(-) rename src/main/kotlin/lsifjava/{GradleInterface.kt => BuildToolInterface.kt} (71%) create mode 100644 src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt delete mode 100644 src/main/kotlin/lsifjava/LanguageUtils.kt diff --git a/build.gradle b/build.gradle index b66b2905..e22ccd02 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,11 @@ repositories { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" - compile files("${System.getProperty('java.home')}/../lib/tools.jar") + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.0' + if(JavaVersion.current() < JavaVersion.VERSION_1_9) { + println "JDK version is ${JavaVersion.current()}, including tools.jar @ ${System.properties['java.home']}/lib/tools.jar" + compile files("${System.properties['java.home']}/../lib/tools.jar") + } compile 'com.google.code.gson:gson:2.8.6' compile 'commons-cli:commons-cli:1.4' compile 'org.apache.commons:commons-lang3:3.11' @@ -29,7 +33,7 @@ dependencies { } application { - mainClassName = 'lsifjava.Main' + mainClassName = 'lsifjava.MainKt' } compileJava { @@ -55,4 +59,10 @@ compileJava { } group = 'com.sourcegraph' -version = '1.0-SNAPSHOT' \ No newline at end of file +version = '1.0-SNAPSHOT' + +compileKotlin { + kotlinOptions { + freeCompilerArgs = ["-Xinline-classes"] + } +} \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/ArgumentParser.kt b/src/main/kotlin/lsifjava/ArgumentParser.kt index 41dd3de8..677bda1f 100644 --- a/src/main/kotlin/lsifjava/ArgumentParser.kt +++ b/src/main/kotlin/lsifjava/ArgumentParser.kt @@ -1,15 +1,19 @@ package lsifjava import org.apache.commons.cli.* -import java.io.IOException +import java.nio.file.Path import java.nio.file.Paths import kotlin.system.exitProcess const val VERSION = "0.1.0" const val PROTOCOL_VERSION = "0.4.0" +inline class CanonicalPath(val path: Path) { + override fun toString(): String = path.toFile().canonicalPath +} + data class Arguments( - val projectRoot: String, + val projectRoot: CanonicalPath, val outFile: String, val verbose: Boolean, val javacOutWriter: String @@ -17,30 +21,26 @@ data class Arguments( fun parse(args: Array): Arguments { val options = createOptions() - val parser = DefaultParser() val formatter = HelpFormatter() - val cmd: CommandLine - cmd = try { - parser.parse(options, args) + + val cmd = try { + DefaultParser().parse(options, args) } catch (e: ParseException) { println(e.message) formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, "") exitProcess(1) } + if (cmd.hasOption("help")) { formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, "") exitProcess(0) } if (cmd.hasOption("version")) { - println("${VERSION}, protocol version $PROTOCOL_VERSION") + println("$VERSION, protocol version $PROTOCOL_VERSION") exitProcess(0) } - - val projectRoot = try { - Paths.get(cmd.getOptionValue("projectRoot", ".")).toFile().canonicalPath - } catch (e: IOException) { - throw RuntimeException(String.format("Directory %s could not be found", cmd.getOptionValue("projectRoot", "."))) - } + + val projectRoot = CanonicalPath(Paths.get(cmd.getOptionValue("projectRoot", "."))) val outFile = cmd.getOptionValue("out", "$projectRoot/dump.lsif") val verbosity = cmd.hasOption("verbose") val javacOutWriter = cmd.getOptionValue("javacOut", "none") @@ -48,31 +48,29 @@ fun parse(args: Array): Arguments { return Arguments(projectRoot, outFile, verbosity, javacOutWriter) } -private fun createOptions(): Options { - val options = Options() - options.addOption(Option( +private fun createOptions() = Options().apply { + addOption(Option( "help", false, "Show help." )) - options.addOption(Option( + addOption(Option( "version", false, "Show version." )) - options.addOption(Option( + addOption(Option( "verbose", false, "Display verbose information." )) - options.addOption(Option( + addOption(Option( "projectRoot", true, "Specifies the project root. Defaults to the current working directory." )) - options.addOption(Option( + addOption(Option( "out", true, "The output file the dump is save to." )) - options.addOption(Option( + addOption(Option( "javacOut", true, - "The output location for output from javac. Options include stdout, stderr or filepath." + "The output location for output from javac. Options include stdout, stderr, none or filepath." )) - return options } diff --git a/src/main/kotlin/lsifjava/GradleInterface.kt b/src/main/kotlin/lsifjava/BuildToolInterface.kt similarity index 71% rename from src/main/kotlin/lsifjava/GradleInterface.kt rename to src/main/kotlin/lsifjava/BuildToolInterface.kt index bf43832c..e9e9ab2c 100644 --- a/src/main/kotlin/lsifjava/GradleInterface.kt +++ b/src/main/kotlin/lsifjava/BuildToolInterface.kt @@ -5,13 +5,17 @@ import org.gradle.tooling.model.eclipse.EclipseProject import java.nio.file.Path import java.nio.file.Paths -// TODO(nsc) exclusions? subprojects? inter-project dependencies? fml -class GradleInterface(private val projectDir: String): AutoCloseable { +interface BuildToolInterface { + fun getClasspaths(): List + fun getSourceDirectories(): List> + fun javaSourceVersions(): List +} + +// TODO(nsc) exclusions? subprojects? lazy eval with tail rec? +class GradleInterface(private val projectDir: CanonicalPath): AutoCloseable, BuildToolInterface { private val projectConnection by lazy { - // TODO(nsc) version override, 6.0 < x < 6.3 seems to be an issue GradleConnector.newConnector() - .forProjectDirectory(Paths.get(projectDir).toFile()) - //.useGradleVersion("6.3") + .forProjectDirectory(projectDir.path.toFile()) .connect() } @@ -19,7 +23,7 @@ class GradleInterface(private val projectDir: String): AutoCloseable { projectConnection.getModel(EclipseProject::class.java) } - fun getClasspaths() = classpath(eclipseModel) + override fun getClasspaths() = classpath(eclipseModel) private fun classpath(project: EclipseProject): List { val classPaths = arrayListOf() @@ -28,7 +32,7 @@ class GradleInterface(private val projectDir: String): AutoCloseable { return classPaths } - fun getSourceDirectories() = sourceDirectories(eclipseModel) + override fun getSourceDirectories() = sourceDirectories(eclipseModel) private fun sourceDirectories(project: EclipseProject): List> { val sourceDirs = arrayListOf>() @@ -37,8 +41,9 @@ class GradleInterface(private val projectDir: String): AutoCloseable { return sourceDirs } - fun javaSourceVersions() = javaSourceVersion(eclipseModel) - + override fun javaSourceVersions() = javaSourceVersion(eclipseModel) + + // get rid of String?, use parent version or else fallback private fun javaSourceVersion(project: EclipseProject): List { val javaVersions = arrayListOf() javaVersions.add(project.javaSourceSettings?.sourceLanguageLevel?.toString()) diff --git a/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt b/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt new file mode 100644 index 00000000..2ae340e1 --- /dev/null +++ b/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt @@ -0,0 +1,47 @@ +package lsifjava + +import java.io.File +import java.io.PrintStream +import java.io.Writer +import javax.tools.Diagnostic +import javax.tools.DiagnosticListener + +fun createJavacDiagnosticListener(javacOutDest: String): CountingDiagnosticListener { + return when(javacOutDest) { + "stdout" -> CountingDiagnosticListener.StdoutWriter + "stderr" -> CountingDiagnosticListener.StderrWriter + "none" -> CountingDiagnosticListener.NullWriter + else -> CountingDiagnosticListener.FileWriter(javacOutDest) + } +} + +sealed class CountingDiagnosticListener(private val out: Writer): DiagnosticListener, AutoCloseable { + var count = 0 + + override fun report(diagnostic: Diagnostic?) { + count++ + out.write(diagnostic.toString()) + out.flush() + } + + override fun close() = out.close() + + class FileWriter(path: String): CountingDiagnosticListener(PrintStream(File(path)).writer()) + + object StdoutWriter: CountingDiagnosticListener(System.out.writer()) { + override fun close() = Unit + } + + object StderrWriter: CountingDiagnosticListener(System.err.writer()) { + override fun close() = Unit + } + + object NullWriter: CountingDiagnosticListener(NoopWriter) +} + +// OutputStreamWriter.nullWriter() CANT USE IN JAVA 8 :( +object NoopWriter: Writer() { + override fun close() = Unit + override fun flush() = Unit + override fun write(p0: CharArray, p1: Int, p2: Int) = Unit +} diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index bb18a10a..2d9b1cb5 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -17,14 +17,13 @@ import kotlin.collections.HashMap import kotlin.collections.HashSet class DocumentIndexer( - private val verbose: Boolean, - private val path: Path, - private val projectId: String, - private val emitter: Emitter, - private val indexers: Map, + private val filepath: CanonicalPath, private val classpath: Classpath, private val javaSourceVersion: String, - private val javacOutput: Writer? + private val indexers: Map, + private val emitter: Emitter, + private val diagnosticListener: CountingDiagnosticListener, + private val verbose: Boolean, ) { companion object { private val systemProvider = JavacTool.create() @@ -47,15 +46,10 @@ class DocumentIndexer( private set private val diagnosticsListener = DiagnosticListener { javacDiagnostics.add(it) } - fun numDefinitions(): Int { - return definitions.size - } - - fun preIndex(fileManager: SourceFileManager) { - this.fileManager = fileManager + init { val args = mapOf( "languageId" to "java", - "uri" to String.format("file://%s", path.toAbsolutePath().toString()) + "uri" to "file://$filepath" ) documentId = emitter.emitVertex("document", args) } @@ -77,11 +71,10 @@ class DocumentIndexer( private fun analyzeFile(): Pair { val context = SimpleContext() - // TODO(nsc) diagnosticsListener not being null seems to interfere with javacOutput val task = systemProvider.getTask( - javacOutput, fileManager, diagnosticsListener, - listOf("-source", "8", "-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString()), - listOf(), listOf(SourceFileObject(path)), context + NoopWriter, fileManager, diagnosticListener, + listOf("-source", "8", "-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString()/* , "--enable-preview" */), + listOf(), listOf(SourceFileObject(filepath.path)), context ) val compUnit = task.parse().iterator().next() task.analyze() @@ -135,7 +128,7 @@ class DocumentIndexer( } private fun emitDefinition(range: Range, hover: String) { - if (verbose) println("DEF " + path.toString() + ":" + humanRange(range)) + if (verbose) println("DEF ${filepath}:${humanRange(range)}") val hoverId = emitter.emitVertex("hoverResult", mapOf( "result" to mapOf( "contents" to mapOf( @@ -159,12 +152,12 @@ class DocumentIndexer( internal fun emitUse(use: Range, def: Range, defPath: Path) { referencesBacklog.add { - val indexer = indexers[defPath] - val link = path.toString() + ":" + humanRange(use) + " -> " + defPath.toString() + ":" + humanRange(def) + val indexer = indexers[defPath] ?: error("expected indexer for $defPath") + val link = "${filepath}:${humanRange(use)} -> ${defPath}:${humanRange(def)}" if (verbose) println("Linking use to definition: $link") - val meta = indexer!!.definitions[def] + val meta = indexer.definitions[def] if (meta == null) { if (verbose) println("WARNING missing definition for: $link") return@add @@ -192,18 +185,17 @@ class DocumentIndexer( } } - private fun mkDoc(signature: String, docComment: String?): String { - return "```java\n$signature\n```" + - if (docComment == null || docComment == "") "" else - "\n---\n${docComment.trim()}" - } + private fun mkDoc(signature: String, docComment: String?) = + "```java\n$signature\n```" + + if (docComment == null || docComment == "") "" + else "\n---\n${docComment.trim()}" /** * Returns the stringified range with the lines+1 to make clicking the links in your editor's terminal * direct to the correct line */ private fun humanRange(r: Range): String { - return (r.start.line+1).toString() + ":" + (r.start.character+1)+ "-" + (r.end.line+1) + ":" + (r.end.character+1) + return "${r.start.line+1}:${r.start.character+1}-${r.end.line+1}:${r.end.character+1}" } private fun createRange(range: Range): Map { diff --git a/src/main/kotlin/lsifjava/FileCollector.kt b/src/main/kotlin/lsifjava/FileCollector.kt index 3c8c5c59..5c41c520 100644 --- a/src/main/kotlin/lsifjava/FileCollector.kt +++ b/src/main/kotlin/lsifjava/FileCollector.kt @@ -1,63 +1,96 @@ package lsifjava -import org.apache.commons.io.output.NullOutputStream -import java.io.* +import kotlinx.coroutines.* import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.Path import java.nio.file.SimpleFileVisitor import java.nio.file.attribute.BasicFileAttributes +import kotlinx.coroutines.channels.Channel +private data class FileBuildInfo(val filepath: Path, val classpath: Classpath, val javaVersion: String?) -class FileCollector(private val projectId: String, private val args: Arguments, private val emitter: Emitter, private val indexers: MutableMap) : SimpleFileVisitor() { - lateinit var classpath: Classpath - var javaSourceVersion: String? = null - private val javacOutStream by lazy { createJavacOutWriter(args) } +/** + * Builds a map of Java filepaths to DocumentIndexers, + * with build info derived from `buildToolInterface` + */ +fun buildIndexerMap( + buildToolInterface: BuildToolInterface, + emitter: Emitter, + verbose: Boolean, + javacDiagListener: CountingDiagnosticListener, +): Map { + val indexers = mutableMapOf() + + val classpaths = buildToolInterface.getClasspaths() + val sourceVersions = buildToolInterface.javaSourceVersions() - override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { - return if (attrs.isSymbolicLink) { - FileVisitResult.SKIP_SUBTREE - } else FileVisitResult.CONTINUE + val fileBuildInfo = Channel() + + runBlocking { + launchFileTreeWalkers( + buildToolInterface, fileBuildInfo, classpaths, sourceVersions, + ).invokeOnCompletion(fileBuildInfo::close) + + launch { + for(info in fileBuildInfo) { + indexers[info.filepath] = DocumentIndexer( + CanonicalPath(info.filepath), info.classpath, info.javaVersion!!, + indexers, emitter, javacDiagListener, verbose, + ) + } + }.join() + } + + return indexers +} + +private fun CoroutineScope.launchFileTreeWalkers( + buildToolInterface: BuildToolInterface, + fileBuildInfoChannel: Channel, + classpaths: List, + sourceVersions: List, +) = launch(Dispatchers.IO) { + buildToolInterface.getSourceDirectories().forEachIndexed { i, paths -> + launch { + val collector = AsyncFileCollector(fileBuildInfoChannel, classpaths[i], sourceVersions[i], this) + paths.forEach { + if(Files.notExists(it)) return@forEach + Files.walkFileTree(it, collector) + } + } } +} + +/** + * A weakly asynchronous file-tree visitor. Sends discovered and valid + * files+build info into the provided channel asynchronously + */ +private class AsyncFileCollector( + private val fileInfoChannel: Channel, + private val classpath: Classpath, + private val sourceVersion: String?, + private val coroutineScope: CoroutineScope, +) : SimpleFileVisitor() { + + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = + if (attrs.isSymbolicLink) FileVisitResult.SKIP_SUBTREE + else FileVisitResult.CONTINUE - // TODO(nsc) uncouple FileCollector + DocumentIndexer override fun visitFile(file: Path, _attrs: BasicFileAttributes): FileVisitResult { if (isJavaFile(file)) { - indexers[file] = DocumentIndexer( - args.verbose, - file, - projectId, - emitter, - indexers, - classpath, - javaSourceVersion!!, - javacOutStream - ) + coroutineScope.launch { + fileInfoChannel.send(FileBuildInfo(file, classpath, sourceVersion)) + } } return FileVisitResult.CONTINUE } companion object { - fun createJavacOutWriter(args: Arguments): Writer? { - return when(args.javacOutWriter) { - "stdout" -> null // getTask interprets this as stdout - "stderr" -> System.err.writer() - //"none" -> OutputStreamWriter.nullWriter() CANT USE IN JAVA 8 :( - "none" -> object : Writer() { - override fun close() {} - override fun flush() {} - override fun write(p0: CharArray, p1: Int, p2: Int) {} - } - else -> PrintStream(File(args.javacOutWriter)).writer() - } - } - fun isJavaFile(file: Path): Boolean { val name = file.fileName.toString() // We hide module-info.java from javac, because when javac sees module-info.java // it goes into "module mode" and starts looking for classes on the module class path. - // This becomes evident when javac starts recompiling *way too much* on each task, - // because it doesn't realize there are already up-to-date .class files. // The better solution would be for java-language server to detect the presence of module-info.java, // and go into its own "module mode" where it infers a module source path and a module class path. return name.endsWith(".java") && !Files.isDirectory(file) && name != "module-info.java" diff --git a/src/main/kotlin/lsifjava/IndexingVisitor.kt b/src/main/kotlin/lsifjava/IndexingVisitor.kt index eb4c2119..05a16d56 100644 --- a/src/main/kotlin/lsifjava/IndexingVisitor.kt +++ b/src/main/kotlin/lsifjava/IndexingVisitor.kt @@ -268,4 +268,19 @@ class IndexingVisitor( matcher.start() } else -1 } + + private fun getTopLevelClass(element: Element?): Element? { + var element = element + var highestClass: Element? = null + while (element != null) { + val kind = element.kind + if (isTopLevel(kind)) { + highestClass = element + } + element = element.enclosingElement + } + return highestClass + } + + private fun isTopLevel(kind: ElementKind) = kind.isClass || kind.isInterface } \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/LanguageUtils.kt b/src/main/kotlin/lsifjava/LanguageUtils.kt deleted file mode 100644 index dce0e3f0..00000000 --- a/src/main/kotlin/lsifjava/LanguageUtils.kt +++ /dev/null @@ -1,20 +0,0 @@ -package lsifjava - -import javax.lang.model.element.* - -fun getTopLevelClass(element: Element?): Element? { - var element = element - var highestClass: Element? = null - while (element != null) { - val kind = element.kind - if (isTopLevel(kind)) { - highestClass = element - } - element = element.enclosingElement - } - return highestClass -} - -fun isTopLevel(kind: ElementKind): Boolean { - return kind.isClass || kind.isInterface -} diff --git a/src/main/kotlin/lsifjava/ProjectIndexer.kt b/src/main/kotlin/lsifjava/ProjectIndexer.kt index b836b834..9fcd855c 100644 --- a/src/main/kotlin/lsifjava/ProjectIndexer.kt +++ b/src/main/kotlin/lsifjava/ProjectIndexer.kt @@ -1,9 +1,5 @@ package lsifjava -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - class ProjectIndexer(private val arguments: Arguments, private val emitter: Emitter) { var numFiles = 0 var numDefinitions = 0 @@ -13,7 +9,7 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit emitter.emitVertex("metaData", mapOf( "version" to "0.4.0", "positionEncoding" to "utf-16", - "projectRoot" to String.format("file://%s", Paths.get(arguments.projectRoot).toFile().canonicalPath), + "projectRoot" to "file://${arguments.projectRoot}", "toolInfo" to mapOf("name" to "lsif-java", "version" to "0.7.0") )) @@ -21,26 +17,17 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit "kind" to "java" )) - val indexers = HashMap() - val collector = FileCollector(projectId, arguments, emitter, indexers) - GradleInterface(arguments.projectRoot).use { gradleInterface -> - val classpaths = gradleInterface.getClasspaths() - val javaSourceVersions = gradleInterface.javaSourceVersions() - gradleInterface.getSourceDirectories().forEachIndexed { i, paths -> - collector.classpath = classpaths[i] - collector.javaSourceVersion = javaSourceVersions[i] - paths.forEach { - if (Files.notExists(it)) return@forEach - Files.walkFileTree(it, collector) - } + createJavacDiagnosticListener(arguments.javacOutWriter).use { javacDiagListener -> + val indexers = GradleInterface(arguments.projectRoot).use { + buildIndexerMap(it, emitter, arguments.verbose, javacDiagListener) } - } - val fileManager = SourceFileManager(indexers.keys) + val fileManager = SourceFileManager(indexers.keys) for (indexer in indexers.values) { indexer.preIndex(fileManager) } + for (indexer in indexers.values) { indexer.index() } @@ -55,6 +42,4 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit numJavacErrors += indexer.javacDiagnostics.size } } -} - - +} \ No newline at end of file From 9068dcabcacd61e8ad127822639c3e02d0285f57 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Wed, 28 Oct 2020 16:45:24 +0000 Subject: [PATCH 2/6] Lazy eval'd index method for DocumentIndexer, some stylistic changes, display total javac errors Using lazy block for the DocumentIndexer index method to mimic Gos sync.Once --- src/main/kotlin/lsifjava/DocumentIndexer.kt | 37 ++++------ src/main/kotlin/lsifjava/Emitter.kt | 4 +- src/main/kotlin/lsifjava/IndexingVisitor.kt | 74 +++++++++++++------ src/main/kotlin/lsifjava/Main.kt | 5 +- src/main/kotlin/lsifjava/ProjectIndexer.kt | 29 ++++---- src/main/kotlin/lsifjava/SourceFileManager.kt | 14 ++-- 6 files changed, 94 insertions(+), 69 deletions(-) diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index 2d9b1cb5..d9bc6185 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -8,11 +8,8 @@ import com.sun.tools.javac.main.JavaCompiler import com.sun.tools.javac.util.Context import org.eclipse.lsp4j.Position import org.eclipse.lsp4j.Range -import java.io.Writer import java.nio.file.Path import java.util.* -import javax.tools.Diagnostic -import javax.tools.DiagnosticListener import kotlin.collections.HashMap import kotlin.collections.HashSet @@ -26,25 +23,24 @@ class DocumentIndexer( private val verbose: Boolean, ) { companion object { - private val systemProvider = JavacTool.create() + private val systemProvider by lazy { JavacTool.create() } } data class DefinitionMeta(val rangeId: String, val resultSetId: String) { var definitionResultId: String? = null - val referenceRangeIds = HashMap>() + val referenceRangeIds: MutableMap> = HashMap() } - private lateinit var documentId: String - private var indexed = false + private var documentId: String private val rangeIds: MutableSet = HashSet() private val definitions: MutableMap = HashMap() - private lateinit var fileManager: SourceFileManager - + + val numDefinitions get() = definitions.size + + lateinit var fileManager: SourceFileManager + private val referencesBacklog: LinkedList<() -> Unit> = LinkedList() - - var javacDiagnostics = LinkedList>() - private set - private val diagnosticsListener = DiagnosticListener { javacDiagnostics.add(it) } + init { val args = mapOf( @@ -54,19 +50,14 @@ class DocumentIndexer( documentId = emitter.emitVertex("document", args) } - @Synchronized - fun index() { - if (indexed) return - indexed = true - + // lazy block ensures synchronized calling and only one invocation + fun index() = lazy { val (task, compUnit) = analyzeFile() IndexingVisitor(task, compUnit, this, indexers).scan(compUnit, null) referencesBacklog.forEach { it() } - //javacDiagnostics.forEach(::println) - - println("finished analyzing and visiting $path") - } + println("finished analyzing and visiting $filepath") + }.value // must reference this to trigger the lazy load private fun analyzeFile(): Pair { val context = SimpleContext() @@ -101,7 +92,7 @@ class DocumentIndexer( } } - fun postIndex() { + fun postIndex(projectId: String) { for (meta in definitions.values) linkUses(meta, documentId) emitter.emitEdge("contains", mapOf("outV" to projectId, "inVs" to arrayOf(documentId))) diff --git a/src/main/kotlin/lsifjava/Emitter.kt b/src/main/kotlin/lsifjava/Emitter.kt index 1d1b25fd..9c0f64b3 100644 --- a/src/main/kotlin/lsifjava/Emitter.kt +++ b/src/main/kotlin/lsifjava/Emitter.kt @@ -9,6 +9,8 @@ class Emitter(private val writer: PrintWriter) { private val gson: Gson = Gson() + fun numElements() = nextId + fun emitVertex(labelName: String, args: Map) = emit("vertex", labelName, args) fun emitEdge(labelName: String, args: Map) = emit("edge", labelName, args) @@ -27,6 +29,4 @@ class Emitter(private val writer: PrintWriter) { return id } - - fun numElements() = nextId } \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/IndexingVisitor.kt b/src/main/kotlin/lsifjava/IndexingVisitor.kt index 05a16d56..e53f6550 100644 --- a/src/main/kotlin/lsifjava/IndexingVisitor.kt +++ b/src/main/kotlin/lsifjava/IndexingVisitor.kt @@ -14,6 +14,7 @@ import java.nio.file.Path import java.nio.file.Paths import java.util.regex.Pattern import javax.lang.model.element.Element +import javax.lang.model.element.ElementKind data class ReferenceData(val useRange: Range, val refRange: Range, val defPath: Path) @@ -29,7 +30,8 @@ class IndexingVisitor( // TODO(nsc) handle 'var' override fun visitVariable(node: VariableTree, p: Unit?): Unit? { // emit var definition - val defRange = findLocation(currentPath, node.name.toString())?.range ?: return super.visitVariable(node, p) + val defRange = findLocation(currentPath, node.name.toString())?.range + ?: return super.visitVariable(node, p) indexer.emitDefinition(defRange, node.toString(), docs.getDocComment(currentPath)) @@ -42,7 +44,8 @@ class IndexingVisitor( // TODO(nsc): records val classOrEnum = if ((node as JCClassDecl).sym.flags() and Flags.ENUM.toLong() != 0L) "enum " else "class " - val range = findLocation(currentPath, node.simpleName.toString())?.range ?: return super.visitClass(node, p) + val range = findLocation(currentPath, node.simpleName.toString())?.range + ?: return super.visitClass(node, p) indexer.emitDefinition( range, @@ -62,7 +65,8 @@ class IndexingVisitor( val returnType = node.returnType?.toString()?.plus(" ") ?: "" - val range = findLocation(currentPath, methodName, node.pos)?.range ?: return super.visitMethod(node, p) + val range = findLocation(currentPath, methodName, node.pos)?.range + ?: return super.visitMethod(node, p) indexer.emitDefinition( range, @@ -78,6 +82,7 @@ class IndexingVisitor( override fun visitNewClass(node: NewClassTree, p: Unit?): Unit? { (node as JCNewClass).constructor ?: return super.visitNewClass(node, p) + // ignore auto-genned constructors (for now) if(node.constructor.flags() and Flags.GENERATEDCONSTR != 0L) return null @@ -103,15 +108,17 @@ class IndexingVisitor( val identPath = TreePath(currentPath, node.identifier) val identElement = node.constructor - val defContainer = getTopLevelClass(identElement) as Symbol.ClassSymbol? ?: return super.visitNewClass(node, p) - val (useRange, refRange, defPath) = findReference(identElement, identName, defContainer, path = identPath) ?: return super.visitNewClass(node, p) + val defContainer = getTopLevelClass(identElement) as Symbol.ClassSymbol? + ?: return super.visitNewClass(node, p) + val (useRange, refRange, defPath) = findReference(identElement, identName, defContainer, path = identPath) + ?: return super.visitNewClass(node, p) indexer.emitUse(useRange, refRange, defPath) return super.visitNewClass(node, p) } - override fun visitMethodInvocation(node: MethodInvocationTree?, p: Unit?): Unit? { + override fun visitMethodInvocation(node: MethodInvocationTree, p: Unit?): Unit? { val symbol = when((node as JCMethodInvocation).meth) { is JCFieldAccess -> (node.meth as JCFieldAccess).sym is JCIdent -> (node.meth as JCIdent).sym @@ -126,12 +133,16 @@ class IndexingVisitor( val name = symbol?.name?.toString()?.let { if(it == "") "this" else it } ?: return super.visitMethodInvocation(node, p) + val methodPath = TreePath(currentPath, node.meth) - val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMethodInvocation(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? + ?: return super.visitMethodInvocation(node, p) // this gives us the start pos of the name of the method, so override defStartPos - val overrideStartOffset = (trees.getPath(symbol)?.leaf as JCMethodDecl?)?.pos ?: return super.visitMethodInvocation(node, p) - val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, path = methodPath, defStartPos = overrideStartOffset) ?: return super.visitMethodInvocation(node, p) + val overrideStartOffset = (trees.getPath(symbol)?.leaf as JCMethodDecl?)?.pos + ?: return super.visitMethodInvocation(node, p) + val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, path = methodPath, defStartPos = overrideStartOffset) + ?: return super.visitMethodInvocation(node, p) indexer.emitUse(useRange, refRange, defPath) @@ -139,24 +150,29 @@ class IndexingVisitor( } // does not match `var` or constructor calls - override fun visitIdentifier(node: IdentifierTree?, p: Unit?): Unit? { - val symbol = (node as JCIdent).sym ?: return super.visitIdentifier(node, p) + override fun visitIdentifier(node: IdentifierTree, p: Unit?): Unit? { + val symbol = (node as JCIdent).sym + ?: return super.visitIdentifier(node, p) if(symbol is Symbol.PackageSymbol) return super.visitIdentifier(node, p) val name = symbol.name.toString() if(name == "") return super.visitIdentifier(node, p) // handled by visitMethodInvocation - val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitIdentifier(node, p) - val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) ?: return super.visitIdentifier(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? + ?: return super.visitIdentifier(node, p) + + val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) + ?: return super.visitIdentifier(node, p) indexer.emitUse(useRange, refRange, defPath) return super.visitIdentifier(node, p) } - override fun visitTypeParameter(node: TypeParameterTree?, p: Unit?): Unit? { - val range = findLocation(currentPath, node!!.name.toString(), (node as JCTypeParameter).pos)?.range ?: return super.visitTypeParameter(node, p) + override fun visitTypeParameter(node: TypeParameterTree, p: Unit?): Unit? { + val range = findLocation(currentPath, node.name.toString(), (node as JCTypeParameter).pos)?.range + ?: return super.visitTypeParameter(node, p) indexer.emitDefinition(range, "type-parameter $node") @@ -164,21 +180,24 @@ class IndexingVisitor( } // function references eg test::myfunc or banana::new - override fun visitMemberReference(node: MemberReferenceTree?, p: Unit?): Unit? { - val symbol = (node as JCMemberReference).sym ?: return super.visitMemberReference(node, p) + override fun visitMemberReference(node: MemberReferenceTree, p: Unit?): Unit? { + val symbol = (node as JCMemberReference).sym + ?: return super.visitMemberReference(node, p) val name = symbol.name.toString() - val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMemberReference(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? + ?: return super.visitMemberReference(node, p) - val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) ?: return super.visitMemberReference(node, p) + val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) + ?: return super.visitMemberReference(node, p) indexer.emitUse(useRange, refRange, defPath) return super.visitMemberReference(node, p) } - override fun visitMemberSelect(node: MemberSelectTree?, p: Unit?): Unit? { + override fun visitMemberSelect(node: MemberSelectTree, p: Unit?): Unit? { if((node as JCFieldAccess).sym == null) return super.visitMemberSelect(node, p) val symbol = when(node.sym) { @@ -189,10 +208,12 @@ class IndexingVisitor( val name = symbol.name.toString() - val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMemberSelect(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? + ?: return super.visitMemberSelect(node, p) // need to narrow down start pos of the field access here, so override refStartPos - val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, refStartPos = node.pos) ?: return super.visitMemberSelect(node, p) + val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, refStartPos = node.pos) + ?: return super.visitMemberSelect(node, p) indexer.emitUse(useRange, refRange, defPath) @@ -207,7 +228,14 @@ class IndexingVisitor( * @param refStartPos document offset to be provided if a node supplies a more accurate start position for a reference/use search * @param defStartPos document offset to be provided if a node supplies a more accurate start position for a definition search */ - private fun findReference(element: Element, symbolName: String, container: Symbol.ClassSymbol, path: TreePath = currentPath, refStartPos: Int? = null, defStartPos: Int? = null): ReferenceData? { + private fun findReference( + element: Element, + symbolName: String, + container: Symbol.ClassSymbol, + path: TreePath = currentPath, + refStartPos: Int? = null, + defStartPos: Int? = null, + ): ReferenceData? { val sourceFilePath = container.sourcefile ?: return null val defPath = Paths.get(sourceFilePath.name) if(sourceFilePath.name != compUnit.sourceFile.name) diff --git a/src/main/kotlin/lsifjava/Main.kt b/src/main/kotlin/lsifjava/Main.kt index 66b522ed..7901796c 100644 --- a/src/main/kotlin/lsifjava/Main.kt +++ b/src/main/kotlin/lsifjava/Main.kt @@ -29,10 +29,11 @@ private fun createWriter(args: Arguments): PrintWriter { private fun displayStats(indexer: ProjectIndexer, emitter: Emitter, start: Long) { System.out.printf( - "%d file(s), %d def(s), %d element(s)\n", + "%d file(s), %d def(s), %d LSIF element(s), %d total javac error(s)\n", indexer.numFiles, indexer.numDefinitions, - emitter.numElements() + emitter.numElements(), + indexer.numJavacErrors ) System.out.printf("Processed in %.0fms", (System.nanoTime() - start) / 1e6) } \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/ProjectIndexer.kt b/src/main/kotlin/lsifjava/ProjectIndexer.kt index 9fcd855c..d7f1b836 100644 --- a/src/main/kotlin/lsifjava/ProjectIndexer.kt +++ b/src/main/kotlin/lsifjava/ProjectIndexer.kt @@ -24,22 +24,25 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit val fileManager = SourceFileManager(indexers.keys) - for (indexer in indexers.values) { - indexer.preIndex(fileManager) - } + // can we do this earlier??? + for (indexer in indexers.values) { + indexer.fileManager = fileManager + } - for (indexer in indexers.values) { - indexer.index() - } + for (indexer in indexers.values) { + indexer.index() + } - for (indexer in indexers.values) { - indexer.postIndex() - } + for (indexer in indexers.values) { + indexer.postIndex(projectId) + } + + for (indexer in indexers.values) { + numFiles++ + numDefinitions += indexer.numDefinitions + } - for (indexer in indexers.values) { - numFiles++ - numDefinitions += indexer.numDefinitions() - numJavacErrors += indexer.javacDiagnostics.size + numJavacErrors = javacDiagListener.count } } } \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/SourceFileManager.kt b/src/main/kotlin/lsifjava/SourceFileManager.kt index 9eb32872..2aca2938 100644 --- a/src/main/kotlin/lsifjava/SourceFileManager.kt +++ b/src/main/kotlin/lsifjava/SourceFileManager.kt @@ -16,7 +16,11 @@ import javax.tools.StandardLocation class SourceFileManager internal constructor(private val paths: Set): JavacFileManager(Context(), false, Charset.defaultCharset()) { override fun list( - location: JavaFileManager.Location, packageName: String, kinds: Set, recurse: Boolean): Iterable { + location: JavaFileManager.Location, + packageName: String, + kinds: Set, + recurse: Boolean + ): Iterable { return if (location === StandardLocation.SOURCE_PATH) { val sourceFileObjectStream = list(packageName) Iterable { sourceFileObjectStream.iterator() } @@ -26,9 +30,9 @@ class SourceFileManager internal constructor(private val paths: Set): Java } private fun list(packageName: String): Stream { - return paths.stream().map { path: Path? -> + return paths.stream().map { path: Path -> try { - return@map SourceFileObject(path!!) + return@map SourceFileObject(path) } catch (e: IOException) { return@map null } @@ -52,9 +56,7 @@ class SourceFileManager internal constructor(private val paths: Set): Java if(it == -1) fileName else fileName.substring(0, it) } - override fun hasLocation(location: JavaFileManager.Location): Boolean { - return location === StandardLocation.SOURCE_PATH || super.hasLocation(location) - } + override fun hasLocation(location: JavaFileManager.Location) = location === StandardLocation.SOURCE_PATH || super.hasLocation(location) override fun getJavaFileForInput(location: JavaFileManager.Location, className: String, kind: JavaFileObject.Kind): JavaFileObject? { // FileStore shadows disk From 16bbcac43502c4e0ecd727b002877dec1bc4aa17 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Wed, 28 Oct 2020 16:50:13 +0000 Subject: [PATCH 3/6] Support wider range of 'class' declaration types in the type name emitted --- src/main/kotlin/lsifjava/DocumentIndexer.kt | 1 - src/main/kotlin/lsifjava/IndexingVisitor.kt | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index d9bc6185..93654b91 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -41,7 +41,6 @@ class DocumentIndexer( private val referencesBacklog: LinkedList<() -> Unit> = LinkedList() - init { val args = mapOf( "languageId" to "java", diff --git a/src/main/kotlin/lsifjava/IndexingVisitor.kt b/src/main/kotlin/lsifjava/IndexingVisitor.kt index e53f6550..121e5e03 100644 --- a/src/main/kotlin/lsifjava/IndexingVisitor.kt +++ b/src/main/kotlin/lsifjava/IndexingVisitor.kt @@ -19,10 +19,10 @@ import javax.lang.model.element.ElementKind data class ReferenceData(val useRange: Range, val refRange: Range, val defPath: Path) class IndexingVisitor( - task: JavacTask, - private val compUnit: CompilationUnitTree, - private val indexer: DocumentIndexer, - private val indexers: Map + task: JavacTask, + private val compUnit: CompilationUnitTree, + private val indexer: DocumentIndexer, + private val indexers: Map ): TreePathScanner() { private val trees: Trees = Trees.instance(task) private val docs: DocTrees = DocTrees.instance(task) @@ -41,8 +41,14 @@ class IndexingVisitor( override fun visitClass(node: ClassTree, p: Unit?): Unit? { val packagePrefix = compUnit.packageName?.toString()?.plus(".") ?: "" - // TODO(nsc): records - val classOrEnum = if ((node as JCClassDecl).sym.flags() and Flags.ENUM.toLong() != 0L) "enum " else "class " + // TODO(nsc): snazzy records hover + val classOrEnum: String = when { + (node as JCClassDecl).sym.isEnum -> "enum " + node.sym.isRecord -> "record " + node.sym.isInterface -> "" // for some reason, 'interface ' is part of the modifiers + node.sym.isAnnotationType -> "" + else -> "class " + } val range = findLocation(currentPath, node.simpleName.toString())?.range ?: return super.visitClass(node, p) From 167fe910aba79d0a95e48963a23531e308b77029 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 29 Oct 2020 13:12:51 +0000 Subject: [PATCH 4/6] Removed tools.jar stuff from build.gradle, JDK15 only pls --- build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.gradle b/build.gradle index e22ccd02..d9bafebe 100644 --- a/build.gradle +++ b/build.gradle @@ -14,10 +14,6 @@ repositories { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.0' - if(JavaVersion.current() < JavaVersion.VERSION_1_9) { - println "JDK version is ${JavaVersion.current()}, including tools.jar @ ${System.properties['java.home']}/lib/tools.jar" - compile files("${System.properties['java.home']}/../lib/tools.jar") - } compile 'com.google.code.gson:gson:2.8.6' compile 'commons-cli:commons-cli:1.4' compile 'org.apache.commons:commons-lang3:3.11' From 822565101ab5725680a17db4bd1586e8128fffd5 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 29 Oct 2020 13:30:35 +0000 Subject: [PATCH 5/6] Moved some utility classes to a separate file --- src/main/kotlin/lsifjava/ArgumentParser.kt | 4 ---- .../kotlin/lsifjava/CountingDiagnosticWriter.kt | 9 +-------- src/main/kotlin/lsifjava/UtilTypes.kt | 15 +++++++++++++++ 3 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 src/main/kotlin/lsifjava/UtilTypes.kt diff --git a/src/main/kotlin/lsifjava/ArgumentParser.kt b/src/main/kotlin/lsifjava/ArgumentParser.kt index 677bda1f..bfc60e09 100644 --- a/src/main/kotlin/lsifjava/ArgumentParser.kt +++ b/src/main/kotlin/lsifjava/ArgumentParser.kt @@ -8,10 +8,6 @@ import kotlin.system.exitProcess const val VERSION = "0.1.0" const val PROTOCOL_VERSION = "0.4.0" -inline class CanonicalPath(val path: Path) { - override fun toString(): String = path.toFile().canonicalPath -} - data class Arguments( val projectRoot: CanonicalPath, val outFile: String, diff --git a/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt b/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt index 2ae340e1..1061c9d2 100644 --- a/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt +++ b/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt @@ -37,11 +37,4 @@ sealed class CountingDiagnosticListener(private val out: Writer): DiagnosticList } object NullWriter: CountingDiagnosticListener(NoopWriter) -} - -// OutputStreamWriter.nullWriter() CANT USE IN JAVA 8 :( -object NoopWriter: Writer() { - override fun close() = Unit - override fun flush() = Unit - override fun write(p0: CharArray, p1: Int, p2: Int) = Unit -} +} \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/UtilTypes.kt b/src/main/kotlin/lsifjava/UtilTypes.kt new file mode 100644 index 00000000..2ba7501d --- /dev/null +++ b/src/main/kotlin/lsifjava/UtilTypes.kt @@ -0,0 +1,15 @@ +package lsifjava + +import java.io.Writer +import java.nio.file.Path + +inline class CanonicalPath(val path: Path) { + override fun toString(): String = path.toFile().canonicalPath +} + +// OutputStreamWriter.nullWriter() CANT USE IN JAVA 8 :( +object NoopWriter: Writer() { + override fun close() = Unit + override fun flush() = Unit + override fun write(p0: CharArray, p1: Int, p2: Int) = Unit +} From 43ae97965fa18b2a1a095944920c7397dc4c5e6a Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 29 Oct 2020 13:42:28 +0000 Subject: [PATCH 6/6] Filtering path sequence in collector, just because it looks nice --- src/main/kotlin/lsifjava/FileCollector.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/kotlin/lsifjava/FileCollector.kt b/src/main/kotlin/lsifjava/FileCollector.kt index 5c41c520..bf251c07 100644 --- a/src/main/kotlin/lsifjava/FileCollector.kt +++ b/src/main/kotlin/lsifjava/FileCollector.kt @@ -54,10 +54,7 @@ private fun CoroutineScope.launchFileTreeWalkers( buildToolInterface.getSourceDirectories().forEachIndexed { i, paths -> launch { val collector = AsyncFileCollector(fileBuildInfoChannel, classpaths[i], sourceVersions[i], this) - paths.forEach { - if(Files.notExists(it)) return@forEach - Files.walkFileTree(it, collector) - } + paths.asSequence().filter { Files.exists(it) }.forEach { Files.walkFileTree(it, collector) } } } }