From 1a3bddfe5e53c7d0cf065fa412cea5e5273a90a3 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 4 Feb 2021 00:19:43 +0000 Subject: [PATCH] WIP Docs --- src/main/kotlin/lsifjava/Compiler.kt | 60 ++++++++++++++++ src/main/kotlin/lsifjava/DocumentIndexer.kt | 72 ++++++------------- src/main/kotlin/lsifjava/ExternalDocs.kt | 2 +- src/main/kotlin/lsifjava/IndexingVisitor.kt | 14 ++-- .../kotlin/lsifjava/JDK8CompatFileManager.kt | 16 ++--- src/main/kotlin/lsifjava/JavaHomeHelper.kt | 32 +++++---- 6 files changed, 113 insertions(+), 83 deletions(-) create mode 100644 src/main/kotlin/lsifjava/Compiler.kt diff --git a/src/main/kotlin/lsifjava/Compiler.kt b/src/main/kotlin/lsifjava/Compiler.kt new file mode 100644 index 00000000..863d6fca --- /dev/null +++ b/src/main/kotlin/lsifjava/Compiler.kt @@ -0,0 +1,60 @@ +package lsifjava + +import com.sun.source.tree.CompilationUnitTree +import com.sun.source.util.JavacTask +import com.sun.tools.javac.api.JavacTool +import com.sun.tools.javac.main.JavaCompiler +import com.sun.tools.javac.util.Context + +fun generateParseTree( + systemProvider: JavacTool, + fileManager: SourceFileManager, + diagnosticListener: CountingDiagnosticListener, + javaSourceVersion: String, + classpath: Classpath, + filepath: CanonicalPath +): Pair { + val task = systemProvider.getTask( + NoopWriter, fileManager, diagnosticListener, + listOf("-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString() /*, "--enable-preview" */), + listOf(), listOf(SourceFileObject(filepath.path)), SimpleContext() + ) + val compUnit = task.parse().iterator().next() + task.analyze() + + return Pair(task, compUnit) +} + +/** + * Context lets us configure some aspects of the parsing process. SimpleContext + * incorporates a SimpleCompiler that configures the parser to generate end positions, + * preserve javadoc and keep line debug info (not sure if we use that currently/ever). + */ +class SimpleContext: Context() { + init { + put(SimpleCompiler.contextKey, SimpleCompiler.factory) + } +} + +private class SimpleCompiler(context: Context?): JavaCompiler(context) { + companion object { + val factory = Context.Factory { context: Context? -> SimpleCompiler(context) } + + val contextKey: Context.Key by lazy { + // compilerKey is not public in Java 8, so we have to kinda: https://media.discordapp.net/attachments/287739410286379019/806651592216281108/unknown.png + if(javaVersion == 8) run { + val field = SimpleCompiler::class.java.superclass.getDeclaredField("compilerKey") + field.isAccessible = true + val key: Context.Key = Context.Key() // dummy value to be overwritten + return@lazy field.get(key) as Context.Key + } + compilerKey + } + } + + init { + genEndPos = true + lineDebugInfo = true + keepComments = true + } +} \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index 8ee11a0c..62aeb520 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -2,7 +2,9 @@ package lsifjava import com.sun.source.tree.CompilationUnitTree +import com.sun.source.util.DocTrees import com.sun.source.util.JavacTask +import com.sun.source.util.Trees import com.sun.tools.javac.api.JavacTool import com.sun.tools.javac.main.JavaCompiler import com.sun.tools.javac.util.Context @@ -14,6 +16,9 @@ import java.util.concurrent.atomic.AtomicBoolean import kotlin.collections.HashMap import kotlin.collections.HashSet +/** + * + */ class DocumentIndexer( private val filepath: CanonicalPath, private val classpath: Classpath, @@ -24,10 +29,6 @@ class DocumentIndexer( private val diagnosticListener: CountingDiagnosticListener, private val verbose: Boolean, ) { - companion object { - private val systemProvider by lazy { JavacTool.create() } - } - data class DefinitionMeta(val definitionRangeId: String, val resultSetId: String) { var definitionResultId: String? = null val referenceRangeIds: MutableMap> = HashMap() @@ -52,61 +53,28 @@ class DocumentIndexer( private var indexed = AtomicBoolean(false) + /** + * Invokes the javac + */ fun index() { try { if(!indexed.compareAndSet(false, true)) return - val (task, compUnit) = analyzeFile() - IndexingVisitor(task, systemProvider, docManager, compUnit, this, indexers).scan(compUnit, null) - referencesBacklog.forEach { it() } + + val systemProvider = JavacTool.create() + val (task, compUnit) = generateParseTree(systemProvider, fileManager, diagnosticListener, javaSourceVersion, classpath, filepath) + + val trees: Trees = Trees.instance(task) + val docs: DocTrees = DocTrees.instance(task) + IndexingVisitor(trees, docs, systemProvider, docManager, compUnit, this, indexers).scan(compUnit, null) } catch (e: Exception) { throw IndexingException(filepath.toString(), e) + } finally { + referencesBacklog.forEach { it() } } println("finished analyzing and visiting $filepath") } - private fun analyzeFile(): Pair { - val context = SimpleContext() - - val task = systemProvider.getTask( - NoopWriter, fileManager, diagnosticListener, - listOf("-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString() /*, "--enable-preview" */), - listOf(), listOf(SourceFileObject(filepath.path)), context - ) - val compUnit = task.parse().iterator().next() - task.analyze() - - return Pair(task, compUnit) - } - - internal class SimpleContext: Context() { - init { - put(SimpleCompiler.contextKey, SimpleCompiler.factory) - } - } - - internal class SimpleCompiler(context: Context?): JavaCompiler(context) { - companion object { - val factory = Context.Factory { context: Context? -> SimpleCompiler(context) } - - val contextKey: Context.Key by lazy { - if(javaVersion == 8) run { - val field = SimpleCompiler::class.java.superclass.getDeclaredField("compilerKey") - field.isAccessible = true - val key: Context.Key = Context.Key() // dummy value to be overwritten - return@lazy field.get(key) as Context.Key - } - compilerKey - } - } - - init { - genEndPos = true - lineDebugInfo = true - keepComments = true - } - } - fun postIndex(projectId: String) { for (meta in definitions.values) linkUses(meta) @@ -164,7 +132,7 @@ class DocumentIndexer( definitions[range] = DefinitionMeta(rangeId, resultSetId) // + contents? } - // TODO(nsc): GLOBAL CACHE SO WE DONT HAVE MULTIPLE REFS FOR SAME DEF + // TODO GLOBAL CACHE SO WE DONT HAVE MULTIPLE REFS FOR SAME DEF internal fun emitExternalHoverAndUse(signature: String, doc: String, use: Range) { val hoverId = emitter.emitVertex("hoverResult", mapOf( "result" to mapOf( @@ -186,6 +154,10 @@ class DocumentIndexer( linkUses(meta) } + /** + * Adds a processing function to a backlog to be executed after the current document and all transitively dependent + * documents are indexed to emit ranges that are linked to def ranges in postIndex. + */ internal fun emitUse(use: Range, def: Range, defPath: Path) { referencesBacklog.add { val indexer = indexers[defPath] ?: error("expected indexer for $defPath") diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index 9c60bcff..95dd3b71 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -37,7 +37,7 @@ class ExternalDocs(private val docPaths: List, private val diagnosticListe * associated with containerClass, which is expected to be the fully qualified name of the class file */ fun findDocForElement(containerClass: String, javac: JavacTool, element: Element): ExternalHoverMeta? { - val context = DocumentIndexer.SimpleContext() + val context = SimpleContext() val (task, compUnit) = fileCache.getOrPut(containerClass) { val fileObject = findFileFromJars(containerClass) ?: return@getOrPut null diff --git a/src/main/kotlin/lsifjava/IndexingVisitor.kt b/src/main/kotlin/lsifjava/IndexingVisitor.kt index 529bb4af..0407d123 100644 --- a/src/main/kotlin/lsifjava/IndexingVisitor.kt +++ b/src/main/kotlin/lsifjava/IndexingVisitor.kt @@ -34,16 +34,14 @@ data class ReferenceData(val useRange: Range, val defRange: Range?, val defPath: fun Type.strip() = this.toString().removePrefix("java.lang.") class IndexingVisitor( - task: JavacTask, + private val trees: Trees, + private val docs: DocTrees, private val tool: JavacTool, private val externalDocManager: ExternalDocs, private val compUnit: CompilationUnitTree, private val indexer: DocumentIndexer, private val indexers: Map ): TreePathScanner() { - private val trees: Trees = Trees.instance(task) - private val docs: DocTrees = DocTrees.instance(task) - // TODO(nsc) handle 'var' override fun visitVariable(node: VariableTree, p: Unit?): Unit? { // emit var definition @@ -74,11 +72,12 @@ class IndexingVisitor( val symbol = (node as JCClassDecl?)?.sym ?: return super.visitClass(node, p) - // TODO(nsc): snazzy records hover + // TODO snazzy records hover val classOrEnum: String = when { symbol.isEnum -> "enum " javaVersion >= 14 && symbol.isRecord -> "record " - symbol.isInterface -> "" // for some reason, 'interface ' is part of the modifiers + // for some reason, 'interface ' is part of the modifiers, so we exclude the string here + symbol.isInterface -> "" // TODO lowest version javaVersion >= 14 && symbol.isAnnotationType -> "" else -> "class " @@ -360,8 +359,7 @@ class IndexingVisitor( // this may fail to find the correct position if start is too far behind the actual start position // that we want and another matching but incorrect symbol matches the regex private fun findNameIn(root: CompilationUnitTree, name: CharSequence, start: Int, end: Int): Int { - val contents: CharSequence - contents = try { + val contents = try { root.sourceFile.getCharContent(true) } catch (e: IOException) { throw RuntimeException(e) diff --git a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt index 5c4c4f10..e7a40ca9 100644 --- a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt +++ b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt @@ -102,20 +102,18 @@ private val JDK_MODULES = listOf( ) /** - * FileManager that falls back to JavacPathFileManager for Java 8. - * Java 8 StandardJavaFileManager doesn't have the setLocationFromPaths - * method, instead it only has setLocation which requires an - * Iterable, which would cause an UnsupportedException - * when trying to turn the ZipFile from the in-memory FileSystem into a File. - * Because JavacPathFileManager doesn't exist beyond Java 8 (9?) and we build with 14+, - * the symbol resolver would fail for that symbol, hence we create an instance via - * reflection if the Java version is 8. God I hate this. + * FileManager that falls back to JavacPathFileManager for Java 8. Java 8 StandardJavaFileManager + * doesn't have the setLocationFromPaths method, instead it only has setLocation which requires an + * Iterable, which would cause an UnsupportedException when trying to turn the + * ZipFile from the in-memory FileSystem into a File. Because JavacPathFileManager doesn't exist + * beyond Java 8 (9?) and we build with 14+, the symbol resolver would fail for that symbol, + * hence we create an instance via reflection if the Java version is 8. God I hate this. */ class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFileManager(getFileManager(manager)) { companion object { /** * returns a different JavaFileManager based on the current java version, with either SOURCE_PATH or MODULE_SOURCE_PATH set to "/" - * denoting the root of the JAR/ZIP file that this file manager is for. If JDK 8, we instantiate a com.sun.tools.javac.nio.JavacPathFileManager + * denoting the root of the JAR/ZIP file that this file manager is for. If JDK 8, we instantiate a com.sun.tools.javac.nio.JavacPathFileManager * via reflection, else we use the passed file manager. See the class doc for the reason why. */ private fun getFileManager(fileManager: StandardJavaFileManager): JavaFileManager { diff --git a/src/main/kotlin/lsifjava/JavaHomeHelper.kt b/src/main/kotlin/lsifjava/JavaHomeHelper.kt index 5739bda3..35adcdee 100644 --- a/src/main/kotlin/lsifjava/JavaHomeHelper.kt +++ b/src/main/kotlin/lsifjava/JavaHomeHelper.kt @@ -13,6 +13,12 @@ import java.util.stream.Collectors // from https://github.com/georgewfraser/java-language-server // bless you george for all the references. Maybe Ill cut this down/refactor +/** + * Attempts to derive the possible JAVA_HOME, either from the set env var or + * through searching various platform dependent directories. If the wrong JAVA_HOME + * is found via search in the case where multiple versions are installed, the env + * var should be used. + */ object JavaHomeHelper { fun javaHome(): Path? { System.getenv("JAVA_HOME")?.let { @@ -37,7 +43,17 @@ object JavaHomeHelper { private fun macJavaHome(): Path? { if (Files.isExecutable(Paths.get("/usr/libexec/java_home"))) { - return execJavaHome() + return try { + val process = ProcessBuilder().command("/usr/libexec/java_home").start() + val out = BufferedReader(InputStreamReader(process.inputStream)) + val line = out.readLine() + process.waitFor(5, TimeUnit.SECONDS) + Paths.get(line) + } catch (e: IOException) { + throw RuntimeException(e) + } catch (e: InterruptedException) { + throw RuntimeException(e) + } } val homes = arrayOf( "/Library/Java/JavaVirtualMachines/Home", @@ -52,20 +68,6 @@ object JavaHomeHelper { return check(*homes) } - private fun execJavaHome(): Path { - return try { - val process = ProcessBuilder().command("/usr/libexec/java_home").start() - val out = BufferedReader(InputStreamReader(process.inputStream)) - val line = out.readLine() - process.waitFor(5, TimeUnit.SECONDS) - Paths.get(line) - } catch (e: IOException) { - throw RuntimeException(e) - } catch (e: InterruptedException) { - throw RuntimeException(e) - } - } - private fun check(vararg roots: String): Path? { for (root in roots) { val list: List = try {