From 3ca6e350016edaf3655d58409ec39ec55d0938dc Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 13 Nov 2020 00:19:49 +0000 Subject: [PATCH 1/8] JDK 8 (+?) Standard Library Hovers --- src/main/kotlin/lsifjava/ExternalDocs.kt | 25 ++- .../kotlin/lsifjava/JDK8CompatFileManager.kt | 164 ++++++++++++++++++ src/main/kotlin/lsifjava/SourceFileManager.kt | 9 +- 3 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 src/main/kotlin/lsifjava/JDK8CompatFileManager.kt diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index 065aab07..d69e2f70 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -5,13 +5,15 @@ import com.sun.source.tree.* import com.sun.source.util.DocTrees import com.sun.source.util.JavacTask import com.sun.source.util.TreePathScanner -import com.sun.tools.javac.util.Context import com.sun.tools.javac.api.JavacTool import com.sun.tools.javac.tree.JCTree.* +import com.sun.tools.javac.util.Context import java.nio.charset.Charset -import java.nio.file.Path +import java.nio.file.* import javax.lang.model.element.Element -import javax.tools.* +import javax.tools.JavaFileObject +import com.sun.tools.javac.file.JavacFileManager +import javax.tools.StandardLocation data class ExternalHoverMeta(val doc: String, val tree: Tree) @@ -19,13 +21,14 @@ class ExternalDocs(private val docPaths: List) { private val fileCache = HashMap?>() - private val fileManager: StandardJavaFileManager by lazy { - val manager = ToolProvider.getSystemJavaCompiler() - .getStandardFileManager(null, null, Charset.defaultCharset()) + private val fileManager = let { + val manager = JavacFileManager(Context(), false, Charset.defaultCharset()) manager.setLocation(StandardLocation.SOURCE_PATH, docPaths.map { it.toFile() }) manager } + private val jdkFileManager = JDK8CompatFileManager(fileManager) + fun findDocForElement(containerClass: String, javac: JavacTool, element: Element): ExternalHoverMeta? { val context = DocumentIndexer.SimpleContext() @@ -35,9 +38,13 @@ class ExternalDocs(private val docPaths: List) { return DocExtractionVisitor(task, element).scan(compUnit, null) } + private fun findFileFromJars(containerClass: String) = + fileManager.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) + ?: jdkFileManager.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) + private fun analyzeFileFromJar(containerClass: String, context: Context, javac: JavacTool): Pair? { - val file = fileManager.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) - ?: return null + val file = findFileFromJars(containerClass) ?: return null + val task = javac.getTask(NoopWriter, fileManager, CountingDiagnosticListener.NullWriter, listOf(), listOf(), listOf(file), context) val compUnit = task.parse().iterator().next() val analyzeResult = runCatching { task.analyze() } @@ -47,7 +54,7 @@ class ExternalDocs(private val docPaths: List) { } return Pair(task, compUnit) } - + private class DocExtractionVisitor(task: JavacTask, private val element: Element): TreePathScanner() { private val docs: DocTrees = DocTrees.instance(task) diff --git a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt new file mode 100644 index 00000000..ebb17c0d --- /dev/null +++ b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt @@ -0,0 +1,164 @@ +@file:Suppress("JAVA_MODULE_DOES_NOT_EXPORT_PACKAGE") +package lsifjava + +import javax.tools.ForwardingJavaFileManager +import javax.tools.StandardJavaFileManager +import com.sun.tools.javac.util.Context +import java.io.BufferedReader +import java.io.File +import java.io.IOException +import java.io.InputStreamReader +import java.nio.charset.Charset +import java.nio.file.* +import java.util.concurrent.TimeUnit +import java.util.stream.Collectors +import javax.tools.JavaFileManager +import javax.tools.StandardLocation + +/** + * FileManager that falls back to JavacPathFileManager for Java 8. + * Java 8 StandardJavaFileManager doesn't have the setLocationFromPaths + * method, instead it only has setLocation which requires an + * Iterable, which would cause an UnsupportedException + * when trying to turn the ZipFile from the in-memory FileSystem into a File. + * Because JavacPathFileManager doesnt exist beyond Java 8 (9?) and we build with 14+, + * the symbol resolver would fail for that symbol, hence we create an instance via + * reflection if the Java version is 8. God I hate this. + */ +class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFileManager(getFileManager(manager)) { + companion object { + private fun getFileManager(fileManager: StandardJavaFileManager): JavaFileManager { + var java8Manager: JavaFileManager? = null + + if(javaVersion == 8) { + java8Manager = Class.forName("com.sun.tools.javac.nio.JavacPathFileManager") + .constructors[0] + .newInstance(Context(), false, Charset.defaultCharset()) + as JavaFileManager? + } + + val srcZip = srcZip() + if(srcZip != null) { + if(javaVersion > 8) { + fileManager.setLocationFromPaths(StandardLocation.MODULE_SOURCE_PATH, setOf(srcZip.getPath("/"))) + } else { + java8Manager!!::class.java + .getMethod("setDefaultFileSystem", FileSystem::class.java) + .invoke(java8Manager, srcZip) + java8Manager::class.java + .getMethod("setLocation", JavaFileManager.Location::class.java, Iterable::class.java) + .invoke(java8Manager, StandardLocation.SOURCE_PATH, setOf(srcZip.getPath("/"))) + } + } + + return java8Manager ?: fileManager + } + + private fun srcZip(): FileSystem? { + val srcZip = findSrcZip() ?: return null + return try { + FileSystems.newFileSystem(srcZip, ExternalDocs::class.java.classLoader) + } catch (e: IOException) { + throw RuntimeException(e) + } + } + + private fun findSrcZip(): Path? { + val javaHome = JavaHomeHelper.javaHome() ?: return null + val locations = arrayOf("lib/src.zip", "src.zip") + for (rel in locations) { + val abs = javaHome.resolve(rel) + if (Files.exists(abs)) { + return abs + } + } + return null + } + } + + // from https://github.com/georgewfraser/java-language-server + // bless you george for all the references. Maybe Ill cut this down/refactor + private object JavaHomeHelper { + fun javaHome(): Path? { + val fromEnv = System.getenv("JAVA_HOME") + if (fromEnv != null) + return Paths.get(fromEnv) + val osName = System.getProperty("os.name") + if (isWindows(osName)) return windowsJavaHome() + if (isMac(osName)) return macJavaHome() + if (isLinux(osName)) return linuxJavaHome() + throw RuntimeException("Unrecognized os.name $osName") + } + + private fun windowsJavaHome(): Path? { + for (root in File.listRoots()) { + val x64 = root.toPath().resolve("Program Files/Java").toString() + val x86 = root.toPath().resolve("Program Files (x86)/Java").toString() + val found = check(x64, x86) + if (found !== null) return found + } + return null + } + + private fun macJavaHome(): Path? { + if (Files.isExecutable(Paths.get("/usr/libexec/java_home"))) { + return execJavaHome() + } + val homes = arrayOf( + "/Library/Java/JavaVirtualMachines/Home", + "/System/Library/Java/JavaVirtualMachines/Home", + "/Library/Java/JavaVirtualMachines/Contents/Home", + "/System/Library/Java/JavaVirtualMachines/Contents/Home") + return check(*homes) + } + + private fun linuxJavaHome(): Path? { + val homes = arrayOf("/usr/java", "/opt/java", "/usr/lib/jvm") + return check(*homes) + } + + private fun execJavaHome(): Path { + return try { + val process = ProcessBuilder().command("/usr/libexec/java_home").start() + val out = BufferedReader(InputStreamReader(process.inputStream)) + val line = out.readLine() + process.waitFor(5, TimeUnit.SECONDS) + Paths.get(line) + } catch (e: IOException) { + throw RuntimeException(e) + } catch (e: InterruptedException) { + throw RuntimeException(e) + } + } + + private fun check(vararg roots: String): Path? { + for (root in roots) { + var list: List = try { + Files.list(Paths.get(root)).collect(Collectors.toList()) + } catch (e: NoSuchFileException) { + continue + } catch (e: IOException) { + throw RuntimeException(e) + } + for (jdk in list) { + if (Files.exists(jdk.resolve("bin/javac")) || Files.exists(jdk.resolve("bin/javac.exe"))) { + return jdk + } + } + } + return null + } + + private fun isWindows(osName: String): Boolean { + return osName.toLowerCase().startsWith("windows") + } + + private fun isMac(osName: String): Boolean { + return osName.toLowerCase().startsWith("mac") + } + + private fun isLinux(osName: String): Boolean { + return osName.toLowerCase().startsWith("linux") + } + } +} diff --git a/src/main/kotlin/lsifjava/SourceFileManager.kt b/src/main/kotlin/lsifjava/SourceFileManager.kt index 8e648f8f..f7bb5a58 100644 --- a/src/main/kotlin/lsifjava/SourceFileManager.kt +++ b/src/main/kotlin/lsifjava/SourceFileManager.kt @@ -4,12 +4,12 @@ package lsifjava import java.io.IOException import java.nio.charset.Charset import java.nio.file.Path -import java.util.* import java.util.regex.Pattern import java.util.stream.Stream import javax.tools.* +import javax.tools.JavaFileObject.Kind -class SourceFileManager internal constructor(private val paths: Set): +class SourceFileManager(private val paths: Set): ForwardingJavaFileManager(getDelegateFileManager) { companion object { @@ -19,10 +19,7 @@ class SourceFileManager internal constructor(private val paths: Set): } override fun list( - location: JavaFileManager.Location, - packageName: String, - kinds: Set, - recurse: Boolean + location: JavaFileManager.Location, packageName: String, kinds: Set, recurse: Boolean ): Iterable { return if (location === StandardLocation.SOURCE_PATH) { val sourceFileObjectStream = list(packageName) From da768edfeac5b3c3d84e3593c1341fb45fb428ea Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 20 Nov 2020 01:09:52 +0000 Subject: [PATCH 2/8] Fixed JDK>8 stdlib source parsing Turns out passing in an "empty" sourcefilemanager keeps the validator happy, which feels like a weird hack But it seems to work? And we dont really need a full file manager as we're only parsing, not analyzing. On that note, we're ditching analyzing. Even if the cost could be offset to making this whole spiel async, the failure rate was too high (to be investigated), and with some more manual work of equality checking, we should be able to get the same results for significantly cheaper and more "reliably" --- src/main/kotlin/lsifjava/ExternalDocs.kt | 19 ++- .../kotlin/lsifjava/JDK8CompatFileManager.kt | 124 ++++++++++++++++-- 2 files changed, 123 insertions(+), 20 deletions(-) diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index d69e2f70..093cc990 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -1,6 +1,7 @@ @file:Suppress("JAVA_MODULE_DOES_NOT_EXPORT_PACKAGE") package lsifjava +import com.sun.tools.javac.code.* import com.sun.source.tree.* import com.sun.source.util.DocTrees import com.sun.source.util.JavacTask @@ -17,8 +18,9 @@ import javax.tools.StandardLocation data class ExternalHoverMeta(val doc: String, val tree: Tree) -class ExternalDocs(private val docPaths: List) { +private val emptyFileManager = SourceFileManager(emptySet()) +class ExternalDocs(private val docPaths: List) { private val fileCache = HashMap?>() private val fileManager = let { @@ -38,20 +40,17 @@ class ExternalDocs(private val docPaths: List) { return DocExtractionVisitor(task, element).scan(compUnit, null) } - private fun findFileFromJars(containerClass: String) = + private fun findFileFromJars(containerClass: String): JavaFileObject? = fileManager.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) - ?: jdkFileManager.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) + ?: jdkFileManager.getJavaFileForInput(containerClass) private fun analyzeFileFromJar(containerClass: String, context: Context, javac: JavacTool): Pair? { val file = findFileFromJars(containerClass) ?: return null - val task = javac.getTask(NoopWriter, fileManager, CountingDiagnosticListener.NullWriter, listOf(), listOf(), listOf(file), context) - val compUnit = task.parse().iterator().next() - val analyzeResult = runCatching { task.analyze() } - analyzeResult.getOrNull() ?: run { - //println("${file.name} threw exception") - return null - } + val task = javac.getTask(NoopWriter, emptyFileManager, CountingDiagnosticListener.NullWriter, listOf(), listOf(), listOf(file), context) + val compUnit = task.parse().iterator().apply { + if(!hasNext()) return null + }.next() return Pair(task, compUnit) } diff --git a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt index ebb17c0d..ac85c348 100644 --- a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt +++ b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt @@ -1,8 +1,6 @@ @file:Suppress("JAVA_MODULE_DOES_NOT_EXPORT_PACKAGE") package lsifjava -import javax.tools.ForwardingJavaFileManager -import javax.tools.StandardJavaFileManager import com.sun.tools.javac.util.Context import java.io.BufferedReader import java.io.File @@ -12,14 +10,106 @@ import java.nio.charset.Charset import java.nio.file.* import java.util.concurrent.TimeUnit import java.util.stream.Collectors -import javax.tools.JavaFileManager -import javax.tools.StandardLocation +import javax.tools.* + +private val JDK_MODULES = listOf( + "java.activation", + "java.base", + "java.compiler", + "java.corba", + "java.datatransfer", + "java.desktop", + "java.instrument", + "java.jnlp", + "java.logging", + "java.management", + "java.management.rmi", + "java.naming", + "java.net.http", + "java.prefs", + "java.rmi", + "java.scripting", + "java.se", + "java.se.ee", + "java.security.jgss", + "java.security.sasl", + "java.smartcardio", + "java.sql", + "java.sql.rowset", + "java.transaction", + "java.transaction.xa", + "java.xml", + "java.xml.bind", + "java.xml.crypto", + "java.xml.ws", + "java.xml.ws.annotation", + "javafx.base", + "javafx.controls", + "javafx.fxml", + "javafx.graphics", + "javafx.media", + "javafx.swing", + "javafx.web", + "jdk.accessibility", + "jdk.aot", + "jdk.attach", + "jdk.charsets", + "jdk.compiler", + "jdk.crypto.cryptoki", + "jdk.crypto.ec", + "jdk.dynalink", + "jdk.editpad", + "jdk.hotspot.agent", + "jdk.httpserver", + "jdk.incubator.httpclient", + "jdk.internal.ed", + "jdk.internal.jvmstat", + "jdk.internal.le", + "jdk.internal.opt", + "jdk.internal.vm.ci", + "jdk.internal.vm.compiler", + "jdk.internal.vm.compiler.management", + "jdk.jartool", + "jdk.javadoc", + "jdk.jcmd", + "jdk.jconsole", + "jdk.jdeps", + "jdk.jdi", + "jdk.jdwp.agent", + "jdk.jfr", + "jdk.jlink", + "jdk.jshell", + "jdk.jsobject", + "jdk.jstatd", + "jdk.localedata", + "jdk.management", + "jdk.management.agent", + "jdk.management.cmm", + "jdk.management.jfr", + "jdk.management.resource", + "jdk.naming.dns", + "jdk.naming.rmi", + "jdk.net", + "jdk.pack", + "jdk.packager.services", + "jdk.rmic", + "jdk.scripting.nashorn", + "jdk.scripting.nashorn.shell", + "jdk.sctp", + "jdk.security.auth", + "jdk.security.jgss", + "jdk.snmp", + "jdk.unsupported", + "jdk.unsupported.desktop", + "jdk.xml.dom", + "jdk.zipfs", +) /** * FileManager that falls back to JavacPathFileManager for Java 8. * Java 8 StandardJavaFileManager doesn't have the setLocationFromPaths * method, instead it only has setLocation which requires an - * Iterable, which would cause an UnsupportedException + * Iterable\, which would cause an UnsupportedException * when trying to turn the ZipFile from the in-memory FileSystem into a File. * Because JavacPathFileManager doesnt exist beyond Java 8 (9?) and we build with 14+, * the symbol resolver would fail for that symbol, hence we create an instance via @@ -37,17 +127,16 @@ class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFil as JavaFileManager? } - val srcZip = srcZip() - if(srcZip != null) { + srcZip()?.also { if(javaVersion > 8) { - fileManager.setLocationFromPaths(StandardLocation.MODULE_SOURCE_PATH, setOf(srcZip.getPath("/"))) + fileManager.setLocationFromPaths(StandardLocation.MODULE_SOURCE_PATH, setOf(it.getPath("/"))) } else { java8Manager!!::class.java .getMethod("setDefaultFileSystem", FileSystem::class.java) - .invoke(java8Manager, srcZip) + .invoke(java8Manager, it) java8Manager::class.java .getMethod("setLocation", JavaFileManager.Location::class.java, Iterable::class.java) - .invoke(java8Manager, StandardLocation.SOURCE_PATH, setOf(srcZip.getPath("/"))) + .invoke(java8Manager, StandardLocation.SOURCE_PATH, setOf(it.getPath("/"))) } } @@ -76,6 +165,21 @@ class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFil } } + fun getJavaFileForInput(containerClass: String): JavaFileObject? { + if(javaVersion == 8) { + return this.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) + } else { + for(module in JDK_MODULES) { + val moduleLocation = this.getLocationForModule(StandardLocation.MODULE_SOURCE_PATH, module) ?: continue + this.getJavaFileForInput(moduleLocation, containerClass, JavaFileObject.Kind.SOURCE)?.run { + return this + } + } + } + + return null + } + // from https://github.com/georgewfraser/java-language-server // bless you george for all the references. Maybe Ill cut this down/refactor private object JavaHomeHelper { From d0a6f5d791bc7045559815ccb0147c64286a6de9 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Sat, 21 Nov 2020 23:49:22 +0000 Subject: [PATCH 3/8] New DocExtractionVisitor per class(esque) declaration As we no longer analyze deps, we dont get symbol info eg the owner symbol Hence we cant deduce what the owning class of a method is to avoid collision scenarios where a single comp unit has multiple identical methods or vars in different nested classes We work around this by having one DocExtractionVisitor per class decl, which holds the ClassTree ref for other visitor methods to reference to check what class we are in --- src/main/kotlin/lsifjava/ExternalDocs.kt | 21 +++++++--- .../kotlin/lsifjava/JDK8CompatFileManager.kt | 38 +++++++++---------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index 093cc990..7a00b8f6 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -54,10 +54,11 @@ class ExternalDocs(private val docPaths: List) { return Pair(task, compUnit) } - private class DocExtractionVisitor(task: JavacTask, private val element: Element): TreePathScanner() { - private val docs: DocTrees = DocTrees.instance(task) + private class DocExtractionVisitor(val task: JavacTask, private val element: Element, private val docs: DocTrees = DocTrees.instance(task)): TreePathScanner() { + private var new: Boolean = true + private lateinit var classDecl: ClassTree - override fun visitMethod(node: MethodTree?, p: Unit?): ExternalHoverMeta? { + override fun visitMethod(node: MethodTree, p: Unit?): ExternalHoverMeta? { (node as JCMethodDecl).sym ?: return null if(node.sym.toString() == element.toString()) { @@ -68,7 +69,7 @@ class ExternalDocs(private val docPaths: List) { return null } - override fun visitVariable(node: VariableTree?, p: Unit?): ExternalHoverMeta? { + override fun visitVariable(node: VariableTree, p: Unit?): ExternalHoverMeta? { (node as JCVariableDecl).sym ?: return null if(node.sym.toString() == element.toString()) { @@ -78,8 +79,16 @@ class ExternalDocs(private val docPaths: List) { return null } - override fun visitClass(node: ClassTree?, p: Unit?): ExternalHoverMeta? { - (node as JCClassDecl).sym ?: return null + override fun visitClass(node: ClassTree, p: Unit?): ExternalHoverMeta? { + // we need this logic here to stop calling scan on the same ClassTree infinitely, but we also + // want to start a new visitor for each nested class decl + if(!new) { + return DocExtractionVisitor(task, element, docs).scan(node, null) + } + new = false + classDecl = node + + (node as JCClassDecl).sym ?: return super.visitClass(node, p) if(node.sym.toString() == element.toString()) { val doc = docs.getDocComment(currentPath) ?: return super.visitClass(node, p) diff --git a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt index ac85c348..5ec010de 100644 --- a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt +++ b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt @@ -109,9 +109,9 @@ private val JDK_MODULES = listOf( * FileManager that falls back to JavacPathFileManager for Java 8. * Java 8 StandardJavaFileManager doesn't have the setLocationFromPaths * method, instead it only has setLocation which requires an - * Iterable\, which would cause an UnsupportedException + * Iterable, which would cause an UnsupportedException * when trying to turn the ZipFile from the in-memory FileSystem into a File. - * Because JavacPathFileManager doesnt exist beyond Java 8 (9?) and we build with 14+, + * Because JavacPathFileManager doesn't exist beyond Java 8 (9?) and we build with 14+, * the symbol resolver would fail for that symbol, hence we create an instance via * reflection if the Java version is 8. God I hate this. */ @@ -165,28 +165,28 @@ class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFil } } - fun getJavaFileForInput(containerClass: String): JavaFileObject? { - if(javaVersion == 8) { - return this.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) - } else { - for(module in JDK_MODULES) { - val moduleLocation = this.getLocationForModule(StandardLocation.MODULE_SOURCE_PATH, module) ?: continue - this.getJavaFileForInput(moduleLocation, containerClass, JavaFileObject.Kind.SOURCE)?.run { - return this - } - } - } + fun getJavaFileForInput(containerClass: String): JavaFileObject? { + if(javaVersion == 8) { + return this.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) + } else { + for(module in JDK_MODULES) { + val moduleLocation = this.getLocationForModule(StandardLocation.MODULE_SOURCE_PATH, module) ?: continue + this.getJavaFileForInput(moduleLocation, containerClass, JavaFileObject.Kind.SOURCE)?.run { + return this + } + } + } - return null - } + return null + } // from https://github.com/georgewfraser/java-language-server // bless you george for all the references. Maybe Ill cut this down/refactor private object JavaHomeHelper { fun javaHome(): Path? { - val fromEnv = System.getenv("JAVA_HOME") - if (fromEnv != null) - return Paths.get(fromEnv) + System.getenv("JAVA_HOME")?.let { + return Paths.get(it) + } val osName = System.getProperty("os.name") if (isWindows(osName)) return windowsJavaHome() if (isMac(osName)) return macJavaHome() @@ -237,7 +237,7 @@ class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFil private fun check(vararg roots: String): Path? { for (root in roots) { - var list: List = try { + val list: List = try { Files.list(Paths.get(root)).collect(Collectors.toList()) } catch (e: NoSuchFileException) { continue From e5f2c9ec2fdfb633ec2fab19ca2598f07f880947 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 26 Nov 2020 22:35:05 +0000 Subject: [PATCH 4/8] Matching class decl+idents by name --- src/main/kotlin/lsifjava/ExternalDocs.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index 7a00b8f6..18072c66 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -87,12 +87,13 @@ class ExternalDocs(private val docPaths: List) { } new = false classDecl = node - - (node as JCClassDecl).sym ?: return super.visitClass(node, p) - - if(node.sym.toString() == element.toString()) { + + if(element !is Symbol.ClassSymbol) return super.visitClass(node, p) + + // Assumption: no class-like name conflicts within a single class file + if((node as JCClassDecl).name.toString() == element.simpleName.toString()) { val doc = docs.getDocComment(currentPath) ?: return super.visitClass(node, p) - return ExternalHoverMeta(doc, node) + return ExternalHoverMeta(doc.trim(), node) } return super.visitClass(node, p) From 8ac5263d724fab07b75eb0ac8137b3daca02be65 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Sun, 29 Nov 2020 01:42:50 +0000 Subject: [PATCH 5/8] Matching method/constructor calls on the individual parts --- src/main/kotlin/lsifjava/ExternalDocs.kt | 31 +++++++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index 18072c66..e1e418c4 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -59,11 +59,34 @@ class ExternalDocs(private val docPaths: List) { private lateinit var classDecl: ClassTree override fun visitMethod(node: MethodTree, p: Unit?): ExternalHoverMeta? { - (node as JCMethodDecl).sym ?: return null - - if(node.sym.toString() == element.toString()) { + if(element !is Symbol.MethodSymbol) return null + + if(element.owner.simpleName.toString() != classDecl.simpleName.toString()) + return null + + if(element.name.toString() != node.name.toString()) return null + + if(element.name.toString() != "" && + element.returnType.toString() != node.returnType.toString()) return null + + val paramsEqual = element.params.size == node.parameters.size && + element.params.foldIndexed(true) { i, acc, sym -> + val paramType = (node.parameters[i] as JCVariableDecl).vartype + val defTypeName = when(paramType) { + is JCPrimitiveTypeTree -> paramType.toString() + is JCTypeApply -> paramType.clazz.toString() + is JCIdent -> paramType.toString() + else -> { + println("param type wasn't JCPrimitiveTypeTree|JCTypeApply|JCIdent, but ${paramType::class.java}") + return null + } + } + acc && sym.type.tsym.simpleName.toString() == defTypeName + } + + if(paramsEqual) { val doc = docs.getDocComment(currentPath) ?: return null - return ExternalHoverMeta(doc, node) + return ExternalHoverMeta(doc.trim(), node) } return null From f34c4e13ab55812f578d58a936b566732d4bfb16 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Sun, 29 Nov 2020 23:44:12 +0000 Subject: [PATCH 6/8] Matching class attributes on the individual parts --- src/main/kotlin/lsifjava/ExternalDocs.kt | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index e1e418c4..be13d1ba 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -55,6 +55,10 @@ class ExternalDocs(private val docPaths: List) { } private class DocExtractionVisitor(val task: JavacTask, private val element: Element, private val docs: DocTrees = DocTrees.instance(task)): TreePathScanner() { + // Basic flag to indicate if this DocExtractionVisitor has visited its assigned class decl yet. + // If set to false, we want to create a new DocExtractionVisitor for the class decl we are visiting. + // This way we can keep track of the owning class decl for methods/variables that are otherwise only + // available with fully resolved symbols, which we don't get with non-full-fat parsing+analyzing private var new: Boolean = true private lateinit var classDecl: ClassTree @@ -93,11 +97,16 @@ class ExternalDocs(private val docPaths: List) { } override fun visitVariable(node: VariableTree, p: Unit?): ExternalHoverMeta? { - (node as JCVariableDecl).sym ?: return null + if(element !is Symbol.VarSymbol) return null - if(node.sym.toString() == element.toString()) { + if(element.owner.simpleName.toString() != classDecl.simpleName.toString()) + return null + if(element.name.toString() == node.name.toString()) { + val doc = docs.getDocComment(currentPath) ?: return null + return ExternalHoverMeta(doc.trim(), node) } + // filter to instance variables return null } From 6616eebf092f4f9db8ecc675fc2926942a8cc69a Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 30 Nov 2020 00:58:52 +0000 Subject: [PATCH 7/8] Parse errors from deps now honour the users chosen output --- src/main/kotlin/lsifjava/ExternalDocs.kt | 4 +- src/main/kotlin/lsifjava/FileCollector.kt | 2 +- .../kotlin/lsifjava/JDK8CompatFileManager.kt | 92 +---------------- src/main/kotlin/lsifjava/JavaHomeHelper.kt | 98 +++++++++++++++++++ src/main/kotlin/lsifjava/Main.kt | 2 +- 5 files changed, 103 insertions(+), 95 deletions(-) create mode 100644 src/main/kotlin/lsifjava/JavaHomeHelper.kt diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index be13d1ba..38c86f73 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -20,7 +20,7 @@ data class ExternalHoverMeta(val doc: String, val tree: Tree) private val emptyFileManager = SourceFileManager(emptySet()) -class ExternalDocs(private val docPaths: List) { +class ExternalDocs(private val docPaths: List, val diagnosticListener: CountingDiagnosticListener) { private val fileCache = HashMap?>() private val fileManager = let { @@ -47,7 +47,7 @@ class ExternalDocs(private val docPaths: List) { private fun analyzeFileFromJar(containerClass: String, context: Context, javac: JavacTool): Pair? { val file = findFileFromJars(containerClass) ?: return null - val task = javac.getTask(NoopWriter, emptyFileManager, CountingDiagnosticListener.NullWriter, listOf(), listOf(), listOf(file), context) + val task = javac.getTask(NoopWriter, emptyFileManager, diagnosticListener, listOf(), listOf(), listOf(file), context) val compUnit = task.parse().iterator().apply { if(!hasNext()) return null }.next() diff --git a/src/main/kotlin/lsifjava/FileCollector.kt b/src/main/kotlin/lsifjava/FileCollector.kt index 02675b60..a6f7eaf8 100644 --- a/src/main/kotlin/lsifjava/FileCollector.kt +++ b/src/main/kotlin/lsifjava/FileCollector.kt @@ -27,7 +27,7 @@ fun buildIndexerMap( val sourceVersions = buildToolInterface.javaSourceVersions // TODO(nsc) where to move this - val externalDocManager = ExternalDocs(buildToolInterface.sourcesList) + val externalDocManager = ExternalDocs(buildToolInterface.sourcesList, javacDiagListener) val fileBuildInfo = Channel() diff --git a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt index 5ec010de..379c9f2c 100644 --- a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt +++ b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt @@ -2,16 +2,12 @@ package lsifjava import com.sun.tools.javac.util.Context -import java.io.BufferedReader -import java.io.File import java.io.IOException -import java.io.InputStreamReader import java.nio.charset.Charset import java.nio.file.* -import java.util.concurrent.TimeUnit -import java.util.stream.Collectors import javax.tools.* +// hard-coded list for convenience. Sorry, George :) private val JDK_MODULES = listOf( "java.activation", "java.base", @@ -179,90 +175,4 @@ class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFil return null } - - // from https://github.com/georgewfraser/java-language-server - // bless you george for all the references. Maybe Ill cut this down/refactor - private object JavaHomeHelper { - fun javaHome(): Path? { - System.getenv("JAVA_HOME")?.let { - return Paths.get(it) - } - val osName = System.getProperty("os.name") - if (isWindows(osName)) return windowsJavaHome() - if (isMac(osName)) return macJavaHome() - if (isLinux(osName)) return linuxJavaHome() - throw RuntimeException("Unrecognized os.name $osName") - } - - private fun windowsJavaHome(): Path? { - for (root in File.listRoots()) { - val x64 = root.toPath().resolve("Program Files/Java").toString() - val x86 = root.toPath().resolve("Program Files (x86)/Java").toString() - val found = check(x64, x86) - if (found !== null) return found - } - return null - } - - private fun macJavaHome(): Path? { - if (Files.isExecutable(Paths.get("/usr/libexec/java_home"))) { - return execJavaHome() - } - val homes = arrayOf( - "/Library/Java/JavaVirtualMachines/Home", - "/System/Library/Java/JavaVirtualMachines/Home", - "/Library/Java/JavaVirtualMachines/Contents/Home", - "/System/Library/Java/JavaVirtualMachines/Contents/Home") - return check(*homes) - } - - private fun linuxJavaHome(): Path? { - val homes = arrayOf("/usr/java", "/opt/java", "/usr/lib/jvm") - return check(*homes) - } - - private fun execJavaHome(): Path { - return try { - val process = ProcessBuilder().command("/usr/libexec/java_home").start() - val out = BufferedReader(InputStreamReader(process.inputStream)) - val line = out.readLine() - process.waitFor(5, TimeUnit.SECONDS) - Paths.get(line) - } catch (e: IOException) { - throw RuntimeException(e) - } catch (e: InterruptedException) { - throw RuntimeException(e) - } - } - - private fun check(vararg roots: String): Path? { - for (root in roots) { - val list: List = try { - Files.list(Paths.get(root)).collect(Collectors.toList()) - } catch (e: NoSuchFileException) { - continue - } catch (e: IOException) { - throw RuntimeException(e) - } - for (jdk in list) { - if (Files.exists(jdk.resolve("bin/javac")) || Files.exists(jdk.resolve("bin/javac.exe"))) { - return jdk - } - } - } - return null - } - - private fun isWindows(osName: String): Boolean { - return osName.toLowerCase().startsWith("windows") - } - - private fun isMac(osName: String): Boolean { - return osName.toLowerCase().startsWith("mac") - } - - private fun isLinux(osName: String): Boolean { - return osName.toLowerCase().startsWith("linux") - } - } } diff --git a/src/main/kotlin/lsifjava/JavaHomeHelper.kt b/src/main/kotlin/lsifjava/JavaHomeHelper.kt new file mode 100644 index 00000000..5739bda3 --- /dev/null +++ b/src/main/kotlin/lsifjava/JavaHomeHelper.kt @@ -0,0 +1,98 @@ +package lsifjava + +import java.io.BufferedReader +import java.io.File +import java.io.IOException +import java.io.InputStreamReader +import java.nio.file.Files +import java.nio.file.NoSuchFileException +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.TimeUnit +import java.util.stream.Collectors + +// from https://github.com/georgewfraser/java-language-server +// bless you george for all the references. Maybe Ill cut this down/refactor +object JavaHomeHelper { + fun javaHome(): Path? { + System.getenv("JAVA_HOME")?.let { + return Paths.get(it) + } + val osName = System.getProperty("os.name") + if (isWindows(osName)) return windowsJavaHome() + if (isMac(osName)) return macJavaHome() + if (isLinux(osName)) return linuxJavaHome() + throw RuntimeException("Unrecognized os.name $osName") + } + + private fun windowsJavaHome(): Path? { + for (root in File.listRoots()) { + val x64 = root.toPath().resolve("Program Files/Java").toString() + val x86 = root.toPath().resolve("Program Files (x86)/Java").toString() + val found = check(x64, x86) + if (found !== null) return found + } + return null + } + + private fun macJavaHome(): Path? { + if (Files.isExecutable(Paths.get("/usr/libexec/java_home"))) { + return execJavaHome() + } + val homes = arrayOf( + "/Library/Java/JavaVirtualMachines/Home", + "/System/Library/Java/JavaVirtualMachines/Home", + "/Library/Java/JavaVirtualMachines/Contents/Home", + "/System/Library/Java/JavaVirtualMachines/Contents/Home") + return check(*homes) + } + + private fun linuxJavaHome(): Path? { + val homes = arrayOf("/usr/java", "/opt/java", "/usr/lib/jvm") + return check(*homes) + } + + private fun execJavaHome(): Path { + return try { + val process = ProcessBuilder().command("/usr/libexec/java_home").start() + val out = BufferedReader(InputStreamReader(process.inputStream)) + val line = out.readLine() + process.waitFor(5, TimeUnit.SECONDS) + Paths.get(line) + } catch (e: IOException) { + throw RuntimeException(e) + } catch (e: InterruptedException) { + throw RuntimeException(e) + } + } + + private fun check(vararg roots: String): Path? { + for (root in roots) { + val list: List = try { + Files.list(Paths.get(root)).collect(Collectors.toList()) + } catch (e: NoSuchFileException) { + continue + } catch (e: IOException) { + throw RuntimeException(e) + } + for (jdk in list) { + if (Files.exists(jdk.resolve("bin/javac")) || Files.exists(jdk.resolve("bin/javac.exe"))) { + return jdk + } + } + } + return null + } + + private fun isWindows(osName: String): Boolean { + return osName.toLowerCase().startsWith("windows") + } + + private fun isMac(osName: String): Boolean { + return osName.toLowerCase().startsWith("mac") + } + + private fun isLinux(osName: String): Boolean { + return osName.toLowerCase().startsWith("linux") + } +} \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/Main.kt b/src/main/kotlin/lsifjava/Main.kt index a6086f54..19167053 100644 --- a/src/main/kotlin/lsifjava/Main.kt +++ b/src/main/kotlin/lsifjava/Main.kt @@ -6,7 +6,7 @@ import java.io.PrintWriter val javaVersion: Int = getVersion() fun main(args: Array) { - println("Running JVM ${System.getProperty("java.version")}") + println("Running JVM ${System.getProperty("java.version")}, JAVA_HOME is set to ${JavaHomeHelper.javaHome() ?: ""}") val arguments = parse(args) val writer = createWriter(arguments) From 258604338e1d3173210f81f6fa9281498cf6adce Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 30 Nov 2020 01:39:01 +0000 Subject: [PATCH 8/8] Added some documentation related to this PR --- src/main/kotlin/lsifjava/ExternalDocs.kt | 28 ++++++++++++++----- .../kotlin/lsifjava/JDK8CompatFileManager.kt | 8 ++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/lsifjava/ExternalDocs.kt b/src/main/kotlin/lsifjava/ExternalDocs.kt index 38c86f73..9c60bcff 100644 --- a/src/main/kotlin/lsifjava/ExternalDocs.kt +++ b/src/main/kotlin/lsifjava/ExternalDocs.kt @@ -20,7 +20,8 @@ data class ExternalHoverMeta(val doc: String, val tree: Tree) private val emptyFileManager = SourceFileManager(emptySet()) -class ExternalDocs(private val docPaths: List, val diagnosticListener: CountingDiagnosticListener) { +class ExternalDocs(private val docPaths: List, private val diagnosticListener: CountingDiagnosticListener) { + // we cache compilation unit trees here to reduce the number of times we have to invoke the parser private val fileCache = HashMap?>() private val fileManager = let { @@ -31,11 +32,17 @@ class ExternalDocs(private val docPaths: List, val diagnosticListener: Cou private val jdkFileManager = JDK8CompatFileManager(fileManager) + /** + * Returns hover metadata for the tree section that matches the given element in the class file + * associated with containerClass, which is expected to be the fully qualified name of the class file + */ fun findDocForElement(containerClass: String, javac: JavacTool, element: Element): ExternalHoverMeta? { val context = DocumentIndexer.SimpleContext() - val (task, compUnit) = fileCache.getOrPut(containerClass) { analyzeFileFromJar(containerClass, context, javac) } - ?: return null + val (task, compUnit) = fileCache.getOrPut(containerClass) { + val fileObject = findFileFromJars(containerClass) ?: return@getOrPut null + parseFileObject(fileObject, context, javac) + } ?: return null return DocExtractionVisitor(task, element).scan(compUnit, null) } @@ -44,17 +51,24 @@ class ExternalDocs(private val docPaths: List, val diagnosticListener: Cou fileManager.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE) ?: jdkFileManager.getJavaFileForInput(containerClass) - private fun analyzeFileFromJar(containerClass: String, context: Context, javac: JavacTool): Pair? { - val file = findFileFromJars(containerClass) ?: return null - + /** + * Performs basic parsing of the external dependency JavaFileObject and returns a (javac task, compilation unit) pair. + * For reasons beyond my knowing (and many a staring at the debugger), an empty file manager + */ + private fun parseFileObject(file: JavaFileObject, context: Context, javac: JavacTool): Pair? { val task = javac.getTask(NoopWriter, emptyFileManager, diagnosticListener, listOf(), listOf(), listOf(file), context) val compUnit = task.parse().iterator().apply { + // bail out of the enclosing function if(!hasNext()) return null }.next() return Pair(task, compUnit) } - private class DocExtractionVisitor(val task: JavacTask, private val element: Element, private val docs: DocTrees = DocTrees.instance(task)): TreePathScanner() { + private class DocExtractionVisitor( + val task: JavacTask, + private val element: Element, + private val docs: DocTrees = DocTrees.instance(task) + ): TreePathScanner() { // Basic flag to indicate if this DocExtractionVisitor has visited its assigned class decl yet. // If set to false, we want to create a new DocExtractionVisitor for the class decl we are visiting. // This way we can keep track of the owning class decl for methods/variables that are otherwise only diff --git a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt index 379c9f2c..5c4c4f10 100644 --- a/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt +++ b/src/main/kotlin/lsifjava/JDK8CompatFileManager.kt @@ -113,6 +113,11 @@ private val JDK_MODULES = listOf( */ class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFileManager(getFileManager(manager)) { companion object { + /** + * returns a different JavaFileManager based on the current java version, with either SOURCE_PATH or MODULE_SOURCE_PATH set to "/" + * denoting the root of the JAR/ZIP file that this file manager is for. If JDK 8, we instantiate a com.sun.tools.javac.nio.JavacPathFileManager + * via reflection, else we use the passed file manager. See the class doc for the reason why. + */ private fun getFileManager(fileManager: StandardJavaFileManager): JavaFileManager { var java8Manager: JavaFileManager? = null @@ -161,6 +166,9 @@ class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFil } } + /** + * Searches for a class either by module or not depending on the current java version. + */ fun getJavaFileForInput(containerClass: String): JavaFileObject? { if(javaVersion == 8) { return this.getJavaFileForInput(StandardLocation.SOURCE_PATH, containerClass, JavaFileObject.Kind.SOURCE)