WIP: Documenting lsif-java ft. minor refactorings

src/main/kotlin/lsifjava/Compiler.kt

@@ -0,0 +1,60 @@
+package lsifjava
+
+import com.sun.source.tree.CompilationUnitTree
+import com.sun.source.util.JavacTask
+import com.sun.tools.javac.api.JavacTool
+import com.sun.tools.javac.main.JavaCompiler
+import com.sun.tools.javac.util.Context
+
+fun generateParseTree(
+    systemProvider: JavacTool,
+    fileManager: SourceFileManager,
+    diagnosticListener: CountingDiagnosticListener,
+    javaSourceVersion: String,
+    classpath: Classpath,
+    filepath: CanonicalPath
+): Pair<JavacTask, CompilationUnitTree> {
+    val task = systemProvider.getTask(
+        NoopWriter, fileManager, diagnosticListener,
+        listOf("-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString() /*, "--enable-preview" */),
+        listOf(), listOf(SourceFileObject(filepath.path)), SimpleContext()
+    )
+    val compUnit = task.parse().iterator().next()
+    task.analyze()
+
+    return Pair(task, compUnit)
+}
+
+/**
+ * Context lets us configure some aspects of the parsing process. SimpleContext
+ * incorporates a SimpleCompiler that configures the parser to generate end positions,
+ * preserve javadoc and keep line debug info (not sure if we use that currently/ever).
+ */
+class SimpleContext: Context() {
+    init {
+        put(SimpleCompiler.contextKey, SimpleCompiler.factory)
+    }
+}
+
+private class SimpleCompiler(context: Context?): JavaCompiler(context) {
+    companion object {
+        val factory = Context.Factory<JavaCompiler> { context: Context? -> SimpleCompiler(context) }
+
+        val contextKey: Context.Key<JavaCompiler> by lazy {
+            // compilerKey is not public in Java 8, so we have to kinda: https://media.discordapp.net/attachments/287739410286379019/806651592216281108/unknown.png
+            if(javaVersion == 8) run {
+                val field = SimpleCompiler::class.java.superclass.getDeclaredField("compilerKey")
+                field.isAccessible = true
+                val key: Context.Key<JavaCompiler> = Context.Key() // dummy value to be overwritten
+                return@lazy field.get(key) as Context.Key<JavaCompiler>
+            }
+            compilerKey
+        }
+    }
+
+    init {
+        genEndPos = true
+        lineDebugInfo = true
+        keepComments = true
+    }
+}

src/main/kotlin/lsifjava/DocumentIndexer.kt

@@ -2,7 +2,9 @@
 package lsifjava
 
 import com.sun.source.tree.CompilationUnitTree
+import com.sun.source.util.DocTrees
 import com.sun.source.util.JavacTask
+import com.sun.source.util.Trees
 import com.sun.tools.javac.api.JavacTool
 import com.sun.tools.javac.main.JavaCompiler
 import com.sun.tools.javac.util.Context
@@ -14,6 +16,9 @@ import java.util.concurrent.atomic.AtomicBoolean
 import kotlin.collections.HashMap
 import kotlin.collections.HashSet
 
+/**
+ *
+ */
 class DocumentIndexer(
     private val filepath: CanonicalPath,
     private val classpath: Classpath,
@@ -24,10 +29,6 @@ class DocumentIndexer(
     private val diagnosticListener: CountingDiagnosticListener,
     private val verbose: Boolean,
 ) {
-    companion object {
-        private val systemProvider by lazy { JavacTool.create() }
-    }
-
     data class DefinitionMeta(val definitionRangeId: String, val resultSetId: String) {
         var definitionResultId: String? = null
         val referenceRangeIds: MutableMap<String, MutableSet<String>> = HashMap()
@@ -52,61 +53,28 @@ class DocumentIndexer(
 
     private var indexed = AtomicBoolean(false)
 
+    /**
+     * Invokes the javac
+     */
     fun index() {
         try {
             if(!indexed.compareAndSet(false, true)) return
-            val (task, compUnit) = analyzeFile()
-            IndexingVisitor(task, systemProvider, docManager, compUnit, this, indexers).scan(compUnit, null)
-            referencesBacklog.forEach { it() }
+
+            val systemProvider = JavacTool.create()
+            val (task, compUnit) = generateParseTree(systemProvider, fileManager, diagnosticListener, javaSourceVersion, classpath, filepath)
+
+            val trees: Trees = Trees.instance(task)
+            val docs: DocTrees = DocTrees.instance(task)
+            IndexingVisitor(trees, docs, systemProvider, docManager, compUnit, this, indexers).scan(compUnit, null)
         } catch (e: Exception) {
             throw IndexingException(filepath.toString(), e)
+        } finally {
+            referencesBacklog.forEach { it() }
         }
 
         println("finished analyzing and visiting $filepath")
     }
 
-    private fun analyzeFile(): Pair<JavacTask, CompilationUnitTree> {
-        val context = SimpleContext()
-
-        val task = systemProvider.getTask(
-            NoopWriter, fileManager, diagnosticListener,
-            listOf("-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString() /*, "--enable-preview" */),
-            listOf(), listOf(SourceFileObject(filepath.path)), context
-        )
-        val compUnit = task.parse().iterator().next()
-        task.analyze()
-
-        return Pair(task, compUnit)
-    }
-
-    internal class SimpleContext: Context() {
-        init {
-            put(SimpleCompiler.contextKey, SimpleCompiler.factory)
-        }
-    }
-
-    internal class SimpleCompiler(context: Context?): JavaCompiler(context) {
-        companion object {
-            val factory = Context.Factory<JavaCompiler> { context: Context? -> SimpleCompiler(context) }
-
-            val contextKey: Context.Key<JavaCompiler> by lazy {
-                if(javaVersion == 8) run {
-                    val field = SimpleCompiler::class.java.superclass.getDeclaredField("compilerKey")
-                    field.isAccessible = true
-                    val key: Context.Key<JavaCompiler> = Context.Key() // dummy value to be overwritten
-                    return@lazy field.get(key) as Context.Key<JavaCompiler>
-                }
-                compilerKey  
-            }
-        }
-
-        init {
-            genEndPos = true
-            lineDebugInfo = true
-            keepComments = true
-        }
-    }
-
     fun postIndex(projectId: String) {
         for (meta in definitions.values)
             linkUses(meta)
@@ -164,7 +132,7 @@ class DocumentIndexer(
         definitions[range] = DefinitionMeta(rangeId, resultSetId) // + contents?
     }
 
-    // TODO(nsc): GLOBAL CACHE SO WE DONT HAVE MULTIPLE REFS FOR SAME DEF
+    // TODO GLOBAL CACHE SO WE DONT HAVE MULTIPLE REFS FOR SAME DEF
     internal fun emitExternalHoverAndUse(signature: String, doc: String, use: Range) {
         val hoverId = emitter.emitVertex("hoverResult", mapOf(
             "result" to mapOf(
@@ -186,6 +154,10 @@ class DocumentIndexer(
         linkUses(meta)
     }
 
+    /**
+     * Adds a processing function to a backlog to be executed after the current document and all transitively dependent
+     * documents are indexed to emit ranges that are linked to def ranges in postIndex.
+     */
     internal fun emitUse(use: Range, def: Range, defPath: Path) {
         referencesBacklog.add {
             val indexer = indexers[defPath] ?: error("expected indexer for $defPath")

src/main/kotlin/lsifjava/ExternalDocs.kt

@@ -37,7 +37,7 @@ class ExternalDocs(private val docPaths: List<Path>, private val diagnosticListe
      * associated with <code>containerClass</code>, which is expected to be the fully qualified name of the class file
      */
     fun findDocForElement(containerClass: String, javac: JavacTool, element: Element): ExternalHoverMeta? {
-        val context = DocumentIndexer.SimpleContext()
+        val context = SimpleContext()
 
         val (task, compUnit) = fileCache.getOrPut(containerClass) {
             val fileObject = findFileFromJars(containerClass) ?: return@getOrPut null

src/main/kotlin/lsifjava/IndexingVisitor.kt

@@ -34,16 +34,14 @@ data class ReferenceData(val useRange: Range, val defRange: Range?, val defPath:
 fun Type.strip() = this.toString().removePrefix("java.lang.")
 
 class IndexingVisitor(
-    task: JavacTask,
+    private val trees: Trees,
+    private val docs: DocTrees,
     private val tool: JavacTool,
     private val externalDocManager: ExternalDocs,
     private val compUnit: CompilationUnitTree,
     private val indexer: DocumentIndexer,
     private val indexers: Map<Path, DocumentIndexer>
 ): TreePathScanner<Unit?, Unit?>() {
-    private val trees: Trees = Trees.instance(task)
-    private val docs: DocTrees = DocTrees.instance(task)
-
     // TODO(nsc) handle 'var'
     override fun visitVariable(node: VariableTree, p: Unit?): Unit? {
         // emit var definition
@@ -74,11 +72,12 @@ class IndexingVisitor(
 
         val symbol = (node as JCClassDecl?)?.sym ?: return super.visitClass(node, p)
 
-        // TODO(nsc): snazzy records hover
+        // TODO snazzy records hover
         val classOrEnum: String = when {
             symbol.isEnum -> "enum "
             javaVersion >= 14 && symbol.isRecord -> "record "
-            symbol.isInterface -> "" // for some reason, 'interface ' is part of the modifiers
+            // for some reason, 'interface ' is part of the modifiers, so we exclude the string here
+            symbol.isInterface -> ""
             // TODO lowest version
             javaVersion >= 14 && symbol.isAnnotationType -> ""
             else -> "class "
@@ -360,8 +359,7 @@ class IndexingVisitor(
     // this may fail to find the correct position if start is too far behind the actual start position
     // that we want and another matching but incorrect symbol matches the regex
     private fun findNameIn(root: CompilationUnitTree, name: CharSequence, start: Int, end: Int): Int {
-        val contents: CharSequence
-        contents = try {
+        val contents = try {
             root.sourceFile.getCharContent(true)
         } catch (e: IOException) {
             throw RuntimeException(e)

src/main/kotlin/lsifjava/JDK8CompatFileManager.kt

@@ -102,20 +102,18 @@ private val JDK_MODULES = listOf(
 )
 
 /**
- * FileManager that falls back to JavacPathFileManager for Java 8.
- * Java 8 StandardJavaFileManager doesn't have the <code>setLocationFromPaths</code>
- * method, instead it only has <code>setLocation</code> which requires an 
- * <code>Iterable<? extends File></code>, which would cause an UnsupportedException
- * when trying to turn the ZipFile from the in-memory FileSystem into a File.
- * Because JavacPathFileManager doesn't exist beyond Java 8 (9?) and we build with 14+,
- * the symbol resolver would fail for that symbol, hence we create an instance via
- * reflection if the Java version is 8. God I hate this.
+ * FileManager that falls back to JavacPathFileManager for Java 8. Java 8 StandardJavaFileManager
+ * doesn't have the setLocationFromPaths method, instead it only has setLocation which requires an
+ * Iterable<? extends File>, which would cause an UnsupportedException when trying to turn the
+ * ZipFile from the in-memory FileSystem into a File. Because JavacPathFileManager doesn't exist
+ * beyond Java 8 (9?) and we build with 14+, the symbol resolver would fail for that symbol,
+ * hence we create an instance via reflection if the Java version is 8. God I hate this.
  */
 class JDK8CompatFileManager(manager: StandardJavaFileManager): ForwardingJavaFileManager<JavaFileManager>(getFileManager(manager)) {
     companion object {
         /**
          * returns a different JavaFileManager based on the current java version, with either SOURCE_PATH or MODULE_SOURCE_PATH set to "/"
-         * denoting the root of the JAR/ZIP file that this file manager is for. If JDK 8, we instantiate a <code>com.sun.tools.javac.nio.JavacPathFileManager</code>
+         * denoting the root of the JAR/ZIP file that this file manager is for. If JDK 8, we instantiate a com.sun.tools.javac.nio.JavacPathFileManager
          * via reflection, else we use the passed file manager. See the class doc for the reason why.
          */
         private fun getFileManager(fileManager: StandardJavaFileManager): JavaFileManager {

src/main/kotlin/lsifjava/JavaHomeHelper.kt

@@ -13,6 +13,12 @@ import java.util.stream.Collectors
 
 // from https://github.com/georgewfraser/java-language-server
 // bless you george for all the references. Maybe Ill cut this down/refactor
+/**
+ * Attempts to derive the possible JAVA_HOME, either from the set env var or
+ * through searching various platform dependent directories. If the wrong JAVA_HOME
+ * is found via search in the case where multiple versions are installed, the env
+ * var should be used.
+ */
 object JavaHomeHelper {
     fun javaHome(): Path? {
         System.getenv("JAVA_HOME")?.let {
@@ -37,7 +43,17 @@ object JavaHomeHelper {
 
     private fun macJavaHome(): Path? {
         if (Files.isExecutable(Paths.get("/usr/libexec/java_home"))) {
-            return execJavaHome()
+            return try {
+                val process = ProcessBuilder().command("/usr/libexec/java_home").start()
+                val out = BufferedReader(InputStreamReader(process.inputStream))
+                val line = out.readLine()
+                process.waitFor(5, TimeUnit.SECONDS)
+                Paths.get(line)
+            } catch (e: IOException) {
+                throw RuntimeException(e)
+            } catch (e: InterruptedException) {
+                throw RuntimeException(e)
+            }
         }
         val homes = arrayOf(
             "/Library/Java/JavaVirtualMachines/Home",
@@ -52,20 +68,6 @@ object JavaHomeHelper {
         return check(*homes)
     }
 
-    private fun execJavaHome(): Path {
-        return try {
-            val process = ProcessBuilder().command("/usr/libexec/java_home").start()
-            val out = BufferedReader(InputStreamReader(process.inputStream))
-            val line = out.readLine()
-            process.waitFor(5, TimeUnit.SECONDS)
-            Paths.get(line)
-        } catch (e: IOException) {
-            throw RuntimeException(e)
-        } catch (e: InterruptedException) {
-            throw RuntimeException(e)
-        }
-    }
-
     private fun check(vararg roots: String): Path? {
         for (root in roots) {
             val list: List<Path> = try {