Decoupled filetree visitor from DocumentIndexer type and tracking javac error count

build.gradle

@@ -13,7 +13,7 @@ repositories {
 
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
-    compile files("${System.getProperty('java.home')}/../lib/tools.jar")
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.0'
     compile 'com.google.code.gson:gson:2.8.6'
     compile 'commons-cli:commons-cli:1.4'
     compile 'org.apache.commons:commons-lang3:3.11'
@@ -29,7 +29,7 @@ dependencies {
 }
 
 application {
-    mainClassName = 'lsifjava.Main'
+    mainClassName = 'lsifjava.MainKt'
 }
 
 compileJava {
@@ -55,4 +55,10 @@ compileJava {
 }
 
 group = 'com.sourcegraph'
-version = '1.0-SNAPSHOT'
+version = '1.0-SNAPSHOT'
+
+compileKotlin {
+    kotlinOptions {
+        freeCompilerArgs = ["-Xinline-classes"]
+    }
+}

src/main/kotlin/lsifjava/ArgumentParser.kt

@@ -1,78 +1,72 @@
 package lsifjava
 
 import org.apache.commons.cli.*
-import java.io.IOException
+import java.nio.file.Path
 import java.nio.file.Paths
 import kotlin.system.exitProcess
 
 const val VERSION = "0.1.0"
 const val PROTOCOL_VERSION = "0.4.0"
 
 data class Arguments(
-    val projectRoot: String,
+    val projectRoot: CanonicalPath,
     val outFile: String,
     val verbose: Boolean,
     val javacOutWriter: String
 )
 
 fun parse(args: Array<String>): Arguments {
     val options = createOptions()
-    val parser = DefaultParser()
     val formatter = HelpFormatter()
-    val cmd: CommandLine
-    cmd = try {
-        parser.parse(options, args)
+
+    val cmd = try {
+        DefaultParser().parse(options, args)
     } catch (e: ParseException) {
         println(e.message)
         formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, "")
         exitProcess(1)
     }
+
     if (cmd.hasOption("help")) {
         formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, "")
         exitProcess(0)
     }
     if (cmd.hasOption("version")) {
-        println("${VERSION}, protocol version $PROTOCOL_VERSION")
+        println("$VERSION, protocol version $PROTOCOL_VERSION")
         exitProcess(0)
     }
-
-    val projectRoot = try {
-        Paths.get(cmd.getOptionValue("projectRoot", ".")).toFile().canonicalPath
-    } catch (e: IOException) {
-        throw RuntimeException(String.format("Directory %s could not be found", cmd.getOptionValue("projectRoot", ".")))
-    }
+
+    val projectRoot = CanonicalPath(Paths.get(cmd.getOptionValue("projectRoot", ".")))
     val outFile = cmd.getOptionValue("out", "$projectRoot/dump.lsif")
     val verbosity = cmd.hasOption("verbose")
     val javacOutWriter = cmd.getOptionValue("javacOut", "none")
 
     return Arguments(projectRoot, outFile, verbosity, javacOutWriter)
 }
 
-private fun createOptions(): Options {
-    val options = Options()
-    options.addOption(Option(
+private fun createOptions() = Options().apply {
+    addOption(Option(
         "help", false,
         "Show help."
     ))
-    options.addOption(Option(
+    addOption(Option(
         "version", false,
         "Show version."
     ))
-    options.addOption(Option(
+    addOption(Option(
         "verbose", false,
         "Display verbose information."
     ))
-    options.addOption(Option(
+    addOption(Option(
         "projectRoot", true,
         "Specifies the project root. Defaults to the current working directory."
     ))
-    options.addOption(Option(
+    addOption(Option(
         "out", true,
         "The output file the dump is save to."
     ))
-    options.addOption(Option(
+    addOption(Option(
         "javacOut", true,
-        "The output location for output from javac. Options include stdout, stderr or filepath."
+        "The output location for output from javac. Options include stdout, stderr, none or filepath."
     ))
-    return options
 }

src/main/kotlin/lsifjava/GradleInterface.kt → src/main/kotlin/lsifjava/BuildToolInterface.kt

@@ -5,21 +5,25 @@ import org.gradle.tooling.model.eclipse.EclipseProject
 import java.nio.file.Path
 import java.nio.file.Paths
 
-// TODO(nsc) exclusions? subprojects? inter-project dependencies? fml
-class GradleInterface(private val projectDir: String): AutoCloseable {
+interface BuildToolInterface {
+    fun getClasspaths(): List<Classpath>
+    fun getSourceDirectories(): List<List<Path>>
+    fun javaSourceVersions(): List<String?>
+}
+
+// TODO(nsc) exclusions? subprojects? lazy eval with tail rec?
+class GradleInterface(private val projectDir: CanonicalPath): AutoCloseable, BuildToolInterface {
     private val projectConnection by lazy {
-        // TODO(nsc) version override, 6.0 < x < 6.3 seems to be an issue
         GradleConnector.newConnector()
-            .forProjectDirectory(Paths.get(projectDir).toFile())
-            //.useGradleVersion("6.3")
+            .forProjectDirectory(projectDir.path.toFile())
             .connect()
     }
 
     private val eclipseModel by lazy {
         projectConnection.getModel(EclipseProject::class.java)
     }
 
-    fun getClasspaths() = classpath(eclipseModel)
+    override fun getClasspaths() = classpath(eclipseModel)
 
     private fun classpath(project: EclipseProject): List<Classpath> {
         val classPaths = arrayListOf<Classpath>()
@@ -28,7 +32,7 @@ class GradleInterface(private val projectDir: String): AutoCloseable {
         return classPaths
     }
 
-    fun getSourceDirectories() = sourceDirectories(eclipseModel)
+    override fun getSourceDirectories() = sourceDirectories(eclipseModel)
 
     private fun sourceDirectories(project: EclipseProject): List<List<Path>> {
         val sourceDirs = arrayListOf<List<Path>>()
@@ -37,8 +41,9 @@ class GradleInterface(private val projectDir: String): AutoCloseable {
         return sourceDirs
     }
 
-    fun javaSourceVersions() = javaSourceVersion(eclipseModel)
-
+    override fun javaSourceVersions() = javaSourceVersion(eclipseModel)
+
+    // get rid of String?, use parent version or else fallback
     private fun javaSourceVersion(project: EclipseProject): List<String?> {
         val javaVersions = arrayListOf<String?>()
         javaVersions.add(project.javaSourceSettings?.sourceLanguageLevel?.toString())

src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt

@@ -0,0 +1,40 @@
+package lsifjava
+
+import java.io.File
+import java.io.PrintStream
+import java.io.Writer
+import javax.tools.Diagnostic
+import javax.tools.DiagnosticListener
+
+fun createJavacDiagnosticListener(javacOutDest: String): CountingDiagnosticListener {
+    return when(javacOutDest) {
+        "stdout" -> CountingDiagnosticListener.StdoutWriter
+        "stderr" -> CountingDiagnosticListener.StderrWriter
+        "none" -> CountingDiagnosticListener.NullWriter
+        else -> CountingDiagnosticListener.FileWriter(javacOutDest)
+    }
+}
+
+sealed class CountingDiagnosticListener(private val out: Writer): DiagnosticListener<Any>, AutoCloseable {
+    var count = 0
+
+    override fun report(diagnostic: Diagnostic<out Any>?) {
+        count++
+        out.write(diagnostic.toString())
+        out.flush()
+    }
+
+    override fun close() = out.close()
+
+    class FileWriter(path: String): CountingDiagnosticListener(PrintStream(File(path)).writer())
+
+    object StdoutWriter: CountingDiagnosticListener(System.out.writer()) {
+        override fun close() = Unit
+    }
+
+    object StderrWriter: CountingDiagnosticListener(System.err.writer()) {
+        override fun close() = Unit
+    }
+
+    object NullWriter: CountingDiagnosticListener(NoopWriter)
+}

src/main/kotlin/lsifjava/DocumentIndexer.kt

@@ -8,80 +8,63 @@ import com.sun.tools.javac.main.JavaCompiler
 import com.sun.tools.javac.util.Context
 import org.eclipse.lsp4j.Position
 import org.eclipse.lsp4j.Range
-import java.io.Writer
 import java.nio.file.Path
 import java.util.*
-import javax.tools.Diagnostic
-import javax.tools.DiagnosticListener
 import kotlin.collections.HashMap
 import kotlin.collections.HashSet
 
 class DocumentIndexer(
-    private val verbose: Boolean,
-    private val path: Path,
-    private val projectId: String,
-    private val emitter: Emitter,
-    private val indexers: Map<Path, DocumentIndexer>,
+    private val filepath: CanonicalPath,
     private val classpath: Classpath,
     private val javaSourceVersion: String,
-    private val javacOutput: Writer?
+    private val indexers: Map<Path, DocumentIndexer>,
+    private val emitter: Emitter,
+    private val diagnosticListener: CountingDiagnosticListener,
+    private val verbose: Boolean,
 ) {
     companion object {
-        private val systemProvider = JavacTool.create()
+        private val systemProvider by lazy { JavacTool.create() }
     }
 
     data class DefinitionMeta(val rangeId: String, val resultSetId: String) {
         var definitionResultId: String? = null
-        val referenceRangeIds = HashMap<String, MutableSet<String>>()
+        val referenceRangeIds: MutableMap<String, MutableSet<String>> = HashMap()
     }
 
-    private lateinit var documentId: String
-    private var indexed = false
+    private var documentId: String
     private val rangeIds: MutableSet<String> = HashSet()
     private val definitions: MutableMap<Range, DefinitionMeta> = HashMap()
-    private lateinit var fileManager: SourceFileManager
-
-    private val referencesBacklog: LinkedList<() -> Unit> = LinkedList()
-
-    var javacDiagnostics = LinkedList<Diagnostic<out Any>>()
-        private set
-    private val diagnosticsListener = DiagnosticListener<Any> { javacDiagnostics.add(it) }
 
-    fun numDefinitions(): Int {
-        return definitions.size
-    }
+    val numDefinitions get() = definitions.size
+
+    lateinit var fileManager: SourceFileManager
+
+    private val referencesBacklog: LinkedList<() -> Unit> = LinkedList()
 
-    fun preIndex(fileManager: SourceFileManager) {
-        this.fileManager = fileManager
+    init {
         val args = mapOf(
             "languageId" to "java",
-            "uri" to String.format("file://%s", path.toAbsolutePath().toString())
+            "uri" to "file://$filepath"
         )
         documentId = emitter.emitVertex("document", args)
     }
 
-    @Synchronized
-    fun index() {
-        if (indexed) return
-        indexed = true
-
+    // lazy block ensures synchronized calling and only one invocation
+    fun index() = lazy {
         val (task, compUnit) = analyzeFile()
         IndexingVisitor(task, compUnit, this, indexers).scan(compUnit, null)
         referencesBacklog.forEach { it() }
 
-        //javacDiagnostics.forEach(::println)
-
-        println("finished analyzing and visiting $path")
-    }
+        println("finished analyzing and visiting $filepath")
+    }.value // must reference this to trigger the lazy load
 
     private fun analyzeFile(): Pair<JavacTask, CompilationUnitTree> {
         val context = SimpleContext()
 
-        // TODO(nsc) diagnosticsListener not being null seems to interfere with javacOutput
         val task = systemProvider.getTask(
-            javacOutput, fileManager, diagnosticsListener,
-            listOf("-source", "8", "-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString()),
-            listOf(), listOf(SourceFileObject(path)), context
+            NoopWriter, fileManager, diagnosticListener,
+            listOf("-source", "8", "-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString()/* , "--enable-preview" */),
+            listOf(), listOf(SourceFileObject(filepath.path)), context
         )
         val compUnit = task.parse().iterator().next()
         task.analyze()
@@ -108,7 +91,7 @@ class DocumentIndexer(
         }
     }
 
-    fun postIndex() {
+    fun postIndex(projectId: String) {
         for (meta in definitions.values)
             linkUses(meta, documentId)
         emitter.emitEdge("contains", mapOf("outV" to projectId, "inVs" to arrayOf(documentId)))
@@ -135,7 +118,7 @@ class DocumentIndexer(
     }
 
     private fun emitDefinition(range: Range, hover: String) {
-        if (verbose) println("DEF " + path.toString() + ":" + humanRange(range))
+        if (verbose) println("DEF ${filepath}:${humanRange(range)}")
         val hoverId = emitter.emitVertex("hoverResult", mapOf(
             "result" to mapOf(
                 "contents" to mapOf(
@@ -159,12 +142,12 @@ class DocumentIndexer(
 
     internal fun emitUse(use: Range, def: Range, defPath: Path) {
         referencesBacklog.add {
-            val indexer = indexers[defPath]
-            val link = path.toString() + ":" + humanRange(use) + " -> " + defPath.toString() + ":" + humanRange(def)
+            val indexer = indexers[defPath] ?: error("expected indexer for $defPath")
+            val link = "${filepath}:${humanRange(use)} -> ${defPath}:${humanRange(def)}"
 
             if (verbose) println("Linking use to definition: $link")
 
-            val meta = indexer!!.definitions[def]
+            val meta = indexer.definitions[def]
             if (meta == null) {
                 if (verbose) println("WARNING missing definition for: $link")
                 return@add
@@ -192,18 +175,17 @@ class DocumentIndexer(
         }
     }
 
-    private fun mkDoc(signature: String, docComment: String?): String {
-        return "```java\n$signature\n```" +
-            if (docComment == null || docComment == "") "" else
-                "\n---\n${docComment.trim()}"
-    }
+    private fun mkDoc(signature: String, docComment: String?) = 
+        "```java\n$signature\n```" +
+        if (docComment == null || docComment == "") "" 
+        else "\n---\n${docComment.trim()}"
 
     /**
      * Returns the stringified range with the lines+1 to make clicking the links in your editor's terminal
      * direct to the correct line
      */
     private fun humanRange(r: Range): String {
-        return (r.start.line+1).toString() + ":" + (r.start.character+1)+ "-" + (r.end.line+1) + ":" + (r.end.character+1)
+        return "${r.start.line+1}:${r.start.character+1}-${r.end.line+1}:${r.end.character+1}"
     }
 
     private fun createRange(range: Range): Map<String, Any> {