diff --git a/build.gradle b/build.gradle index b66b2905..d9bafebe 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ repositories { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" - compile files("${System.getProperty('java.home')}/../lib/tools.jar") + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.0' compile 'com.google.code.gson:gson:2.8.6' compile 'commons-cli:commons-cli:1.4' compile 'org.apache.commons:commons-lang3:3.11' @@ -29,7 +29,7 @@ dependencies { } application { - mainClassName = 'lsifjava.Main' + mainClassName = 'lsifjava.MainKt' } compileJava { @@ -55,4 +55,10 @@ compileJava { } group = 'com.sourcegraph' -version = '1.0-SNAPSHOT' \ No newline at end of file +version = '1.0-SNAPSHOT' + +compileKotlin { + kotlinOptions { + freeCompilerArgs = ["-Xinline-classes"] + } +} \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/ArgumentParser.kt b/src/main/kotlin/lsifjava/ArgumentParser.kt index 41dd3de8..bfc60e09 100644 --- a/src/main/kotlin/lsifjava/ArgumentParser.kt +++ b/src/main/kotlin/lsifjava/ArgumentParser.kt @@ -1,7 +1,7 @@ package lsifjava import org.apache.commons.cli.* -import java.io.IOException +import java.nio.file.Path import java.nio.file.Paths import kotlin.system.exitProcess @@ -9,7 +9,7 @@ const val VERSION = "0.1.0" const val PROTOCOL_VERSION = "0.4.0" data class Arguments( - val projectRoot: String, + val projectRoot: CanonicalPath, val outFile: String, val verbose: Boolean, val javacOutWriter: String @@ -17,30 +17,26 @@ data class Arguments( fun parse(args: Array): Arguments { val options = createOptions() - val parser = DefaultParser() val formatter = HelpFormatter() - val cmd: CommandLine - cmd = try { - parser.parse(options, args) + + val cmd = try { + DefaultParser().parse(options, args) } catch (e: ParseException) { println(e.message) formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, "") exitProcess(1) } + if (cmd.hasOption("help")) { formatter.printHelp("lsif-java", "lsif-java is an LSIF indexer for Java.\n\n", options, "") exitProcess(0) } if (cmd.hasOption("version")) { - println("${VERSION}, protocol version $PROTOCOL_VERSION") + println("$VERSION, protocol version $PROTOCOL_VERSION") exitProcess(0) } - - val projectRoot = try { - Paths.get(cmd.getOptionValue("projectRoot", ".")).toFile().canonicalPath - } catch (e: IOException) { - throw RuntimeException(String.format("Directory %s could not be found", cmd.getOptionValue("projectRoot", "."))) - } + + val projectRoot = CanonicalPath(Paths.get(cmd.getOptionValue("projectRoot", "."))) val outFile = cmd.getOptionValue("out", "$projectRoot/dump.lsif") val verbosity = cmd.hasOption("verbose") val javacOutWriter = cmd.getOptionValue("javacOut", "none") @@ -48,31 +44,29 @@ fun parse(args: Array): Arguments { return Arguments(projectRoot, outFile, verbosity, javacOutWriter) } -private fun createOptions(): Options { - val options = Options() - options.addOption(Option( +private fun createOptions() = Options().apply { + addOption(Option( "help", false, "Show help." )) - options.addOption(Option( + addOption(Option( "version", false, "Show version." )) - options.addOption(Option( + addOption(Option( "verbose", false, "Display verbose information." )) - options.addOption(Option( + addOption(Option( "projectRoot", true, "Specifies the project root. Defaults to the current working directory." )) - options.addOption(Option( + addOption(Option( "out", true, "The output file the dump is save to." )) - options.addOption(Option( + addOption(Option( "javacOut", true, - "The output location for output from javac. Options include stdout, stderr or filepath." + "The output location for output from javac. Options include stdout, stderr, none or filepath." )) - return options } diff --git a/src/main/kotlin/lsifjava/GradleInterface.kt b/src/main/kotlin/lsifjava/BuildToolInterface.kt similarity index 71% rename from src/main/kotlin/lsifjava/GradleInterface.kt rename to src/main/kotlin/lsifjava/BuildToolInterface.kt index bf43832c..e9e9ab2c 100644 --- a/src/main/kotlin/lsifjava/GradleInterface.kt +++ b/src/main/kotlin/lsifjava/BuildToolInterface.kt @@ -5,13 +5,17 @@ import org.gradle.tooling.model.eclipse.EclipseProject import java.nio.file.Path import java.nio.file.Paths -// TODO(nsc) exclusions? subprojects? inter-project dependencies? fml -class GradleInterface(private val projectDir: String): AutoCloseable { +interface BuildToolInterface { + fun getClasspaths(): List + fun getSourceDirectories(): List> + fun javaSourceVersions(): List +} + +// TODO(nsc) exclusions? subprojects? lazy eval with tail rec? +class GradleInterface(private val projectDir: CanonicalPath): AutoCloseable, BuildToolInterface { private val projectConnection by lazy { - // TODO(nsc) version override, 6.0 < x < 6.3 seems to be an issue GradleConnector.newConnector() - .forProjectDirectory(Paths.get(projectDir).toFile()) - //.useGradleVersion("6.3") + .forProjectDirectory(projectDir.path.toFile()) .connect() } @@ -19,7 +23,7 @@ class GradleInterface(private val projectDir: String): AutoCloseable { projectConnection.getModel(EclipseProject::class.java) } - fun getClasspaths() = classpath(eclipseModel) + override fun getClasspaths() = classpath(eclipseModel) private fun classpath(project: EclipseProject): List { val classPaths = arrayListOf() @@ -28,7 +32,7 @@ class GradleInterface(private val projectDir: String): AutoCloseable { return classPaths } - fun getSourceDirectories() = sourceDirectories(eclipseModel) + override fun getSourceDirectories() = sourceDirectories(eclipseModel) private fun sourceDirectories(project: EclipseProject): List> { val sourceDirs = arrayListOf>() @@ -37,8 +41,9 @@ class GradleInterface(private val projectDir: String): AutoCloseable { return sourceDirs } - fun javaSourceVersions() = javaSourceVersion(eclipseModel) - + override fun javaSourceVersions() = javaSourceVersion(eclipseModel) + + // get rid of String?, use parent version or else fallback private fun javaSourceVersion(project: EclipseProject): List { val javaVersions = arrayListOf() javaVersions.add(project.javaSourceSettings?.sourceLanguageLevel?.toString()) diff --git a/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt b/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt new file mode 100644 index 00000000..1061c9d2 --- /dev/null +++ b/src/main/kotlin/lsifjava/CountingDiagnosticWriter.kt @@ -0,0 +1,40 @@ +package lsifjava + +import java.io.File +import java.io.PrintStream +import java.io.Writer +import javax.tools.Diagnostic +import javax.tools.DiagnosticListener + +fun createJavacDiagnosticListener(javacOutDest: String): CountingDiagnosticListener { + return when(javacOutDest) { + "stdout" -> CountingDiagnosticListener.StdoutWriter + "stderr" -> CountingDiagnosticListener.StderrWriter + "none" -> CountingDiagnosticListener.NullWriter + else -> CountingDiagnosticListener.FileWriter(javacOutDest) + } +} + +sealed class CountingDiagnosticListener(private val out: Writer): DiagnosticListener, AutoCloseable { + var count = 0 + + override fun report(diagnostic: Diagnostic?) { + count++ + out.write(diagnostic.toString()) + out.flush() + } + + override fun close() = out.close() + + class FileWriter(path: String): CountingDiagnosticListener(PrintStream(File(path)).writer()) + + object StdoutWriter: CountingDiagnosticListener(System.out.writer()) { + override fun close() = Unit + } + + object StderrWriter: CountingDiagnosticListener(System.err.writer()) { + override fun close() = Unit + } + + object NullWriter: CountingDiagnosticListener(NoopWriter) +} \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/DocumentIndexer.kt b/src/main/kotlin/lsifjava/DocumentIndexer.kt index bb18a10a..93654b91 100644 --- a/src/main/kotlin/lsifjava/DocumentIndexer.kt +++ b/src/main/kotlin/lsifjava/DocumentIndexer.kt @@ -8,80 +8,63 @@ import com.sun.tools.javac.main.JavaCompiler import com.sun.tools.javac.util.Context import org.eclipse.lsp4j.Position import org.eclipse.lsp4j.Range -import java.io.Writer import java.nio.file.Path import java.util.* -import javax.tools.Diagnostic -import javax.tools.DiagnosticListener import kotlin.collections.HashMap import kotlin.collections.HashSet class DocumentIndexer( - private val verbose: Boolean, - private val path: Path, - private val projectId: String, - private val emitter: Emitter, - private val indexers: Map, + private val filepath: CanonicalPath, private val classpath: Classpath, private val javaSourceVersion: String, - private val javacOutput: Writer? + private val indexers: Map, + private val emitter: Emitter, + private val diagnosticListener: CountingDiagnosticListener, + private val verbose: Boolean, ) { companion object { - private val systemProvider = JavacTool.create() + private val systemProvider by lazy { JavacTool.create() } } data class DefinitionMeta(val rangeId: String, val resultSetId: String) { var definitionResultId: String? = null - val referenceRangeIds = HashMap>() + val referenceRangeIds: MutableMap> = HashMap() } - private lateinit var documentId: String - private var indexed = false + private var documentId: String private val rangeIds: MutableSet = HashSet() private val definitions: MutableMap = HashMap() - private lateinit var fileManager: SourceFileManager - - private val referencesBacklog: LinkedList<() -> Unit> = LinkedList() - - var javacDiagnostics = LinkedList>() - private set - private val diagnosticsListener = DiagnosticListener { javacDiagnostics.add(it) } - fun numDefinitions(): Int { - return definitions.size - } + val numDefinitions get() = definitions.size + + lateinit var fileManager: SourceFileManager + + private val referencesBacklog: LinkedList<() -> Unit> = LinkedList() - fun preIndex(fileManager: SourceFileManager) { - this.fileManager = fileManager + init { val args = mapOf( "languageId" to "java", - "uri" to String.format("file://%s", path.toAbsolutePath().toString()) + "uri" to "file://$filepath" ) documentId = emitter.emitVertex("document", args) } - @Synchronized - fun index() { - if (indexed) return - indexed = true - + // lazy block ensures synchronized calling and only one invocation + fun index() = lazy { val (task, compUnit) = analyzeFile() IndexingVisitor(task, compUnit, this, indexers).scan(compUnit, null) referencesBacklog.forEach { it() } - //javacDiagnostics.forEach(::println) - - println("finished analyzing and visiting $path") - } + println("finished analyzing and visiting $filepath") + }.value // must reference this to trigger the lazy load private fun analyzeFile(): Pair { val context = SimpleContext() - // TODO(nsc) diagnosticsListener not being null seems to interfere with javacOutput val task = systemProvider.getTask( - javacOutput, fileManager, diagnosticsListener, - listOf("-source", "8", "-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString()), - listOf(), listOf(SourceFileObject(path)), context + NoopWriter, fileManager, diagnosticListener, + listOf("-source", "8", "-proc:none", "-nowarn", "-source", javaSourceVersion, "-classpath", classpath.toString()/* , "--enable-preview" */), + listOf(), listOf(SourceFileObject(filepath.path)), context ) val compUnit = task.parse().iterator().next() task.analyze() @@ -108,7 +91,7 @@ class DocumentIndexer( } } - fun postIndex() { + fun postIndex(projectId: String) { for (meta in definitions.values) linkUses(meta, documentId) emitter.emitEdge("contains", mapOf("outV" to projectId, "inVs" to arrayOf(documentId))) @@ -135,7 +118,7 @@ class DocumentIndexer( } private fun emitDefinition(range: Range, hover: String) { - if (verbose) println("DEF " + path.toString() + ":" + humanRange(range)) + if (verbose) println("DEF ${filepath}:${humanRange(range)}") val hoverId = emitter.emitVertex("hoverResult", mapOf( "result" to mapOf( "contents" to mapOf( @@ -159,12 +142,12 @@ class DocumentIndexer( internal fun emitUse(use: Range, def: Range, defPath: Path) { referencesBacklog.add { - val indexer = indexers[defPath] - val link = path.toString() + ":" + humanRange(use) + " -> " + defPath.toString() + ":" + humanRange(def) + val indexer = indexers[defPath] ?: error("expected indexer for $defPath") + val link = "${filepath}:${humanRange(use)} -> ${defPath}:${humanRange(def)}" if (verbose) println("Linking use to definition: $link") - val meta = indexer!!.definitions[def] + val meta = indexer.definitions[def] if (meta == null) { if (verbose) println("WARNING missing definition for: $link") return@add @@ -192,18 +175,17 @@ class DocumentIndexer( } } - private fun mkDoc(signature: String, docComment: String?): String { - return "```java\n$signature\n```" + - if (docComment == null || docComment == "") "" else - "\n---\n${docComment.trim()}" - } + private fun mkDoc(signature: String, docComment: String?) = + "```java\n$signature\n```" + + if (docComment == null || docComment == "") "" + else "\n---\n${docComment.trim()}" /** * Returns the stringified range with the lines+1 to make clicking the links in your editor's terminal * direct to the correct line */ private fun humanRange(r: Range): String { - return (r.start.line+1).toString() + ":" + (r.start.character+1)+ "-" + (r.end.line+1) + ":" + (r.end.character+1) + return "${r.start.line+1}:${r.start.character+1}-${r.end.line+1}:${r.end.character+1}" } private fun createRange(range: Range): Map { diff --git a/src/main/kotlin/lsifjava/Emitter.kt b/src/main/kotlin/lsifjava/Emitter.kt index 1d1b25fd..9c0f64b3 100644 --- a/src/main/kotlin/lsifjava/Emitter.kt +++ b/src/main/kotlin/lsifjava/Emitter.kt @@ -9,6 +9,8 @@ class Emitter(private val writer: PrintWriter) { private val gson: Gson = Gson() + fun numElements() = nextId + fun emitVertex(labelName: String, args: Map) = emit("vertex", labelName, args) fun emitEdge(labelName: String, args: Map) = emit("edge", labelName, args) @@ -27,6 +29,4 @@ class Emitter(private val writer: PrintWriter) { return id } - - fun numElements() = nextId } \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/FileCollector.kt b/src/main/kotlin/lsifjava/FileCollector.kt index 3c8c5c59..bf251c07 100644 --- a/src/main/kotlin/lsifjava/FileCollector.kt +++ b/src/main/kotlin/lsifjava/FileCollector.kt @@ -1,63 +1,93 @@ package lsifjava -import org.apache.commons.io.output.NullOutputStream -import java.io.* +import kotlinx.coroutines.* import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.Path import java.nio.file.SimpleFileVisitor import java.nio.file.attribute.BasicFileAttributes +import kotlinx.coroutines.channels.Channel +private data class FileBuildInfo(val filepath: Path, val classpath: Classpath, val javaVersion: String?) -class FileCollector(private val projectId: String, private val args: Arguments, private val emitter: Emitter, private val indexers: MutableMap) : SimpleFileVisitor() { - lateinit var classpath: Classpath - var javaSourceVersion: String? = null - private val javacOutStream by lazy { createJavacOutWriter(args) } +/** + * Builds a map of Java filepaths to DocumentIndexers, + * with build info derived from `buildToolInterface` + */ +fun buildIndexerMap( + buildToolInterface: BuildToolInterface, + emitter: Emitter, + verbose: Boolean, + javacDiagListener: CountingDiagnosticListener, +): Map { + val indexers = mutableMapOf() + + val classpaths = buildToolInterface.getClasspaths() + val sourceVersions = buildToolInterface.javaSourceVersions() - override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult { - return if (attrs.isSymbolicLink) { - FileVisitResult.SKIP_SUBTREE - } else FileVisitResult.CONTINUE + val fileBuildInfo = Channel() + + runBlocking { + launchFileTreeWalkers( + buildToolInterface, fileBuildInfo, classpaths, sourceVersions, + ).invokeOnCompletion(fileBuildInfo::close) + + launch { + for(info in fileBuildInfo) { + indexers[info.filepath] = DocumentIndexer( + CanonicalPath(info.filepath), info.classpath, info.javaVersion!!, + indexers, emitter, javacDiagListener, verbose, + ) + } + }.join() } - // TODO(nsc) uncouple FileCollector + DocumentIndexer + return indexers +} + +private fun CoroutineScope.launchFileTreeWalkers( + buildToolInterface: BuildToolInterface, + fileBuildInfoChannel: Channel, + classpaths: List, + sourceVersions: List, +) = launch(Dispatchers.IO) { + buildToolInterface.getSourceDirectories().forEachIndexed { i, paths -> + launch { + val collector = AsyncFileCollector(fileBuildInfoChannel, classpaths[i], sourceVersions[i], this) + paths.asSequence().filter { Files.exists(it) }.forEach { Files.walkFileTree(it, collector) } + } + } +} + +/** + * A weakly asynchronous file-tree visitor. Sends discovered and valid + * files+build info into the provided channel asynchronously + */ +private class AsyncFileCollector( + private val fileInfoChannel: Channel, + private val classpath: Classpath, + private val sourceVersion: String?, + private val coroutineScope: CoroutineScope, +) : SimpleFileVisitor() { + + override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = + if (attrs.isSymbolicLink) FileVisitResult.SKIP_SUBTREE + else FileVisitResult.CONTINUE + override fun visitFile(file: Path, _attrs: BasicFileAttributes): FileVisitResult { if (isJavaFile(file)) { - indexers[file] = DocumentIndexer( - args.verbose, - file, - projectId, - emitter, - indexers, - classpath, - javaSourceVersion!!, - javacOutStream - ) + coroutineScope.launch { + fileInfoChannel.send(FileBuildInfo(file, classpath, sourceVersion)) + } } return FileVisitResult.CONTINUE } companion object { - fun createJavacOutWriter(args: Arguments): Writer? { - return when(args.javacOutWriter) { - "stdout" -> null // getTask interprets this as stdout - "stderr" -> System.err.writer() - //"none" -> OutputStreamWriter.nullWriter() CANT USE IN JAVA 8 :( - "none" -> object : Writer() { - override fun close() {} - override fun flush() {} - override fun write(p0: CharArray, p1: Int, p2: Int) {} - } - else -> PrintStream(File(args.javacOutWriter)).writer() - } - } - fun isJavaFile(file: Path): Boolean { val name = file.fileName.toString() // We hide module-info.java from javac, because when javac sees module-info.java // it goes into "module mode" and starts looking for classes on the module class path. - // This becomes evident when javac starts recompiling *way too much* on each task, - // because it doesn't realize there are already up-to-date .class files. // The better solution would be for java-language server to detect the presence of module-info.java, // and go into its own "module mode" where it infers a module source path and a module class path. return name.endsWith(".java") && !Files.isDirectory(file) && name != "module-info.java" diff --git a/src/main/kotlin/lsifjava/IndexingVisitor.kt b/src/main/kotlin/lsifjava/IndexingVisitor.kt index eb4c2119..121e5e03 100644 --- a/src/main/kotlin/lsifjava/IndexingVisitor.kt +++ b/src/main/kotlin/lsifjava/IndexingVisitor.kt @@ -14,14 +14,15 @@ import java.nio.file.Path import java.nio.file.Paths import java.util.regex.Pattern import javax.lang.model.element.Element +import javax.lang.model.element.ElementKind data class ReferenceData(val useRange: Range, val refRange: Range, val defPath: Path) class IndexingVisitor( - task: JavacTask, - private val compUnit: CompilationUnitTree, - private val indexer: DocumentIndexer, - private val indexers: Map + task: JavacTask, + private val compUnit: CompilationUnitTree, + private val indexer: DocumentIndexer, + private val indexers: Map ): TreePathScanner() { private val trees: Trees = Trees.instance(task) private val docs: DocTrees = DocTrees.instance(task) @@ -29,7 +30,8 @@ class IndexingVisitor( // TODO(nsc) handle 'var' override fun visitVariable(node: VariableTree, p: Unit?): Unit? { // emit var definition - val defRange = findLocation(currentPath, node.name.toString())?.range ?: return super.visitVariable(node, p) + val defRange = findLocation(currentPath, node.name.toString())?.range + ?: return super.visitVariable(node, p) indexer.emitDefinition(defRange, node.toString(), docs.getDocComment(currentPath)) @@ -39,10 +41,17 @@ class IndexingVisitor( override fun visitClass(node: ClassTree, p: Unit?): Unit? { val packagePrefix = compUnit.packageName?.toString()?.plus(".") ?: "" - // TODO(nsc): records - val classOrEnum = if ((node as JCClassDecl).sym.flags() and Flags.ENUM.toLong() != 0L) "enum " else "class " + // TODO(nsc): snazzy records hover + val classOrEnum: String = when { + (node as JCClassDecl).sym.isEnum -> "enum " + node.sym.isRecord -> "record " + node.sym.isInterface -> "" // for some reason, 'interface ' is part of the modifiers + node.sym.isAnnotationType -> "" + else -> "class " + } - val range = findLocation(currentPath, node.simpleName.toString())?.range ?: return super.visitClass(node, p) + val range = findLocation(currentPath, node.simpleName.toString())?.range + ?: return super.visitClass(node, p) indexer.emitDefinition( range, @@ -62,7 +71,8 @@ class IndexingVisitor( val returnType = node.returnType?.toString()?.plus(" ") ?: "" - val range = findLocation(currentPath, methodName, node.pos)?.range ?: return super.visitMethod(node, p) + val range = findLocation(currentPath, methodName, node.pos)?.range + ?: return super.visitMethod(node, p) indexer.emitDefinition( range, @@ -78,6 +88,7 @@ class IndexingVisitor( override fun visitNewClass(node: NewClassTree, p: Unit?): Unit? { (node as JCNewClass).constructor ?: return super.visitNewClass(node, p) + // ignore auto-genned constructors (for now) if(node.constructor.flags() and Flags.GENERATEDCONSTR != 0L) return null @@ -103,15 +114,17 @@ class IndexingVisitor( val identPath = TreePath(currentPath, node.identifier) val identElement = node.constructor - val defContainer = getTopLevelClass(identElement) as Symbol.ClassSymbol? ?: return super.visitNewClass(node, p) - val (useRange, refRange, defPath) = findReference(identElement, identName, defContainer, path = identPath) ?: return super.visitNewClass(node, p) + val defContainer = getTopLevelClass(identElement) as Symbol.ClassSymbol? + ?: return super.visitNewClass(node, p) + val (useRange, refRange, defPath) = findReference(identElement, identName, defContainer, path = identPath) + ?: return super.visitNewClass(node, p) indexer.emitUse(useRange, refRange, defPath) return super.visitNewClass(node, p) } - override fun visitMethodInvocation(node: MethodInvocationTree?, p: Unit?): Unit? { + override fun visitMethodInvocation(node: MethodInvocationTree, p: Unit?): Unit? { val symbol = when((node as JCMethodInvocation).meth) { is JCFieldAccess -> (node.meth as JCFieldAccess).sym is JCIdent -> (node.meth as JCIdent).sym @@ -126,12 +139,16 @@ class IndexingVisitor( val name = symbol?.name?.toString()?.let { if(it == "") "this" else it } ?: return super.visitMethodInvocation(node, p) + val methodPath = TreePath(currentPath, node.meth) - val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMethodInvocation(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? + ?: return super.visitMethodInvocation(node, p) // this gives us the start pos of the name of the method, so override defStartPos - val overrideStartOffset = (trees.getPath(symbol)?.leaf as JCMethodDecl?)?.pos ?: return super.visitMethodInvocation(node, p) - val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, path = methodPath, defStartPos = overrideStartOffset) ?: return super.visitMethodInvocation(node, p) + val overrideStartOffset = (trees.getPath(symbol)?.leaf as JCMethodDecl?)?.pos + ?: return super.visitMethodInvocation(node, p) + val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, path = methodPath, defStartPos = overrideStartOffset) + ?: return super.visitMethodInvocation(node, p) indexer.emitUse(useRange, refRange, defPath) @@ -139,24 +156,29 @@ class IndexingVisitor( } // does not match `var` or constructor calls - override fun visitIdentifier(node: IdentifierTree?, p: Unit?): Unit? { - val symbol = (node as JCIdent).sym ?: return super.visitIdentifier(node, p) + override fun visitIdentifier(node: IdentifierTree, p: Unit?): Unit? { + val symbol = (node as JCIdent).sym + ?: return super.visitIdentifier(node, p) if(symbol is Symbol.PackageSymbol) return super.visitIdentifier(node, p) val name = symbol.name.toString() if(name == "") return super.visitIdentifier(node, p) // handled by visitMethodInvocation - val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitIdentifier(node, p) - val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) ?: return super.visitIdentifier(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? + ?: return super.visitIdentifier(node, p) + + val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) + ?: return super.visitIdentifier(node, p) indexer.emitUse(useRange, refRange, defPath) return super.visitIdentifier(node, p) } - override fun visitTypeParameter(node: TypeParameterTree?, p: Unit?): Unit? { - val range = findLocation(currentPath, node!!.name.toString(), (node as JCTypeParameter).pos)?.range ?: return super.visitTypeParameter(node, p) + override fun visitTypeParameter(node: TypeParameterTree, p: Unit?): Unit? { + val range = findLocation(currentPath, node.name.toString(), (node as JCTypeParameter).pos)?.range + ?: return super.visitTypeParameter(node, p) indexer.emitDefinition(range, "type-parameter $node") @@ -164,21 +186,24 @@ class IndexingVisitor( } // function references eg test::myfunc or banana::new - override fun visitMemberReference(node: MemberReferenceTree?, p: Unit?): Unit? { - val symbol = (node as JCMemberReference).sym ?: return super.visitMemberReference(node, p) + override fun visitMemberReference(node: MemberReferenceTree, p: Unit?): Unit? { + val symbol = (node as JCMemberReference).sym + ?: return super.visitMemberReference(node, p) val name = symbol.name.toString() - val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMemberReference(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? + ?: return super.visitMemberReference(node, p) - val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) ?: return super.visitMemberReference(node, p) + val (useRange, refRange, defPath) = findReference(symbol, name, defContainer) + ?: return super.visitMemberReference(node, p) indexer.emitUse(useRange, refRange, defPath) return super.visitMemberReference(node, p) } - override fun visitMemberSelect(node: MemberSelectTree?, p: Unit?): Unit? { + override fun visitMemberSelect(node: MemberSelectTree, p: Unit?): Unit? { if((node as JCFieldAccess).sym == null) return super.visitMemberSelect(node, p) val symbol = when(node.sym) { @@ -189,10 +214,12 @@ class IndexingVisitor( val name = symbol.name.toString() - val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? ?: return super.visitMemberSelect(node, p) + val defContainer = getTopLevelClass(symbol) as Symbol.ClassSymbol? + ?: return super.visitMemberSelect(node, p) // need to narrow down start pos of the field access here, so override refStartPos - val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, refStartPos = node.pos) ?: return super.visitMemberSelect(node, p) + val (useRange, refRange, defPath) = findReference(symbol, name, defContainer, refStartPos = node.pos) + ?: return super.visitMemberSelect(node, p) indexer.emitUse(useRange, refRange, defPath) @@ -207,7 +234,14 @@ class IndexingVisitor( * @param refStartPos document offset to be provided if a node supplies a more accurate start position for a reference/use search * @param defStartPos document offset to be provided if a node supplies a more accurate start position for a definition search */ - private fun findReference(element: Element, symbolName: String, container: Symbol.ClassSymbol, path: TreePath = currentPath, refStartPos: Int? = null, defStartPos: Int? = null): ReferenceData? { + private fun findReference( + element: Element, + symbolName: String, + container: Symbol.ClassSymbol, + path: TreePath = currentPath, + refStartPos: Int? = null, + defStartPos: Int? = null, + ): ReferenceData? { val sourceFilePath = container.sourcefile ?: return null val defPath = Paths.get(sourceFilePath.name) if(sourceFilePath.name != compUnit.sourceFile.name) @@ -268,4 +302,19 @@ class IndexingVisitor( matcher.start() } else -1 } + + private fun getTopLevelClass(element: Element?): Element? { + var element = element + var highestClass: Element? = null + while (element != null) { + val kind = element.kind + if (isTopLevel(kind)) { + highestClass = element + } + element = element.enclosingElement + } + return highestClass + } + + private fun isTopLevel(kind: ElementKind) = kind.isClass || kind.isInterface } \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/LanguageUtils.kt b/src/main/kotlin/lsifjava/LanguageUtils.kt deleted file mode 100644 index dce0e3f0..00000000 --- a/src/main/kotlin/lsifjava/LanguageUtils.kt +++ /dev/null @@ -1,20 +0,0 @@ -package lsifjava - -import javax.lang.model.element.* - -fun getTopLevelClass(element: Element?): Element? { - var element = element - var highestClass: Element? = null - while (element != null) { - val kind = element.kind - if (isTopLevel(kind)) { - highestClass = element - } - element = element.enclosingElement - } - return highestClass -} - -fun isTopLevel(kind: ElementKind): Boolean { - return kind.isClass || kind.isInterface -} diff --git a/src/main/kotlin/lsifjava/Main.kt b/src/main/kotlin/lsifjava/Main.kt index 66b522ed..7901796c 100644 --- a/src/main/kotlin/lsifjava/Main.kt +++ b/src/main/kotlin/lsifjava/Main.kt @@ -29,10 +29,11 @@ private fun createWriter(args: Arguments): PrintWriter { private fun displayStats(indexer: ProjectIndexer, emitter: Emitter, start: Long) { System.out.printf( - "%d file(s), %d def(s), %d element(s)\n", + "%d file(s), %d def(s), %d LSIF element(s), %d total javac error(s)\n", indexer.numFiles, indexer.numDefinitions, - emitter.numElements() + emitter.numElements(), + indexer.numJavacErrors ) System.out.printf("Processed in %.0fms", (System.nanoTime() - start) / 1e6) } \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/ProjectIndexer.kt b/src/main/kotlin/lsifjava/ProjectIndexer.kt index b836b834..d7f1b836 100644 --- a/src/main/kotlin/lsifjava/ProjectIndexer.kt +++ b/src/main/kotlin/lsifjava/ProjectIndexer.kt @@ -1,9 +1,5 @@ package lsifjava -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - class ProjectIndexer(private val arguments: Arguments, private val emitter: Emitter) { var numFiles = 0 var numDefinitions = 0 @@ -13,7 +9,7 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit emitter.emitVertex("metaData", mapOf( "version" to "0.4.0", "positionEncoding" to "utf-16", - "projectRoot" to String.format("file://%s", Paths.get(arguments.projectRoot).toFile().canonicalPath), + "projectRoot" to "file://${arguments.projectRoot}", "toolInfo" to mapOf("name" to "lsif-java", "version" to "0.7.0") )) @@ -21,40 +17,32 @@ class ProjectIndexer(private val arguments: Arguments, private val emitter: Emit "kind" to "java" )) - val indexers = HashMap() - val collector = FileCollector(projectId, arguments, emitter, indexers) - GradleInterface(arguments.projectRoot).use { gradleInterface -> - val classpaths = gradleInterface.getClasspaths() - val javaSourceVersions = gradleInterface.javaSourceVersions() - gradleInterface.getSourceDirectories().forEachIndexed { i, paths -> - collector.classpath = classpaths[i] - collector.javaSourceVersion = javaSourceVersions[i] - paths.forEach { - if (Files.notExists(it)) return@forEach - Files.walkFileTree(it, collector) - } + createJavacDiagnosticListener(arguments.javacOutWriter).use { javacDiagListener -> + val indexers = GradleInterface(arguments.projectRoot).use { + buildIndexerMap(it, emitter, arguments.verbose, javacDiagListener) } - } - val fileManager = SourceFileManager(indexers.keys) + val fileManager = SourceFileManager(indexers.keys) - for (indexer in indexers.values) { - indexer.preIndex(fileManager) - } - for (indexer in indexers.values) { - indexer.index() - } + // can we do this earlier??? + for (indexer in indexers.values) { + indexer.fileManager = fileManager + } - for (indexer in indexers.values) { - indexer.postIndex() - } + for (indexer in indexers.values) { + indexer.index() + } - for (indexer in indexers.values) { - numFiles++ - numDefinitions += indexer.numDefinitions() - numJavacErrors += indexer.javacDiagnostics.size - } - } -} + for (indexer in indexers.values) { + indexer.postIndex(projectId) + } + for (indexer in indexers.values) { + numFiles++ + numDefinitions += indexer.numDefinitions + } + numJavacErrors = javacDiagListener.count + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/lsifjava/SourceFileManager.kt b/src/main/kotlin/lsifjava/SourceFileManager.kt index 9eb32872..2aca2938 100644 --- a/src/main/kotlin/lsifjava/SourceFileManager.kt +++ b/src/main/kotlin/lsifjava/SourceFileManager.kt @@ -16,7 +16,11 @@ import javax.tools.StandardLocation class SourceFileManager internal constructor(private val paths: Set): JavacFileManager(Context(), false, Charset.defaultCharset()) { override fun list( - location: JavaFileManager.Location, packageName: String, kinds: Set, recurse: Boolean): Iterable { + location: JavaFileManager.Location, + packageName: String, + kinds: Set, + recurse: Boolean + ): Iterable { return if (location === StandardLocation.SOURCE_PATH) { val sourceFileObjectStream = list(packageName) Iterable { sourceFileObjectStream.iterator() } @@ -26,9 +30,9 @@ class SourceFileManager internal constructor(private val paths: Set): Java } private fun list(packageName: String): Stream { - return paths.stream().map { path: Path? -> + return paths.stream().map { path: Path -> try { - return@map SourceFileObject(path!!) + return@map SourceFileObject(path) } catch (e: IOException) { return@map null } @@ -52,9 +56,7 @@ class SourceFileManager internal constructor(private val paths: Set): Java if(it == -1) fileName else fileName.substring(0, it) } - override fun hasLocation(location: JavaFileManager.Location): Boolean { - return location === StandardLocation.SOURCE_PATH || super.hasLocation(location) - } + override fun hasLocation(location: JavaFileManager.Location) = location === StandardLocation.SOURCE_PATH || super.hasLocation(location) override fun getJavaFileForInput(location: JavaFileManager.Location, className: String, kind: JavaFileObject.Kind): JavaFileObject? { // FileStore shadows disk diff --git a/src/main/kotlin/lsifjava/UtilTypes.kt b/src/main/kotlin/lsifjava/UtilTypes.kt new file mode 100644 index 00000000..2ba7501d --- /dev/null +++ b/src/main/kotlin/lsifjava/UtilTypes.kt @@ -0,0 +1,15 @@ +package lsifjava + +import java.io.Writer +import java.nio.file.Path + +inline class CanonicalPath(val path: Path) { + override fun toString(): String = path.toFile().canonicalPath +} + +// OutputStreamWriter.nullWriter() CANT USE IN JAVA 8 :( +object NoopWriter: Writer() { + override fun close() = Unit + override fun flush() = Unit + override fun write(p0: CharArray, p1: Int, p2: Int) = Unit +}