Go to definition in classpath

build.sbt

@@ -26,6 +26,7 @@ lazy val metaserver = project
       "io.monix" %% "monix" % "2.3.0",
       "com.lihaoyi" %% "pprint" % "0.5.3",
       "com.github.javaparser" % "javaparser-core" % "3.4.3",
+      "com.thoughtworks.qdox" % "qdox" % "2.0-M7",
       "io.get-coursier" %% "coursier" % coursier.util.Properties.version,
       "io.get-coursier" %% "coursier-cache" % coursier.util.Properties.version,
       "ch.epfl.scala" % "scalafix-cli" % "0.5.3" cross CrossVersion.full,

metaserver/src/main/scala/scala/meta/languageserver/Compiler.scala

@@ -1,6 +1,5 @@
 package scala.meta.languageserver
 
-import java.io.File
 import java.io.PrintStream
 import java.util.concurrent.ConcurrentHashMap
 import scala.collection.mutable
@@ -12,6 +11,7 @@ import scala.tools.nsc.reporters.StoreReporter
 import com.typesafe.scalalogging.LazyLogging
 import langserver.core.Connection
 import langserver.messages.MessageType
+import monix.eval.Task
 import monix.execution.Scheduler
 import monix.reactive.MulticastStrategy
 import monix.reactive.Observable
@@ -30,13 +30,15 @@ class Compiler(
   val documentPublisher: Observable[Document] = myDocumentPublisher
   private val indexedJars: ConcurrentHashMap[AbsolutePath, Unit] =
     new ConcurrentHashMap[AbsolutePath, Unit]()
-  val onNewCompilerConfig: Observable[Unit] =
+  val onNewCompilerConfig: Observable[
+    (Effects.InstallPresentationCompiler, Effects.IndexSourcesClasspath)
+  ] =
     config
       .map(path => CompilerConfig.fromPath(path))
       .flatMap { config =>
-        Observable.merge(
-          Observable.delay(loadNewCompilerGlobals(config)),
-          Observable.delay(indexDependencyClasspath(config))
+        Observable.fromTask(
+          Task(loadNewCompilerGlobals(config))
+            .zip(Task(indexDependencyClasspath(config)))
         )
       }
 
@@ -79,7 +81,9 @@ class Compiler(
   }
 
   private val compilerByPath = mutable.Map.empty[AbsolutePath, Global]
-  private def loadNewCompilerGlobals(config: CompilerConfig): Unit = {
+  private def loadNewCompilerGlobals(
+      config: CompilerConfig
+  ): Effects.InstallPresentationCompiler = {
     logger.info(s"Loading new compiler from config $config")
     val vd = new io.VirtualDirectory("(memory)", None)
     val settings = new Settings
@@ -93,10 +97,13 @@ class Compiler(
       // TODO(olafur) garbage collect compilers from removed files.
       compilerByPath(path) = compiler
     }
+    Effects.InstallPresentationCompiler
   }
-  private def indexDependencyClasspath(config: CompilerConfig): Unit = {
+  private def indexDependencyClasspath(
+      config: CompilerConfig
+  ): Effects.IndexSourcesClasspath = {
     val buf = List.newBuilder[AbsolutePath]
-    val sourceJars = Jars.fetch(config.libraryDependencies, out, sources = true)
+    val sourceJars = config.sourceJars
     sourceJars.foreach { jar =>
       // ensure we only index each jar once even under race conditions.
       indexedJars.computeIfAbsent(
@@ -107,12 +114,14 @@ class Compiler(
     val sourcesClasspath = buf.result()
     if (sourcesClasspath.nonEmpty) {
       logger.info(
-        s"Indexing classpath ${sourcesClasspath.mkString(File.pathSeparator)}"
+        s"Indexing classpath with ${sourcesClasspath.length} entries..."
       )
     }
     ctags.Ctags.index(sourcesClasspath) { doc =>
       documentSubscriber.onNext(doc)
     }
+    import scala.collection.JavaConverters._
+    Effects.IndexSourcesClasspath
   }
   private def noCompletions: List[(String, String)] = {
     connection.showMessage(

metaserver/src/main/scala/scala/meta/languageserver/CompilerConfig.scala

@@ -5,38 +5,78 @@ import java.nio.file.Files
 import java.util.Properties
 import com.typesafe.scalalogging.LazyLogging
 import org.langmeta.io.AbsolutePath
+import org.langmeta.io.Classpath
 
+/**
+ * Configuration to load up a presentation compiler.
+ *
+ * In sbt, one compiler config typically corresponds to one project+config.
+ * For example one sbt project with test/main/it configurations has three
+ * CompilerConfig.
+ *
+ * @param sources list of source files for this project
+ * @param scalacOptions space separated list of flags to pass to the Scala compiler
+ * @param dependencyClasspath File.pathSeparated list of *.jar and classDirectories.
+ *                  Includes both dependencyClasspath and classDirectory.
+ * @param classDirectory The output directory where *.class files are emitted
+ *                       for this project.
+ * @param sourceJars File.pathSeparated list of *-sources.jar from the
+ *                   dependencyClasspath.
+ */
 case class CompilerConfig(
     sources: List[AbsolutePath],
     scalacOptions: List[String],
-    classpath: String,
-    libraryDependencies: List[ModuleID]
-)
+    classDirectory: AbsolutePath,
+    dependencyClasspath: List[AbsolutePath],
+    sourceJars: List[AbsolutePath]
+) {
+  override def toString: String =
+    s"CompilerConfig(" +
+      s"sources={+${sources.length}}, " +
+      s"scalacOptions=${scalacOptions.mkString(" ")}, " +
+      s"dependencyClasspath={+${dependencyClasspath.length}}, " +
+      s"classDirectory=$classDirectory, " +
+      s"sourceJars={+${sourceJars.length}})"
+  def classpath: String =
+    (classDirectory :: dependencyClasspath).mkString(File.pathSeparator)
+}
 
 object CompilerConfig extends LazyLogging {
+
   def fromPath(
       path: AbsolutePath
   )(implicit cwd: AbsolutePath): CompilerConfig = {
     val input = Files.newInputStream(path.toNIO)
     try {
       val props = new Properties()
       props.load(input)
-      val sources = props
-        .getProperty("sources")
-        .split(File.pathSeparator)
-        .iterator
-        .map(AbsolutePath(_))
-        .toList
-      val scalacOptions = props.getProperty("scalacOptions").split(" ").toList
-      val classpath = props.getProperty("classpath")
-      val libraryDependencies =
-        ModuleID.fromString(props.getProperty("libraryDependencies"))
-      CompilerConfig(
-        sources,
-        scalacOptions,
-        classpath,
-        libraryDependencies.toList
-      )
+      fromProperties(props)
     } finally input.close()
   }
+
+  def fromProperties(
+      props: Properties
+  )(implicit cwd: AbsolutePath): CompilerConfig = {
+    val sources = props
+      .getProperty("sources")
+      .split(File.pathSeparator)
+      .iterator
+      .map(AbsolutePath(_))
+      .toList
+    val scalacOptions =
+      props.getProperty("scalacOptions").split(" ").toList
+    val dependencyClasspath =
+      Classpath(props.getProperty("dependencyClasspath")).shallow
+    val sourceJars =
+      Classpath(props.getProperty("sourceJars")).shallow
+    val classDirectory =
+      AbsolutePath(props.getProperty("classDirectory"))
+    CompilerConfig(
+      sources,
+      scalacOptions,
+      classDirectory,
+      dependencyClasspath,
+      sourceJars
+    )
+  }
 }

metaserver/src/main/scala/scala/meta/languageserver/Effects.scala

@@ -0,0 +1,18 @@
+package scala.meta.languageserver
+
+/**
+ * The ScalametaLanguageServer effects.
+ *
+ * Observable[Unit] is not descriptive of what the observable represents.
+ * Instead, we create Unit-like types to better document what effects are
+ * flowing through our application.
+ */
+sealed abstract class Effects
+object Effects {
+  final class IndexSemanticdb extends Effects
+  final val IndexSemanticdb = new IndexSemanticdb
+  final class IndexSourcesClasspath extends Effects
+  final val IndexSourcesClasspath = new IndexSourcesClasspath
+  final class InstallPresentationCompiler extends Effects
+  final val InstallPresentationCompiler = new InstallPresentationCompiler
+}

metaserver/src/main/scala/scala/meta/languageserver/SymbolIndexer.scala

@@ -16,7 +16,7 @@ import langserver.messages.MessageType
 
 // NOTE(olafur) it would make a lot of sense to use tries where Symbol is key.
 class SymbolIndexer(
-    val indexer: Observable[Unit],
+    val indexer: Observable[Effects.IndexSemanticdb],
     logger: Logger,
     connection: Connection,
     buffers: Buffers,
@@ -259,7 +259,10 @@ object SymbolIndexer {
       )
     }
 
-    val indexer = semanticdbs.map(db => db.documents.foreach(indexDocument))
+    val indexer = semanticdbs.map { db =>
+      db.documents.foreach(indexDocument)
+      Effects.IndexSemanticdb
+    }
 
     new SymbolIndexer(
       indexer,

metaserver/src/main/scala/scala/meta/languageserver/ctags/Ctags.scala

@@ -7,10 +7,16 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.SimpleFileVisitor
 import java.nio.file.attribute.BasicFileAttributes
+import java.text.DecimalFormat
+import java.text.NumberFormat
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicInteger
 import java.util.zip.ZipInputStream
 import scala.collection.GenSeq
 import scala.collection.parallel.mutable.ParArray
 import scala.meta.parsers.ParseException
+import scala.reflect.ClassTag
+import scala.util.Sorting
 import scala.util.control.NonFatal
 import com.typesafe.scalalogging.LazyLogging
 import org.langmeta.inputs.Input
@@ -37,22 +43,47 @@ object Ctags extends LazyLogging {
    * Build an index from a classpath of -sources.jar
    *
    * @param shouldIndex An optional filter to skip particular relative filenames.
-   * @param inParallel If true, use parallel collection to index using all
-   *                   available CPU power. If false, uses single-threaded
-   *                   collection.
    * @param callback A callback that is called as soon as a document has been
    *                 indexed.
    */
   def index(
       classpath: List[AbsolutePath],
-      shouldIndex: RelativePath => Boolean = _ => true,
-      inParallel: Boolean = true
+      shouldIndex: RelativePath => Boolean = _ => true
   )(callback: Document => Unit): Unit = {
-    val fragments = allClasspathFragments(classpath, inParallel)
+    val fragments = allClasspathFragments(classpath)
+    val totalIndexedFiles = new AtomicInteger()
+    val totalIndexedLines = new AtomicInteger()
+    val start = System.nanoTime()
+    def elapsed: Long =
+      TimeUnit.SECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS)
+    val decimal = new DecimalFormat("###.###")
+    val N = fragments.length
+    def updateTotalLines(doc: Document): Unit = doc.input match {
+      case Input.VirtualFile(_, contents) =>
+        // NOTE(olafur) it would be interesting to have separate statistics for
+        // Java/Scala lines/s but that would require a more sophisticated setup.
+        totalIndexedLines.addAndGet(countLines(contents))
+      case _ =>
+    }
+    def reportProgress(indexedFiles: Int): Unit = {
+      val percentage = ((indexedFiles / N.toDouble) * 100).toInt
+      val loc = decimal.format(totalIndexedLines.get() / elapsed)
+      logger.info(
+        s"Progress $percentage%, ${decimal.format(indexedFiles)} files indexed " +
+          s"out of total ${decimal.format(fragments.length)} ($loc loc/s)"
+      )
+    }
+    logger.info(s"Indexing $N source files")
     fragments.foreach { fragment =>
       try {
+        val indexedFiles = totalIndexedFiles.incrementAndGet()
+        if (indexedFiles % 200 == 0) {
+          reportProgress(indexedFiles)
+        }
         if (shouldIndex(fragment.name)) {
-          callback(index(fragment))
+          val doc = index(fragment)
+          updateTotalLines(doc)
+          callback(doc)
         }
       } catch {
         case _: ParseException => // nothing
@@ -79,7 +110,7 @@ object Ctags extends LazyLogging {
     logger.trace(s"Indexing ${input.path} with length ${input.value.length}")
     val indexer: CtagsIndexer =
       if (isScala(input.path)) ScalaCtags.index(input)
-      else if (isJava(input.path)) JavaCtags.index(input)
+      else if (isJava(input.path)) QDoxCtags.index(input)
       else {
         throw new IllegalArgumentException(
           s"Unknown file extension ${input.path}"
@@ -96,7 +127,9 @@ object Ctags extends LazyLogging {
     )
   }
 
-  private def canIndex(path: String): Boolean = isScala(path) || isJava(path)
+  private def canIndex(path: String): Boolean =
+//    isScala(path) ||
+    isJava(path)
   private def isJava(path: String): Boolean = path.endsWith(".java")
   private def isScala(path: String): Boolean = path.endsWith(".scala")
   private def isScala(path: Path): Boolean = PathIO.extension(path) == "scala"
@@ -112,11 +145,8 @@ object Ctags extends LazyLogging {
    */
   private def allClasspathFragments(
       classpath: List[AbsolutePath],
-      inParallel: Boolean
-  ): GenSeq[Fragment] = {
-    var buf =
-      if (inParallel) ParArray.newBuilder[Fragment]
-      else List.newBuilder[Fragment]
+  ): ParArray[Fragment] = {
+    var buf = ParArray.newBuilder[Fragment]
     classpath.foreach { base =>
       def exploreJar(base: AbsolutePath): Unit = {
         val stream = Files.newInputStream(base.toNIO)
@@ -167,6 +197,27 @@ object Ctags extends LazyLogging {
         // Skip
       }
     }
-    buf.result()
+    val result = buf.result()
+    Sorting.stableSort(result.arrayseq)(
+      implicitly[ClassTag[Fragment]],
+      Ordering.by { fragment =>
+        PathIO.extension(fragment.name.toNIO) match {
+          case "scala" => 1
+          case "java" => 2
+          case _ => 3
+        }
+      }
+    )
+    result
+  }
+
+  private def countLines(string: String): Int = {
+    var i = 0
+    var lines = 0
+    while (i < string.length) {
+      if (string.charAt(i) == '\n') lines += 1
+      i += 1
+    }
+    lines
   }
 }

metaserver/src/main/scala/scala/meta/languageserver/ctags/CtagsIndexer.scala

@@ -10,6 +10,9 @@ trait CtagsIndexer {
     indexRoot()
     names.result() -> symbols.result()
   }
+  def owner(isStatic: Boolean): Symbol.Global =
+    if (isStatic) currentOwner.toTerm
+    else currentOwner
   def withOwner[A](owner: Symbol.Global = currentOwner)(thunk: => A): A = {
     val old = currentOwner
     currentOwner = owner