BSP: Improve support for mill-build module and the build.sc (#2291)

This change improves the editing experience for `build.sc` files in Metals. We now rudimentary detect ammonite includes (`$file` and `$ivy`). We somewhat break the BSP protocol now, as we return `sources` as paths to single files instead of directories. It is known, that IDEA does not handle it well, but Metals seems to have no problem with it. We don't report directories as the `build.sc` is located in the project top-level directory, and it would contain all other project files too, which is definitely not what we want. Also I refined the module dependencies: `compileIvyDeps` now returns Mill-provided fixed dependencies, `ivyDeps` returns all extra resources included via Ammonite `$ivy` imports, and `scalacPluginIvyDeps` now includes Mill own compiler plugins which among other things make the `override` keyword optional. I also restructured some source files in `mill.scalalib.bsp` package. Fixed the overridden-check for `compiledClassesAndSemanticDbFiles`. This should significantly speed things up for larger projects. Pull request: #2291
com-lihaoyi · Jan 29, 2023 · b6c53f4 · b6c53f4
2 parents fad7b62 + 1fb5d43
commit b6c53f4
Show file tree

Hide file tree

Showing 12 changed files with 299 additions and 161 deletions.
diff --git a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala
@@ -62,8 +62,9 @@ import mill.define.{BaseModule, Discover, ExternalModule, Module, Segments, Task
 import mill.eval.Evaluator
 import mill.main.{BspServerResult, EvaluatorScopt, MainModule}
 import mill.scalalib.{JavaModule, SemanticDbJavaModule, TestModule}
-import mill.scalalib.bsp.{BspModule, JvmBuildTarget, MillBuildTarget, ScalaBuildTarget}
+import mill.scalalib.bsp.{BspModule, JvmBuildTarget, MillBuildModule, ScalaBuildTarget}
 import mill.scalalib.internal.ModuleUtils
+import os.Path
 
 import java.io.PrintStream
 import java.util.concurrent.CompletableFuture
@@ -105,7 +106,7 @@ class MillBuildServer(
             val modules: Seq[Module] =
               ModuleUtils.transitiveModules(evaluator.rootModule) ++ Seq(`mill-build`)
             val map = modules.collect {
-              case m: MillBuildTarget =>
+              case m: MillBuildModule =>
                 val uri = sanitizeUri(m.millSourcePath) +
                   m.bspBuildTarget.displayName.map(n => s"?id=${n}").getOrElse("")
                 val id = new BuildTargetIdentifier(uri)
@@ -129,9 +130,9 @@ class MillBuildServer(
 
     }
 
-    lazy val `mill-build`: MillBuildTarget = {
-      object `mill-build` extends MillBuildTarget {
-        override protected def rootModule: BaseModule = evaluator.rootModule
+    lazy val `mill-build`: MillBuildModule = {
+      object `mill-build` extends MillBuildModule {
+        override protected def projectPath: Path = evaluator.rootModule.millSourcePath
       }
       `mill-build`
     }
@@ -154,8 +155,6 @@ class MillBuildServer(
   private[this] var statePromise: Promise[State] = Promise[State]()
   initialEvaluator.foreach(e => statePromise.success(new State(e)))
 
-//  private[this] def stateFuture: Future[State] = statePromise.future
-
   def updateEvaluator(evaluator: Option[Evaluator]): Unit = {
     log.debug(s"Updating Evaluator: ${evaluator}")
     if (statePromise.isCompleted) {
@@ -374,7 +373,7 @@ class MillBuildServer(
         targetIds = sourcesParams.getTargets.asScala.toSeq,
         agg = (items: Seq[SourcesItem]) => new SourcesResult(items.asJava)
       ) {
-        case (id, module: MillBuildTarget) if clientIsIntelliJ =>
+        case (id, module: MillBuildModule) if clientIsIntelliJ =>
           T.task {
             val sources = new SourcesItem(
               id,
@@ -733,28 +732,29 @@ class MillBuildServer(
   )(f: State => V): CompletableFuture[V] = {
     log.debug(s"Entered ${hint}")
     val start = System.currentTimeMillis()
+    val prefix = hint.split(" ").head
     def took =
-      log.debug(s"${hint.split("[ ]").head} took ${System.currentTimeMillis() - start} msec")
+      log.debug(s"${prefix} took ${System.currentTimeMillis() - start} msec")
 
     val future = new CompletableFuture[V]()
 
     if (checkInitialized && !initialized) {
       future.completeExceptionally(
-        new Exception("Can not respond to any request before receiving the `initialize` request.")
+        new Exception(s"Can not respond to ${prefix} request before receiving the `initialize` request.")
       )
     } else {
       statePromise.future.onComplete {
         case Success(state) =>
           try {
             val v = f(state)
-            log.debug(s"${hint.split("[ ]").head} result: ${v}")
             took
+            log.debug(s"${prefix} result: ${v}")
             future.complete(v)
           } catch {
             case e: Exception =>
-              logStream.println(s"Caught exception: ${e}")
-              e.printStackTrace(logStream)
               took
+              logStream.println(s"${prefix} caught exception: ${e}")
+              e.printStackTrace(logStream)
               future.completeExceptionally(e)
           }
         case Failure(exception) =>
@@ -772,26 +772,27 @@ class MillBuildServer(
   )(f: => V): CompletableFuture[V] = {
     log.debug(s"Entered ${hint}")
     val start = System.currentTimeMillis()
+    val prefix = hint.split(" ").head
     def took =
-      log.debug(s"${hint.split("[ ]").head} took ${System.currentTimeMillis() - start} msec")
+      log.debug(s"${prefix} took ${System.currentTimeMillis() - start} msec")
 
     val future = new CompletableFuture[V]()
 
     if (checkInitialized && !initialized) {
       future.completeExceptionally(
-        new Exception("Can not respond to any request before receiving the `initialize` request.")
+        new Exception(s"Can not respond to ${prefix} request before receiving the `initialize` request.")
       )
     } else {
       try {
         val v = f
-        log.debug(s"${hint.split("[ ]").head} result: ${v}")
         took
+        log.debug(s"${prefix} result: ${v}")
         future.complete(v)
       } catch {
         case e: Exception =>
-          logStream.println(s"Caugh exception: ${e}")
-          e.printStackTrace(logStream)
           took
+          logStream.println(s"${prefix} caught exception: ${e}")
+          e.printStackTrace(logStream)
           future.completeExceptionally(e)
       }
     }

diff --git a/bsp/worker/src/mill/bsp/worker/MillJavaBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillJavaBuildServer.scala
@@ -10,6 +10,7 @@ import scala.jdk.CollectionConverters._
 
 @internal
 trait MillJavaBuildServer extends JavaBuildServer { this: MillBuildServer =>
+
   override def buildTargetJavacOptions(javacOptionsParams: JavacOptionsParams)
       : CompletableFuture[JavacOptionsResult] =
     completable(s"buildTargetJavacOptions ${javacOptionsParams}") { state =>

diff --git a/build.sc b/build.sc
@@ -148,7 +148,8 @@ object Deps {
   val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.19.0"
   val osLib = ivy"com.lihaoyi::os-lib:0.9.0"
   val millModuledefsVersion = "0.10.9"
-  val millModuledefs = ivy"com.lihaoyi::mill-moduledefs:${millModuledefsVersion}"
+  val millModuledefsString = s"com.lihaoyi::mill-moduledefs:${millModuledefsVersion}"
+  val millModuledefs = ivy"${millModuledefsString}"
   val millModuledefsPlugin =
     ivy"com.lihaoyi:::scalac-mill-moduledefs-plugin:${millModuledefsVersion}"
   val testng = ivy"org.testng:testng:7.5"
@@ -418,6 +419,10 @@ object main extends MillModule {
            |  val millEmbeddedDeps = ${artifacts.map(artifact =>
             s""""${artifact.group}:${artifact.id}:${artifact.version}""""
           )}
+           |  /** Scalac compiler plugin dependencies to compile the build script. */
+           |  val millScalacPluginDeps = Seq(
+           |    "${Deps.millModuledefsString}"
+           |  )
            |  /** Mill documentation url. */
            |  val millDocUrl = "${Settings.docUrl}"
            |}

diff --git a/scalalib/src/mill/scalalib/SemanticDbJavaModule.scala b/scalalib/src/mill/scalalib/SemanticDbJavaModule.scala
@@ -224,7 +224,7 @@ trait SemanticDbJavaModule extends CoursierModule { hostModule: JavaModule =>
   }
 
   // keep in sync with bspCompiledClassesAndSemanticDbFiles
-  def compiledClassesAndSemanticDbFiles = T {
+  def compiledClassesAndSemanticDbFiles: Target[PathRef] = T {
     val dest = T.dest
     val classes = compile().classes.path
     val sems = semanticDbData().path
@@ -236,7 +236,7 @@ trait SemanticDbJavaModule extends CoursierModule { hostModule: JavaModule =>
   // keep in sync with compiledClassesAndSemanticDbFiles
   def bspCompiledClassesAndSemanticDbFiles: Target[UnresolvedPath] = {
     if (
-      compile.ctx.enclosing == s"${classOf[SemanticDbJavaModule].getName}#compiledClassesAndSemanticDbFiles"
+      compiledClassesAndSemanticDbFiles.ctx.enclosing == s"${classOf[SemanticDbJavaModule].getName}#compiledClassesAndSemanticDbFiles"
     ) {
       T {
         T.log.debug(

diff --git a/scalalib/src/mill/scalalib/bsp/BspBuildTarget.scala b/scalalib/src/mill/scalalib/bsp/BspBuildTarget.scala
@@ -0,0 +1,12 @@
+package mill.scalalib.bsp
+
+case class BspBuildTarget(
+    displayName: Option[String],
+    baseDirectory: Option[os.Path],
+    tags: Seq[String],
+    languageIds: Seq[String],
+    canCompile: Boolean,
+    canTest: Boolean,
+    canRun: Boolean,
+    canDebug: Boolean
+)
diff --git a/scalalib/src/mill/scalalib/bsp/BspModule.scala b/scalalib/src/mill/scalalib/bsp/BspModule.scala
@@ -26,18 +26,26 @@ trait BspModule extends Module {
     canDebug = false
   )
 
-  /** Use to populate the `BuildTarget.{dataKind,data}` fields. */
+  /**
+   * Use to populate the `BuildTarget.{dataKind,data}` fields.
+   *
+   * Mill specific implementations:
+   * - [[JvmBuildTarget]]
+   * - [[ScalaBuildTarget]]
+   */
   @internal
   def bspBuildTargetData: Task[Option[(String, AnyRef)]] = T.task { None }
 
 }
 
 object BspModule {
+  /** Used to define the [[BspBuildTarget.languageIds]] field. */
   object LanguageId {
     val Java = "java"
     val Scala = "scala"
   }
 
+  /** Used to define the [[BspBuildTarget.tags]] field. */
   object Tag {
     val Library = "library"
     val Application = "application"
@@ -48,142 +56,3 @@ object BspModule {
     val Manual = "manual"
   }
 }
-
-case class BspBuildTarget(
-    displayName: Option[String],
-    baseDirectory: Option[os.Path],
-    tags: Seq[String],
-    languageIds: Seq[String],
-    canCompile: Boolean,
-    canTest: Boolean,
-    canRun: Boolean,
-    canDebug: Boolean
-)
-
-case class BspBuildTargetId(id: BspUri)
-
-case class BspUri(uri: String)
-
-object BspUri {
-  def apply(path: os.Path): BspUri = BspUri(path.toNIO.toUri.toString)
-}
-
-case class JvmBuildTarget(
-    javaHome: Option[BspUri],
-    javaVersion: Option[String]
-)
-
-object JvmBuildTarget {
-  val dataKind: String = "jvm"
-}
-
-case class ScalaBuildTarget(
-    /** The Scala organization that is used for a target. */
-    scalaOrganization: String,
-    /** The scala version to compile this target */
-    scalaVersion: String,
-    /**
-     * The binary version of scalaVersion.
-     * For example, 2.12 if scalaVersion is 2.12.4.
-     */
-    scalaBinaryVersion: String,
-    /** The target platform for this target */
-    platform: ScalaPlatform,
-    /** A sequence of Scala jars such as scala-library, scala-compiler and scala-reflect. */
-    jars: Seq[String],
-    /** The jvm build target describing jdk to be used */
-    jvmBuildTarget: Option[JvmBuildTarget]
-)
-
-object ScalaBuildTarget {
-  val dataKind: String = "scala"
-}
-
-abstract class ScalaPlatform(val number: Int)
-object ScalaPlatform {
-  case object JVM extends ScalaPlatform(1)
-  case object JS extends ScalaPlatform(2)
-  case object Native extends ScalaPlatform(3)
-}
-
-/**
- * Synthetic module representing the mill-build project itself in a BSP context.
- * @param rootModule
- * @param outerCtx0
- */
-@internal
-trait MillBuildTarget
-    extends ScalaModule {
-  protected def rootModule: BaseModule
-  override def millSourcePath: os.Path = rootModule.millSourcePath
-  override def scalaVersion: T[String] = BuildInfo.scalaVersion
-  override def compileIvyDeps: T[Agg[Dep]] = T {
-    T.log.errorStream.println(s"ivyDeps: ${T.dest}")
-    Agg.from(BuildInfo.millEmbeddedDeps.map(d => ivy"${d}"))
-  }
-
-  /**
-   * We need to add all resources from Ammonites cache,
-   * which typically also include resolved `ivy`-imports and compiled `$file`-imports.
-   */
-  override def unmanagedClasspath: T[Agg[PathRef]] = T {
-    super.unmanagedClasspath() ++ (
-      rootModule.getClass.getClassLoader match {
-        case cl: SpecialClassLoader =>
-          cl.allJars.map(url => PathRef(os.Path(java.nio.file.Paths.get(url.toURI))))
-        case _ => Seq()
-      }
-    )
-  }
-
-  // The buildfile and single source of truth
-  def buildScFile = T.source(millSourcePath / "build.sc")
-  def ammoniteFiles = T {
-    T.log.errorStream.println(s"ammoniteFiles: ${T.dest}")
-    // we depend on buildScFile, to recompute whenever build.sc changes
-    findSources(Seq(millSourcePath), excludes = Seq(millSourcePath / "out"))
-  }
-  // We need to be careful here to not include the out/ directory
-  override def sources: Sources = T.sources {
-    T.log.errorStream.println(s"sources: ${T.dest}")
-    val sources = ammoniteFiles()
-    T.log.errorStream.println(s"sources: ${sources}")
-    sources
-  }
-  override def allSourceFiles: T[Seq[PathRef]] = T {
-    findSources(sources().map(_.path))
-  }
-  def findSources(paths: Seq[os.Path], excludes: Seq[os.Path] = Seq()): Seq[PathRef] = {
-    def isHiddenFile(path: os.Path) = path.last.startsWith(".")
-    (for {
-      root <- paths
-      if os.exists(root) && !excludes.exists(excl => root.startsWith(excl))
-      path <- if (os.isDir(root)) os.walk(root) else Seq(root)
-      if os.isFile(path) && ((path.ext == "sc") && !isHiddenFile(path))
-    } yield PathRef(path)).distinct
-  }
-  override def bspBuildTarget: BspBuildTarget = super.bspBuildTarget.copy(
-    displayName = Some("mill-build"),
-    baseDirectory = Some(rootModule.millSourcePath),
-    languageIds = Seq(BspModule.LanguageId.Scala),
-    canRun = false,
-    canCompile = false,
-    canTest = false,
-    canDebug = false,
-    tags = Seq(BspModule.Tag.Library, BspModule.Tag.Application)
-  )
-  override def compile: T[CompilationResult] = T {
-    T.log.errorStream.println(s"compile: ${T.dest}")
-    os.write(T.dest / "dummy", "")
-    os.makeDir(T.dest / "classes")
-    CompilationResult(T.dest / "dummy", PathRef(T.dest / "classes"))
-  }
-
-  override def semanticDbData: T[PathRef] = T {
-    T.log.errorStream.println(s"semanticDbData: ${T.dest}")
-    PathRef(T.dest)
-  }
-
-  /** Used in BSP IntelliJ, which can only work with directories */
-  def dummySources: Sources = T.sources(T.dest)
-}
diff --git a/scalalib/src/mill/scalalib/bsp/BspUri.scala b/scalalib/src/mill/scalalib/bsp/BspUri.scala
@@ -0,0 +1,7 @@
+package mill.scalalib.bsp
+
+case class BspUri(uri: String)
+
+object BspUri {
+  def apply(path: os.Path): BspUri = BspUri(path.toNIO.toUri.toString)
+}