Improved MillBuildModule and parsing; disabled using directives

Using directives parser is currently not able to handle larger files,
see VirtusLab/using_directives#41

Also improved BSP support.
BSP mill-build modules should now be able to compile build files.

main/buildfile/src/ammoniteParser.scala

@@ -1,41 +1,64 @@
 package mill.buildfile
 
+import java.util.regex.Pattern
 import scala.collection.mutable
+import scala.util.matching.Regex
 
 object AmmoniteParser {
 
-  def parseAddionalAmmoniteImports(parsedMillSetup: ParsedMillSetup) = {
+  private val MatchDep = """^import ([$]ivy[.]`([^`]+)`)""".r
+  private val MatchFile: Regex = """^import ([$]file[.]([^,]+)).*""".r
+
+  def parseAmmoniteImports(parsedMillSetup: ParsedMillSetup): ParsedMillSetup = {
     val queue = mutable.Queue.empty[os.Path]
     queue.enqueueAll(parsedMillSetup.buildScript.toSeq ++ parsedMillSetup.includedSourceFiles)
 
-    val seenFiles = mutable.Set.empty[os.Path]
-    val newDirectives = mutable.Seq.empty[MillUsingDirective]
+    var seenFiles = Set.empty[os.Path]
+    var newDirectives = Seq.empty[MillUsingDirective]
 
     while (queue.nonEmpty) {
       val file = queue.dequeue()
-      val MatchDep = """^import [$]ivy[.]`([^`]+)`""".r
-      val MatchFile = """^import [$]file[.]([^,]+)""".r
       val directives = os.read.lines(file).collect {
-        case MatchDep(dep) => MillUsingDirective.Dep(dep, file)
-        case MatchFile(include) =>
+        case m @ MatchDep(full, dep) => MillUsingDirective.Dep(dep, file)
+        case m @ MatchFile(full, include) =>
           val pat = include.split("[.]").map {
             case "^" => os.up
             case x => os.rel / x
           }
           val incFile0: os.Path = file / os.up / pat
           val incFile = incFile0 / os.up / s"${incFile0.last}.sc"
 
-          if(!seenFiles.contains(incFile)) {
-            seenFiles.add(incFile)
+          if (!seenFiles.contains(incFile)) {
+            seenFiles += incFile
             queue.enqueue(incFile)
           }
 
           MillUsingDirective.File(incFile.toString(), file)
       }
-      newDirectives.appendedAll(directives)
+
+      newDirectives = newDirectives ++ directives
     }
 
-    parsedMillSetup.copy(directives = (parsedMillSetup.directives ++ newDirectives.toList).distinct)
+    parsedMillSetup.copy(
+      directives = (parsedMillSetup.directives ++ newDirectives).distinct
+    )
+  }
+
+  def replaceAmmoniteImports(parsedMillSetup: ParsedMillSetup): Map[os.Path, Seq[String]] = {
+    (parsedMillSetup.buildScript.toSeq ++ parsedMillSetup.includedSourceFiles)
+      .map(f => (f, f))
+      .toMap.view.mapValues { file =>
+        os.read.lines(file).map {
+          case m @ MatchDep(full, dep) =>
+            m.replaceFirst(Pattern.quote(full), "java.lang.Object")
+
+          case m @ MatchFile(full, include) =>
+            m.replaceFirst(Pattern.quote(full), "java.lang.Object")
+
+          case x => x
+        }
+      }
+      .toMap
   }
 
 }

main/buildfile/src/usingParser.scala

@@ -18,8 +18,8 @@ case class ParsedMillSetup(
     buildScript: Option[os.Path]
 ) {
   lazy val includedSourceFiles: Seq[os.Path] = directives.collect {
-    case MillUsingDirective.File(file, src) => projectDir / file
-  }
+    case MillUsingDirective.File(file, src) => os.Path(file, projectDir)
+  }.distinct
   lazy val millVersion: Option[String] = directives.collect {
     case MillUsingDirective.MillVersion(version, src) => version
   }.headOption

scalalib/src/mill/scalalib/bsp/BspModule.scala

@@ -2,13 +2,13 @@ package mill.scalalib.bsp
 
 import ammonite.runtime.SpecialClassLoader
 import mill.api.{Loose, PathRef, internal}
-import mill.define.{BaseModule, Segments, Sources, Task}
-import mill.eval.EvaluatorPathsResolver
-import mill.modules.Jvm
+import mill.define.{BaseModule, Input, Sources, Target, Task}
 import mill.scalalib.api.CompilationResult
+import mill.scalalib.buildfile.{MillBuildModule, MillSetupScannerModule}
 import mill.scalalib.internal.ModuleUtils
 import mill.scalalib.{Dep, DepSyntax, ScalaModule}
 import mill.{Agg, BuildInfo, Module, T}
+import upickle.default.{ReadWriter, macroRW}
 
 trait BspModule extends Module {
   import BspModule._
@@ -75,15 +75,24 @@ object BspUri {
  * @param outerCtx0
  */
 @internal
-trait MillBuildTarget // (rootModule: BaseModule, ctx0: mill.define.Ctx)
-    extends // Module()(ctx0) with
-    ScalaModule {
+trait MillBuildTarget
+    extends ScalaModule
+    with MillSetupScannerModule {
   protected def rootModule: BaseModule
   override def millSourcePath: os.Path = rootModule.millSourcePath
   override def scalaVersion: T[String] = BuildInfo.scalaVersion
+
+  protected def millEmbeddedDeps: Input[Seq[String]] = T.input {
+    BuildInfo.millEmbeddedDeps
+  }
+
+  override def compileIvyDeps: T[Agg[Dep]] = T {
+    Agg.from(millEmbeddedDeps().map(d => parseDeps(d)))
+  }
+
   override def ivyDeps: T[Agg[Dep]] = T {
-    T.log.errorStream.println(s"ivyDeps: ${T.dest}")
-    Agg.from(BuildInfo.millEmbeddedDeps.map(d => ivy"${d}"))
+    val deps = parsedMillSetup().ivyDeps
+    Agg.from(deps.map(d => parseDeps(d)))
   }
 
   /**
@@ -100,24 +109,22 @@ trait MillBuildTarget // (rootModule: BaseModule, ctx0: mill.define.Ctx)
     )
   }
 
-  // The buildfile and single source of truth
-  def buildScFile = T.source(millSourcePath / "build.sc")
-  def ammoniteFiles = T {
-    T.log.errorStream.println(s"ammoniteFiles: ${T.dest}")
-    // we depend on buildScFile, to recompute whenever build.sc changes
-    findSources(Seq(millSourcePath), excludes = Seq(millSourcePath / "out"))
-  }
-  // We need to be careful here to not include the out/ directory
   override def sources: Sources = T.sources {
-    T.log.errorStream.println(s"sources: ${T.dest}")
-    val sources = ammoniteFiles()
-    T.log.errorStream.println(s"sources: ${sources}")
-    sources
+    val optsFileName =
+      T.env.get("MILL_JVM_OPTS_PATH").filter(!_.isEmpty).getOrElse(".mill-jvm-opts")
+
+    Seq(
+      PathRef(millSourcePath / ".mill-version"),
+      PathRef(millSourcePath / optsFileName),
+      buildScFile()
+    ) ++ includedSourceFiles()
   }
+
   override def allSourceFiles: T[Seq[PathRef]] = T {
     findSources(sources().map(_.path))
   }
-  def findSources(paths: Seq[os.Path], excludes: Seq[os.Path] = Seq()): Seq[PathRef] = {
+
+  protected def findSources(paths: Seq[os.Path], excludes: Seq[os.Path] = Seq()): Seq[PathRef] = {
     def isHiddenFile(path: os.Path) = path.last.startsWith(".")
     (for {
       root <- paths
@@ -126,22 +133,27 @@ trait MillBuildTarget // (rootModule: BaseModule, ctx0: mill.define.Ctx)
       if os.isFile(path) && ((path.ext == "sc") && !isHiddenFile(path))
     } yield PathRef(path)).distinct
   }
+
+  override def scalacPluginIvyDeps: Target[Loose.Agg[Dep]] = T{
+    super.scalacPluginIvyDeps() ++ Agg(Deps.millModuledefs)
+  }
+
   override def bspBuildTarget: BspBuildTarget = super.bspBuildTarget.copy(
     displayName = Some("mill-build"),
     baseDirectory = Some(rootModule.millSourcePath),
     languageIds = Seq(BspModule.LanguageId.Scala),
     canRun = false,
-    canCompile = false,
+//    canCompile = false,
     canTest = false,
     canDebug = false,
     tags = Seq(BspModule.Tag.Library, BspModule.Tag.Application)
   )
-  override def compile: T[CompilationResult] = T {
-    T.log.errorStream.println(s"compile: ${T.dest}")
-    os.write(T.dest / "dummy", "")
-    os.makeDir(T.dest / "classes")
-    CompilationResult(T.dest / "dummy", PathRef(T.dest / "classes"))
-  }
+//  override def compile: T[CompilationResult] = T {
+//    T.log.errorStream.println(s"compile: ${T.dest}")
+//    os.write(T.dest / "dummy", "")
+//    os.makeDir(T.dest / "classes")
+//    CompilationResult(T.dest / "dummy", PathRef(T.dest / "classes"))
+//  }
 
   /** Used in BSP IntelliJ, which can only work with directories */
   def dummySources: Sources = T.sources(T.dest)