Virtualize compile task

main/src/main/scala/sbt/Defaults.scala

@@ -241,7 +241,7 @@ object Defaults extends BuildCommon {
     Seq(
       internalConfigurationMap :== Configurations.internalMap _,
       credentials :== SysProp.sbtCredentialsEnv.toList,
-      exportJars :== false,
+      exportJars :== true,
       trackInternalDependencies :== TrackLevel.TrackAlways,
       exportToInternal :== TrackLevel.TrackAlways,
       useCoursier :== SysProp.defaultUseCoursier,
@@ -946,7 +946,6 @@ object Defaults extends BuildCommon {
       },
       internalDependencyConfigurations := InternalDependencies.configurations.value,
       manipulateBytecode := compileSplit.value,
-      compileIncremental := compileIncrementalTask.tag(Tags.Compile, Tags.CPU).value,
       printWarnings := printWarningsTask.value,
       compileAnalysisFilename := {
         // Here, if the user wants cross-scala-versioning, we also append it
@@ -1041,7 +1040,9 @@ object Defaults extends BuildCommon {
       // note that we use the same runner and mainClass as plain run
       mainBgRunMainTaskForConfig(This),
       mainBgRunTaskForConfig(This)
-    ) ++ inTask(run)(runnerSettings ++ newRunnerSettings)
+    ) ++ inTask(run)(runnerSettings ++ newRunnerSettings) ++ compileIncrementalTaskSettings(
+      compileIncremental
+    )
 
   private[this] lazy val configGlobal = globalDefaults(
     Seq(
@@ -1937,17 +1938,6 @@ object Defaults extends BuildCommon {
       }
     }
 
-  @deprecated("The configuration(s) should not be decided based on the classifier.", "1.0.0")
-  def artifactConfigurations(
-      base: Artifact,
-      scope: Configuration,
-      classifier: Option[String]
-  ): Iterable[Configuration] =
-    classifier match {
-      case Some(c) => Artifact.classifierConf(c) :: Nil
-      case None    => scope :: Nil
-    }
-
   def packageTaskSettings(
       key: TaskKey[HashedVirtualFileRef],
       mappingsTask: Initialize[Task[Seq[(HashedVirtualFileRef, String)]]]
@@ -2370,20 +2360,22 @@ object Defaults extends BuildCommon {
   private[sbt] def compileScalaBackendTask: Initialize[Task[CompileResult]] = Def.task {
     val setup: Setup = compileIncSetup.value
     val useBinary: Boolean = enableBinaryCompileAnalysis.value
-    val analysisResult: CompileResult = compileIncremental.value
+    val _ = compileIncremental.value
     val exportP = exportPipelining.value
     // Save analysis midway if pipelining is enabled
-    if (analysisResult.hasModified && exportP) {
-      val store =
-        MixedAnalyzingCompiler.staticCachedStore(setup.cacheFile.toPath, !useBinary)
-      val contents = AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
-      store.set(contents)
+    val store = MixedAnalyzingCompiler.staticCachedStore(setup.cachePath, !useBinary)
+    val contents = store.unsafeGet()
+    if (exportP) {
       // this stores the eary analysis (again) in case the subproject contains a macro
       setup.earlyAnalysisStore.toOption map { earlyStore =>
         earlyStore.set(contents)
       }
     }
-    analysisResult
+    CompileResult.of(
+      contents.getAnalysis(),
+      contents.getMiniSetup(),
+      contents.getAnalysis().readCompilations().getAllCompilations().nonEmpty
+    )
   }
 
   /**
@@ -2407,6 +2399,7 @@ object Defaults extends BuildCommon {
       compile.value
     }
   }
+
   def compileTask: Initialize[Task[CompileAnalysis]] = Def.task {
     val setup: Setup = compileIncSetup.value
     val useBinary: Boolean = enableBinaryCompileAnalysis.value
@@ -2427,17 +2420,73 @@ object Defaults extends BuildCommon {
     }
     analysis
   }
-  def compileIncrementalTask = Def.task {
-    val s = streams.value
-    val ci = (compile / compileInputs).value
-    val ping = earlyOutputPing.value
-    val reporter = (compile / bspReporter).value
-    BspCompileTask.compute(bspTargetIdentifier.value, thisProjectRef.value, configuration.value) {
-      task =>
-        // TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too?
-        compileIncrementalTaskImpl(task, s, ci, ping, reporter)
-    }
-  }
+
+  def compileIncrementalTaskSettings(key: TaskKey[(Boolean, HashedVirtualFileRef)]) =
+    inTask(key)(
+      Seq(
+        (TaskZero / key) := (Def
+          .cachedTask {
+            val s = streams.value
+            val ci = (compile / compileInputs).value
+            // This is a cacheable version
+            val ci2 = (compile / compileInputs2).value
+            val ping = (TaskZero / earlyOutputPing).value
+            val reporter = (compile / bspReporter).value
+            val setup: Setup = (TaskZero / compileIncSetup).value
+            val useBinary: Boolean = enableBinaryCompileAnalysis.value
+            val c = fileConverter.value
+            val analysisResult: CompileResult =
+              BspCompileTask
+                .compute(bspTargetIdentifier.value, thisProjectRef.value, configuration.value) {
+                  bspTask =>
+                    // TODO - Should readAnalysis + saveAnalysis be scoped by the compile task too?
+                    compileIncrementalTaskImpl(bspTask, s, ci, ping, reporter)
+                }
+            val analysisOut = c.toVirtualFile(setup.cachePath())
+            val store =
+              MixedAnalyzingCompiler.staticCachedStore(setup.cachePath, !useBinary)
+            val contents =
+              AnalysisContents.create(analysisResult.analysis(), analysisResult.setup())
+            store.set(contents)
+            Def.declareOutput(analysisOut)
+            val dir = classDirectory.value
+            if (dir / "META-INF").exists then IO.delete(dir)
+            // inline mappings
+            val mappings = Path
+              .allSubpaths(dir)
+              .filter(_._1.isFile())
+              .map { case (p, path) =>
+                val vf = c.toVirtualFile(p.toPath())
+                (vf: HashedVirtualFileRef) -> path
+              }
+              .toSeq
+            // inlined to avoid caching mappings
+            val pkgConfig = Pkg.Configuration(
+              mappings,
+              artifactPath.value,
+              packageOptions.value,
+            )
+            val out = Pkg(
+              pkgConfig,
+              c,
+              s.log,
+              Pkg.timeFromConfiguration(pkgConfig)
+            )
+            Def.declareOutput(out)
+            analysisResult.hasModified() -> (out: HashedVirtualFileRef)
+          })
+          .tag(Tags.Compile, Tags.CPU)
+          .value,
+        packagedArtifact := {
+          val (hasModified, out) = key.value
+          artifact.value -> out
+        },
+        artifact := artifactSetting.value,
+        artifactClassifier := Some("noresources"),
+        artifactPath := artifactPathSetting(artifact).value,
+      )
+    )
+
   private val incCompiler = ZincUtil.defaultIncrementalCompiler
   private[sbt] def compileJavaTask: Initialize[Task[CompileResult]] = Def.task {
     val s = streams.value
@@ -2459,6 +2508,7 @@ object Defaults extends BuildCommon {
         throw e
     }
   }
+
   private[this] def compileIncrementalTaskImpl(
       task: BspCompileTask,
       s: TaskStreams,
@@ -2467,43 +2517,35 @@ object Defaults extends BuildCommon {
       reporter: BuildServerReporter,
   ): CompileResult = {
     lazy val x = s.text(ExportStream)
-    def onArgs(cs: Compilers) = {
+    def onArgs(cs: Compilers) =
       cs.withScalac(
-        cs.scalac match {
+        cs.scalac match
           case ac: AnalyzingCompiler => ac.onArgs(exported(x, "scalac"))
           case x                     => x
-        }
       )
-    }
-    def onProgress(s: Setup) = {
+    def onProgress(s: Setup) =
       val cp = new BspCompileProgress(task, s.progress.asScala)
       s.withProgress(cp)
-    }
     val compilers: Compilers = ci.compilers
     val setup: Setup = ci.setup
-    val i = ci
-      .withCompilers(onArgs(compilers))
-      .withSetup(onProgress(setup))
-    try {
+    val i = ci.withCompilers(onArgs(compilers)).withSetup(onProgress(setup))
+    try
       val result = incCompiler.compile(i, s.log)
       reporter.sendSuccessReport(result.getAnalysis)
       result
-    } catch {
+    catch
       case e: Throwable =>
-        if (!promise.isCompleted) {
+        if !promise.isCompleted then
           promise.failure(e)
           ConcurrentRestrictions.cancelAllSentinels()
-        }
         reporter.sendFailureReport(ci.options.sources)
-
         throw e
-    } finally {
-      x.close() // workaround for #937
-    }
+    finally x.close() // workaround for #937
   }
+
   def compileIncSetupTask = Def.task {
     val cp = dependencyPicklePath.value
-    val lookup = new PerClasspathEntryLookup {
+    val lookup = new PerClasspathEntryLookup:
       private val cachedAnalysisMap: VirtualFile => Option[CompileAnalysis] =
         analysisMap(cp)
       private val cachedPerEntryDefinesClassLookup: VirtualFile => DefinesClass =
@@ -2512,12 +2554,12 @@ object Defaults extends BuildCommon {
         cachedAnalysisMap(classpathEntry).toOptional
       override def definesClass(classpathEntry: VirtualFile): DefinesClass =
         cachedPerEntryDefinesClassLookup(classpathEntry)
-    }
     val extra = extraIncOptions.value.map(t2)
     val useBinary: Boolean = enableBinaryCompileAnalysis.value
     val eapath = earlyCompileAnalysisFile.value.toPath
     val eaOpt =
-      if (exportPipelining.value) Some(MixedAnalyzingCompiler.staticCachedStore(eapath, !useBinary))
+      if exportPipelining.value then
+        Some(MixedAnalyzingCompiler.staticCachedStore(eapath, !useBinary))
       else None
     Setup.of(
       lookup,
@@ -2531,6 +2573,7 @@ object Defaults extends BuildCommon {
       extra.toArray,
     )
   }
+
   def compileInputsSettings: Seq[Setting[_]] =
     compileInputsSettings(dependencyPicklePath)
   def compileInputsSettings(classpathTask: TaskKey[VirtualClasspath]): Seq[Setting[_]] = {
@@ -2580,7 +2623,17 @@ object Defaults extends BuildCommon {
           setup,
           prev
         )
-      }
+      },
+      // todo: Zinc's hashing should automatically handle directories
+      compileInputs2 := {
+        val c = fileConverter.value
+        val cp0 = classpathTask.value
+        val inputs = compileInputs.value
+        CompileInputs2(
+          data(cp0).toVector,
+          inputs.options.sources,
+        )
+      },
     )
   }
 
@@ -4147,8 +4200,12 @@ object Classpaths {
     val c = fileConverter.value
     Def.unit(copyResources.value)
     Def.unit(compile.value)
-
-    c.toPath(backendOutput.value).toFile :: Nil
+    val dir = c.toPath(backendOutput.value)
+    val rawJar = compileIncremental.value._2
+    val rawJarPath = c.toPath(rawJar)
+    IO.unzip(rawJarPath.toFile, dir.toFile)
+    IO.delete(dir.toFile / "META-INF")
+    dir.toFile :: Nil
   }
 
   private[sbt] def makePickleProducts: Initialize[Task[Seq[VirtualFile]]] = Def.task {

main/src/main/scala/sbt/Keys.scala

@@ -84,8 +84,12 @@ object Keys {
   val buildDependencies = settingKey[BuildDependencies]("Definitive source of inter-project dependencies for compilation and dependency management.\n\tThis is populated by default by the dependencies declared on Project instances, but may be modified.\n\tThe main restriction is that new builds may not be introduced.").withRank(DSetting)
   val appConfiguration = settingKey[xsbti.AppConfiguration]("Provides access to the launched sbt configuration, including the ScalaProvider, Launcher, and GlobalLock.").withRank(DSetting)
   val thisProject = settingKey[ResolvedProject]("Provides the current project for the referencing scope.").withRank(CSetting)
+
+  @cacheOptOut(reason = "contains file path")
   val thisProjectRef = settingKey[ProjectRef]("Provides a fully-resolved reference to the current project for the referencing scope.").withRank(CSetting)
   val configurationStr = StringAttributeKey("configuration")
+
+  @cacheOptOut("")
   val configuration = settingKey[Configuration]("Provides the current configuration of the referencing scope.").withRank(CSetting)
   val commands = settingKey[Seq[Command]]("Defines commands to be registered when this project or build is the current selected one.").withRank(CSetting)
   val initialize = settingKey[Unit]("A convenience setting for performing side-effects during initialization.").withRank(BSetting)
@@ -166,6 +170,7 @@ object Keys {
   val resources = taskKey[Seq[File]]("All resource files, both managed and unmanaged.").withRank(BTask)
 
   // Output paths
+  @cacheOptOut(reason = "File is machine-specific")
   val classDirectory = settingKey[File]("Directory for compiled classes and copied resources.").withRank(AMinusSetting)
   val earlyOutput = settingKey[VirtualFile]("JAR file for pickles used for build pipelining")
   val backendOutput = settingKey[VirtualFile]("Directory or JAR file for compiled classes and copied resources")
@@ -191,7 +196,10 @@ object Keys {
   val cleanupCommands = settingKey[String]("Commands to execute before the Scala interpreter exits.").withRank(BMinusSetting)
   val asciiGraphWidth = settingKey[Int]("Determines maximum width of the settings graph in ASCII mode").withRank(AMinusSetting)
   val compileOptions = taskKey[CompileOptions]("Collects basic options to configure compilers").withRank(DTask)
+
+  @cacheOptOut(reason = "contains uncachable settings")
   val compileInputs = taskKey[Inputs]("Collects all inputs needed for compilation.").withRank(DTask)
+  val compileInputs2 = taskKey[CompileInputs2]("")
   val scalaHome = settingKey[Option[File]]("If Some, defines the local Scala installation to use for compilation, running, and testing.").withRank(ASetting)
   val scalaInstance = taskKey[ScalaInstance]("Defines the Scala instance to use for compilation, running, and testing.").withRank(DTask)
   val scalaOrganization = settingKey[String]("Organization/group ID of the Scala used in the project. Default value is 'org.scala-lang'. This is an advanced setting used for clones of the Scala Language. It should be disregarded in standard use cases.").withRank(CSetting)
@@ -229,11 +237,13 @@ object Keys {
   val consoleProject = taskKey[Unit]("Starts the Scala interpreter with the sbt and the build definition on the classpath and useful imports.").withRank(AMinusTask)
   val compile = taskKey[CompileAnalysis]("Compiles sources.").withRank(APlusTask)
   val manipulateBytecode = taskKey[CompileResult]("Manipulates generated bytecode").withRank(BTask)
-  val compileIncremental = taskKey[CompileResult]("Actually runs the incremental compilation").withRank(DTask)
+  val compileIncremental = taskKey[(Boolean, HashedVirtualFileRef)]("Actually runs the incremental compilation").withRank(DTask)
   val previousCompile = taskKey[PreviousResult]("Read the incremental compiler analysis from disk").withRank(DTask)
   val tastyFiles = taskKey[Seq[File]]("Returns the TASTy files produced by compilation").withRank(DTask)
   private[sbt] val compileScalaBackend = taskKey[CompileResult]("Compiles only Scala sources if pipelining is enabled. Compiles both Scala and Java sources otherwise").withRank(Invisible)
   private[sbt] val compileEarly = taskKey[CompileAnalysis]("Compiles only Scala sources if pipelining is enabled, and produce an early output (pickle JAR)").withRank(Invisible)
+
+  @cacheOptOut("this is just for timing signal, so no need to cache")
   private[sbt] val earlyOutputPing = taskKey[PromiseWrap[Boolean]]("When pipelining is enabled, this returns true when early output (pickle JAR) is created; false otherwise").withRank(Invisible)
   private[sbt] val compileJava = taskKey[CompileResult]("Compiles only Java sources (called only for pipelining)").withRank(Invisible)
   private[sbt] val compileSplit = taskKey[CompileResult]("When pipelining is enabled, compile Scala then Java; otherwise compile both").withRank(Invisible)
@@ -245,6 +255,8 @@ object Keys {
   val earlyCompileAnalysisTargetRoot = settingKey[File]("The output directory to produce Zinc Analysis files").withRank(DSetting)
   val compileAnalysisFile = taskKey[File]("Zinc analysis storage.").withRank(DSetting)
   val earlyCompileAnalysisFile = taskKey[File]("Zinc analysis storage for early compilation").withRank(DSetting)
+
+  @cacheOptOut
   val compileIncSetup = taskKey[Setup]("Configures aspects of incremental compilation.").withRank(DTask)
   val compilerCache = taskKey[GlobalsCache]("Cache of scala.tools.nsc.Global instances.  This should typically be cached so that it isn't recreated every task run.").withRank(DTask)
   val stateCompilerCache = AttributeKey[GlobalsCache]("stateCompilerCache", "Internal use: Global cache.")
@@ -410,6 +422,8 @@ object Keys {
   val bspConfig = taskKey[Unit]("Create or update the BSP connection files").withRank(DSetting)
   val bspEnabled = SettingKey[Boolean](BasicKeys.bspEnabled)
   val bspSbtEnabled = settingKey[Boolean]("Should BSP export meta-targets for the SBT build itself?")
+
+  @cacheOptOut(reason = "target identifier includes file path")
   val bspTargetIdentifier = settingKey[BuildTargetIdentifier]("Build target identifier of a project and configuration.").withRank(DSetting)
   val bspWorkspace = settingKey[Map[BuildTargetIdentifier, Scope]]("Mapping of BSP build targets to sbt scopes").withRank(DSetting)
   private[sbt] val bspFullWorkspace = settingKey[BspFullWorkspace]("Mapping of BSP build targets to sbt scopes and meta-targets for the SBT build itself").withRank(DSetting)
@@ -440,6 +454,8 @@ object Keys {
   val bspScalaTestClassesItem = taskKey[Seq[ScalaTestClassesItem]]("").withRank(DTask)
   val bspScalaMainClasses = inputKey[Unit]("Corresponds to buildTarget/scalaMainClasses request").withRank(DTask)
   val bspScalaMainClassesItem = taskKey[ScalaMainClassesItem]("").withRank(DTask)
+
+  @cacheOptOut(reason = "no need to invalidate based on this")
   val bspReporter = taskKey[BuildServerReporter]("").withRank(DTask)
 
   val useCoursier = settingKey[Boolean]("Use Coursier for dependency resolution.").withRank(BSetting)

main/src/main/scala/sbt/internal/CompileInputs2.scala

@@ -0,0 +1,29 @@
+package sbt.internal
+
+import scala.reflect.ClassTag
+import sjsonnew.*
+import xsbti.HashedVirtualFileRef
+
+// CompileOption has the list of sources etc
+case class CompileInputs2(
+    classpath: Seq[HashedVirtualFileRef],
+    sources: Seq[HashedVirtualFileRef],
+)
+
+object CompileInputs2:
+  import sbt.util.CacheImplicits.given
+
+  given IsoLList.Aux[
+    CompileInputs2,
+    Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: LNil
+  ] =
+    LList.iso(
+      { (v: CompileInputs2) =>
+        ("classpath", v.classpath.toVector) :*: ("sources", v.sources.toVector) :*: LNil
+      },
+      { (in: Vector[HashedVirtualFileRef] :*: Vector[HashedVirtualFileRef] :*: LNil) =>
+        CompileInputs2(in.head, in.tail.head)
+      }
+    )
+  given JsonFormat[CompileInputs2] = summon
+end CompileInputs2