Granular cache invalidation with multiple build files

integration/test/resources/invalidation-foreign/build.sc

@@ -0,0 +1,3 @@
+def taskC = T {
+  println("c")
+}

integration/test/resources/invalidation-foreign/foreignA/build.sc

@@ -0,0 +1,12 @@
+import $file.^.foreignB.build
+import $file.^.build
+
+def taskA = T {
+  ^.foreignB.build.taskB()
+  println("a")
+}
+
+def taskD = T {
+  ^.build.taskC()
+  println("d")
+}

integration/test/resources/invalidation-foreign/foreignB/build.sc

@@ -0,0 +1,3 @@
+def taskB = T {
+  println("b")
+}

integration/test/resources/invalidation/a/inputA.sc

@@ -0,0 +1,3 @@
+def input = T {
+  println("a")
+}

integration/test/resources/invalidation/b/inputB.sc

@@ -0,0 +1,5 @@
+import $file.inputD
+def input = T {
+  inputD.method()
+  println("b")
+}

integration/test/resources/invalidation/b/inputD.sc

@@ -0,0 +1,3 @@
+def method() = {
+  println("d")
+}

integration/test/resources/invalidation/build.sc

@@ -0,0 +1,25 @@
+import $file.a.inputA
+import $file.b.{inputB => inputBRenamed}
+import $file.inputC
+import $ivy.`org.scalaj::scalaj-http:2.4.2`
+import $file.e.inputE
+
+def task = T {
+  inputA.input()
+  inputBRenamed.input()
+  inputC.input()
+}
+
+object module extends Module {
+  def task = T {
+    println("task")
+    inputA.input()
+    inputBRenamed.input()
+    inputC.input()
+  }
+}
+
+def taskE = T {
+  println("taskE")
+  inputE.input()
+}

integration/test/resources/invalidation/e/inputE.sc

@@ -0,0 +1,6 @@
+import $file.^.a.inputA
+
+def input = T {
+  println("e")
+  inputA.input()
+}

integration/test/resources/invalidation/inputC.sc

@@ -0,0 +1,3 @@
+def input = T {
+  println("c")
+}

integration/test/src/ScriptsInvalidationForeignTests.scala

@@ -0,0 +1,71 @@
+package mill.integration
+
+import mill.util.ScriptTestSuite
+import utest._
+
+import scala.collection.mutable
+
+class ScriptsInvalidationForeignTests(fork: Boolean) extends ScriptTestSuite(fork) {
+  def workspaceSlug: String = "invalidation-foreign"
+  def scriptSourcePath: os.Path = os.pwd / "integration" / "test" / "resources" / workspaceSlug
+  override def buildPath = os.sub / "foreignA" / "build.sc"
+
+  def runTask(task: String) = {
+    val (successful, stdout) = evalStdout(task)
+    assert(successful)
+    stdout.map(_.trim)
+  }
+
+  val tests = Tests {
+    test("should handle foreign modules") {
+      test("first run") {
+        initWorkspace()
+
+        val result = runTask("taskA")
+
+        val expected = Seq("b", "a")
+
+        assert(result == expected)
+      }
+
+      test("second run modifying script") {
+        val oldContent = os.read(scriptSourcePath / buildPath)
+        val newContent = s"""$oldContent
+                            |def newTask = T { }
+                            |""".stripMargin
+        os.write.over(workspacePath / buildPath, newContent)
+
+        val result = runTask("taskA")
+
+        val expected = Seq("a")
+
+        assert(result == expected)
+      }
+    }
+    test("should handle imports in higher level than top level") {
+      test("first run") {
+        initWorkspace()
+
+        val result = runTask("taskD")
+
+        val expected = Seq("c", "d")
+
+        assert(result == expected)
+      }
+
+      test("second run modifying script") {
+        val oldContent = os.read(scriptSourcePath / buildPath)
+        val newContent = s"""$oldContent
+                            |def newTask = T { }
+                            |""".stripMargin
+        os.write.over(workspacePath / buildPath, newContent)
+
+        val result = runTask("taskD")
+
+        val expected = Seq("d")
+
+        assert(result == expected)
+      }
+    }
+  }
+}

integration/test/src/ScriptsInvalidationTests.scala

@@ -0,0 +1,113 @@
+package mill.integration
+
+import mill.util.ScriptTestSuite
+import utest._
+
+import scala.collection.mutable
+
+class ScriptsInvalidationTests(fork: Boolean) extends ScriptTestSuite(fork) {
+  def workspaceSlug: String = "invalidation"
+  def scriptSourcePath: os.Path = os.pwd / "integration" / "test" / "resources" / workspaceSlug
+
+  def runTask(task: String) = {
+    val (successful, stdout) = evalStdout(task)
+    assert(successful)
+    stdout.map(_.trim)
+  }
+
+  val tests = Tests {
+    test("should not invalidate tasks in different untouched sc files") {
+      test("first run") {
+        initWorkspace()
+
+        val result = runTask("task")
+
+        val expected = Seq("a", "d", "b", "c")
+
+        assert(result == expected)
+      }
+
+      test("second run modifying script") {
+        val oldContent = os.read(scriptSourcePath / buildPath)
+        val newContent = s"""$oldContent
+                            |def newTask = T { }
+                            |""".stripMargin
+        os.write.over(workspacePath / buildPath, newContent)
+
+        val stdout = runTask("task")
+
+        assert(stdout.isEmpty)
+      }
+    }
+    test("should invalidate tasks if leaf file is changed") {
+      test("first run") {
+        initWorkspace()
+
+        val result = runTask("task")
+        val expected = Seq("a", "d", "b", "c")
+
+        assert(result == expected)
+      }
+
+      test("second run modifying script") {
+        val inputD = os.sub / "b" / "inputD.sc"
+        val oldContent = os.read(scriptSourcePath / inputD)
+        val newContent = s"""$oldContent
+                            |def newTask = T { }
+                            |""".stripMargin
+        os.write.over(workspacePath / inputD, newContent)
+
+        val result = runTask("task")
+        val expected = Seq("d", "b")
+
+        assert(result == expected)
+      }
+    }
+    test("should handle submodules in scripts") {
+      test("first run") {
+        initWorkspace()
+
+        val result = runTask("module.task")
+        val expected = Seq("a", "d", "b", "c", "task")
+
+        assert(result == expected)
+      }
+
+      test("second run modifying script") {
+        val oldContent = os.read(scriptSourcePath / buildPath)
+        val newContent = s"""$oldContent
+                            |def newTask = T { }
+                            |""".stripMargin
+        os.write.over(workspacePath / buildPath, newContent)
+
+        val result = runTask("module.task")
+        val expected = Seq("task")
+
+        assert(result == expected)
+      }
+    }
+    test("should handle ammonite ^ imports") {
+      test("first run") {
+        initWorkspace()
+
+        val result = runTask("taskE")
+        val expected = Seq("a", "e", "taskE")
+
+        assert(result == expected)
+      }
+
+      test("second run modifying script") {
+        val oldContent = os.read(scriptSourcePath / buildPath)
+        val newContent = s"""$oldContent
+                            |def newTask = T { }
+                            |""".stripMargin
+        os.write.over(workspacePath / buildPath, newContent)
+
+        val result = runTask("taskE")
+        val expected = Seq("taskE")
+
+        assert(result == expected)
+      }
+    }
+  }
+}

integration/test/src/forked/Tests.scala

@@ -9,3 +9,6 @@ object UpickleTests extends mill.integration.UpickleTests(fork = true)
 object PlayJsonTests extends mill.integration.PlayJsonTests(fork = true)
 object CaffeineTests extends mill.integration.CaffeineTests(fork = true)
 object DocAnnotationsTests extends mill.integration.DocAnnotationsTests(fork = true)
+object ScriptsInvalidationTests extends mill.integration.ScriptsInvalidationTests(fork = true)
+object ScriptsInvalidationForeignTests
+    extends mill.integration.ScriptsInvalidationForeignTests(fork = true)

integration/test/src/local/Tests.scala

@@ -10,3 +10,6 @@ object UpickleTests extends mill.integration.UpickleTests(fork = false)
 object PlayJsonTests extends mill.integration.PlayJsonTests(fork = false)
 object CaffeineTests extends mill.integration.CaffeineTests(fork = false)
 object DocAnnotationsTests extends mill.integration.DocAnnotationsTests(fork = false)
+object ScriptsInvalidationTests extends mill.integration.ScriptsInvalidationTests(fork = false)
+object ScriptsInvalidationForeignTests
+    extends mill.integration.ScriptsInvalidationForeignTests(fork = false)

main/core/src/mill/define/Graph.scala

@@ -41,17 +41,25 @@ object Graph {
    * including the given targets.
    */
   def transitiveTargets(sourceTargets: Agg[Task[_]]): Agg[Task[_]] = {
-    val transitiveTargets = new Agg.Mutable[Task[_]]
-    def rec(t: Task[_]): Unit = {
-      if (transitiveTargets.contains(t)) () // do nothing
+    transitiveNodes(sourceTargets)
+  }
+
+  /**
+   * Collects all transitive dependencies (nodes) of the given nodes,
+   * including the given nodes.
+   */
+  def transitiveNodes[T <: GraphNode[T]](sourceNodes: Agg[T]): Agg[T] = {
+    val transitiveNodes = new Agg.Mutable[T]
+    def rec(t: T): Unit = {
+      if (transitiveNodes.contains(t)) () // do nothing
       else {
-        transitiveTargets.append(t)
+        transitiveNodes.append(t)
         t.inputs.foreach(rec)
       }
     }
 
-    sourceTargets.items.foreach(rec)
-    transitiveTargets
+    sourceNodes.items.foreach(rec)
+    transitiveNodes
   }
 
   /**

main/core/src/mill/define/GraphNode.scala

@@ -0,0 +1,9 @@
+package mill.define
+
+trait GraphNode[T] {
+
+  /**
+   * What other nodes does this node depend on?
+   */
+  def inputs: Seq[T]
+}

main/core/src/mill/define/ScriptNode.scala

@@ -0,0 +1,3 @@
+package mill.define
+
+case class ScriptNode(cls: String, inputs: Seq[ScriptNode]) extends GraphNode[ScriptNode]

main/core/src/mill/define/Task.scala

@@ -15,7 +15,7 @@ import scala.reflect.macros.blackbox.Context
  * Generally not instantiated manually, but instead constructed via the
  * [[Target.apply]] & similar macros.
  */
-abstract class Task[+T] extends Task.Ops[T] with Applyable[Task, T] {
+abstract class Task[+T] extends Task.Ops[T] with Applyable[Task, T] with GraphNode[Task[_]] {
 
   /**
    * What other Targets does this Target depend on?