invalidate cache as required

src/Terrabuild/Core/Build.fs

@@ -59,7 +59,7 @@ let run (configuration: Configuration.Workspace) (graph: Graph.Workspace) (cache
     let targetLabel = if graph.Targets.Count > 1 then "targets" else "target"
     $"{Ansi.Emojis.rocket} Running {targetLabel} {targets}" |> Terminal.writeLine
 
-    let nodesToRun = graph.Nodes |> Seq.filter (fun (KeyValue(_, node)) -> node.Required) |> Seq.length
+    let nodesToRun = graph.Nodes |> Map.filter (fun nodeId node -> node.Required) |> Map.count
     $" {Ansi.Styles.green}{Ansi.Emojis.checkmark}{Ansi.Styles.reset} {nodesToRun} tasks to run" |> Terminal.writeLine
 
     let startedAt = DateTime.UtcNow
@@ -72,10 +72,6 @@ let run (configuration: Configuration.Workspace) (graph: Graph.Workspace) (cache
 
     let containerInfos = Concurrent.ConcurrentDictionary<string, string>()
 
-    let cacheMode =
-        if configuration.SourceControl.CI then Cacheability.Always
-        else Cacheability.Remote
-
     let isBuildSuccess = function
         | NodeStatus.Success _ -> true
         | _ -> false
@@ -99,8 +95,6 @@ let run (configuration: Configuration.Workspace) (graph: Graph.Workspace) (cache
 
 
     let buildNode (node: Graph.Node) =
-        // determine if step node can be reused or not
-        let useRemoteCache = Cacheability.Never <> (node.Cache &&& cacheMode)
         let cacheEntryId = $"{node.ProjectHash}/{node.Target}/{node.Hash}"
 
         notification.NodeDownloading node
@@ -112,14 +106,7 @@ let run (configuration: Configuration.Workspace) (graph: Graph.Workspace) (cache
             | _ -> "."
 
         // check first if it's possible to restore previously built state
-        let summary =
-            if options.Force || node.Cache = Cacheability.Never then None
-            else
-                // get task execution summary & take care of retrying failed tasks
-                match cache.TryGetSummary useRemoteCache cacheEntryId with
-                | Some summary when summary.Status = Cache.TaskStatus.Failure && options.Retry -> None
-                | Some summary -> Some summary
-                | _ -> None
+        let summary = node.BuildSummary
 
         match summary with
         | Some summary ->

src/Terrabuild/Core/Graph.fs

@@ -22,6 +22,7 @@ type Node = {
     Cache: Cacheability
     IsLeaf: bool
     Required: bool
+    BuildSummary: Cache.TargetSummary option
     Batched: bool
 
     TargetHash: string
@@ -155,6 +156,7 @@ let create (configuration: Configuration.Workspace) (targets: string set) =
                              Cache = cache
                              IsLeaf = isLeaf
                              Required = false
+                             BuildSummary = None
                              TargetHash = target.Hash
                              Batched = false }
 
@@ -181,14 +183,90 @@ let create (configuration: Configuration.Workspace) (targets: string set) =
 
 
 
-let trim (configuration: Configuration.Workspace) (graph: Workspace) (cache: Cache.ICache)  (options: Configuration.Options) =
+
+
+let enforceConsistency (configuration: Configuration.Workspace) (graph: Workspace) (cache: Cache.ICache)  (options: Configuration.Options) =
     let startedAt = DateTime.UtcNow
 
     let cacheMode =
         if configuration.SourceControl.CI then Cacheability.Always
         else Cacheability.Remote
 
-    let shallRebuild (node: Node) =
+    let allNodes = ConcurrentDictionary<string, Node>()
+    let hub = Hub.Create(options.MaxConcurrency)
+    for (KeyValue(nodeId, node)) in graph.Nodes do
+        let nodeComputed = hub.CreateComputed<bool*DateTime> nodeId
+
+        // await dependencies
+        let awaitedDependencies =
+            node.Dependencies
+            |> Seq.map (fun awaitedProjectId -> hub.GetComputed<bool*DateTime> awaitedProjectId)
+            |> Array.ofSeq
+
+        let awaitedSignals = awaitedDependencies |> Array.map (fun entry -> entry :> ISignal)
+        hub.Subscribe awaitedSignals (fun () ->
+
+            // children can ask for a build
+            let childrenRebuild, childrenLastBuild =
+                awaitedDependencies |> Seq.fold (fun (childrenRebuild, childrenLastBuild) childComputed ->
+                    let childRebuild, childLastBuild = childComputed.Value
+                    let nodeRebuild = childrenRebuild || childRebuild
+                    let nodeLastBuild = max childrenLastBuild childLastBuild
+                    nodeRebuild, nodeLastBuild) (false, DateTime.MinValue)
+
+            let summary, nodeRebuild, nodeLastBuild =
+                if childrenRebuild then
+                    None, true, DateTime.MaxValue
+
+                else
+                    let useRemoteCache = Cacheability.Never <> (node.Cache &&& cacheMode)
+                    let cacheEntryId = $"{node.ProjectHash}/{node.Target}/{node.Hash}"
+
+                    // check first if it's possible to restore previously built state
+                    let summary =
+                        if options.Force || node.Cache = Cacheability.Never then None
+                        else
+                            // get task execution summary & take care of retrying failed tasks
+                            match cache.TryGetSummary useRemoteCache cacheEntryId with
+                            | Some summary when summary.Status = Cache.TaskStatus.Failure && options.Retry -> None
+                            | Some summary -> Some summary
+                            | _ -> None
+
+                    match summary with
+                    | Some summary ->
+                        if summary.StartedAt < childrenLastBuild then None, true, DateTime.MaxValue
+                        else (Some summary), false, summary.EndedAt
+                    | _ ->
+                        None, true, DateTime.MaxValue
+
+            let node = { node with BuildSummary = summary
+                                   Required = summary |> Option.isNone }
+            allNodes.TryAdd(nodeId, node) |> ignore
+
+            nodeComputed.Value <- (nodeRebuild, nodeLastBuild)
+        )
+
+    let status = hub.WaitCompletion()
+
+    let endedAt = DateTime.UtcNow
+
+    let trimDuration = endedAt - startedAt
+    Log.Debug("Consistency: {duration}", trimDuration)
+    { graph with Nodes = allNodes |> Map.ofDict }
+
+
+
+
+
+let markRequired (configuration: Configuration.Workspace) (graph: Workspace) (cache: Cache.ICache)  (options: Configuration.Options) =
+    let startedAt = DateTime.UtcNow
+
+    // first compute if a node's outputs are required
+    let cacheMode =
+        if configuration.SourceControl.CI then Cacheability.Always
+        else Cacheability.Remote
+
+    let outputConsumed (node: Node) =
         let useRemoteCache = Cacheability.Never <> (node.Cache &&& cacheMode)
         let cacheEntryId = $"{node.ProjectHash}/{node.Target}/{node.Hash}"
         if options.Force || node.Cache = Cacheability.Never then
@@ -214,44 +292,41 @@ let trim (configuration: Configuration.Workspace) (graph: Workspace) (cache: Cac
         |> Map.addMap reversedEdges
 
     let allNodes = ConcurrentDictionary<string, Node>()
-
-    let hub = Hub.Create(options.MaxConcurrency)
+    let hubOutputs = Hub.Create(options.MaxConcurrency)
     for (KeyValue(depNodeId, nodeIds)) in reversedDependencies do
-        let nodeComputed = hub.CreateComputed<bool> depNodeId
+        let nodeComputed = hubOutputs.CreateComputed<bool> depNodeId
 
         // await dependencies
         let awaitedDependencies =
             nodeIds
-            |> Seq.map (fun awaitedProjectId -> hub.GetComputed<bool> awaitedProjectId)
+            |> Seq.map (fun awaitedProjectId -> hubOutputs.GetComputed<bool> awaitedProjectId)
             |> Array.ofSeq
 
         let awaitedSignals = awaitedDependencies |> Array.map (fun entry -> entry :> ISignal)
-        hub.Subscribe awaitedSignals (fun () ->
+        hubOutputs.Subscribe awaitedSignals (fun () ->
             let node = graph.Nodes[depNodeId]
-            let parentRequired = awaitedDependencies |> Seq.fold (fun parentRequired dep -> parentRequired || dep.Value) false
-            let childRequired = parentRequired || shallRebuild node
-
+            let parentRequired =
+                awaitedDependencies |> Seq.fold (fun parentRequired dep -> parentRequired || dep.Value) (node.BuildSummary |> Option.isNone)
+            let childRequired = parentRequired || outputConsumed node
             let node = { node with Required = childRequired }
             allNodes.TryAdd(depNodeId, node) |> ignore
             nodeComputed.Value <- childRequired)
 
-    let status = hub.WaitCompletion()
+    let status = hubOutputs.WaitCompletion()
     match status with
     | Status.Ok -> ()
     | Status.SubcriptionNotRaised projectId -> TerrabuildException.Raise($"Node {projectId} is unknown")
     | Status.SubscriptionError exn -> TerrabuildException.Raise("Optimization error", exn)
 
     let endedAt = DateTime.UtcNow
 
-    let trimDuration = endedAt - startedAt
-    Log.Debug("Trim: {duration}", trimDuration)
+    let requiredDuration = endedAt - startedAt
 
+    Log.Debug("Required: {duration}", requiredDuration)
     { graph with Nodes = allNodes |> Map.ofDict }
 
 
 
-
-
 let optimize (configuration: Configuration.Workspace) (graph: Workspace) (cache: Cache.ICache)  (options: Configuration.Options) =
     let startedAt = DateTime.UtcNow
 
@@ -431,6 +506,7 @@ let optimize (configuration: Configuration.Workspace) (graph: Workspace) (cache:
                     IsLeaf = oneNode.IsLeaf
                     Batched = false
                     Required = true
+                    BuildSummary = None
 
                     TargetHash = cluster
                     CommandLines = optimizedActions

src/Terrabuild/Program.fs

@@ -52,6 +52,15 @@ let processCommandLine (parser: ArgumentParser<TerrabuildArgs>) (result: ParseRe
         Log.Debug("Log created")
 
     let runTarget wsDir target configuration note tag labels variables localOnly logs (options: Configuration.Options) =
+        let logGraph graph name =
+            graph
+            |> Json.Serialize
+            |> IO.writeTextFile (logFile $"{name}-graph.json")
+            graph
+            |> Graph.graph
+            |> String.join "\n"
+            |> IO.writeTextFile (logFile $"{name}-graph.mermaid")
+
         let wsDir = wsDir |> FS.fullPath
         Environment.CurrentDirectory <- wsDir
         Log.Debug("Changing current directory to {directory}", wsDir)
@@ -82,25 +91,16 @@ let processCommandLine (parser: ArgumentParser<TerrabuildArgs>) (result: ParseRe
         let cache = Cache.Cache(storage) :> Cache.ICache
 
         let graph = Graph.create config target
-        if options.Debug then
-            let jsonGraph = Json.Serialize graph
-            jsonGraph |> IO.writeTextFile (logFile "config-graph.json")
-            let mermaid = Graph.graph graph |> String.join "\n"
-            mermaid |> IO.writeTextFile (logFile "config-graph.mermaid")
+        if options.Debug then logGraph graph "config"
 
-        let trimGraph = Graph.trim config graph cache options
-        if options.Debug then
-            let jsonTrimGraph = Json.Serialize trimGraph
-            jsonTrimGraph |> IO.writeTextFile (logFile "trim-graph.json")
-            let mermaid = Graph.graph trimGraph |> String.join "\n"
-            mermaid |> IO.writeTextFile (logFile "trim-graph.mermaid")
+        let consistentGraph = Graph.enforceConsistency config graph cache options
+        if options.Debug then logGraph consistentGraph "consistent"
 
-        let buildGraph = Graph.optimize config trimGraph cache options
-        if options.Debug then
-            let jsonBuildGraph = Json.Serialize buildGraph
-            jsonBuildGraph |> IO.writeTextFile (logFile "build-graph.json")
-            let mermaid = Graph.graph buildGraph |> String.join "\n"
-            mermaid |> IO.writeTextFile (logFile "build-graph.mermaid")
+        let requiredGraph = Graph.markRequired config consistentGraph cache options
+        if options.Debug then logGraph requiredGraph "required"
+
+        let buildGraph = Graph.optimize config requiredGraph cache options
+        if options.Debug then logGraph buildGraph "build"
 
         if options.WhatIf then
             if logs then