feat: batch build

.gitignore

@@ -420,3 +420,5 @@ terrabuild-debug.md
 .mono
 *.zip
 *.fsyacc.output
+
+!**/results/terrabuild-debug.*

.vscode/launch.json

@@ -10,12 +10,14 @@
             // self build
             // "args": ["run", "build", "test", "dist", "--parallel", "1", "--debug", "--log", "--retry", "--local-only" ],
             // "args": ["run", "build", "test", "--workspace", "tests/simple", "--parallel", "1", "--debug", "--log" ],
-            "args": ["run", "build", "test", "plan", "apply", "--workspace", "tests/indirect-target", "--parallel", "1", "--debug", "--log" ],
-
+            // "args": ["run", "build", "test", "plan", "apply", "--workspace", "tests/indirect-target", "--parallel", "1", "--debug", "--log" ],
+
+            "args": ["run", "build", "--workspace", "../../matis/matis-next", "--configuration", "local", "--retry", "--debug", "--local-only" ],
+
             // "args": ["run", "build", "--workspace", "tests/cluster-layers", "--force", "--debug", "-p", "1", "--whatif" ],
             // "args": ["run", "build", "--workspace", "tests/cluster-layers", "--force", "--debug", "-p", "1", "--whatif" ],
             // "args": ["run", "plan", "dist", "build", "test", "--workspace", "../insights", "--log", "-c", "dev", "--debug", "-p", "1", "--whatif" ],
-            // "args": ["run", "build", "test", "plan", "apply", "--log", "--debug", "--parallel", "1" ],
+            // "args": ["run", "build", "test", "dist", "--log", "--debug", "--parallel", "1", "--retry", "--local-only", "-p", "terrabuild" ],
             // "args": [ "run", "build", "-w", "tests/basic", "--whatif", "-p", "1" ],
             // "args": ["run", "build", "-w", "tests/simple", "--debug", "-p", "1", "--force" ],
             // "cwd": "${workspaceFolder}/tests/indirect-target",

Makefile

@@ -73,7 +73,7 @@ docs:
 	dotnet run --project tools/DocGen -- src/Terrabuild.Extensions/bin/$(config)/net9.0/Terrabuild.Extensions.xml ../terrabuild.io/content/docs/extensions
 
 self: clean publish
-	$(PWD)/.out/dotnet/terrabuild run build test dist --configuration $(config) --debug --log --retry --local-only
+	$(PWD)/.out/dotnet/terrabuild run build --configuration $(config) --debug --log --retry --local-only -p terrabuild
 
 self-logs:
 	$(PWD)/.out/dotnet/terrabuild logs build test dist --configuration $(config) --debug --log --local-only
@@ -151,6 +151,8 @@ endef
 define diff_results
 	$(call diff_file,$(1),terrabuild-debug.config.json)
 	$(call diff_file,$(1),terrabuild-debug.build-graph.json)
+	$(call diff_file,$(1),terrabuild-debug.action-graph.json)
+	$(call diff_file,$(1),terrabuild-debug.cluster-graph.json)
 	$(call diff_file,$(1),terrabuild-debug.info.md)
 endef
 

WORKSPACE

@@ -9,11 +9,6 @@ locals {
     configuration = local.is_prod ? "Release" : "Debug"
 }
 
-target install {
-    cache = "local"
-    idempotent = true
-}
-
 target build {
     depends_on = [ target.install
                    target.^build ]
@@ -33,6 +28,7 @@ target publish {
 
 extension @dotnet {
     container = "mcr.microsoft.com/dotnet/sdk:9.0.304"
+    batch = true
     defaults {
         configuration = local.configuration
     }

src/Terrabuild.Common.Tests/PROJECT

@@ -3,12 +3,9 @@ project {
     @dotnet { }
 }
 
-target install {
-    @dotnet restore { }
-}
-
 target build {
-    @dotnet build { }
+    @dotnet restore { dependencies = true }
+    @dotnet build { dependencies = true }
 }
 
 target test {

src/Terrabuild.Common/PROJECT

@@ -3,10 +3,7 @@ project {
     @dotnet { }
 }
 
-target install {
-    @dotnet restore { }
-}
-
 target build {
-    @dotnet build { }
+    @dotnet restore { dependencies = true }
+    @dotnet build { dependencies = true }
 }

src/Terrabuild.Configuration.Tests/PROJECT

@@ -3,12 +3,9 @@ project {
     @dotnet { }
 }
 
-target install {
-    @dotnet restore { }
-}
-
 target build {
-    @dotnet build { }
+    @dotnet restore { dependencies = true }
+    @dotnet build { dependencies = true }
 }
 
 target test {

src/Terrabuild.Configuration.Tests/Project.fs

@@ -27,42 +27,42 @@ let parseProject() =
               Platform = None
               Variables = None
               Script = None
-              Defaults = Map [ "configuration", Expr.Variable "var.configuration" ] |> Some }        
+              Defaults = Map [ "configuration", Expr.Variable "var.configuration" ] |> Some
+              Batch = Some Expr.True }        
         let extDocker =
             { Container = None
               Platform = None
               Variables = Expr.List [ Expr.String "ARM_TENANT_ID" ] |> Some
               Script = None
               Defaults = Map [ "configuration", Expr.Variable "local.configuration"
-                               "image", Expr.String "ghcr.io/magnusopera/dotnet-app" ] |> Some }
+                               "image", Expr.String "ghcr.io/magnusopera/dotnet-app" ] |> Some
+              Batch = None }
         let extDummy =
             { Container = None
               Platform = None
               Variables = None
               Script = "dummy.fsx" |> Expr.String |> Some
-              Defaults = None }
+              Defaults = None
+              Batch = None }
 
         let targetBuild = 
             { TargetBlock.DependsOn = Set [ "dist" ] |> Some
               TargetBlock.Rebuild = None
               TargetBlock.Outputs = None
               TargetBlock.Cache = None
-              TargetBlock.Idempotent = Expr.True |> Some
               TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty } ] }
         let targetDist =
             { TargetBlock.DependsOn = None
               TargetBlock.Rebuild = None
               TargetBlock.Outputs = None
               TargetBlock.Cache = None
-              TargetBlock.Idempotent = None
               TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty }
                                     { Extension = "@dotnet"; Command = "publish"; Parameters = Map.empty } ] }
         let targetDocker =
             { TargetBlock.DependsOn = None
               TargetBlock.Rebuild = Some (Expr.Bool false)
               TargetBlock.Outputs = None
               TargetBlock.Cache = "always" |> Expr.String |> Some
-              TargetBlock.Idempotent = None
               TargetBlock.Steps = [ { Extension = "@shell"; Command = "echo"
                                       Parameters = Map [ "arguments", Expr.Function (Function.Trim,
                                                                                      [ Expr.Function (Function.Plus,
@@ -108,7 +108,8 @@ let parseProject2() =
               Platform = None
               Variables = None
               Script = None
-              Defaults = None }        
+              Defaults = None
+              Batch = None }        
 
         let buildTarget = 
             { TargetBlock.Rebuild = Expr.Bool true |> Some
@@ -118,7 +119,6 @@ let parseProject2() =
                                                                  Expr.String ".dll" ])] |> Some
               TargetBlock.DependsOn = None
               TargetBlock.Cache = None
-              TargetBlock.Idempotent = None
               TargetBlock.Steps = [ { Extension = "@dotnet"; Command = "build"; Parameters = Map.empty } ] }
 
         let locals = 

src/Terrabuild.Configuration.Tests/TestFiles/Error_Project_DuplicatedTarget

@@ -1,11 +1,11 @@
 target build {
   depends_on = [ target.dist ]
 
-  @dotnet build { }
+  @dotnet build { dependencies = true }
 }
 
 target build {
   depends_on = [ target.dist ]
 
-  @dotnet build { }
+  @dotnet build { dependencies = true }
 }

src/Terrabuild.Configuration.Tests/TestFiles/Success_PROJECT

@@ -1,6 +1,7 @@
 # comment at top of file
 
 extension @dotnet {
+  batch = true
   defaults { # comment at end of line
     configuration = var.configuration
   }
@@ -32,7 +33,6 @@ project id {
 
 target build {
   depends_on = [ target.dist ]
-  idempotent = true
 
   @dotnet build { }
 }

src/Terrabuild.Configuration.Tests/TestFiles/Success_WORKSPACE

@@ -7,7 +7,6 @@ workspace {
 
 target build { # comment at end of line
   depends_on = [ target.^build ]
-  idempotent = false
 }
 
 # comment in the middle of the file
@@ -22,6 +21,7 @@ target dummy { }
 
 extension dotnet {
   container = "mcr.microsoft.com/dotnet/sdk:8.0.101"
+  batch = false
   defaults {
     configuration = local.configuration
   }

src/Terrabuild.Configuration.Tests/Workspace.fs

@@ -14,37 +14,37 @@ let parseWorkspace() =
         let targetBuild = 
             { TargetBlock.DependsOn = Set [ "^build" ] |> Some
               TargetBlock.Rebuild = None
-              TargetBlock.Cache = None
-              TargetBlock.Idempotent = Expr.False |> Some }
+              TargetBlock.Cache = None }
         let targetDist =
             { TargetBlock.DependsOn = Set [ "build" ] |> Some
               TargetBlock.Rebuild = Expr.Bool true |> Some
-              TargetBlock.Cache = "never" |> Expr.String |> Some
-              TargetBlock.Idempotent = None }
+              TargetBlock.Cache = "never" |> Expr.String |> Some }
         let targetDummy =
             { TargetBlock.DependsOn = None
               TargetBlock.Rebuild = None
-              TargetBlock.Cache = None
-              TargetBlock.Idempotent = None }
+              TargetBlock.Cache = None }
 
         let extDotnet =
             { Container = Some (Expr.String "mcr.microsoft.com/dotnet/sdk:8.0.101")
               Platform = None
               Variables = None
               Script = None
-              Defaults = Map [ "configuration", Expr.Variable "local.configuration" ] |> Some }
+              Defaults = Map [ "configuration", Expr.Variable "local.configuration" ] |> Some
+              Batch = Some Expr.False }
         let extDocker =
             { Container = None
               Platform = None
               Variables = None
               Script = None
-              Defaults = None }
+              Defaults = None
+              Batch = None }
         let extNpm =
             { Container = Some (Expr.String "node:20")
               Platform = None
               Variables = None
               Script = "scripts/npm.fsx" |> Expr.String |> Some
-              Defaults = None }
+              Defaults = None
+              Batch = None }
 
         { WorkspaceFile.Workspace = { Id = "d7528db2-83e0-4164-8c8e-1e0d6d6357ca" |> Some
                                       Ignores = Set [ "**/node_modules" ] |> Some
@@ -71,18 +71,15 @@ let parseWorkspace2() =
         let targetBuild = 
             { TargetBlock.DependsOn = Set [ "^build" ] |> Some
               TargetBlock.Rebuild = None
-              TargetBlock.Cache = None
-              TargetBlock.Idempotent = None }
+              TargetBlock.Cache = None }
         let targetDist =
             { TargetBlock.DependsOn = Set [ "build" ] |> Some
               TargetBlock.Rebuild = Expr.Bool true |> Some
-              TargetBlock.Cache = None
-              TargetBlock.Idempotent = None }
+              TargetBlock.Cache = None }
         let targetDummy =
             { TargetBlock.DependsOn = None
               TargetBlock.Rebuild = None
-              TargetBlock.Cache = None
-              TargetBlock.Idempotent = None }
+              TargetBlock.Cache = None }
 
         let extDotnet =
             { Container = Expr.String "mcr.microsoft.com/dotnet/sdk:8.0.101" |> Some
@@ -91,13 +88,15 @@ let parseWorkspace2() =
               Script = None
               Defaults = Map [ "configuration1", Expr.Function (Function.Item, [Expr.Variable "var.map"; Expr.String "toto"])
                                "configuration2", Expr.Function (Function.Item, [Expr.Variable "var.map"; Expr.String "titi"])
-                               "configuration3", Expr.Function (Function.Replace, [Expr.String "toto titi"; Expr.String "toto"; Expr.String "titi"]) ] |> Some }
+                               "configuration3", Expr.Function (Function.Replace, [Expr.String "toto titi"; Expr.String "toto"; Expr.String "titi"]) ] |> Some
+              Batch = None }
         let extDocker =
             { Container = None
               Platform = None
               Variables = None
               Script = None
-              Defaults = None }
+              Defaults = None
+              Batch = None }
 
         { WorkspaceFile.Workspace = { Id = None; Ignores = None; Version = None }
           WorkspaceFile.Targets = Map [ "build", targetBuild

src/Terrabuild.Configuration/AST/Common.fs

@@ -6,5 +6,5 @@ type ExtensionBlock =
       Platform: Expr option
       Variables: Expr option
       Script: Expr option
-      Defaults: Map<string, Expr> option }
-
+      Defaults: Map<string, Expr> option
+      Batch: Expr option }

src/Terrabuild.Configuration/AST/Project.fs

@@ -26,7 +26,6 @@ type TargetBlock =
       DependsOn: Set<string> option
       Rebuild: Expr option
       Cache: Expr option
-      Idempotent: Expr option
       Steps: Step list }
 
 [<RequireQualifiedAccess>]

src/Terrabuild.Configuration/AST/Workspace.fs

@@ -14,8 +14,7 @@ type WorkspaceBlock =
 type TargetBlock =
     { DependsOn: Set<string> option
       Rebuild: Expr option
-      Cache: Expr option
-      Idempotent: Expr option }
+      Cache: Expr option }
 
 [<RequireQualifiedAccess>]
 type WorkspaceFile =

src/Terrabuild.Configuration/PROJECT

@@ -3,10 +3,7 @@ project {
     @dotnet { }
 }
 
-target install {
-    @dotnet restore { }
-}
-
 target build {
-    @dotnet build { }
+    @dotnet restore { dependencies = true }
+    @dotnet build { dependencies = true }
 }

src/Terrabuild.Configuration/Transpiler/Common.fs

@@ -5,14 +5,15 @@ open Helpers
 
 let toExtension (block: Block) =
     block
-    |> checkAllowedAttributes ["container"; "platform"; "variables"; "script"; "defaults"]
+    |> checkAllowedAttributes ["container"; "platform"; "variables"; "script"; "defaults"; "batch"]
     |> checkAllowedNestedBlocks ["defaults"]
     |> ignore
 
     let container = block |> tryFindAttribute "container"
     let platform = block |> tryFindAttribute "platform"
     let variables = block |> tryFindAttribute "variables"
     let script = block |> tryFindAttribute "script"
+    let batch = block |> tryFindAttribute "batch"
     let defaults =
         block
         |> tryFindBlock "defaults"
@@ -29,7 +30,8 @@ let toExtension (block: Block) =
       ExtensionBlock.Platform = platform
       ExtensionBlock.Variables = variables
       ExtensionBlock.Script = script
-      ExtensionBlock.Defaults = defaults } 
+      ExtensionBlock.Defaults = defaults
+      ExtensionBlock.Batch = batch } 
 
 let toLocals (block: Block) =
     block

src/Terrabuild.Configuration/Transpiler/Project.fs

@@ -71,7 +71,7 @@ let toProject (block: Block) =
 
 let toTarget (block: Block) =
     block
-    |> checkAllowedAttributes ["outputs"; "depends_on"; "rebuild"; "cache"; "idempotent"]
+    |> checkAllowedAttributes ["outputs"; "depends_on"; "rebuild"; "cache"]
     |> ignore
 
     let outputs = block |> tryFindAttribute "outputs"
@@ -85,7 +85,6 @@ let toTarget (block: Block) =
                 | _ -> raiseInvalidArg $"Invalid target dependency '{dependency}'"))
     let rebuild = block |> tryFindAttribute "rebuild"
     let cache = block |> tryFindAttribute "cache"
-    let idempotent = block |> tryFindAttribute "idempotent"
     let steps =
         block.Blocks
         |> List.map (fun step ->
@@ -111,7 +110,6 @@ let toTarget (block: Block) =
       TargetBlock.DependsOn = dependsOn
       TargetBlock.Rebuild = rebuild
       TargetBlock.Cache = cache
-      TargetBlock.Idempotent = idempotent
       TargetBlock.Steps = steps }