diff --git a/.vscode/launch.json b/.vscode/launch.json index afb641bd..169b8b6b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,13 +7,16 @@ "request": "launch", "preLaunchTask": "build", "program": "${workspaceFolder}/src/Terrabuild/bin/Debug/net9.0/Terrabuild.dll", - "args": ["run", "build", "test", "--workspace", "tests/simple", "--parallel", "1", "--debug", "--log" ], + // self build + // "args": ["run", "build", "test", "dist", "--parallel", "1", "--debug", "--log", "--retry", "--local-only" ], + // "args": ["run", "build", "test", "--workspace", "tests/simple", "--parallel", "1", "--debug", "--log" ], + "args": ["run", "build", "test", "plan", "apply", "--workspace", "tests/indirect-target", "--parallel", "1", "--debug", "--log" ], + // "args": ["run", "build", "--workspace", "tests/cluster-layers", "--force", "--debug", "-p", "1", "--whatif" ], // "args": ["run", "build", "--workspace", "tests/cluster-layers", "--force", "--debug", "-p", "1", "--whatif" ], // "args": ["run", "plan", "dist", "build", "test", "--workspace", "../insights", "--log", "-c", "dev", "--debug", "-p", "1", "--whatif" ], // "args": ["run", "build", "test", "plan", "apply", "--log", "--debug", "--parallel", "1" ], // "args": [ "run", "build", "-w", "tests/basic", "--whatif", "-p", "1" ], - // "args": ["logs", "build" ], // "args": ["run", "build", "-w", "tests/simple", "--debug", "-p", "1", "--force" ], // "cwd": "${workspaceFolder}/tests/indirect-target", "stopAtEntry": false, diff --git a/src/Terrabuild/Core/Builder.fs b/src/Terrabuild/Core/Builder.fs index 68442b0d..a7c2f3c9 100644 --- a/src/Terrabuild/Core/Builder.fs +++ b/src/Terrabuild/Core/Builder.fs @@ -16,9 +16,8 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa $"{Ansi.Emojis.eyes} Building graph" |> Terminal.writeLine - let processedNodes = ConcurrentDictionary() let allNodes = ConcurrentDictionary() - let node2children = ConcurrentDictionary>() + let processedNodes = ConcurrentDictionary() // first check all targets exist in WORKSPACE match options.Targets |> Seq.tryFind (fun targetName -> configuration.Targets |> Map.containsKey targetName |> not) with @@ -26,154 +25,167 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa | _ -> () - let rec buildTarget targetName project = + let buildDependsOn (projectConfig: Configuration.Project) target = + let buildDependsOn = + configuration.Targets + |> Map.tryFind target + |> Option.defaultValue Set.empty + let projDependsOn = + projectConfig.Targets + |> Map.tryFind target + |> Option.map (fun ct -> ct.DependsOn) + |> Option.defaultValue Set.empty + let dependsOns = buildDependsOn + projDependsOn + dependsOns + + + let buildOuterTargets (projectConfig: Configuration.Project) target = [ + let dependsOns = buildDependsOn projectConfig target + for dependsOn in dependsOns do + match dependsOn with + | String.Regex "^\^(.+)$" [ depTarget ] -> + for depProject in projectConfig.Dependencies do + let depConfig = configuration.Projects[depProject] + if depConfig.Targets |> Map.containsKey depTarget then (depProject, depTarget) + | _ -> () + ] + + + let rec buildNode project target = let projectConfig = configuration.Projects[project] - let nodeId = $"{project}:{targetName}" - - let processNode () = - // merge targets requirements - let buildDependsOn = - configuration.Targets - |> Map.tryFind targetName - |> Option.defaultValue Set.empty - let projDependsOn = - projectConfig.Targets - |> Map.tryFind targetName - |> Option.map (fun ct -> ct.DependsOn) - |> Option.defaultValue Set.empty - let dependsOns = buildDependsOn + projDependsOn - - // apply on each dependency - let inChildren, outChildren = - dependsOns |> Set.fold (fun (accInChildren, accOutChildren) dependsOn -> - match dependsOn with - | String.Regex "^\^(.+)$" [ parentDependsOn ] -> - accInChildren, accOutChildren + Set.collect (buildTarget parentDependsOn) projectConfig.Dependencies - | String.Regex "^(.+)$" [ dependsOn ] -> - accInChildren + buildTarget dependsOn project, accOutChildren - | _ -> raiseBugError "Invalid target dependency format") (Set.empty, Set.empty) + let targetConfig = projectConfig.Targets[target] + let nodeId = $"{project}:{target}" + + + let rec buildInnerTargets target = [ + let dependsOns = buildDependsOn projectConfig target + yield! dependsOns |> Seq.collect (fun dependsOn -> + match dependsOn with + | String.Regex "^\^(.+)$" _ -> [] + | target -> + if projectConfig.Targets |> Map.containsKey target then [ (project, target) ] + else buildInnerTargets target) + ] + + + + let processNode() = + let outerDeps = buildOuterTargets projectConfig target |> Set.ofSeq + let innerDeps = buildInnerTargets target |> Set.ofSeq // NOTE: a node is considered a leaf (within this project only) if the target has no internal dependencies detected - let isLeaf = inChildren |> Set.isEmpty - - let children = inChildren + outChildren - - // only generate computation node - that is node that generate something - // barrier nodes are just discarded and dependencies lift level up - match projectConfig.Targets |> Map.tryFind targetName with - | Some target -> - let cache, ops = - target.Operations |> List.fold (fun (cache, ops) operation -> - let optContext = { - Terrabuild.Extensibility.ActionContext.Debug = options.Debug - Terrabuild.Extensibility.ActionContext.CI = options.Run.IsSome - Terrabuild.Extensibility.ActionContext.Command = operation.Command - Terrabuild.Extensibility.ActionContext.Hash = projectConfig.Hash - } - - let parameters = - match operation.Context with - | Terrabuild.Expressions.Value.Map map -> - map - |> Map.add "context" (Terrabuild.Expressions.Value.Object optContext) - |> Terrabuild.Expressions.Value.Map - | _ -> raiseBugError "Failed to get context (internal error)" - - Log.Debug($"{hash}: Invoking extension '{operation.Extension}::{operation.Command}' with args {parameters}") - - let cacheability = - match Extensions.getScriptAttribute optContext.Command (Some operation.Script) with - | Some attr -> attr.Cacheability - | _ -> raiseBugError $"Failed to get cacheability for command {operation.Extension} {optContext.Command}" - - let shellOperations = - match Extensions.invokeScriptMethod optContext.Command parameters (Some operation.Script) with - | Extensions.InvocationResult.Success executionRequest -> executionRequest - | Extensions.InvocationResult.ErrorTarget ex -> forwardExternalError($"{hash}: Failed to get shell operation (extension error)", ex) - | _ -> raiseExternalError $"{hash}: Failed to get shell operation (extension error)" - - let newops = - shellOperations |> List.map (fun shellOperation -> { - ContaineredShellOperation.Container = operation.Container - ContaineredShellOperation.ContainerPlatform = operation.Platform - ContaineredShellOperation.ContainerVariables = operation.ContainerVariables - ContaineredShellOperation.MetaCommand = $"{operation.Extension} {operation.Command}" - ContaineredShellOperation.Command = shellOperation.Command - ContaineredShellOperation.Arguments = shellOperation.Arguments |> String.normalizeShellArgs }) - - let cache = - match cacheability, options.LocalOnly with - | Cacheability.Never, _ -> Cacheability.Never - | Cacheability.Local, _ -> Cacheability.Local - | Cacheability.Remote, true -> Cacheability.Local - | Cacheability.Remote, false -> Cacheability.Remote - - cache, ops @ newops - ) (Cacheability.Never, []) - - let opsCmds = ops |> List.map Json.Serialize - - let hashContent = opsCmds @ [ - yield projectConfig.Hash - yield target.Hash - yield! children |> Seq.map (fun nodeId -> allNodes[nodeId].TargetHash) - ] - - let hash = hashContent |> Hash.sha256strings - - Log.Debug($"Node {nodeId} has ProjectHash {projectConfig.Hash} and TargetHash {hash}") - - // cacheability can be overriden by the target - let cache = target.Cache |> Option.defaultValue cache - - // no rebuild by default unless force - let rebuild = target.Rebuild |> Option.defaultValue options.Force - - let idempotent = target.Idempotent |> Option.defaultValue false - - let targetOutput = target.Outputs - - let node = - { Node.Id = nodeId - - Node.ProjectId = projectConfig.Id - Node.ProjectDir = projectConfig.Directory - Node.Target = targetName - Node.Operations = ops - Node.Cache = cache - Node.Rebuild = rebuild - Node.Idempotent = idempotent - - Node.Dependencies = children - Node.Outputs = targetOutput - - Node.ProjectHash = projectConfig.Hash - Node.TargetHash = hash - - Node.IsLeaf = isLeaf } - - if allNodes.TryAdd(nodeId, node) |> not then raiseBugError "Unexpected graph building race" - Set.singleton nodeId - | _ -> - outChildren - - if processedNodes.TryAdd(nodeId, true) then - let children = processNode() - if node2children.TryAdd(nodeId, children) |> not then raiseBugError "Unexpected graph building race" - Log.Debug($"Node {nodeId} has children: {children}") - children - else - node2children[nodeId] + let isLeaf = innerDeps |> Set.isEmpty + + let allDeps = innerDeps + outerDeps + let children = allDeps |> Set.map (fun (project, target) -> $"{project}:{target}") + + // ensure children exist + for (project, target) in allDeps do + buildNode project target + + let cache, ops = + targetConfig.Operations |> List.fold (fun (cache, ops) operation -> + let optContext = { + Terrabuild.Extensibility.ActionContext.Debug = options.Debug + Terrabuild.Extensibility.ActionContext.CI = options.Run.IsSome + Terrabuild.Extensibility.ActionContext.Command = operation.Command + Terrabuild.Extensibility.ActionContext.Hash = projectConfig.Hash + } + + let parameters = + match operation.Context with + | Terrabuild.Expressions.Value.Map map -> + map + |> Map.add "context" (Terrabuild.Expressions.Value.Object optContext) + |> Terrabuild.Expressions.Value.Map + | _ -> raiseBugError "Failed to get context (internal error)" + + Log.Debug($"{hash}: Invoking extension '{operation.Extension}::{operation.Command}' with args {parameters}") + + let cacheability = + match Extensions.getScriptAttribute optContext.Command (Some operation.Script) with + | Some attr -> attr.Cacheability + | _ -> raiseBugError $"Failed to get cacheability for command {operation.Extension} {optContext.Command}" + + let shellOperations = + match Extensions.invokeScriptMethod optContext.Command parameters (Some operation.Script) with + | Extensions.InvocationResult.Success executionRequest -> executionRequest + | Extensions.InvocationResult.ErrorTarget ex -> forwardExternalError($"{hash}: Failed to get shell operation (extension error)", ex) + | _ -> raiseExternalError $"{hash}: Failed to get shell operation (extension error)" + + let newops = + shellOperations |> List.map (fun shellOperation -> { + ContaineredShellOperation.Container = operation.Container + ContaineredShellOperation.ContainerPlatform = operation.Platform + ContaineredShellOperation.ContainerVariables = operation.ContainerVariables + ContaineredShellOperation.MetaCommand = $"{operation.Extension} {operation.Command}" + ContaineredShellOperation.Command = shellOperation.Command + ContaineredShellOperation.Arguments = shellOperation.Arguments |> String.normalizeShellArgs }) + + let cache = + match cacheability, options.LocalOnly with + | Cacheability.Never, _ -> Cacheability.Never + | Cacheability.Local, _ -> Cacheability.Local + | Cacheability.Remote, true -> Cacheability.Local + | Cacheability.Remote, false -> Cacheability.Remote + + cache, ops @ newops + ) (Cacheability.Never, []) + + let opsCmds = ops |> List.map Json.Serialize + + let hashContent = opsCmds @ [ + yield projectConfig.Hash + yield targetConfig.Hash + yield! children |> Seq.map (fun nodeId -> allNodes[nodeId].TargetHash) + ] + + let hash = hashContent |> Hash.sha256strings + + Log.Debug($"Node {nodeId} has ProjectHash {projectConfig.Hash} and TargetHash {hash}") + + // cacheability can be overriden by the target + let cache = targetConfig.Cache |> Option.defaultValue cache + + // no rebuild by default unless force + let rebuild = targetConfig.Rebuild |> Option.defaultValue options.Force + + let idempotent = targetConfig.Idempotent |> Option.defaultValue false + + let targetOutput = targetConfig.Outputs + + let node = + { Node.Id = nodeId + + Node.ProjectId = projectConfig.Id + Node.ProjectDir = projectConfig.Directory + Node.Target = target + Node.Operations = ops + Node.Cache = cache + Node.Rebuild = rebuild + Node.Idempotent = idempotent + + Node.Dependencies = children + Node.Outputs = targetOutput + + Node.ProjectHash = projectConfig.Hash + Node.TargetHash = hash + + Node.IsLeaf = isLeaf } + if allNodes.TryAdd(nodeId, node) |> not then raiseBugError "Unexpected graph building race" + + if processedNodes.TryAdd(nodeId, true) then processNode() + + configuration.SelectedProjects |> Seq.iter (fun project -> + options.Targets |> Seq.iter (fun target -> + configuration.Projects + |> Map.tryFind project + |> Option.iter (fun projectConfig -> if projectConfig.Targets |> Map.containsKey target then buildNode project target))) let rootNodes = - configuration.SelectedProjects |> Seq.collect (fun project -> - options.Targets |> Seq.choose (fun target -> - buildTarget target project |> ignore - - // identify root target - configuration.Projects[project].Targets |> Map.tryFind target - |> Option.map (fun _ -> $"{project}:{target}"))) - |> Set + let allNodeIds = allNodes.Keys |> Set + let allDependencyIds = allNodes.Values |> Seq.collect (fun node -> node.Dependencies) |> Set.ofSeq + allNodeIds - allDependencyIds let endedAt = DateTime.UtcNow let buildDuration = endedAt - startedAt diff --git a/tests/cluster-layers/results/terrabuild-debug.build-graph.json b/tests/cluster-layers/results/terrabuild-debug.build-graph.json index cdf1eb66..697a95e2 100644 --- a/tests/cluster-layers/results/terrabuild-debug.build-graph.json +++ b/tests/cluster-layers/results/terrabuild-debug.build-graph.json @@ -269,11 +269,6 @@ } }, "rootNodes": [ - "a:build", - "b:build", - "c:build", - "d:build", - "e:build", "f:build", "g:build" ] diff --git a/tests/indirect-target/results/terrabuild-debug.build-graph.json b/tests/indirect-target/results/terrabuild-debug.build-graph.json index d938b72b..1421454b 100644 --- a/tests/indirect-target/results/terrabuild-debug.build-graph.json +++ b/tests/indirect-target/results/terrabuild-debug.build-graph.json @@ -22,30 +22,6 @@ "idempotent": false, "isLeaf": true }, - "a:dist": { - "id": "a:dist", - "projectId": "a", - "projectDir": "A", - "target": "dist", - "dependencies": [ - "a:build" - ], - "outputs": [], - "projectHash": "9C785D55C11F07C0562A258ED07F84A7D8F048C0642BB37FDD51CA79B1BA7477", - "targetHash": "50F5F31C8F99596F8E1E420A060A506196FCD6248BF7AD58EE006D3EC84A570B", - "operations": [ - { - "containerVariables": [], - "metaCommand": "@shell echo", - "command": "echo", - "arguments": "dist A" - } - ], - "cache": "never", - "rebuild": true, - "idempotent": false, - "isLeaf": false - }, "b:apply": { "id": "b:apply", "projectId": "b", @@ -56,7 +32,7 @@ ], "outputs": [], "projectHash": "216C66023FFF22BF42082547C0CA6B8558703DE2E94147B86DDE8E0EB9140214", - "targetHash": "5B1E3A6B67518D428A8866B441FD4BD50A7599BB73514A1C8CB1D5A3D362DB9D", + "targetHash": "4C1DC6E6563A97E0B27E8752BB0AD2812BF57E870EB0E867E66720CC60477C86", "operations": [ { "containerVariables": [], @@ -75,12 +51,10 @@ "projectId": "b", "projectDir": "B", "target": "plan", - "dependencies": [ - "a:dist" - ], + "dependencies": [], "outputs": [], "projectHash": "216C66023FFF22BF42082547C0CA6B8558703DE2E94147B86DDE8E0EB9140214", - "targetHash": "7B87E97A91FEB653E59685702413244279D3C23DCAD0BB317C4F24F0D69382BB", + "targetHash": "C02332643A2557B7E0269B00CEA6B1607A00B886453073360C33AB68C64DB318", "operations": [ { "containerVariables": [], @@ -92,7 +66,7 @@ "cache": "never", "rebuild": false, "idempotent": false, - "isLeaf": false + "isLeaf": true }, "c:build": { "id": "c:build", @@ -115,36 +89,11 @@ "rebuild": true, "idempotent": false, "isLeaf": true - }, - "c:dist": { - "id": "c:dist", - "projectId": "c", - "projectDir": "C", - "target": "dist", - "dependencies": [ - "c:build" - ], - "outputs": [], - "projectHash": "5140E561832E315D1BDE9C88BC59A0C76757647DFB07F409BA73535ABDB5EC16", - "targetHash": "530F9C5E56668D23CBFF3BE3917BF657EE0E1A4DCB587721669C2C0927E7AA5C", - "operations": [ - { - "containerVariables": [], - "metaCommand": "@shell echo", - "command": "echo", - "arguments": "dist C" - } - ], - "cache": "never", - "rebuild": true, - "idempotent": false, - "isLeaf": false } }, "rootNodes": [ "a:build", "b:apply", - "b:plan", "c:build" ] } \ No newline at end of file diff --git a/tests/indirect-target/results/terrabuild-debug.info.md b/tests/indirect-target/results/terrabuild-debug.info.md index 896d4046..26a420b8 100644 --- a/tests/indirect-target/results/terrabuild-debug.info.md +++ b/tests/indirect-target/results/terrabuild-debug.info.md @@ -18,25 +18,16 @@ classDef restore stroke:orange,stroke-width:3px classDef ignore stroke:black,stroke-width:3px a:build("build a A") -a:dist("dist a -A") b:apply("apply b B") b:plan("plan b B") c:build("build c C") -c:dist("dist c -C") class a:build ignore -a:dist --> a:build -class a:dist ignore b:apply --> b:plan class b:apply ignore -b:plan --> a:dist class b:plan ignore class c:build ignore -c:dist --> c:build -class c:dist ignore ``` diff --git a/tests/multirefs/results/terrabuild-debug.build-graph.json b/tests/multirefs/results/terrabuild-debug.build-graph.json index 2b5d9bcf..2b5255cb 100644 --- a/tests/multirefs/results/terrabuild-debug.build-graph.json +++ b/tests/multirefs/results/terrabuild-debug.build-graph.json @@ -70,8 +70,6 @@ } }, "rootNodes": [ - "a:build", - "b:build", - "c:build" + "a:build" ] } \ No newline at end of file diff --git a/tests/simple/results/terrabuild-debug.build-graph.json b/tests/simple/results/terrabuild-debug.build-graph.json index ff3dcb71..2753761d 100644 --- a/tests/simple/results/terrabuild-debug.build-graph.json +++ b/tests/simple/results/terrabuild-debug.build-graph.json @@ -321,12 +321,7 @@ }, "rootNodes": [ "deployments/terraform-deploy:build", - "libraries/dotnet-lib:build", - "libraries/npm-lib:build", - "libraries/shell-lib:build", - "projects/dotnet-app:build", "projects/make-app:build", - "projects/npm-app:build", "projects/open-api:build", "projects/rust-app:build", "tests/playwright:test"