Skip to content

feat: rewrite builder #237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pchalamet merged 4 commits into from
Aug 20, 2025
Merged

feat: rewrite builder #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d270b97
rewrite builder
pchalamet Aug 19, 2025
a0ffca4
verified builder
pchalamet Aug 19, 2025
713038e
Merge branch 'main' into feature/rewrite-builder
pchalamet Aug 20, 2025
d498b8e
Merge branch 'main' into feature/rewrite-builder
pchalamet Aug 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .vscode/launch.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/Terrabuild/bin/Debug/net9.0/Terrabuild.dll",
"args": ["run", "build", "test", "--workspace", "tests/simple", "--parallel", "1", "--debug", "--log" ],
// self build
// "args": ["run", "build", "test", "dist", "--parallel", "1", "--debug", "--log", "--retry", "--local-only" ],
// "args": ["run", "build", "test", "--workspace", "tests/simple", "--parallel", "1", "--debug", "--log" ],
"args": ["run", "build", "test", "plan", "apply", "--workspace", "tests/indirect-target", "--parallel", "1", "--debug", "--log" ],

// "args": ["run", "build", "--workspace", "tests/cluster-layers", "--force", "--debug", "-p", "1", "--whatif" ],
// "args": ["run", "build", "--workspace", "tests/cluster-layers", "--force", "--debug", "-p", "1", "--whatif" ],
// "args": ["run", "plan", "dist", "build", "test", "--workspace", "../insights", "--log", "-c", "dev", "--debug", "-p", "1", "--whatif" ],
// "args": ["run", "build", "test", "plan", "apply", "--log", "--debug", "--parallel", "1" ],
// "args": [ "run", "build", "-w", "tests/basic", "--whatif", "-p", "1" ],
// "args": ["logs", "build" ],
// "args": ["run", "build", "-w", "tests/simple", "--debug", "-p", "1", "--force" ],
// "cwd": "${workspaceFolder}/tests/indirect-target",
"stopAtEntry": false,
Expand Down
302 changes: 157 additions & 145 deletions src/Terrabuild/Core/Builder.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,164 +16,176 @@ let build (options: ConfigOptions.Options) (configuration: Configuration.Workspa

$"{Ansi.Emojis.eyes} Building graph" |> Terminal.writeLine

let processedNodes = ConcurrentDictionary<string, bool>()
let allNodes = ConcurrentDictionary<string, Node>()
let node2children = ConcurrentDictionary<string, Set<string>>()
let processedNodes = ConcurrentDictionary<string, bool>()

// first check all targets exist in WORKSPACE
match options.Targets |> Seq.tryFind (fun targetName -> configuration.Targets |> Map.containsKey targetName |> not) with
| Some undefinedTarget -> raiseSymbolError $"Target {undefinedTarget} is not defined in WORKSPACE"
| _ -> ()


let rec buildTarget targetName project =
let buildDependsOn (projectConfig: Configuration.Project) target =
let buildDependsOn =
configuration.Targets
|> Map.tryFind target
|> Option.defaultValue Set.empty
let projDependsOn =
projectConfig.Targets
|> Map.tryFind target
|> Option.map (fun ct -> ct.DependsOn)
|> Option.defaultValue Set.empty
let dependsOns = buildDependsOn + projDependsOn
dependsOns


let buildOuterTargets (projectConfig: Configuration.Project) target = [
let dependsOns = buildDependsOn projectConfig target
for dependsOn in dependsOns do
match dependsOn with
| String.Regex "^\^(.+)$" [ depTarget ] ->
for depProject in projectConfig.Dependencies do
let depConfig = configuration.Projects[depProject]
if depConfig.Targets |> Map.containsKey depTarget then (depProject, depTarget)
| _ -> ()
]


let rec buildNode project target =
let projectConfig = configuration.Projects[project]
let nodeId = $"{project}:{targetName}"

let processNode () =
// merge targets requirements
let buildDependsOn =
configuration.Targets
|> Map.tryFind targetName
|> Option.defaultValue Set.empty
let projDependsOn =
projectConfig.Targets
|> Map.tryFind targetName
|> Option.map (fun ct -> ct.DependsOn)
|> Option.defaultValue Set.empty
let dependsOns = buildDependsOn + projDependsOn

// apply on each dependency
let inChildren, outChildren =
dependsOns |> Set.fold (fun (accInChildren, accOutChildren) dependsOn ->
match dependsOn with
| String.Regex "^\^(.+)$" [ parentDependsOn ] ->
accInChildren, accOutChildren + Set.collect (buildTarget parentDependsOn) projectConfig.Dependencies
| String.Regex "^(.+)$" [ dependsOn ] ->
accInChildren + buildTarget dependsOn project, accOutChildren
| _ -> raiseBugError "Invalid target dependency format") (Set.empty, Set.empty)
let targetConfig = projectConfig.Targets[target]
let nodeId = $"{project}:{target}"


let rec buildInnerTargets target = [
let dependsOns = buildDependsOn projectConfig target
yield! dependsOns |> Seq.collect (fun dependsOn ->
match dependsOn with
| String.Regex "^\^(.+)$" _ -> []
| target ->
if projectConfig.Targets |> Map.containsKey target then [ (project, target) ]
else buildInnerTargets target)
]



let processNode() =
let outerDeps = buildOuterTargets projectConfig target |> Set.ofSeq
let innerDeps = buildInnerTargets target |> Set.ofSeq

// NOTE: a node is considered a leaf (within this project only) if the target has no internal dependencies detected
let isLeaf = inChildren |> Set.isEmpty

let children = inChildren + outChildren

// only generate computation node - that is node that generate something
// barrier nodes are just discarded and dependencies lift level up
match projectConfig.Targets |> Map.tryFind targetName with
| Some target ->
let cache, ops =
target.Operations |> List.fold (fun (cache, ops) operation ->
let optContext = {
Terrabuild.Extensibility.ActionContext.Debug = options.Debug
Terrabuild.Extensibility.ActionContext.CI = options.Run.IsSome
Terrabuild.Extensibility.ActionContext.Command = operation.Command
Terrabuild.Extensibility.ActionContext.Hash = projectConfig.Hash
}

let parameters =
match operation.Context with
| Terrabuild.Expressions.Value.Map map ->
map
|> Map.add "context" (Terrabuild.Expressions.Value.Object optContext)
|> Terrabuild.Expressions.Value.Map
| _ -> raiseBugError "Failed to get context (internal error)"

Log.Debug($"{hash}: Invoking extension '{operation.Extension}::{operation.Command}' with args {parameters}")

let cacheability =
match Extensions.getScriptAttribute<CacheableAttribute> optContext.Command (Some operation.Script) with
| Some attr -> attr.Cacheability
| _ -> raiseBugError $"Failed to get cacheability for command {operation.Extension} {optContext.Command}"

let shellOperations =
match Extensions.invokeScriptMethod<Terrabuild.Extensibility.ShellOperations> optContext.Command parameters (Some operation.Script) with
| Extensions.InvocationResult.Success executionRequest -> executionRequest
| Extensions.InvocationResult.ErrorTarget ex -> forwardExternalError($"{hash}: Failed to get shell operation (extension error)", ex)
| _ -> raiseExternalError $"{hash}: Failed to get shell operation (extension error)"

let newops =
shellOperations |> List.map (fun shellOperation -> {
ContaineredShellOperation.Container = operation.Container
ContaineredShellOperation.ContainerPlatform = operation.Platform
ContaineredShellOperation.ContainerVariables = operation.ContainerVariables
ContaineredShellOperation.MetaCommand = $"{operation.Extension} {operation.Command}"
ContaineredShellOperation.Command = shellOperation.Command
ContaineredShellOperation.Arguments = shellOperation.Arguments |> String.normalizeShellArgs })

let cache =
match cacheability, options.LocalOnly with
| Cacheability.Never, _ -> Cacheability.Never
| Cacheability.Local, _ -> Cacheability.Local
| Cacheability.Remote, true -> Cacheability.Local
| Cacheability.Remote, false -> Cacheability.Remote

cache, ops @ newops
) (Cacheability.Never, [])

let opsCmds = ops |> List.map Json.Serialize

let hashContent = opsCmds @ [
yield projectConfig.Hash
yield target.Hash
yield! children |> Seq.map (fun nodeId -> allNodes[nodeId].TargetHash)
]

let hash = hashContent |> Hash.sha256strings

Log.Debug($"Node {nodeId} has ProjectHash {projectConfig.Hash} and TargetHash {hash}")

// cacheability can be overriden by the target
let cache = target.Cache |> Option.defaultValue cache

// no rebuild by default unless force
let rebuild = target.Rebuild |> Option.defaultValue options.Force

let idempotent = target.Idempotent |> Option.defaultValue false

let targetOutput = target.Outputs

let node =
{ Node.Id = nodeId

Node.ProjectId = projectConfig.Id
Node.ProjectDir = projectConfig.Directory
Node.Target = targetName
Node.Operations = ops
Node.Cache = cache
Node.Rebuild = rebuild
Node.Idempotent = idempotent

Node.Dependencies = children
Node.Outputs = targetOutput

Node.ProjectHash = projectConfig.Hash
Node.TargetHash = hash

Node.IsLeaf = isLeaf }

if allNodes.TryAdd(nodeId, node) |> not then raiseBugError "Unexpected graph building race"
Set.singleton nodeId
| _ ->
outChildren

if processedNodes.TryAdd(nodeId, true) then
let children = processNode()
if node2children.TryAdd(nodeId, children) |> not then raiseBugError "Unexpected graph building race"
Log.Debug($"Node {nodeId} has children: {children}")
children
else
node2children[nodeId]
let isLeaf = innerDeps |> Set.isEmpty

let allDeps = innerDeps + outerDeps
let children = allDeps |> Set.map (fun (project, target) -> $"{project}:{target}")

// ensure children exist
for (project, target) in allDeps do
buildNode project target

let cache, ops =
targetConfig.Operations |> List.fold (fun (cache, ops) operation ->
let optContext = {
Terrabuild.Extensibility.ActionContext.Debug = options.Debug
Terrabuild.Extensibility.ActionContext.CI = options.Run.IsSome
Terrabuild.Extensibility.ActionContext.Command = operation.Command
Terrabuild.Extensibility.ActionContext.Hash = projectConfig.Hash
}

let parameters =
match operation.Context with
| Terrabuild.Expressions.Value.Map map ->
map
|> Map.add "context" (Terrabuild.Expressions.Value.Object optContext)
|> Terrabuild.Expressions.Value.Map
| _ -> raiseBugError "Failed to get context (internal error)"

Log.Debug($"{hash}: Invoking extension '{operation.Extension}::{operation.Command}' with args {parameters}")

let cacheability =
match Extensions.getScriptAttribute<CacheableAttribute> optContext.Command (Some operation.Script) with
| Some attr -> attr.Cacheability
| _ -> raiseBugError $"Failed to get cacheability for command {operation.Extension} {optContext.Command}"

let shellOperations =
match Extensions.invokeScriptMethod<Terrabuild.Extensibility.ShellOperations> optContext.Command parameters (Some operation.Script) with
| Extensions.InvocationResult.Success executionRequest -> executionRequest
| Extensions.InvocationResult.ErrorTarget ex -> forwardExternalError($"{hash}: Failed to get shell operation (extension error)", ex)
| _ -> raiseExternalError $"{hash}: Failed to get shell operation (extension error)"

let newops =
shellOperations |> List.map (fun shellOperation -> {
ContaineredShellOperation.Container = operation.Container
ContaineredShellOperation.ContainerPlatform = operation.Platform
ContaineredShellOperation.ContainerVariables = operation.ContainerVariables
ContaineredShellOperation.MetaCommand = $"{operation.Extension} {operation.Command}"
ContaineredShellOperation.Command = shellOperation.Command
ContaineredShellOperation.Arguments = shellOperation.Arguments |> String.normalizeShellArgs })

let cache =
match cacheability, options.LocalOnly with
| Cacheability.Never, _ -> Cacheability.Never
| Cacheability.Local, _ -> Cacheability.Local
| Cacheability.Remote, true -> Cacheability.Local
| Cacheability.Remote, false -> Cacheability.Remote

cache, ops @ newops
) (Cacheability.Never, [])

let opsCmds = ops |> List.map Json.Serialize

let hashContent = opsCmds @ [
yield projectConfig.Hash
yield targetConfig.Hash
yield! children |> Seq.map (fun nodeId -> allNodes[nodeId].TargetHash)
]

let hash = hashContent |> Hash.sha256strings

Log.Debug($"Node {nodeId} has ProjectHash {projectConfig.Hash} and TargetHash {hash}")

// cacheability can be overriden by the target
let cache = targetConfig.Cache |> Option.defaultValue cache

// no rebuild by default unless force
let rebuild = targetConfig.Rebuild |> Option.defaultValue options.Force

let idempotent = targetConfig.Idempotent |> Option.defaultValue false

let targetOutput = targetConfig.Outputs

let node =
{ Node.Id = nodeId

Node.ProjectId = projectConfig.Id
Node.ProjectDir = projectConfig.Directory
Node.Target = target
Node.Operations = ops
Node.Cache = cache
Node.Rebuild = rebuild
Node.Idempotent = idempotent

Node.Dependencies = children
Node.Outputs = targetOutput

Node.ProjectHash = projectConfig.Hash
Node.TargetHash = hash

Node.IsLeaf = isLeaf }
if allNodes.TryAdd(nodeId, node) |> not then raiseBugError "Unexpected graph building race"

if processedNodes.TryAdd(nodeId, true) then processNode()

configuration.SelectedProjects |> Seq.iter (fun project ->
options.Targets |> Seq.iter (fun target ->
configuration.Projects
|> Map.tryFind project
|> Option.iter (fun projectConfig -> if projectConfig.Targets |> Map.containsKey target then buildNode project target)))

let rootNodes =
configuration.SelectedProjects |> Seq.collect (fun project ->
options.Targets |> Seq.choose (fun target ->
buildTarget target project |> ignore

// identify root target
configuration.Projects[project].Targets |> Map.tryFind target
|> Option.map (fun _ -> $"{project}:{target}")))
|> Set
let allNodeIds = allNodes.Keys |> Set
let allDependencyIds = allNodes.Values |> Seq.collect (fun node -> node.Dependencies) |> Set.ofSeq
allNodeIds - allDependencyIds

let endedAt = DateTime.UtcNow
let buildDuration = endedAt - startedAt
Expand Down
5 changes: 0 additions & 5 deletions tests/cluster-layers/results/terrabuild-debug.build-graph.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,11 +269,6 @@
}
},
"rootNodes": [
"a:build",
"b:build",
"c:build",
"d:build",
"e:build",
"f:build",
"g:build"
]
Expand Down
Loading