diff --git a/src/Terrabuild.Common.Tests/packages.lock.json b/src/Terrabuild.Common.Tests/packages.lock.json index 17f18982..f456a612 100644 --- a/src/Terrabuild.Common.Tests/packages.lock.json +++ b/src/Terrabuild.Common.Tests/packages.lock.json @@ -59,11 +59,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.ApplicationInsights": { "type": "Transitive", "resolved": "2.23.0", @@ -174,7 +169,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Common/IO.fs b/src/Terrabuild.Common/IO.fs index 98b555ce..81751432 100644 --- a/src/Terrabuild.Common/IO.fs +++ b/src/Terrabuild.Common/IO.fs @@ -3,7 +3,6 @@ open System.IO open Microsoft.Extensions.FileSystemGlobbing open Collections open System -open Ignore let chmod permissions (path: string) = File.SetUnixFileMode(path, permissions) @@ -108,40 +107,3 @@ let createSnapshot outputs projectDirectory = |> Map { TimestampedFiles = files } - - -let loadIgnoreFile dir = - let ignoreAccumulator = Ignore() - let rec combineIgnoreFiles dir = - if FS.combinePath dir "WORKSPACE" |> exists |> not then - match dir |> FS.parentDirectory with - | Some dir -> dir |> combineIgnoreFiles - | _ -> () - - let ignoreFile = FS.combinePath dir ".gitignore" - if FS.fileExists ignoreFile then - let content = File.ReadAllLines ignoreFile - content |> ignoreAccumulator.Add |> ignore - - dir |> combineIgnoreFiles - ignoreAccumulator - -let enumeratedCommittedFiles projectDir = - - let rec enumeratedCommittedFiles (dirIgnore: Ignore) dir = [ - // enumerate whitelisted files - yield! dir |> enumerateFiles |> List.filter (not << dirIgnore.IsIgnored) - - // enumerate whitelisted directories - let dirs = dir |> enumerateDirs |> List.filter (not << dirIgnore.IsIgnored) - for subdir in dirs do - // update ignore if new .gitignore file discovered - let ignoreFile = FS.combinePath dir ".gitignore" - let subDirIgnore = - if FS.fileExists ignoreFile then Ignore().Add(dirIgnore.OriginalRules).Add(File.ReadAllLines ignoreFile) - else dirIgnore - yield! enumeratedCommittedFiles subDirIgnore subdir - ] - - let projectDirIgnore = loadIgnoreFile projectDir - enumeratedCommittedFiles projectDirIgnore projectDir diff --git a/src/Terrabuild.Common/Terrabuild.Common.fsproj b/src/Terrabuild.Common/Terrabuild.Common.fsproj index d9230f17..c5fb3ae8 100644 --- a/src/Terrabuild.Common/Terrabuild.Common.fsproj +++ b/src/Terrabuild.Common/Terrabuild.Common.fsproj @@ -19,7 +19,6 @@ - diff --git a/src/Terrabuild.Common/packages.lock.json b/src/Terrabuild.Common/packages.lock.json index 70e2666e..c574b85c 100644 --- a/src/Terrabuild.Common/packages.lock.json +++ b/src/Terrabuild.Common/packages.lock.json @@ -18,12 +18,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Direct", - "requested": "[0.2.1, )", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.Extensions.FileSystemGlobbing": { "type": "Direct", "requested": "[9.0.8, )", diff --git a/src/Terrabuild.Configuration.Tests/packages.lock.json b/src/Terrabuild.Configuration.Tests/packages.lock.json index e332e2c6..b74a94ec 100644 --- a/src/Terrabuild.Configuration.Tests/packages.lock.json +++ b/src/Terrabuild.Configuration.Tests/packages.lock.json @@ -76,11 +76,6 @@ "FSharp.Core": "4.6.2" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.ApplicationInsights": { "type": "Transitive", "resolved": "2.23.0", @@ -191,7 +186,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Configuration/packages.lock.json b/src/Terrabuild.Configuration/packages.lock.json index 626b06b2..87c0fce1 100644 --- a/src/Terrabuild.Configuration/packages.lock.json +++ b/src/Terrabuild.Configuration/packages.lock.json @@ -40,11 +40,6 @@ "FSharp.Core": "4.6.2" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.Extensions.FileSystemGlobbing": { "type": "Transitive", "resolved": "9.0.8", @@ -55,7 +50,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Expressions.Tests/packages.lock.json b/src/Terrabuild.Expressions.Tests/packages.lock.json index 9db3cfc0..b592499c 100644 --- a/src/Terrabuild.Expressions.Tests/packages.lock.json +++ b/src/Terrabuild.Expressions.Tests/packages.lock.json @@ -59,11 +59,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.ApplicationInsights": { "type": "Transitive", "resolved": "2.23.0", @@ -174,7 +169,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Expressions/packages.lock.json b/src/Terrabuild.Expressions/packages.lock.json index bed218d9..936693f3 100644 --- a/src/Terrabuild.Expressions/packages.lock.json +++ b/src/Terrabuild.Expressions/packages.lock.json @@ -23,11 +23,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.Extensions.FileSystemGlobbing": { "type": "Transitive", "resolved": "9.0.8", @@ -38,7 +33,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Extensions.Tests/packages.lock.json b/src/Terrabuild.Extensions.Tests/packages.lock.json index 418c607b..5e291628 100644 --- a/src/Terrabuild.Extensions.Tests/packages.lock.json +++ b/src/Terrabuild.Extensions.Tests/packages.lock.json @@ -59,11 +59,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.ApplicationInsights": { "type": "Transitive", "resolved": "2.23.0", @@ -174,7 +169,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Extensions/packages.lock.json b/src/Terrabuild.Extensions/packages.lock.json index 14e6d52d..9a85070e 100644 --- a/src/Terrabuild.Extensions/packages.lock.json +++ b/src/Terrabuild.Extensions/packages.lock.json @@ -23,11 +23,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.Extensions.FileSystemGlobbing": { "type": "Transitive", "resolved": "9.0.8", @@ -38,7 +33,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Lang.Tests/packages.lock.json b/src/Terrabuild.Lang.Tests/packages.lock.json index 61ed7a28..8393d5bb 100644 --- a/src/Terrabuild.Lang.Tests/packages.lock.json +++ b/src/Terrabuild.Lang.Tests/packages.lock.json @@ -76,11 +76,6 @@ "FSharp.Core": "4.6.2" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.ApplicationInsights": { "type": "Transitive", "resolved": "2.23.0", @@ -191,7 +186,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Lang/packages.lock.json b/src/Terrabuild.Lang/packages.lock.json index 11fc92fe..eb0cc9f1 100644 --- a/src/Terrabuild.Lang/packages.lock.json +++ b/src/Terrabuild.Lang/packages.lock.json @@ -41,11 +41,6 @@ "FSharp.Core": "4.6.2" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.Extensions.FileSystemGlobbing": { "type": "Transitive", "resolved": "9.0.8", @@ -56,7 +51,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.PubSub.Tests/packages.lock.json b/src/Terrabuild.PubSub.Tests/packages.lock.json index 9f028e0c..23bed5c2 100644 --- a/src/Terrabuild.PubSub.Tests/packages.lock.json +++ b/src/Terrabuild.PubSub.Tests/packages.lock.json @@ -59,11 +59,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.ApplicationInsights": { "type": "Transitive", "resolved": "2.23.0", @@ -174,7 +169,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.PubSub/packages.lock.json b/src/Terrabuild.PubSub/packages.lock.json index bed218d9..936693f3 100644 --- a/src/Terrabuild.PubSub/packages.lock.json +++ b/src/Terrabuild.PubSub/packages.lock.json @@ -23,11 +23,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.Extensions.FileSystemGlobbing": { "type": "Transitive", "resolved": "9.0.8", @@ -38,7 +33,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Scripting.Tests/packages.lock.json b/src/Terrabuild.Scripting.Tests/packages.lock.json index ea9e3d4d..5eafce6d 100644 --- a/src/Terrabuild.Scripting.Tests/packages.lock.json +++ b/src/Terrabuild.Scripting.Tests/packages.lock.json @@ -74,11 +74,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.ApplicationInsights": { "type": "Transitive", "resolved": "2.23.0", @@ -206,7 +201,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Scripting/packages.lock.json b/src/Terrabuild.Scripting/packages.lock.json index 9a7002ab..0bed39a3 100644 --- a/src/Terrabuild.Scripting/packages.lock.json +++ b/src/Terrabuild.Scripting/packages.lock.json @@ -39,11 +39,6 @@ "System.Text.Json": "6.0.10" } }, - "Ignore": { - "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" - }, "Microsoft.Extensions.FileSystemGlobbing": { "type": "Transitive", "resolved": "9.0.8", @@ -89,7 +84,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } diff --git a/src/Terrabuild.Tests/packages.lock.json b/src/Terrabuild.Tests/packages.lock.json index cddea73c..3980d020 100644 --- a/src/Terrabuild.Tests/packages.lock.json +++ b/src/Terrabuild.Tests/packages.lock.json @@ -228,10 +228,18 @@ "FSharp.Core": "4.6.2" } }, - "Ignore": { + "LibGit2Sharp": { "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" + "resolved": "0.31.0", + "contentHash": "b3+UfV7LjKMjAHWwl7VawejiOv2gJIC6dTCA/S0puLTHACAA/Oeb5JJmWUQMeyH/T/WR/LaIK8bk2RbdFnrZvg==", + "dependencies": { + "LibGit2Sharp.NativeBinaries": "[2.0.323]" + } + }, + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" }, "Microsoft.ApplicationInsights": { "type": "Transitive", @@ -1345,6 +1353,7 @@ "FSharp.Core": "[9.0.303, )", "FSharp.Data": "[6.6.0, )", "FSharp.SystemTextJson": "[1.4.36, )", + "LibGit2Sharp": "[0.31.0, )", "SemanticVersioning": "[3.0.0, )", "Sentry": "[5.14.1, )", "Serilog.Sinks.File": "[7.0.0, )", @@ -1362,7 +1371,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } @@ -1429,6 +1437,11 @@ } }, "net9.0/linux-arm64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -2291,6 +2304,11 @@ } }, "net9.0/linux-x64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -3153,6 +3171,11 @@ } }, "net9.0/osx-arm64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -4015,6 +4038,11 @@ } }, "net9.0/osx-x64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -4877,6 +4905,11 @@ } }, "net9.0/win-arm64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -5713,6 +5746,11 @@ } }, "net9.0/win-x64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", diff --git a/src/Terrabuild/Contracts/BuildProgress.fs b/src/Terrabuild/Contracts/BuildProgress.fs new file mode 100644 index 00000000..537f5d93 --- /dev/null +++ b/src/Terrabuild/Contracts/BuildProgress.fs @@ -0,0 +1,15 @@ + +namespace BuildProgress + + + + +type IBuildProgress = + abstract BuildStarted: unit -> unit + abstract BuildCompleted: unit -> unit + + abstract TaskScheduled: taskId:string -> label:string -> unit + abstract TaskDownloading: taskId:string -> unit + abstract TaskBuilding: taskId:string -> unit + abstract TaskUploading: taskId:string -> unit + abstract TaskCompleted: taskId:string -> restore:bool -> success:bool -> unit diff --git a/src/Terrabuild/Core/Build.fs b/src/Terrabuild/Core/Build.fs index 51cb6b79..f84b8ef3 100644 --- a/src/Terrabuild/Core/Build.fs +++ b/src/Terrabuild/Core/Build.fs @@ -42,18 +42,6 @@ type Summary = { -type IBuildNotification = - abstract WaitCompletion: unit -> unit - - abstract BuildStarted: graph:GraphDef.Graph -> unit - abstract BuildCompleted: summary:Summary -> unit - - abstract TaskScheduled: taskId:string -> label:string -> unit - abstract TaskDownloading: taskId:string -> unit - abstract TaskBuilding: taskId:string -> unit - abstract TaskUploading: taskId:string -> unit - abstract TaskCompleted: taskId:string -> restore:bool -> success:bool -> unit - let private containerInfos = Concurrent.ConcurrentDictionary() @@ -152,11 +140,12 @@ let execCommands (node: GraphDef.Node) (cacheEntry: Cache.IEntry) (options: Conf -let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.IApiClient option) (notification: IBuildNotification) (graph: GraphDef.Graph) = +let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.IApiClient option) (graph: GraphDef.Graph) = let startedAt = DateTime.UtcNow $"{Ansi.Emojis.rocket} Processing tasks" |> Terminal.writeLine - notification.BuildStarted graph + let buildProgress = Notification.BuildNotification() :> BuildProgress.IBuildProgress + buildProgress.BuildStarted() api |> Option.iter (fun api -> api.StartBuild()) let allowRemoteCache = options.LocalOnly |> not @@ -180,8 +169,9 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I hub.GetSignal $"{projectId}+exec") |> List.ofSeq - let execRestore() = - notification.TaskDownloading node.Id + buildProgress.TaskScheduled node.Id $"{node.Target} {node.ProjectDir}" + hub.Subscribe $"{node.Id} restore" execDependencies (fun () -> + buildProgress.TaskDownloading node.Id let projectDirectory = match node.ProjectDir with @@ -215,12 +205,10 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I match status with | TaskStatus.Success completionDate -> nodeExecSignal.Set completionDate - notification.TaskCompleted node.Id true true + buildProgress.TaskCompleted node.Id true true | _ -> - notification.TaskCompleted node.Id true false + buildProgress.TaskCompleted node.Id true false) - notification.TaskScheduled node.Id $"{node.Target} {node.ProjectDir}" - hub.Subscribe $"{node.Id} restore" execDependencies execRestore and buildNode (node: GraphDef.Node) = @@ -232,9 +220,10 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I hub.GetSignal $"{projectId}+exec") |> List.ofSeq - let execDependenciesCompleted() = + buildProgress.TaskScheduled node.Id $"{node.Target} {node.ProjectDir}" + hub.Subscribe $"{node.Id} build" execDependencies (fun () -> let startedAt = DateTime.UtcNow - notification.TaskBuilding node.Id + buildProgress.TaskBuilding node.Id let projectDirectory = match node.ProjectDir with @@ -268,7 +257,7 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I Cache.TargetSummary.Duration = endedAt - startedAt Cache.TargetSummary.Cache = node.Cache } - notification.TaskUploading node.Id + buildProgress.TaskUploading node.Id // create an archive with new files Log.Debug("{NodeId}: Building '{Project}/{Target}' with {Hash}", node.Id, node.ProjectDir, node.Target, node.TargetHash) @@ -289,12 +278,9 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I let nodeStatusSignal = hub.GetSignal $"{node.Id}+status" nodeStatusSignal.Set completionDate - notification.TaskCompleted node.Id false true + buildProgress.TaskCompleted node.Id false true | _ -> - notification.TaskCompleted node.Id false false - - notification.TaskScheduled node.Id $"{node.Target} {node.ProjectDir}" - hub.Subscribe $"{node.Id} build" execDependencies execDependenciesCompleted + buildProgress.TaskCompleted node.Id false false) and buildOrRestoreNode (node: GraphDef.Node) = if node.Idempotent then buildNode node @@ -346,7 +332,7 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I scheduleNodeStatus projectId hub.GetSignal $"{projectId}+status") |> List.ofSeq - let onDependencyCompleted() = + hub.Subscribe $"{nodeId} status" dependencyStatus (fun () -> let nodeStatusSignal = hub.GetSignal $"{nodeId}+status" // now decide what to do @@ -366,14 +352,14 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I else buildNode node | (TaskRequest.Restore, Some buildDate) -> nodeStatusSignal.Set buildDate - | _ -> raiseBugError $"Unexpected compute action: {buildRequest}" - - hub.Subscribe $"{nodeId} status" dependencyStatus onDependencyCompleted + | _ -> raiseBugError $"Unexpected compute action: {buildRequest}") graph.RootNodes |> Seq.iter scheduleNodeStatus let status = hub.WaitCompletion() + buildProgress.BuildCompleted() + match status with | Status.Ok -> Log.Debug("Build successful") @@ -424,7 +410,6 @@ let run (options: ConfigOptions.Options) (cache: Cache.ICache) (api: Contracts.I Summary.Targets = options.Targets Summary.Nodes = nodeStatus } - notification.BuildCompleted buildInfo api |> Option.iter (fun api -> api.CompleteBuild buildInfo.IsSuccess) buildInfo diff --git a/src/Terrabuild/Core/Configuration.fs b/src/Terrabuild/Core/Configuration.fs index 4865f23c..db508a37 100644 --- a/src/Terrabuild/Core/Configuration.fs +++ b/src/Terrabuild/Core/Configuration.fs @@ -343,11 +343,12 @@ let private loadProjectDef (options: ConfigOptions.Options) (workspaceConfig: AS // this is the final stage: create targets and create the project -let private finalizeProject projectDir evaluationContext (projectDef: LoadedProject) (projectDependencies: Map) = +let private finalizeProject workspaceDir projectDir evaluationContext (projectDef: LoadedProject) (projectDependencies: Map) = + let startFinalize = DateTime.UtcNow let projectId = projectDir |> String.toLower // get dependencies on files - let committedFiles = IO.enumeratedCommittedFiles projectDir |> Set.ofList + let committedFiles = Git.enumeratedCommittedFiles workspaceDir projectDir |> Set.ofList let additionalFiles = projectDir |> IO.enumerateFilesBut projectDef.Includes (projectDef.Outputs + projectDef.Ignores) @@ -558,6 +559,9 @@ let private finalizeProject projectDir evaluationContext (projectDef: LoadedProj let projectDependencies = projectDependencies.Keys |> Seq.map String.toLower |> Set.ofSeq + let endFinalize = DateTime.UtcNow + Log.Debug("Finalized project '{ProjectId}' for {Duration}", projectDir, endFinalize - startFinalize) + { Project.Id = projectDef.Id Project.Directory = projectDir Project.Hash = projectHash @@ -632,6 +636,7 @@ let read (options: ConfigOptions.Options) = let rec loadProject projectDir = let projectPathId = projectDir |> String.toLower if projectLoading.TryAdd(projectPathId, true) then + // parallel load of projects hub.Subscribe projectDir [] (fun () -> let loadedProject = @@ -673,16 +678,14 @@ let read (options: ConfigOptions.Options) = |> Seq.map (fun projectDependency -> projectDependency.Get().Directory, projectDependency.Get()) |> Map.ofSeq - let project = finalizeProject projectDir evaluationContext loadedProject dependsOnProjects + let project = finalizeProject options.Workspace projectDir evaluationContext loadedProject dependsOnProjects if projects.TryAdd(projectPathId, project) |> not then raiseBugError "Unexpected error" - Log.Debug($"Signaling projectPath '{projectPathId}") let loadedProjectPathIdSignal = hub.GetSignal projectPathId loadedProjectPathIdSignal.Set(project) match loadedProject.Id with | Some projectId -> - Log.Debug($"Signaling projectId '{projectId}") let loadedProjectIdSignal = hub.GetSignal $"project.{projectId}" loadedProjectIdSignal.Set(project) | _ -> () @@ -703,6 +706,7 @@ let read (options: ConfigOptions.Options) = findDependencies true options.Workspace let status = hub.WaitCompletion() + match status with | Status.Ok -> projects |> Map.ofDict diff --git a/src/Terrabuild/Core/GraphDef.fs b/src/Terrabuild/Core/GraphDef.fs index afb4dc0c..fe7a2a6c 100644 --- a/src/Terrabuild/Core/GraphDef.fs +++ b/src/Terrabuild/Core/GraphDef.fs @@ -42,3 +42,4 @@ type Graph = { let buildCacheKey (node: Node) = $"{node.ProjectHash}/{node.Target}/{node.TargetHash}" + diff --git a/src/Terrabuild/Core/Notification.fs b/src/Terrabuild/Core/Notification.fs index ac4240f9..5105f136 100644 --- a/src/Terrabuild/Core/Notification.fs +++ b/src/Terrabuild/Core/Notification.fs @@ -24,8 +24,8 @@ type TaskStatus = [] type PrinterProtocol = - | BuildStarted of graph:GraphDef.Graph - | BuildCompleted of summary:Build.Summary + | BuildStarted + | BuildCompleted | TaskScheduled of taskId:string * label:string | TaskStatusChanged of taskId:string * status:TaskStatus | TaskCompleted of taskId:string * restore:bool * success:bool @@ -50,10 +50,10 @@ type BuildNotification() = let rec messageLoop () = async { let! msg = inbox.Receive() match msg with - | PrinterProtocol.BuildStarted graph -> + | PrinterProtocol.BuildStarted -> return! messageLoop () - | PrinterProtocol.BuildCompleted summary -> + | PrinterProtocol.BuildCompleted -> cts.Cancel() renderer.Refresh () buildComplete.Set() |> ignore @@ -86,17 +86,15 @@ type BuildNotification() = let printerAgent = MailboxProcessor.Start(handler) - interface Build.IBuildNotification with - member _.WaitCompletion(): unit = - buildComplete.WaitOne() |> ignore - - member _.BuildStarted graph = - PrinterProtocol.BuildStarted graph + interface BuildProgress.IBuildProgress with + member _.BuildStarted () = + PrinterProtocol.BuildStarted |> printerAgent.Post - member _.BuildCompleted (summary: Build.Summary) = - PrinterProtocol.BuildCompleted summary + member _.BuildCompleted () = + PrinterProtocol.BuildCompleted |> printerAgent.Post + buildComplete.WaitOne() |> ignore member _.TaskScheduled (taskId:string) (label:string) = PrinterProtocol.TaskScheduled (taskId, label) diff --git a/src/Terrabuild/Helpers/Git.fs b/src/Terrabuild/Helpers/Git.fs index ca5df5c6..b8507a4e 100644 --- a/src/Terrabuild/Helpers/Git.fs +++ b/src/Terrabuild/Helpers/Git.fs @@ -1,6 +1,8 @@ module Git open Errors open System +open LibGit2Sharp +open System.IO let getBranchOrTag (dir: string) = // https://stackoverflow.com/questions/18659425/get-git-current-branch-tag-name @@ -29,3 +31,38 @@ let getCommitLog (dir: string) = |> Seq.map (fun arr -> {| Sha = arr[0]; Subject = arr[1]; Author = arr[2]; Email = arr[3]; Timestamp = DateTime.Parse(arr[4]) |}) |> List.ofSeq | _ -> raiseExternalError "Failed to get commit log" + +// workspaceDir: absolute path anywhere inside the repo (ok if it's a nested "workspace") +// projectDir: path relative to workspaceDir +// returns: absolute file paths in the working tree that are NOT ignored by git +let enumeratedCommittedFiles (workspaceDir: string) (projectDir: string) : string list = + use repo = new Repository(workspaceDir |> Repository.Discover) + let repoDir = repo.Info.WorkingDirectory + + let startDir = FS.combinePath workspaceDir projectDir |> Path.GetFullPath + + let isIgnored (absPath: string) = + let rel = FS.relativePath repoDir absPath + let relForGit = + if Directory.Exists absPath then (if rel.EndsWith "/" then rel else rel + "/") else rel + repo.Ignore.IsPathIgnored(relForGit) + + let results = ResizeArray() + let stack = Collections.Generic.Stack() + stack.Push(startDir) + + while stack.Count > 0 do + let dir = stack.Pop() + + if String.Equals(Path.GetFileName(dir), ".git") then + () // never descend into .git + elif not (isIgnored dir) then + // files + for f in Directory.EnumerateFiles(dir) do + if not (isIgnored f) then + results.Add(Path.GetFullPath f) + // subdirs + for d in Directory.EnumerateDirectories(dir) do + stack.Push d + + results |> Seq.toList diff --git a/src/Terrabuild/Helpers/Terminal.fs b/src/Terrabuild/Helpers/Terminal.fs index 607c3f71..d50d5ff2 100644 --- a/src/Terrabuild/Helpers/Terminal.fs +++ b/src/Terrabuild/Helpers/Terminal.fs @@ -41,12 +41,10 @@ let writeLine (str: string) = Console.Out.WriteLine(str) let hideCursor() = - if supportAnsi then - Ansi.Styles.cursorHide |> write |> flush + if supportAnsi then Ansi.Styles.cursorHide |> write let showCursor() = - if supportAnsi then - Ansi.Styles.cursorShow |> write |> flush + if supportAnsi then Ansi.Styles.cursorShow |> write let autoflush() = if supportAnsi then diff --git a/src/Terrabuild/Program.fs b/src/Terrabuild/Program.fs index 6b9a8428..02319d7c 100644 --- a/src/Terrabuild/Program.fs +++ b/src/Terrabuild/Program.fs @@ -158,11 +158,7 @@ let processCommandLine (parser: ArgumentParser) (result: ParseRe let errCode = if options.WhatIf then 0 else - let summary = - let buildNotification = Notification.BuildNotification() :> Build.IBuildNotification - let summary = Build.run options cache api buildNotification buildGraph - buildNotification.WaitCompletion() - summary + let summary = Build.run options cache api buildGraph if options.Debug then let jsonBuild = Json.Serialize summary diff --git a/src/Terrabuild/Terrabuild.fsproj b/src/Terrabuild/Terrabuild.fsproj index cccc9517..112d5a05 100644 --- a/src/Terrabuild/Terrabuild.fsproj +++ b/src/Terrabuild/Terrabuild.fsproj @@ -12,6 +12,7 @@ + @@ -21,6 +22,7 @@ + @@ -28,7 +30,6 @@ - @@ -59,6 +60,7 @@ + diff --git a/src/Terrabuild/packages.lock.json b/src/Terrabuild/packages.lock.json index 4c68cbb0..0de5114b 100644 --- a/src/Terrabuild/packages.lock.json +++ b/src/Terrabuild/packages.lock.json @@ -67,6 +67,15 @@ "System.Text.Json": "6.0.10" } }, + "LibGit2Sharp": { + "type": "Direct", + "requested": "[0.31.0, )", + "resolved": "0.31.0", + "contentHash": "b3+UfV7LjKMjAHWwl7VawejiOv2gJIC6dTCA/S0puLTHACAA/Oeb5JJmWUQMeyH/T/WR/LaIK8bk2RbdFnrZvg==", + "dependencies": { + "LibGit2Sharp.NativeBinaries": "[2.0.323]" + } + }, "SemanticVersioning": { "type": "Direct", "requested": "[3.0.0, )", @@ -218,10 +227,10 @@ "FSharp.Core": "4.6.2" } }, - "Ignore": { + "LibGit2Sharp.NativeBinaries": { "type": "Transitive", - "resolved": "0.2.1", - "contentHash": "Qw3s0pTwK3o6Iv6kTMjmxzOt91pczU533OmtAvFRsJ7PdCVMhGCRyUkMsCOI7ejxOtHJdRsj141HeZWeedlqkQ==" + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", @@ -1231,7 +1240,6 @@ "dependencies": { "FSharp.Core": "[9.0.303, )", "FSharp.SystemTextJson": "[1.4.36, )", - "Ignore": "[0.2.1, )", "Microsoft.Extensions.FileSystemGlobbing": "[9.0.8, )", "System.Text.Json": "[9.0.0, )" } @@ -1298,6 +1306,11 @@ } }, "net9.0/linux-arm64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -2160,6 +2173,11 @@ } }, "net9.0/linux-x64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -3022,6 +3040,11 @@ } }, "net9.0/osx-arm64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -3884,6 +3907,11 @@ } }, "net9.0/osx-x64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -4746,6 +4774,11 @@ } }, "net9.0/win-arm64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", @@ -5582,6 +5615,11 @@ } }, "net9.0/win-x64": { + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, "Microsoft.Win32.Primitives": { "type": "Transitive", "resolved": "4.3.0", diff --git a/tests/simple/results/terrabuild-debug.build-graph.json b/tests/simple/results/terrabuild-debug.build-graph.json index df4b5fd4..05289bb0 100644 --- a/tests/simple/results/terrabuild-debug.build-graph.json +++ b/tests/simple/results/terrabuild-debug.build-graph.json @@ -11,8 +11,8 @@ "outputs": [ "*.planfile" ], - "projectHash": "66D01BA3879BB7926D41E7A60D780C74920BF79068DAD91771B86C198B32EF1D", - "targetHash": "76734C3F501CC560F668ACA1AF33F3569DB055FFF0E1BEAD39AA8060512F2CFE", + "projectHash": "BFA50CC4701C93ED862F3608316CBE8BAE0D569DD18A060C7129A1D4DF7BCFB7", + "targetHash": "C47A717B43D162309032AF12B1E2A681BC951B6DD86BBAFD51691571C9D7C8F2", "operations": [ { "container": "hashicorp/terraform:1.10", @@ -28,7 +28,7 @@ "containerVariables": [], "metaCommand": "@terraform plan", "command": "terraform", - "arguments": "plan -out=terrabuild.planfile -var=\u0022dotnet_app_version=26E3A1BECCD5D71B2FCF43FD9A22B931AAE58622995B126AF60C3EC553D1C04A\u0022 -var=\u0022npm_app_version=37ED3ADC0FCE859AB887474C73CCC1A2CD45535746F562F664DF766EF6C1E9B6\u0022" + "arguments": "plan -out=terrabuild.planfile -var=\u0022dotnet_app_version=AF7DA885096F5E8E14B8A8FAB101958472F0ABE049250D5D7366DBA89C3A5771\u0022 -var=\u0022npm_app_version=37ED3ADC0FCE859AB887474C73CCC1A2CD45535746F562F664DF766EF6C1E9B6\u0022" } ], "cache": "local", @@ -144,8 +144,8 @@ "obj/*.props", "obj/*.targets" ], - "projectHash": "26E3A1BECCD5D71B2FCF43FD9A22B931AAE58622995B126AF60C3EC553D1C04A", - "targetHash": "4ADE680B3DFD1A990C0337225AD933446F38EF52636DACC30CD5303974C0A533", + "projectHash": "AF7DA885096F5E8E14B8A8FAB101958472F0ABE049250D5D7366DBA89C3A5771", + "targetHash": "C7E15B98D6E39C4DADD65464B9EFBA34BF35F26DE6982FFB6B23C0FE871E4AE7", "operations": [ { "container": "mcr.microsoft.com/dotnet/sdk:9.0.202", diff --git a/tests/simple/results/terrabuild-debug.config.json b/tests/simple/results/terrabuild-debug.config.json index 809c1bbd..354f41e8 100644 --- a/tests/simple/results/terrabuild-debug.config.json +++ b/tests/simple/results/terrabuild-debug.config.json @@ -34,7 +34,7 @@ "projects": { "deployments/terraform-deploy": { "directory": "deployments/terraform-deploy", - "hash": "66D01BA3879BB7926D41E7A60D780C74920BF79068DAD91771B86C198B32EF1D", + "hash": "BFA50CC4701C93ED862F3608316CBE8BAE0D569DD18A060C7129A1D4DF7BCFB7", "dependencies": [ "projects/dotnet-app", "projects/npm-app" @@ -91,7 +91,7 @@ { "dotnet_app_version": [ "string", - "26E3A1BECCD5D71B2FCF43FD9A22B931AAE58622995B126AF60C3EC553D1C04A" + "AF7DA885096F5E8E14B8A8FAB101958472F0ABE049250D5D7366DBA89C3A5771" ], "npm_app_version": [ "string", @@ -330,7 +330,7 @@ "projects/dotnet-app": { "id": "dotnet_app", "directory": "projects/dotnet-app", - "hash": "26E3A1BECCD5D71B2FCF43FD9A22B931AAE58622995B126AF60C3EC553D1C04A", + "hash": "AF7DA885096F5E8E14B8A8FAB101958472F0ABE049250D5D7366DBA89C3A5771", "dependencies": [ "libraries/dotnet-lib" ],