Initial scaffolding of the docs-assembler binary

build/Targets.fs

@@ -57,6 +57,7 @@ let private pristineCheck (arguments:ParseResults<Build>) =
 let private publishBinaries _ =
     exec { run "dotnet" "publish" "src/docs-builder/docs-builder.csproj" }
     exec { run "dotnet" "publish" "src/docs-generator/docs-generator.csproj" }
+    exec { run "dotnet" "publish" "src/docs-assembler/docs-assembler.csproj" }
     Zip.zip
         ".artifacts/publish/docs-builder/release"
         $"docs-builder-%s{OS.Name}-{OS.Arch}.zip"
@@ -103,6 +104,7 @@ let private publishContainers _ =
         exec { run "dotnet" (args @ registry) }
     createImage "docs-builder"
     createImage "docs-generator"
+    createImage "docs-assembler"
 
 let private runTests _ =
     exec {

docs-builder.sln

@@ -48,6 +48,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "publish-vercel", "publish-v
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "build", "build\build.fsproj", "{10857974-6CF1-42B5-B793-AAA988BD7348}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler", "src\docs-assembler\docs-assembler.csproj", "{28350800-B44B-479B-86E2-1D39E321C0B4}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -82,6 +84,10 @@ Global
 		{10857974-6CF1-42B5-B793-AAA988BD7348}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{10857974-6CF1-42B5-B793-AAA988BD7348}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{10857974-6CF1-42B5-B793-AAA988BD7348}.Release|Any CPU.Build.0 = Release|Any CPU
+		{28350800-B44B-479B-86E2-1D39E321C0B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{28350800-B44B-479B-86E2-1D39E321C0B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{28350800-B44B-479B-86E2-1D39E321C0B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{28350800-B44B-479B-86E2-1D39E321C0B4}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -91,5 +97,6 @@ Global
 		{1C340CCF-9AAC-4163-A7BB-60528076E98B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{CD2887E3-BDA9-434B-A5BF-9ED38DE20332} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{A2A34BBC-CB5E-4100-9529-A12B6ECB769C} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
+		{28350800-B44B-479B-86E2-1D39E321C0B4} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 	EndGlobalSection
 EndGlobal

src/docs-assembler/AssemblyConfiguration.cs

@@ -0,0 +1,49 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using YamlDotNet.Serialization;
+
+namespace Documentation.Assembler;
+
+[YamlStaticContext]
+[YamlSerializable(typeof(AssemblyConfiguration))]
+[YamlSerializable(typeof(Repository))]
+public partial class YamlStaticContext;
+
+public record AssemblyConfiguration
+{
+	public static AssemblyConfiguration Deserialize(string yaml)
+	{
+		var input = new StringReader(yaml);
+
+		var deserializer = new StaticDeserializerBuilder(new YamlStaticContext())
+			.IgnoreUnmatchedProperties()
+			.Build();
+
+		try
+		{
+			var config = deserializer.Deserialize<AssemblyConfiguration>(input);
+			return config;
+		}
+		catch (Exception e)
+		{
+			Console.WriteLine(e);
+			Console.WriteLine(e.InnerException);
+			throw;
+		}
+	}
+
+	[YamlMember(Alias = "repos")]
+	public Dictionary<string, Repository> Repositories { get; set; } = new();
+}
+
+public record Repository
+{
+	[YamlMember(Alias = "repo")]
+	public string Origin { get; set; } = string.Empty;
+
+	[YamlMember(Alias = "branch")]
+	public string? Branch { get; set; }
+
+}

src/docs-assembler/Cli/Filters.cs

@@ -0,0 +1,51 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Diagnostics;
+using ConsoleAppFramework;
+
+namespace Documentation.Assembler.Cli;
+
+internal class StopwatchFilter(ConsoleAppFilter next) : ConsoleAppFilter(next)
+{
+	public override async Task InvokeAsync(ConsoleAppContext context, Cancel ctx)
+	{
+		var isHelpOrVersion = context.Arguments.Any(a => a is "--help" or "-h" or "--version");
+		var name = string.IsNullOrWhiteSpace(context.CommandName) ? "generate" : context.CommandName;
+		var startTime = Stopwatch.GetTimestamp();
+		if (!isHelpOrVersion)
+			ConsoleApp.Log($"{name} :: Starting...");
+		try
+		{
+			await Next.InvokeAsync(context, ctx);
+		}
+		finally
+		{
+			var endTime = Stopwatch.GetElapsedTime(startTime);
+			if (!isHelpOrVersion)
+				ConsoleApp.Log($"{name} :: Finished in '{endTime}");
+		}
+	}
+}
+
+internal class CatchExceptionFilter(ConsoleAppFilter next) : ConsoleAppFilter(next)
+{
+	public override async Task InvokeAsync(ConsoleAppContext context, CancellationToken cancellationToken)
+	{
+		try
+		{
+			await Next.InvokeAsync(context, cancellationToken);
+		}
+		catch (Exception ex)
+		{
+			if (ex is OperationCanceledException)
+			{
+				ConsoleApp.Log("Cancellation requested, exiting.");
+				return;
+			}
+
+			ConsoleApp.LogError(ex.ToString()); // .ToString() shows stacktrace, .Message can avoid showing stacktrace to user.
+		}
+	}
+}

src/docs-assembler/Program.cs

@@ -0,0 +1,91 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using ConsoleAppFramework;
+using Documentation.Assembler;
+using Documentation.Assembler.Cli;
+using Elastic.Markdown.IO;
+using ProcNet;
+using ProcNet.Std;
+
+var configFile = Path.Combine(Paths.Root.FullName, "src/docs-assembler/conf.yml");
+var config = AssemblyConfiguration.Deserialize(File.ReadAllText(configFile));
+
+var app = ConsoleApp.Create();
+app.UseFilter<StopwatchFilter>();
+app.UseFilter<CatchExceptionFilter>();
+
+// would love to use libgit2 so there is no git dependency but
+// libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674
+app.Add("clone-all", async Task (CancellationToken ctx) =>
+{
+	Console.WriteLine(config.Repositories.Count);
+	var dict = new ConcurrentDictionary<string, Stopwatch>();
+	await Parallel.ForEachAsync(config.Repositories, new ParallelOptions
+	{
+		CancellationToken = ctx,
+		MaxDegreeOfParallelism = Environment.ProcessorCount / 4
+	}, async (kv, c) =>
+	{
+		await Task.Run(() =>
+		{
+			var name = kv.Key;
+			var repository = kv.Value;
+			var checkoutFolder = Path.Combine(Paths.Root.FullName, $".artifacts/assembly/{name}");
+
+			var sw = Stopwatch.StartNew();
+			dict.AddOrUpdate(name, sw, (_, _) => sw);
+			Console.WriteLine($"Checkout: {name}\t{repository}\t{checkoutFolder}");
+			var branch = repository.Branch ?? "main";
+			var args = new StartArguments(
+				"git", "clone", repository.Origin, checkoutFolder, "--depth", "1"
+				, "--single-branch", "--branch", branch
+			);
+			Proc.StartRedirected(args, new ConsoleLineHandler(name));
+			sw.Stop();
+		}, c);
+	}).ConfigureAwait(false);
+
+	foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed))
+		Console.WriteLine($"-> {kv.Key}\ttook: {kv.Value.Elapsed}");
+});
+app.Add("list", async Task (CancellationToken ctx) =>
+{
+
+	var assemblyPath = Path.Combine(Paths.Root.FullName, $".artifacts/assembly");
+	var dir = new DirectoryInfo(assemblyPath);
+	var dictionary = new Dictionary<string, string>();
+	foreach (var d in dir.GetDirectories())
+	{
+		var checkoutFolder = Path.Combine(assemblyPath, d.Name);
+
+		var capture = Proc.Start(
+			new StartArguments("git", "rev-parse", "--abbrev-ref", "HEAD")
+			{
+				WorkingDirectory = checkoutFolder
+			}
+		);
+		dictionary.Add(d.Name, capture.ConsoleOut.FirstOrDefault()?.Line ?? "unknown");
+	}
+	foreach (var kv in dictionary.OrderBy(kv => kv.Value))
+		Console.WriteLine($"-> {kv.Key}\tbranch: {kv.Value}");
+
+	await Task.CompletedTask;
+});
+
+await app.RunAsync(args);
+
+namespace Documentation.Assembler
+{
+	public class ConsoleLineHandler(string prefix) : IConsoleLineHandler
+	{
+		public void Handle(LineOut lineOut) => lineOut.CharsOrString(
+			r => Console.Write(prefix + ": " + r),
+			l => Console.WriteLine(prefix + ": " + l));
+
+		public void Handle(Exception e) { }
+	}
+}