Initial v0 of docs-assembler repos build-all command.

src/Elastic.Markdown/BuildContext.cs

@@ -29,17 +29,16 @@ public record BuildContext
 
 	public bool Force { get; init; }
 
+	// This property is used to determine if the site should be indexed by search engines
+	public bool AllowIndexing { get; init; }
+
+	private readonly string? _urlPathPrefix;
 	public string? UrlPathPrefix
 	{
 		get => string.IsNullOrWhiteSpace(_urlPathPrefix) ? "" : $"/{_urlPathPrefix.Trim('/')}";
 		init => _urlPathPrefix = value;
 	}
 
-	// This property is used to determine if the site should be indexed by search engines
-	public bool AllowIndexing { get; init; }
-
-	private readonly string? _urlPathPrefix;
-
 	public BuildContext(IFileSystem fileSystem)
 		: this(new DiagnosticsCollector([]), fileSystem, fileSystem, null, null) { }
 

src/docs-assembler/AssembleContext.cs

@@ -21,9 +21,16 @@ public class AssembleContext
 
 	public IFileInfo ConfigurationPath { get; }
 
+	public IDirectoryInfo CheckoutDirectory { get; set; }
+
 	public IDirectoryInfo OutputDirectory { get; set; }
 
-	public AssembleContext(DiagnosticsCollector collector, IFileSystem readFileSystem, IFileSystem writeFileSystem, string? output)
+	public bool Force { get; init; }
+
+	// This property is used to determine if the site should be indexed by search engines
+	public bool AllowIndexing { get; init; }
+
+	public AssembleContext(DiagnosticsCollector collector, IFileSystem readFileSystem, IFileSystem writeFileSystem, string? checkoutDirectory, string? output)
 	{
 		Collector = collector;
 		ReadFileSystem = readFileSystem;
@@ -37,6 +44,7 @@ public AssembleContext(DiagnosticsCollector collector, IFileSystem readFileSyste
 
 		ConfigurationPath = ReadFileSystem.FileInfo.New(configPath);
 		Configuration = AssemblyConfiguration.Deserialize(ReadFileSystem.File.ReadAllText(ConfigurationPath.FullName));
+		CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? ".artifacts/checkouts");
 		OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? ".artifacts/assembly");
 	}
 
@@ -55,7 +63,5 @@ private void ExtractAssemblerConfiguration(string configPath)
 			outputFile.Directory.Create();
 		using var stream = outputFile.OpenWrite();
 		resourceStream.CopyTo(stream);
-
-
 	}
 }

src/docs-assembler/Building/AssemblerBuilder.cs

@@ -0,0 +1,53 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Documentation.Assembler.Sourcing;
+using Elastic.Markdown;
+using Elastic.Markdown.IO;
+using Microsoft.Extensions.Logging;
+
+namespace Documentation.Assembler.Building;
+
+public class AssemblerBuilder(ILoggerFactory logger, AssembleContext context)
+{
+	private readonly ILogger<AssemblerBuilder> _logger = logger.CreateLogger<AssemblerBuilder>();
+
+	public async Task BuildAllAsync(IReadOnlyCollection<Checkout> checkouts, Cancel ctx)
+	{
+		foreach (var checkout in checkouts)
+		{
+			try
+			{
+				await BuildAsync(checkout, ctx);
+			}
+			catch (Exception e) when (e.Message.Contains("Can not locate docset.yml file in"))
+			{
+				// TODO: we should only ignore this temporarily while migration is ongoing
+				_logger.LogWarning("Skipping {Checkout} as its not yet been migrated to V3", checkout.Directory.FullName);
+			}
+			catch (Exception e)
+			{
+				Console.WriteLine(e);
+				throw;
+			}
+		}
+	}
+
+	private async Task BuildAsync(Checkout checkout, Cancel ctx)
+	{
+		var path = checkout.Directory.FullName;
+		var pathPrefix = checkout.Repository.PathPrefix;
+		var output = pathPrefix != null ? Path.Combine(context.OutputDirectory.FullName, pathPrefix) : context.OutputDirectory.FullName;
+
+		var buildContext = new BuildContext(context.Collector, context.ReadFileSystem, context.WriteFileSystem, path, output)
+		{
+			UrlPathPrefix = pathPrefix,
+			Force = true,
+			AllowIndexing = true
+		};
+		var set = new DocumentationSet(buildContext, logger);
+		var generator = new DocumentationGenerator(set, logger);
+		await generator.GenerateAll(ctx);
+	}
+}

src/docs-assembler/Cli/RepositoryCommands.cs

@@ -6,6 +6,7 @@
 using System.IO.Abstractions;
 using Actions.Core.Services;
 using ConsoleAppFramework;
+using Documentation.Assembler.Building;
 using Documentation.Assembler.Sourcing;
 using Elastic.Documentation.Tooling.Diagnostics.Console;
 using Microsoft.Extensions.Logging;
@@ -25,18 +26,56 @@ private void AssignOutputLogger()
 	// would love to use libgit2 so there is no git dependency but
 	// libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674
 	/// <summary> Clones all repositories </summary>
+	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
 	/// <param name="ctx"></param>
 	[Command("clone-all")]
-	public async Task CloneAll(Cancel ctx = default)
+	public async Task<int> CloneAll(bool? strict = null, Cancel ctx = default)
 	{
 		AssignOutputLogger();
 
 		await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService);
 
-		var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null);
-		var cloner = new RepositoryCloner(logger, assembleContext);
-		await cloner.CloneAll(ctx);
+		var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null);
+		var cloner = new RepositoryCheckoutProvider(logger, assembleContext);
+		_ = await cloner.AcquireAllLatest(ctx);
+
+		if (strict ?? false)
+			return collector.Errors + collector.Warnings;
+		return collector.Errors;
 	}
 
+	/// <summary> Builds all repositories </summary>
+	/// <param name="force"> Force a full rebuild of the destination folder</param>
+	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
+	/// <param name="allowIndexing"> Allow indexing and following of html files</param>
+	/// <param name="ctx"></param>
+	[Command("build-all")]
+	public async Task<int> BuildAll(
+		bool? force = null,
+		bool? strict = null,
+		bool? allowIndexing = null,
+		Cancel ctx = default)
+	{
+		AssignOutputLogger();
 
+		await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService);
+		_ = collector.StartAsync(ctx);
+
+		var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null)
+		{
+			Force = force ?? false,
+			AllowIndexing = allowIndexing ?? false,
+		};
+		var cloner = new RepositoryCheckoutProvider(logger, assembleContext);
+		var checkouts = cloner.GetAll().ToArray();
+		if (checkouts.Length == 0)
+			throw new Exception("No checkouts found");
+
+		var builder = new AssemblerBuilder(logger, assembleContext);
+		await builder.BuildAllAsync(checkouts, ctx);
+
+		if (strict ?? false)
+			return collector.Errors + collector.Warnings;
+		return collector.Errors;
+	}
 }

src/docs-assembler/Configuration/AssemblyConfiguration.cs

@@ -25,6 +25,12 @@ public static AssemblyConfiguration Deserialize(string yaml)
 		try
 		{
 			var config = deserializer.Deserialize<AssemblyConfiguration>(input);
+			foreach (var (name, r) in config.ReferenceRepositories)
+			{
+				var repository = RepositoryDefaults(r, name);
+				config.ReferenceRepositories[name] = repository;
+			}
+			config.Narrative = RepositoryDefaults(config.Narrative, NarrativeRepository.RepositoryName);
 			return config;
 		}
 		catch (Exception e)
@@ -35,27 +41,34 @@ public static AssemblyConfiguration Deserialize(string yaml)
 		}
 	}
 
+	private static TRepository RepositoryDefaults<TRepository>(TRepository r, string name)
+		where TRepository : Repository, new()
+	{
+		// ReSharper disable NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
+		var repository = r ?? new TRepository();
+		// ReSharper restore NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
+		repository.Name = name;
+		if (string.IsNullOrEmpty(repository.CurrentBranch))
+			repository.CurrentBranch = "main";
+		if (string.IsNullOrEmpty(repository.Origin))
+		{
+			if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")))
+			{
+				var token = Environment.GetEnvironmentVariable("GITHUB_TOKEN");
+				repository.Origin = !string.IsNullOrEmpty(token)
+					? $"https://oath2:{token}@github.com/elastic/{name}.git"
+					: $"https://github.com/elastic/{name}.git";
+			}
+			else
+				repository.Origin = $"git@github.com:elastic/{name}.git";
+		}
+
+		return repository;
+	}
+
 	[YamlMember(Alias = "narrative")]
 	public NarrativeRepository Narrative { get; set; } = new();
 
 	[YamlMember(Alias = "references")]
-	public Dictionary<string, Repository?> ReferenceRepositories { get; set; } = [];
-}
-
-public record NarrativeRepository : Repository
-{
-	public static string Name { get; } = "docs-content";
-}
-
-public record Repository
-{
-	[YamlMember(Alias = "repo")]
-	public string? Origin { get; set; }
-
-	[YamlMember(Alias = "current")]
-	public string? CurrentBranch { get; set; }
-
-	[YamlMember(Alias = "checkout_strategy")]
-	public string CheckoutStrategy { get; set; } = "partial";
-
+	public Dictionary<string, Repository> ReferenceRepositories { get; set; } = [];
 }

src/docs-assembler/Configuration/Repository.cs

@@ -0,0 +1,37 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using YamlDotNet.Serialization;
+
+namespace Documentation.Assembler.Configuration;
+
+public record NarrativeRepository : Repository
+{
+	public static string RepositoryName { get; } = "docs-content";
+	public override string Name { get; set; } = RepositoryName;
+	public override string? PathPrefix { get; set; }
+}
+
+public record Repository
+{
+	[YamlIgnore]
+	public virtual string Name { get; set; } = string.Empty;
+
+	[YamlMember(Alias = "repo")]
+	public string Origin { get; set; } = string.Empty;
+
+	[YamlMember(Alias = "current")]
+	public string CurrentBranch { get; set; } = "main";
+
+	[YamlMember(Alias = "checkout_strategy")]
+	public string CheckoutStrategy { get; set; } = "partial";
+
+	private string? _pathPrefix;
+	[YamlMember(Alias = "path_prefix")]
+	public virtual string? PathPrefix
+	{
+		get => _pathPrefix ?? $"reference/{Name}";
+		set => _pathPrefix = value;
+	}
+}

src/docs-assembler/Sourcing/Checkout.cs

@@ -0,0 +1,15 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using Documentation.Assembler.Configuration;
+
+namespace Documentation.Assembler.Sourcing;
+
+public record Checkout
+{
+	public required Repository Repository { get; init; }
+	public required string HeadReference { get; init; }
+	public required IDirectoryInfo Directory { get; init; }
+}