Add content_source (current/next) to assembler

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -59,7 +59,7 @@
     <PackageReference Include="RazorSlices" Version="0.8.1" />
     <PackageReference Include="Slugify.Core" Version="4.0.1" />
     <PackageReference Include="Utf8StreamReader" Version="1.3.2"/>
-    <PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" Version="16.1.3" />
+    <PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" Version="16.1.3" PrivateAssets="All"/>
     <PackageReference Include="YamlDotNet" Version="16.3.0" />
     <PackageReference Include="System.IO.Abstractions" Version="21.0.29" />
   </ItemGroup>

src/Elastic.Markdown/Myst/YamlSerialization.cs

@@ -39,5 +39,4 @@ public static T Deserialize<T>(string yaml)
 [YamlSerializable(typeof(Setting))]
 [YamlSerializable(typeof(AllowedValue))]
 [YamlSerializable(typeof(SettingMutability))]
-
 public partial class DocsBuilderYamlStaticContext;

src/docs-assembler/AssembleContext.cs

@@ -67,13 +67,15 @@ public AssembleContext(
 			ExtractAssemblerConfiguration(historyMappingPath, "historymapping.yml");
 		HistoryMappingPath = ReadFileSystem.FileInfo.New(historyMappingPath);
 
-		CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? ".artifacts/checkouts");
-		OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? ".artifacts/assembly");
-
-
 		if (!Configuration.Environments.TryGetValue(environment, out var env))
 			throw new Exception($"Could not find environment {environment}");
 		Environment = env;
+
+		var contentSource = Environment.ContentSource.ToStringFast();
+		CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? Path.Combine(".artifacts", "checkouts", contentSource));
+		OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? Path.Combine(".artifacts", "assembly"));
+
+
 	}
 
 	private void ExtractAssemblerConfiguration(string configPath, string file)

src/docs-assembler/Cli/RepositoryCommands.cs

@@ -29,15 +29,22 @@ private void AssignOutputLogger()
 	// libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674
 	/// <summary> Clones all repositories </summary>
 	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
+	/// <param name="environment"> The environment to build</param>
 	/// <param name="ctx"></param>
 	[Command("clone-all")]
-	public async Task<int> CloneAll(bool? strict = null, Cancel ctx = default)
+	public async Task<int> CloneAll(
+		bool? strict = null,
+		string? environment = null,
+		Cancel ctx = default
+	)
 	{
 		AssignOutputLogger();
+		var githubEnvironmentInput = githubActionsService.GetInput("environment");
+		environment ??= !string.IsNullOrEmpty(githubEnvironmentInput) ? githubEnvironmentInput : "dev";
 
 		await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService);
 
-		var assembleContext = new AssembleContext("dev", collector, new FileSystem(), new FileSystem(), null, null);
+		var assembleContext = new AssembleContext(environment, collector, new FileSystem(), new FileSystem(), null, null);
 		var cloner = new AssemblerRepositorySourcer(logger, assembleContext);
 		_ = await cloner.AcquireAllLatest(ctx);
 
@@ -50,7 +57,7 @@ public async Task<int> CloneAll(bool? strict = null, Cancel ctx = default)
 	/// <param name="force"> Force a full rebuild of the destination folder</param>
 	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
 	/// <param name="allowIndexing"> Allow indexing and following of html files</param>
-	/// <param name="environment"> The environment to resolve links to</param>
+	/// <param name="environment"> The environment to build</param>
 	/// <param name="ctx"></param>
 	[Command("build-all")]
 	public async Task<int> BuildAll(

src/docs-assembler/Configuration/AssemblyConfiguration.cs

@@ -2,6 +2,8 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.ComponentModel.DataAnnotations;
+using NetEscapades.EnumGenerators;
 using YamlDotNet.Serialization;
 
 namespace Documentation.Assembler.Configuration;
@@ -24,6 +26,7 @@ public static AssemblyConfiguration Deserialize(string yaml)
 				var repository = RepositoryDefaults(r, name);
 				config.ReferenceRepositories[name] = repository;
 			}
+
 			foreach (var (name, env) in config.Environments)
 				env.Name = name;
 			config.Narrative = RepositoryDefaults(config.Narrative, NarrativeRepository.RepositoryName);
@@ -44,8 +47,10 @@ private static TRepository RepositoryDefaults<TRepository>(TRepository r, string
 		var repository = r ?? new TRepository();
 		// ReSharper restore NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
 		repository.Name = name;
-		if (string.IsNullOrEmpty(repository.CurrentBranch))
-			repository.CurrentBranch = "main";
+		if (string.IsNullOrEmpty(repository.GitReferenceCurrent))
+			repository.GitReferenceCurrent = "main";
+		if (string.IsNullOrEmpty(repository.GitReferenceNext))
+			repository.GitReferenceNext = "main";
 		if (string.IsNullOrEmpty(repository.Origin))
 		{
 			if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")))
@@ -70,6 +75,9 @@ private static TRepository RepositoryDefaults<TRepository>(TRepository r, string
 
 	[YamlMember(Alias = "environments")]
 	public Dictionary<string, PublishEnvironment> Environments { get; set; } = [];
+
+	[YamlMember(Alias = "named_git_references")]
+	public Dictionary<string, string> NamedGitReferences { get; set; } = [];
 }
 
 public record PublishEnvironment
@@ -86,16 +94,30 @@ public record PublishEnvironment
 	[YamlMember(Alias = "allow_indexing")]
 	public bool AllowIndexing { get; set; }
 
+	[YamlMember(Alias = "content_source")]
+	public ContentSource ContentSource { get; set; }
+
 	[YamlMember(Alias = "google_tag_manager")]
 	public GoogleTagManager GoogleTagManager { get; set; } = new();
 }
 
+[EnumExtensions]
+public enum ContentSource
+{
+	[Display(Name = "next")]
+	Next,
+
+	[Display(Name = "current")]
+	Current
+}
+
 public record GoogleTagManager
 {
 	[YamlMember(Alias = "enabled")]
 	public bool Enabled { get; set; }
 
 	private string _id = string.Empty;
+
 	[YamlMember(Alias = "id")]
 	public string Id
 	{
@@ -107,6 +129,7 @@ public string Id
 			_id = value;
 		}
 	}
+
 	[YamlMember(Alias = "auth")]
 	public string? Auth { get; set; }
 

src/docs-assembler/Configuration/Repository.cs

@@ -21,7 +21,10 @@ public record Repository
 	public string Origin { get; set; } = string.Empty;
 
 	[YamlMember(Alias = "current")]
-	public string CurrentBranch { get; set; } = "main";
+	public string GitReferenceCurrent { get; set; } = "main";
+
+	[YamlMember(Alias = "next")]
+	public string GitReferenceNext { get; set; } = "main";
 
 	[YamlMember(Alias = "checkout_strategy")]
 	public string CheckoutStrategy { get; set; } = "partial";

src/docs-assembler/Navigation/AssemblerDocumentationSet.cs

@@ -44,7 +44,7 @@ public AssemblerDocumentationSet(
 			RepositoryName = checkout.Repository.Name,
 			Ref = checkout.HeadReference,
 			Remote = $"elastic/${checkout.Repository.Name}",
-			Branch = checkout.Repository.CurrentBranch
+			Branch = checkout.Repository.GitReferenceCurrent
 		};
 
 		var buildContext = new BuildContext(

src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs

@@ -4,6 +4,7 @@
 
 using System.Collections.Concurrent;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.IO.Abstractions;
 using Documentation.Assembler.Configuration;
 using Elastic.Markdown.IO;
@@ -18,6 +19,7 @@ public class AssemblerRepositorySourcer(ILoggerFactory logger, AssembleContext c
 	private readonly ILogger<AssemblerRepositorySourcer> _logger = logger.CreateLogger<AssemblerRepositorySourcer>();
 
 	private AssemblyConfiguration Configuration => context.Configuration;
+	private PublishEnvironment PublishEnvironment => context.Environment;
 
 	public IReadOnlyCollection<Checkout> GetAll()
 	{
@@ -27,20 +29,28 @@ public IReadOnlyCollection<Checkout> GetAll()
 		foreach (var repo in repositories)
 		{
 			var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, repo.Name));
-			//var head = Capture(checkoutFolder, "git", "rev-parse", "HEAD");
 			var checkout = new Checkout
 			{
 				Repository = repo,
 				Directory = checkoutFolder,
+				//TODO read from links.json and ensure we check out exactly that git reference
+				//+ validate that git reference belongs to the appropriate branch
 				HeadReference = Guid.NewGuid().ToString("N")
 			};
 			checkouts.Add(checkout);
 		}
+
 		return checkouts;
 	}
 
 	public async Task<IReadOnlyCollection<Checkout>> AcquireAllLatest(Cancel ctx = default)
 	{
+		_logger.LogInformation(
+			"Cloning all repositories for environment {EnvironmentName} using '{ContentSourceStrategy}' content sourcing strategy",
+			PublishEnvironment.Name,
+			PublishEnvironment.ContentSource.ToStringFast()
+		);
+
 		var dict = new ConcurrentDictionary<string, Stopwatch>();
 		var checkouts = new ConcurrentBag<Checkout>();
 
@@ -65,7 +75,7 @@ await Task.Run(() =>
 			}).ConfigureAwait(false);
 
 		foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed))
-			_logger.LogInformation("-> {Repository}\ttook: {Elapsed}", kv.Key, kv.Value.Elapsed);
+			_logger.LogInformation("-> took: {Elapsed}\t{RepositoryBranch}", kv.Key, kv.Value.Elapsed);
 
 		return checkouts.ToList().AsReadOnly();
 	}
@@ -76,39 +86,20 @@ private Checkout CloneOrUpdateRepository(Repository repository, string name, Con
 		var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, name));
 		var relativePath = Path.GetRelativePath(Paths.WorkingDirectoryRoot.FullName, checkoutFolder.FullName);
 		var sw = Stopwatch.StartNew();
-		_ = dict.AddOrUpdate(name, sw, (_, _) => sw);
-		var head = string.Empty;
+		var branch = PublishEnvironment.ContentSource == ContentSource.Next
+			? repository.GitReferenceNext
+			: repository.GitReferenceCurrent;
+
+		_ = dict.AddOrUpdate($"{name} ({branch})", sw, (_, _) => sw);
+
+		string? head;
 		if (checkoutFolder.Exists)
 		{
-			_logger.LogInformation("Pull: {Name}\t{Repository}\t{RelativePath}", name, repository, relativePath);
-			// --allow-unrelated-histories due to shallow clones not finding a common ancestor
-			ExecIn(checkoutFolder, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff");
-			//head = Capture(checkoutFolder, "git", "rev-parse", "HEAD");
-			head = Guid.NewGuid().ToString("N");
+			if (!TryUpdateSource(name, branch, relativePath, checkoutFolder, out head))
+				head = CheckoutFromScratch(repository, name, branch, relativePath, checkoutFolder);
 		}
 		else
-		{
-			_logger.LogInformation("Checkout: {Name}\t{Repository}\t{RelativePath}", name, repository, relativePath);
-			if (repository.CheckoutStrategy == "full")
-			{
-				Exec("git", "clone", repository.Origin, checkoutFolder.FullName,
-					"--depth", "1", "--single-branch",
-					"--branch", repository.CurrentBranch
-				);
-			}
-			else if (repository.CheckoutStrategy == "partial")
-			{
-				Exec(
-					"git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName
-				);
-
-				ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone");
-				ExecIn(checkoutFolder, "git", "checkout", repository.CurrentBranch);
-				ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs");
-				//head = Capture(checkoutFolder, "git", "rev-parse", "HEAD");
-				head = Guid.NewGuid().ToString("N");
-			}
-		}
+			head = CheckoutFromScratch(repository, name, branch, relativePath, checkoutFolder);
 
 		sw.Stop();
 
@@ -120,6 +111,55 @@ private Checkout CloneOrUpdateRepository(Repository repository, string name, Con
 		};
 	}
 
+	private bool TryUpdateSource(string name, string branch, string relativePath, IDirectoryInfo checkoutFolder, [NotNullWhen(true)] out string? head)
+	{
+		head = null;
+		try
+		{
+			_logger.LogInformation("Pull: {Name}\t{Branch}\t{RelativePath}", name, branch, relativePath);
+			// --allow-unrelated-histories due to shallow clones not finding a common ancestor
+			ExecIn(checkoutFolder, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff");
+			head = Capture(checkoutFolder, "git", "rev-parse", "HEAD");
+			return true;
+		}
+		catch (Exception e)
+		{
+			_logger.LogError(e, "Failed to update {Name} from {RelativePath}, falling back to recreating from scratch", name, relativePath);
+			if (checkoutFolder.Exists)
+			{
+				checkoutFolder.Delete(true);
+				checkoutFolder.Refresh();
+			}
+		}
+
+		return false;
+	}
+
+	private string CheckoutFromScratch(Repository repository, string name, string branch, string relativePath,
+		IDirectoryInfo checkoutFolder)
+	{
+		_logger.LogInformation("Checkout: {Name}\t{Branch}\t{RelativePath}", name, branch, relativePath);
+		if (repository.CheckoutStrategy == "full")
+		{
+			Exec("git", "clone", repository.Origin, checkoutFolder.FullName,
+				"--depth", "1", "--single-branch",
+				"--branch", branch
+			);
+		}
+		else if (repository.CheckoutStrategy == "partial")
+		{
+			Exec(
+				"git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName
+			);
+
+			ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone");
+			ExecIn(checkoutFolder, "git", "checkout", branch);
+			ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs");
+		}
+
+		return Capture(checkoutFolder, "git", "rev-parse", "HEAD");
+	}
+
 	private void Exec(string binary, params string[] args) => ExecIn(null, binary, args);
 
 	private void ExecIn(IDirectoryInfo? workingDirectory, string binary, params string[] args)
@@ -136,19 +176,36 @@ private void ExecIn(IDirectoryInfo? workingDirectory, string binary, params stri
 	// ReSharper disable once UnusedMember.Local
 	private string Capture(IDirectoryInfo? workingDirectory, string binary, params string[] args)
 	{
-		var arguments = new StartArguments(binary, args)
+		// Try 10 times to capture the output of the command, if it fails we'll throw an exception on the last try
+		for (var i = 0; i < 9; i++)
 		{
-			WorkingDirectory = workingDirectory?.FullName,
-			//WaitForStreamReadersTimeout = TimeSpan.FromSeconds(3),
-			Timeout = TimeSpan.FromSeconds(3),
-			WaitForExit = TimeSpan.FromSeconds(3),
-			ConsoleOutWriter = NoopConsoleWriter.Instance
-		};
-		var result = Proc.Start(arguments);
-		if (result.ExitCode != 0)
-			context.Collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}");
-		var line = result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}");
-		return line;
+			try
+			{
+				return CaptureOutput();
+			}
+			catch
+			{
+				// ignored
+			}
+		}
+		return CaptureOutput();
+
+		string CaptureOutput()
+		{
+			var arguments = new StartArguments(binary, args)
+			{
+				WorkingDirectory = workingDirectory?.FullName,
+				//WaitForStreamReadersTimeout = TimeSpan.FromSeconds(3),
+				Timeout = TimeSpan.FromSeconds(3),
+				WaitForExit = TimeSpan.FromSeconds(3),
+				ConsoleOutWriter = NoopConsoleWriter.Instance
+			};
+			var result = Proc.Start(arguments);
+			if (result.ExitCode != 0)
+				context.Collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}");
+			var line = result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}");
+			return line;
+		}
 	}
 }
 

src/docs-assembler/YamlStaticContext.cs

@@ -13,4 +13,5 @@ namespace Documentation.Assembler;
 [YamlSerializable(typeof(NarrativeRepository))]
 [YamlSerializable(typeof(PublishEnvironment))]
 [YamlSerializable(typeof(GoogleTagManager))]
+[YamlSerializable(typeof(ContentSource))]
 public partial class YamlStaticContext;