Add update-link-index-all command

src/docs-assembler/Cli/RepositoryCommands.cs

@@ -2,15 +2,25 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Collections.Concurrent;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.IO.Abstractions;
+using System.Net.Mime;
 using Actions.Core.Services;
+using Amazon.S3;
+using Amazon.S3.Model;
 using ConsoleAppFramework;
 using Documentation.Assembler.Building;
+using Documentation.Assembler.Configuration;
 using Documentation.Assembler.Mapping;
 using Documentation.Assembler.Navigation;
 using Documentation.Assembler.Sourcing;
 using Elastic.Documentation.Tooling.Diagnostics.Console;
+using Elastic.Markdown;
+using Elastic.Markdown.Exporters;
+using Elastic.Markdown.IO;
+using Elastic.Markdown.IO.State;
 using Microsoft.Extensions.Logging;
 
 namespace Documentation.Assembler.Cli;
@@ -117,4 +127,68 @@ public async Task<int> BuildAll(
 			return collector.Errors + collector.Warnings;
 		return collector.Errors;
 	}
+
+	/// <param name="contentSource"> The content source. "current" or "next"</param>
+	/// <param name="ctx"></param>
+	[Command("update-all-link-reference")]
+	public async Task<int> UpdateLinkIndexAll(ContentSource contentSource, Cancel ctx = default)
+	{
+		var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService);
+		// The environment ist not relevant here.
+		// It's only used to get the list of repositories.
+		var assembleContext = new AssembleContext("prod", collector, new FileSystem(), new FileSystem(), null, null);
+		var cloner = new RepositorySourcer(logger, assembleContext.CheckoutDirectory, new FileSystem(), collector);
+		var dict = new ConcurrentDictionary<string, Stopwatch>();
+		var repositories = new Dictionary<string, Repository>(assembleContext.Configuration.ReferenceRepositories)
+		{
+			{ NarrativeRepository.RepositoryName, assembleContext.Configuration.Narrative }
+		};
+		await Parallel.ForEachAsync(repositories,
+			new ParallelOptions
+			{
+				CancellationToken = ctx,
+				MaxDegreeOfParallelism = Environment.ProcessorCount
+			}, async (kv, c) =>
+			{
+				try
+				{
+					var name = kv.Key.Trim();
+					var checkout = cloner.CloneOrUpdateRepository(kv.Value, name, kv.Value.GetBranch(contentSource), dict);
+
+					var docsMetadataPath = Path.Combine(checkout.Directory.FullName, ".docs-metadata");
+
+					var context = new BuildContext(
+						collector,
+						new FileSystem(),
+						new FileSystem(),
+						checkout.Directory.FullName,
+						docsMetadataPath
+					);
+					var set = new DocumentationSet(context, logger);
+					var generator = new DocumentationGenerator(set, logger, null, null, new NoopDocumentationFileExporter());
+					await generator.GenerateAll(c);
+
+					IAmazonS3 s3Client = new AmazonS3Client();
+					const string bucketName = "elastic-docs-link-index";
+					var linksJsonPath = Path.Combine(docsMetadataPath, "links.json");
+					var content = await File.ReadAllTextAsync(linksJsonPath, c);
+					var putObjectRequest = new PutObjectRequest
+					{
+						BucketName = bucketName,
+						Key = $"elastic/{checkout.Repository.Name}/{checkout.Repository.GetBranch(contentSource)}/links.json",
+						ContentBody = content,
+						ContentType = MediaTypeNames.Application.Json,
+						ChecksumAlgorithm = ChecksumAlgorithm.SHA256
+					};
+					var response = await s3Client.PutObjectAsync(putObjectRequest, c);
+					if (response.HttpStatusCode != System.Net.HttpStatusCode.OK)
+						collector.EmitError(linksJsonPath, $"Failed to upload {putObjectRequest.Key} to S3");
+				}
+				catch (Exception e)
+				{
+					collector.EmitError(kv.Key, $"Failed to update link index for {kv.Key}: {e.Message}", e);
+				}
+			}).ConfigureAwait(false);
+		return collector.Errors > 0 ? 1 : 0;
+	}
 }

src/docs-assembler/Configuration/Repository.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using Elastic.Markdown.IO.State;
 using YamlDotNet.Serialization;
 
 namespace Documentation.Assembler.Configuration;
@@ -31,4 +32,11 @@ public record Repository
 
 	[YamlMember(Alias = "skip")]
 	public bool Skip { get; set; }
+
+	public string GetBranch(ContentSource contentSource) => contentSource switch
+	{
+		ContentSource.Current => GitReferenceCurrent,
+		ContentSource.Next => GitReferenceNext,
+		_ => throw new ArgumentException($"The content source {contentSource} is not supported.", nameof(contentSource))
+	};
 }

src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs

@@ -7,6 +7,8 @@
 using System.Diagnostics.CodeAnalysis;
 using System.IO.Abstractions;
 using Documentation.Assembler.Configuration;
+using Elastic.Documentation.Tooling.Diagnostics.Console;
+using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.State;
 using Microsoft.Extensions.Logging;
@@ -22,6 +24,8 @@ public class AssemblerRepositorySourcer(ILoggerFactory logger, AssembleContext c
 	private AssemblyConfiguration Configuration => context.Configuration;
 	private PublishEnvironment PublishEnvironment => context.Environment;
 
+	private RepositorySourcer RepositorySourcer => new(logger, context.CheckoutDirectory, context.ReadFileSystem, context.Collector);
+
 	public IReadOnlyCollection<Checkout> GetAll()
 	{
 		var fs = context.ReadFileSystem;
@@ -52,15 +56,23 @@ public async Task<IReadOnlyCollection<Checkout>> AcquireAllLatest(Cancel ctx = d
 			PublishEnvironment.ContentSource.ToStringFast(true)
 		);
 
-		var dict = new ConcurrentDictionary<string, Stopwatch>();
-		var checkouts = new ConcurrentBag<Checkout>();
+		var repositories = new Dictionary<string, Repository>(Configuration.ReferenceRepositories)
+		{
+			{ NarrativeRepository.RepositoryName, Configuration.Narrative }
+		};
+		return await RepositorySourcer.AcquireAllLatest(repositories, PublishEnvironment.ContentSource, ctx);
+	}
+}
 
-		_logger.LogInformation("Cloning narrative content: {Repository}", NarrativeRepository.RepositoryName);
-		var checkout = CloneOrUpdateRepository(Configuration.Narrative, NarrativeRepository.RepositoryName, dict);
-		checkouts.Add(checkout);
+public class RepositorySourcer(ILoggerFactory logger, IDirectoryInfo checkoutDirectory, IFileSystem readFileSystem, DiagnosticsCollector collector)
+{
+	private readonly ILogger<RepositorySourcer> _logger = logger.CreateLogger<RepositorySourcer>();
 
-		_logger.LogInformation("Cloning {ReferenceRepositoryCount} repositories", Configuration.ReferenceRepositories.Count);
-		await Parallel.ForEachAsync(Configuration.ReferenceRepositories,
+	public async Task<IReadOnlyCollection<Checkout>> AcquireAllLatest(Dictionary<string, Repository> repositories, ContentSource source, Cancel ctx = default)
+	{
+		var dict = new ConcurrentDictionary<string, Stopwatch>();
+		var checkouts = new ConcurrentBag<Checkout>();
+		await Parallel.ForEachAsync(repositories,
 			new ParallelOptions
 			{
 				CancellationToken = ctx,
@@ -70,26 +82,21 @@ await Parallel.ForEachAsync(Configuration.ReferenceRepositories,
 				await Task.Run(() =>
 				{
 					var name = kv.Key.Trim();
-					var clone = CloneOrUpdateRepository(kv.Value, name, dict);
+					var repo = kv.Value;
+					var clone = CloneOrUpdateRepository(kv.Value, name, repo.GetBranch(source), dict);
 					checkouts.Add(clone);
 				}, c);
 			}).ConfigureAwait(false);
 
-		foreach (var kv in dict.OrderBy(kv => kv.Value.Elapsed))
-			_logger.LogInformation("-> took: {Elapsed}\t{RepositoryBranch}", kv.Key, kv.Value.Elapsed);
-
 		return checkouts.ToList().AsReadOnly();
 	}
 
-	private Checkout CloneOrUpdateRepository(Repository repository, string name, ConcurrentDictionary<string, Stopwatch> dict)
+	public Checkout CloneOrUpdateRepository(Repository repository, string name, string branch, ConcurrentDictionary<string, Stopwatch> dict)
 	{
-		var fs = context.ReadFileSystem;
-		var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, name));
+		var fs = readFileSystem;
+		var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, name));
 		var relativePath = Path.GetRelativePath(Paths.WorkingDirectoryRoot.FullName, checkoutFolder.FullName);
 		var sw = Stopwatch.StartNew();
-		var branch = PublishEnvironment.ContentSource == ContentSource.Next
-			? repository.GitReferenceNext
-			: repository.GitReferenceCurrent;
 
 		_ = dict.AddOrUpdate($"{name} ({branch})", sw, (_, _) => sw);
 
@@ -140,22 +147,23 @@ private string CheckoutFromScratch(Repository repository, string name, string br
 		IDirectoryInfo checkoutFolder)
 	{
 		_logger.LogInformation("Checkout: {Name}\t{Branch}\t{RelativePath}", name, branch, relativePath);
-		if (repository.CheckoutStrategy == "full")
+		switch (repository.CheckoutStrategy)
 		{
-			Exec("git", "clone", repository.Origin, checkoutFolder.FullName,
-				"--depth", "1", "--single-branch",
-				"--branch", branch
-			);
-		}
-		else if (repository.CheckoutStrategy == "partial")
-		{
-			Exec(
-				"git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName
-			);
-
-			ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone");
-			ExecIn(checkoutFolder, "git", "checkout", branch);
-			ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs");
+			case "full":
+				Exec("git", "clone", repository.Origin, checkoutFolder.FullName,
+					"--depth", "1", "--single-branch",
+					"--branch", branch
+				);
+				break;
+			case "partial":
+				Exec(
+					"git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName
+				);
+
+				ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone");
+				ExecIn(checkoutFolder, "git", "checkout", branch);
+				ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs");
+				break;
 		}
 
 		return Capture(checkoutFolder, "git", "rev-parse", "HEAD");
@@ -171,7 +179,7 @@ private void ExecIn(IDirectoryInfo? workingDirectory, string binary, params stri
 		};
 		var result = Proc.Exec(arguments);
 		if (result != 0)
-			context.Collector.EmitError("", $"Exit code: {result} while executing {binary} {string.Join(" ", args)} in {workingDirectory}");
+			collector.EmitError("", $"Exit code: {result} while executing {binary} {string.Join(" ", args)} in {workingDirectory}");
 	}
 
 	// ReSharper disable once UnusedMember.Local
@@ -203,11 +211,12 @@ string CaptureOutput()
 			};
 			var result = Proc.Start(arguments);
 			if (result.ExitCode != 0)
-				context.Collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}");
+				collector.EmitError("", $"Exit code: {result.ExitCode} while executing {binary} {string.Join(" ", args)} in {workingDirectory}");
 			var line = result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}");
 			return line;
 		}
 	}
+
 }
 
 public class NoopConsoleWriter : IConsoleOutWriter

src/docs-builder/Cli/Commands.cs

@@ -102,7 +102,7 @@ public async Task<int> Generate(
 		var runningOnCi = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS"));
 		BuildContext context;
 
-		Uri? canonicalBaseUri = null;
+		Uri? canonicalBaseUri;
 
 		if (canonicalBaseUrl is null)
 			canonicalBaseUri = new Uri("https://docs-v3-preview.elastic.dev");