Add links create-index command to the assembler

src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs

@@ -5,12 +5,41 @@
 using System.Collections.Frozen;
 using System.Diagnostics.CodeAnalysis;
 using System.Text.Json;
+using System.Text.Json.Serialization;
 using Elastic.Markdown.IO.Configuration;
 using Elastic.Markdown.IO.State;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown.CrossLinks;
 
+
+public record LinkIndex
+{
+	[JsonPropertyName("repositories")]
+	public required Dictionary<string, Dictionary<string, LinkIndexEntry>> Repositories { get; init; }
+
+	public static LinkIndex Deserialize(string json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkIndex)!;
+
+	public static string Serialize(LinkIndex index) =>
+		JsonSerializer.Serialize(index, SourceGenerationContext.Default.LinkIndex);
+}
+
+public record LinkIndexEntry
+{
+	[JsonPropertyName("repository")]
+	public required string Repository { get; init; }
+
+	[JsonPropertyName("path")]
+	public required string Path { get; init; }
+
+	[JsonPropertyName("branch")]
+	public required string Branch { get; init; }
+
+	[JsonPropertyName("etag")]
+	public required string ETag { get; init; }
+}
+
 public interface ICrossLinkResolver
 {
 	Task FetchLinks();

src/Elastic.Markdown/SourceGenerationContext.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using System.Text.Json.Serialization;
+using Elastic.Markdown.CrossLinks;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.Discovery;
 using Elastic.Markdown.IO.State;
@@ -15,4 +16,6 @@ namespace Elastic.Markdown;
 [JsonSerializable(typeof(GenerationState))]
 [JsonSerializable(typeof(LinkReference))]
 [JsonSerializable(typeof(GitCheckoutInformation))]
+[JsonSerializable(typeof(LinkIndex))]
+[JsonSerializable(typeof(LinkIndexEntry))]
 internal partial class SourceGenerationContext : JsonSerializerContext;

src/docs-assembler/Cli/LinkCommands.cs

@@ -0,0 +1,87 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text;
+using Actions.Core.Services;
+using Amazon.S3;
+using Amazon.S3.Model;
+using ConsoleAppFramework;
+using Elastic.Markdown.CrossLinks;
+using Microsoft.Extensions.Logging;
+
+namespace Documentation.Assembler.Cli;
+
+internal class LinkCommands(ILoggerFactory logger)
+{
+	private void AssignOutputLogger()
+	{
+		var log = logger.CreateLogger<Program>();
+		ConsoleApp.Log = msg => log.LogInformation(msg);
+		ConsoleApp.LogError = msg => log.LogError(msg);
+	}
+
+	/// <summary>
+	/// Create an index.json file from all discovered links.json files in our S3 bucket
+	/// </summary>
+	/// <param name="ctx"></param>
+	[Command("links create-index")]
+	public async Task CreateLinkIndex(Cancel ctx = default)
+	{
+		AssignOutputLogger();
+
+		IAmazonS3 client = new AmazonS3Client();
+		var bucketName = "elastic-docs-link-index";
+		var request = new ListObjectsV2Request { BucketName = bucketName, MaxKeys = 5 };
+
+		Console.WriteLine("--------------------------------------");
+		Console.WriteLine($"Listing the contents of {bucketName}:");
+		Console.WriteLine("--------------------------------------");
+
+
+		var linkIndex = new LinkIndex { Repositories = new Dictionary<string, Dictionary<string, LinkIndexEntry>>() };
+		try
+		{
+			ListObjectsV2Response response;
+			do
+			{
+				response = await client.ListObjectsV2Async(request, ctx);
+				foreach (var obj in response.S3Objects)
+				{
+					if (!obj.Key.StartsWith("elastic/"))
+						continue;
+
+					var tokens = obj.Key.Split('/');
+					if (tokens.Length < 3)
+						continue;
+
+					var repository = tokens[1];
+					var branch = tokens[2];
+
+					var entry = new LinkIndexEntry { Repository = repository, Branch = branch, ETag = obj.ETag.Trim('"'), Path = obj.Key };
+					if (linkIndex.Repositories.TryGetValue(repository, out var existingEntry))
+						existingEntry[branch] = entry;
+					else
+						linkIndex.Repositories.Add(repository, new Dictionary<string, LinkIndexEntry> { { branch, entry } });
+					Console.WriteLine(entry);
+				}
+
+				// If the response is truncated, set the request ContinuationToken
+				// from the NextContinuationToken property of the response.
+				request.ContinuationToken = response.NextContinuationToken;
+			} while (response.IsTruncated);
+		}
+		catch (AmazonS3Exception ex)
+		{
+			Console.WriteLine($"Error encountered on server. Message:'{ex.Message}' getting list of objects.");
+		}
+
+		var json = LinkIndex.Serialize(linkIndex);
+		Console.WriteLine(json);
+
+		using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
+		await client.UploadObjectFromStreamAsync(bucketName, "link-index.json", stream, new Dictionary<string, object>(), ctx);
+
+		Console.WriteLine("Uploaded latest link-index.json");
+	}
+}

src/docs-assembler/Program.cs

@@ -4,20 +4,47 @@
 
 using System.Collections.Concurrent;
 using System.Diagnostics;
+using Actions.Core.Extensions;
 using ConsoleAppFramework;
 using Documentation.Assembler;
 using Documentation.Assembler.Cli;
+using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.IO;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using ProcNet;
 using ProcNet.Std;
 
 var configFile = Path.Combine(Paths.Root.FullName, "src/docs-assembler/conf.yml");
 var config = AssemblyConfiguration.Deserialize(File.ReadAllText(configFile));
 
+var services = new ServiceCollection();
+services.AddGitHubActionsCore();
+services.AddLogging(x =>
+{
+	x.ClearProviders();
+	x.SetMinimumLevel(LogLevel.Information);
+	x.AddSimpleConsole(c =>
+	{
+		c.SingleLine = true;
+		c.IncludeScopes = true;
+		c.UseUtcTimestamp = true;
+		c.TimestampFormat = Environment.UserInteractive ? ":: " : "[yyyy-MM-ddTHH:mm:ss] ";
+	});
+});
+services.AddSingleton<DiagnosticsChannel>();
+services.AddSingleton<DiagnosticsCollector>();
+
+await using var serviceProvider = services.BuildServiceProvider();
+ConsoleApp.ServiceProvider = serviceProvider;
+
+
 var app = ConsoleApp.Create();
 app.UseFilter<StopwatchFilter>();
 app.UseFilter<CatchExceptionFilter>();
 
+app.Add<LinkCommands>();
+
 // would love to use libgit2 so there is no git dependency but
 // libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674
 app.Add("clone-all", async Task (CancellationToken ctx) =>

src/docs-assembler/docs-assembler.csproj

@@ -16,6 +16,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.7.414.5" />
     <PackageReference Include="ConsoleAppFramework" Version="5.3.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>