Update link registry listings to include git reference

src/Elastic.Markdown/Assets/styles.css

@@ -115,7 +115,6 @@
 			color: var(--color-yellow-80);
 		}
 
-
 		.applicable-info {
 			padding: calc(var(--spacing) * 0.5);
 			padding-left: calc(var(--spacing) * 2);

src/Elastic.Markdown/BuildContext.cs

@@ -9,6 +9,7 @@
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.Configuration;
 using Elastic.Markdown.IO.Discovery;
+using Elastic.Markdown.IO.State;
 
 namespace Elastic.Markdown;
 

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -62,6 +62,8 @@
     <PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" Version="16.1.3" PrivateAssets="All"/>
     <PackageReference Include="YamlDotNet" Version="16.3.0" />
     <PackageReference Include="System.IO.Abstractions" Version="21.0.29" />
+    <PackageReference Include="NetEscapades.EnumGenerators" Version="1.0.0-beta12"
+                      PrivateAssets="all" ExcludeAssets="runtime" />
   </ItemGroup>
 
 </Project>

src/Elastic.Markdown/IO/State/ContentSource.cs

@@ -0,0 +1,21 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.ComponentModel.DataAnnotations;
+using System.Text.Json.Serialization;
+using NetEscapades.EnumGenerators;
+
+namespace Elastic.Markdown.IO.State;
+
+[EnumExtensions]
+public enum ContentSource
+{
+	[Display(Name = "next")]
+	[JsonStringEnumMemberName("next")]
+	Next,
+
+	[JsonStringEnumMemberName("current")]
+	[Display(Name = "current")]
+	Current
+}

src/Elastic.Markdown/IO/State/LinkReference.cs

@@ -62,9 +62,15 @@ public record LinkReference
 	public static string SerializeRedirects(Dictionary<string, LinkRedirect>? redirects) =>
 		JsonSerializer.Serialize(redirects, SourceGenerationContext.Default.DictionaryStringLinkRedirect);
 
+	public static LinkReference Deserialize(Stream json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
+
 	public static LinkReference Deserialize(string json) =>
 		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
 
+	public static string Serialize(LinkReference reference) =>
+		JsonSerializer.Serialize(reference, SourceGenerationContext.Default.LinkReference);
+
 	public static LinkReference Create(DocumentationSet set)
 	{
 		var redirects = set.Configuration.Redirects;

src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs

@@ -21,7 +21,7 @@ public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 			_ = declaredRepositories.Add(repository);
 			try
 			{
-				var linkReference = await Fetch(repository, ctx);
+				var linkReference = await Fetch(repository, ["main", "master"], ctx);
 				linkReferences.Add(repository, linkReference);
 				var linkIndexReference = await GetLinkIndexEntry(repository, ctx);
 				linkIndexEntries.Add(repository, linkIndexReference);

src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs

@@ -57,23 +57,34 @@ protected async Task<LinkIndex> FetchLinkIndex(Cancel ctx)
 	protected async Task<LinkIndexEntry> GetLinkIndexEntry(string repository, Cancel ctx)
 	{
 		var linkIndex = await FetchLinkIndex(ctx);
-		if (linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
-			return repositoryLinks.First().Value;
-		throw new Exception($"Repository {repository} not found in link index");
+		if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
+			throw new Exception($"Repository {repository} not found in link index");
+		return GetNextContentSourceLinkIndexEntry(repositoryLinks, repository);
 	}
 
-	protected async Task<LinkReference> Fetch(string repository, Cancel ctx)
+	protected static LinkIndexEntry GetNextContentSourceLinkIndexEntry(IDictionary<string, LinkIndexEntry> repositoryLinks, string repository)
+	{
+		var linkIndexEntry =
+			(repositoryLinks.TryGetValue("main", out var link)
+				? link
+				: repositoryLinks.TryGetValue("master", out link) ? link : null)
+				?? throw new Exception($"Repository {repository} found in link index, but no main or master branch found");
+		return linkIndexEntry;
+	}
+
+	protected async Task<LinkReference> Fetch(string repository, string[] keys, Cancel ctx)
 	{
 		var linkIndex = await FetchLinkIndex(ctx);
 		if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
 			throw new Exception($"Repository {repository} not found in link index");
 
-		if (repositoryLinks.TryGetValue("main", out var linkIndexEntry))
-			return await FetchLinkIndexEntry(repository, linkIndexEntry, ctx);
-		if (repositoryLinks.TryGetValue("master", out linkIndexEntry))
-			return await FetchLinkIndexEntry(repository, linkIndexEntry, ctx);
-		throw new Exception($"Repository {repository} not found in link index");
+		foreach (var key in keys)
+		{
+			if (repositoryLinks.TryGetValue(key, out var linkIndexEntry))
+				return await FetchLinkIndexEntry(repository, linkIndexEntry, ctx);
+		}
 
+		throw new Exception($"Repository found in link index however none of: '{string.Join(", ", keys)}' branches found");
 	}
 
 	protected async Task<LinkReference> FetchLinkIndexEntry(string repository, LinkIndexEntry linkIndexEntry, Cancel ctx)

src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs

@@ -14,6 +14,9 @@ public record LinkIndex
 {
 	[JsonPropertyName("repositories")] public required Dictionary<string, Dictionary<string, LinkIndexEntry>> Repositories { get; init; }
 
+	public static LinkIndex Deserialize(Stream json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkIndex)!;
+
 	public static LinkIndex Deserialize(string json) =>
 		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkIndex)!;
 
@@ -34,6 +37,10 @@ public record LinkIndexEntry
 
 	[JsonPropertyName("etag")]
 	public required string ETag { get; init; }
+
+	// TODO can be made required after all doc_sets have published again
+	[JsonPropertyName("ref")]
+	public string GitReference { get; init; } = "unknown";
 }
 
 public interface ICrossLinkResolver

src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs

@@ -19,7 +19,8 @@ public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 		var linkIndex = await FetchLinkIndex(ctx);
 		foreach (var (repository, value) in linkIndex.Repositories)
 		{
-			var linkIndexEntry = value.First().Value;
+			var linkIndexEntry = GetNextContentSourceLinkIndexEntry(value, repository);
+
 			linkEntries.Add(repository, linkIndexEntry);
 			var linkReference = await FetchLinkIndexEntry(repository, linkIndexEntry, ctx);
 			linkReferences.Add(repository, linkReference);
@@ -34,4 +35,5 @@ public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 			FromConfiguration = false
 		};
 	}
+
 }

src/Elastic.Markdown/SourceGenerationContext.cs

@@ -11,7 +11,7 @@ namespace Elastic.Markdown;
 
 // This configures the source generation for json (de)serialization.
 
-[JsonSourceGenerationOptions(WriteIndented = true)]
+[JsonSourceGenerationOptions(WriteIndented = true, UseStringEnumConverter = true)]
 [JsonSerializable(typeof(GenerationState))]
 [JsonSerializable(typeof(LinkReference))]
 [JsonSerializable(typeof(GitCheckoutInformation))]

src/docs-assembler/AssembleContext.cs

@@ -7,6 +7,7 @@
 using Documentation.Assembler.Configuration;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.IO;
+using Elastic.Markdown.IO.State;
 
 namespace Documentation.Assembler;
 

src/docs-assembler/AssembleSources.cs

@@ -59,7 +59,7 @@ private AssembleSources(AssembleContext assembleContext, Checkout[] checkouts)
 		TocTopLevelMappings = GetConfiguredSources(assembleContext);
 		HistoryMappings = GetHistoryMapping(assembleContext);
 
-		var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration);
+		var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration, assembleContext.Environment);
 		UriResolver = new PublishEnvironmentUriResolver(TocTopLevelMappings, assembleContext.Environment);
 		var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, UriResolver);
 		AssembleSets = checkouts

src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs

@@ -10,7 +10,7 @@
 
 namespace Documentation.Assembler.Building;
 
-public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfiguration configuration) : CrossLinkFetcher(logger)
+public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment) : CrossLinkFetcher(logger)
 {
 	public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 	{
@@ -27,7 +27,11 @@ public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 			if (repository.Skip)
 				continue;
 
-			var linkReference = await Fetch(repositoryName, ctx);
+			var branch = publishEnvironment.ContentSource == ContentSource.Current
+				? repository.GitReferenceCurrent
+				: repository.GitReferenceNext;
+
+			var linkReference = await Fetch(repositoryName, [branch], ctx);
 			linkReferences.Add(repositoryName, linkReference);
 			var linkIndexReference = await GetLinkIndexEntry(repositoryName, ctx);
 			linkIndexEntries.Add(repositoryName, linkIndexReference);

src/docs-assembler/Configuration/AssemblyConfiguration.cs

@@ -2,8 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-using System.ComponentModel.DataAnnotations;
-using NetEscapades.EnumGenerators;
+using Elastic.Markdown.IO.State;
 using YamlDotNet.Serialization;
 
 namespace Documentation.Assembler.Configuration;
@@ -101,16 +100,6 @@ public record PublishEnvironment
 	public GoogleTagManager GoogleTagManager { get; set; } = new();
 }
 
-[EnumExtensions]
-public enum ContentSource
-{
-	[Display(Name = "next")]
-	Next,
-
-	[Display(Name = "current")]
-	Current
-}
-
 public record GoogleTagManager
 {
 	[YamlMember(Alias = "enabled")]

src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs

@@ -8,6 +8,7 @@
 using System.IO.Abstractions;
 using Documentation.Assembler.Configuration;
 using Elastic.Markdown.IO;
+using Elastic.Markdown.IO.State;
 using Microsoft.Extensions.Logging;
 using ProcNet;
 using ProcNet.Std;

src/docs-assembler/YamlStaticContext.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using Documentation.Assembler.Configuration;
+using Elastic.Markdown.IO.State;
 using YamlDotNet.Serialization;
 
 namespace Documentation.Assembler;

src/docs-assembler/docs-assembler.csproj

@@ -23,8 +23,6 @@
     </PackageReference>
     <PackageReference Include="Proc" Version="0.9.1" />
     <PackageReference Include="YamlDotNet" Version="16.3.0" />
-    <PackageReference Include="NetEscapades.EnumGenerators" Version="1.0.0-beta12"
-                      PrivateAssets="all" ExcludeAssets="runtime" />
     <PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" Version="16.1.3"
                       PrivateAssets="all" ExcludeAssets="runtime"/>
   </ItemGroup>

src/infra/docs-lambda-index-publisher/Program.cs

@@ -8,6 +8,7 @@
 using Amazon.Lambda.RuntimeSupport;
 using Amazon.S3;
 using Amazon.S3.Model;
+using Elastic.Markdown.IO.State;
 using Elastic.Markdown.Links.CrossLinks;
 
 await LambdaBootstrapBuilder.Create(Handler)
@@ -44,6 +45,8 @@ static async Task<string> Handler(ILambdaContext context)
 				if (tokens.Length < 3)
 					continue;
 
+				var gitReference = await ReadLinkReferenceSha(client, obj);
+
 				var repository = tokens[1];
 				var branch = tokens[2];
 
@@ -52,7 +55,8 @@ static async Task<string> Handler(ILambdaContext context)
 					Repository = repository,
 					Branch = branch,
 					ETag = obj.ETag.Trim('"'),
-					Path = obj.Key
+					Path = obj.Key,
+					GitReference = gitReference
 				};
 				if (linkIndex.Repositories.TryGetValue(repository, out var existingEntry))
 					existingEntry[branch] = entry;
@@ -83,3 +87,21 @@ static async Task<string> Handler(ILambdaContext context)
 	await client.UploadObjectFromStreamAsync(bucketName, "link-index.json", stream, new Dictionary<string, object>(), CancellationToken.None);
 	return $"Finished in {sw}";
 }
+
+static async Task<string> ReadLinkReferenceSha(IAmazonS3 client, S3Object obj)
+{
+	try
+	{
+		var contents = await client.GetObjectAsync(obj.Key, obj.Key, CancellationToken.None);
+		await using var s = contents.ResponseStream;
+		var linkReference = LinkReference.Deserialize(s);
+		return linkReference.Origin.Ref;
+	}
+	catch (Exception e)
+	{
+		Console.WriteLine(e);
+		// it's important we don't fail here we need to fallback gracefully from this so we can fix the root cause
+		// of why a repository is not reporting its git reference properly
+		return "unknown";
+	}
+}

tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs

@@ -26,6 +26,7 @@ public void EmitsLinks() =>
 	[Fact]
 	public void ShouldNotIncludeSnippets() =>
 		Reference.Links.Should().NotContain(l => l.Key.Contains("_snippets/"));
+
 }
 
 public class GitCheckoutInformationTests(ITestOutputHelper output) : NavigationTestsBase(output)
@@ -47,3 +48,64 @@ public void Create()
 		git.Remote.Should().NotContain(".git");
 	}
 }
+
+public class LinkReferenceSerializationTests
+{
+	[Fact]
+	public void SerializesCurrent()
+	{
+		var linkReference = new LinkReference
+		{
+			Origin = new GitCheckoutInformation
+			{
+				Branch = "branch",
+				Remote = "remote",
+				Ref = "ref"
+			},
+			UrlPathPrefix = "",
+			Links = [],
+			CrossLinks = [],
+		};
+		var json = LinkReference.Serialize(linkReference);
+		// language=json
+		json.Should().Be(
+			"""
+			{
+			  "origin": {
+			    "branch": "branch",
+			    "remote": "remote",
+			    "ref": "ref",
+			    "name": "unavailable"
+			  },
+			  "url_path_prefix": "",
+			  "links": {},
+			  "cross_links": [],
+			  "redirects": null
+			}
+			""");
+	}
+
+	[Fact]
+	public void Deserializes()
+	{
+		// language=json
+		var json =
+			"""
+			{
+			  "origin": {
+			    "branch": "branch",
+			    "remote": "remote",
+			    "ref": "ref",
+			    "name": "unavailable"
+			  },
+			  "url_path_prefix": "",
+			  "links": {},
+			  "cross_links": [],
+			  "redirects": null
+			}
+			""";
+		var linkReference = LinkReference.Deserialize(json);
+		linkReference.Origin.Ref.Should().Be("ref");
+	}
+
+}

tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs

@@ -22,6 +22,7 @@ public Task<FetchedCrossLinks> FetchLinks(Cancel ctx)
 		// language=json
 		var json = """
 		           {
+		              "content_source": "current",
 		           	  "origin": {
 		           		"branch": "main",
 		           		"remote": " https://github.com/elastic/docs-content",
@@ -53,7 +54,8 @@ public Task<FetchedCrossLinks> FetchLinks(Cancel ctx)
 			Repository = e.Key,
 			Path = $"elastic/asciidocalypse/{e.Key}/links.json",
 			Branch = "main",
-			ETag = Guid.NewGuid().ToString()
+			ETag = Guid.NewGuid().ToString(),
+			GitReference = Guid.NewGuid().ToString()
 		});
 		_crossLinks = new FetchedCrossLinks
 		{