Remove hardcoded links to link references, use advertised path from the registry

docs/testing/cross-links.md

@@ -4,4 +4,4 @@
 
 [Kibana][1]
 
-[1]: docs-content://index.md
+[1]: docs-content://index.md

src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs

@@ -13,15 +13,18 @@ public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILog
 {
 	public override async Task<FetchedCrossLinks> Fetch()
 	{
-		var dictionary = new Dictionary<string, LinkReference>();
+		var linkReferences = new Dictionary<string, LinkReference>();
+		var linkIndexEntries = new Dictionary<string, LinkIndexEntry>();
 		var declaredRepositories = new HashSet<string>();
 		foreach (var repository in configuration.CrossLinkRepositories)
 		{
 			_ = declaredRepositories.Add(repository);
 			try
 			{
 				var linkReference = await Fetch(repository);
-				dictionary.Add(repository, linkReference);
+				linkReferences.Add(repository, linkReference);
+				var linkIndexReference = await GetLinkIndexEntry(repository);
+				linkIndexEntries.Add(repository, linkIndexReference);
 			}
 			catch when (repository == "docs-content")
 			{
@@ -36,7 +39,8 @@ public override async Task<FetchedCrossLinks> Fetch()
 		return new FetchedCrossLinks
 		{
 			DeclaredRepositories = declaredRepositories,
-			LinkReferences = dictionary.ToFrozenDictionary(),
+			LinkReferences = linkReferences.ToFrozenDictionary(),
+			LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(),
 			FromConfiguration = true
 		};
 	}

src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs

@@ -18,11 +18,14 @@ public record FetchedCrossLinks
 
 	public required bool FromConfiguration { get; init; }
 
+	public required FrozenDictionary<string, LinkIndexEntry> LinkIndexEntries { get; init; }
+
 	public static FetchedCrossLinks Empty { get; } = new()
 	{
 		DeclaredRepositories = [],
 		LinkReferences = new Dictionary<string, LinkReference>().ToFrozenDictionary(),
-		FromConfiguration = false
+		FromConfiguration = false,
+		LinkIndexEntries = new Dictionary<string, LinkIndexEntry>().ToFrozenDictionary()
 	};
 }
 
@@ -41,7 +44,7 @@ protected async Task<LinkIndex> FetchLinkIndex()
 	{
 		if (_linkIndex is not null)
 		{
-			_logger.LogInformation("Using cached link index");
+			_logger.LogTrace("Using cached link index");
 			return _linkIndex;
 		}
 		var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json";
@@ -51,16 +54,26 @@ protected async Task<LinkIndex> FetchLinkIndex()
 		return _linkIndex;
 	}
 
+	protected async Task<LinkIndexEntry> GetLinkIndexEntry(string repository)
+	{
+		var linkIndex = await FetchLinkIndex();
+		if (linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
+			return repositoryLinks.First().Value;
+		throw new Exception($"Repository {repository} not found in link index");
+	}
+
 	protected async Task<LinkReference> Fetch(string repository)
 	{
 		var linkIndex = await FetchLinkIndex();
 		if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
 			throw new Exception($"Repository {repository} not found in link index");
 
-		if (!repositoryLinks.TryGetValue("main", out var linkIndexEntry))
-			throw new Exception($"Repository {repository} not found in link index");
+		if (repositoryLinks.TryGetValue("main", out var linkIndexEntry))
+			return await FetchLinkIndexEntry(repository, linkIndexEntry);
+		if (repositoryLinks.TryGetValue("master", out linkIndexEntry))
+			return await FetchLinkIndexEntry(repository, linkIndexEntry);
+		throw new Exception($"Repository {repository} not found in link index");
 
-		return await FetchLinkIndexEntry(repository, linkIndexEntry);
 	}
 
 	protected async Task<LinkReference> FetchLinkIndexEntry(string repository, LinkIndexEntry linkIndexEntry)
@@ -69,7 +82,7 @@ protected async Task<LinkReference> FetchLinkIndexEntry(string repository, LinkI
 		if (linkReference is not null)
 			return linkReference;
 
-		var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{repository}/main/links.json";
+		var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkIndexEntry.Path}";
 		_logger.LogInformation("Fetching links.json for '{Repository}': {Url}", repository, url);
 		var json = await _client.GetStringAsync(url);
 		linkReference = Deserialize(json);
@@ -79,7 +92,7 @@ protected async Task<LinkReference> FetchLinkIndexEntry(string repository, LinkI
 
 	private void WriteLinksJsonCachedFile(string repository, LinkIndexEntry linkIndexEntry, string json)
 	{
-		var cachedFileName = $"links-elastic-{repository}-main-{linkIndexEntry.ETag}.json";
+		var cachedFileName = $"links-elastic-{repository}-{linkIndexEntry.Branch}-{linkIndexEntry.ETag}.json";
 		var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "links", cachedFileName);
 		if (File.Exists(cachedPath))
 			return;

src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs

@@ -79,7 +79,7 @@ public static bool TryResolve(
 		resolvedUri = null;
 		var lookup = fetchedCrossLinks.LinkReferences;
 		if (crossLinkUri.Scheme != "asciidocalypse" && lookup.TryGetValue(crossLinkUri.Scheme, out var linkReference))
-			return TryFullyValidate(errorEmitter, uriResolver, linkReference, crossLinkUri, out resolvedUri);
+			return TryFullyValidate(errorEmitter, uriResolver, fetchedCrossLinks, linkReference, crossLinkUri, out resolvedUri);
 
 		// TODO this is temporary while we wait for all links.json to be published
 		// Here we just silently rewrite the cross_link to the url
@@ -103,20 +103,19 @@ public static bool TryResolve(
 		return true;
 	}
 
-	private static bool TryFullyValidate(
-		Action<string> errorEmitter,
+	private static bool TryFullyValidate(Action<string> errorEmitter,
 		IUriEnvironmentResolver uriResolver,
+		FetchedCrossLinks fetchedCrossLinks,
 		LinkReference linkReference,
 		Uri crossLinkUri,
-		[NotNullWhen(true)] out Uri? resolvedUri
-	)
+		[NotNullWhen(true)] out Uri? resolvedUri)
 	{
 		resolvedUri = null;
 		var lookupPath = (crossLinkUri.Host + '/' + crossLinkUri.AbsolutePath.TrimStart('/')).Trim('/');
 		if (string.IsNullOrEmpty(lookupPath) && crossLinkUri.Host.EndsWith(".md"))
 			lookupPath = crossLinkUri.Host;
 
-		if (!LookupLink(errorEmitter, linkReference, crossLinkUri, ref lookupPath, out var link, out var lookupFragment))
+		if (!LookupLink(errorEmitter, fetchedCrossLinks, linkReference, crossLinkUri, ref lookupPath, out var link, out var lookupFragment))
 			return false;
 
 		var path = ToTargetUrlPath(lookupPath);
@@ -142,14 +141,13 @@ private static bool TryFullyValidate(
 		return true;
 	}
 
-	private static bool LookupLink(
-		Action<string> errorEmitter,
+	private static bool LookupLink(Action<string> errorEmitter,
+		FetchedCrossLinks crossLinks,
 		LinkReference linkReference,
 		Uri crossLinkUri,
 		ref string lookupPath,
 		[NotNullWhen(true)] out LinkMetadata? link,
-		[NotNullWhen(true)] out string? lookupFragment
-	)
+		[NotNullWhen(true)] out string? lookupFragment)
 	{
 		lookupFragment = null;
 
@@ -171,6 +169,9 @@ private static bool LookupLink(
 		}
 
 		var linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{crossLinkUri.Scheme}/main/links.json";
+		if (crossLinks.LinkIndexEntries.TryGetValue(crossLinkUri.Scheme, out var linkIndexEntry))
+			linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkIndexEntry.Path}";
+
 		errorEmitter($"'{lookupPath}' is not a valid link in the '{crossLinkUri.Scheme}' cross link index: {linksJson}");
 		return false;
 	}

src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs

@@ -21,11 +21,12 @@ public Uri Resolve(Uri crossLinkUri, string path)
 
 	/// Hardcoding these for now, we'll have an index.json pointing to all links.json files
 	/// at some point from which we can query the branch soon.
-	private static string GetBranch(Uri crossLinkUri)
+	public static string GetBranch(Uri crossLinkUri)
 	{
 		var branch = crossLinkUri.Scheme switch
 		{
 			"docs-content" => "main",
+			"cloud" => "master",
 			_ => "main"
 		};
 		return branch;

src/Elastic.Markdown/InboundLinks/LinkIndexCrossLinkFetcher.cs

@@ -13,21 +13,24 @@ public class LinksIndexCrossLinkFetcher(ILoggerFactory logger) : CrossLinkFetche
 {
 	public override async Task<FetchedCrossLinks> Fetch()
 	{
-		var dictionary = new Dictionary<string, LinkReference>();
+		var linkReferences = new Dictionary<string, LinkReference>();
+		var linkEntries = new Dictionary<string, LinkIndexEntry>();
 		var declaredRepositories = new HashSet<string>();
 		var linkIndex = await FetchLinkIndex();
 		foreach (var (repository, value) in linkIndex.Repositories)
 		{
 			var linkIndexEntry = value.First().Value;
+			linkEntries.Add(repository, linkIndexEntry);
 			var linkReference = await FetchLinkIndexEntry(repository, linkIndexEntry);
-			dictionary.Add(repository, linkReference);
+			linkReferences.Add(repository, linkReference);
 			_ = declaredRepositories.Add(repository);
 		}
 
 		return new FetchedCrossLinks
 		{
 			DeclaredRepositories = declaredRepositories,
-			LinkReferences = dictionary.ToFrozenDictionary(),
+			LinkReferences = linkReferences.ToFrozenDictionary(),
+			LinkIndexEntries = linkEntries.ToFrozenDictionary(),
 			FromConfiguration = false
 		};
 	}

src/Elastic.Markdown/InboundLinks/LinkIndexLinkChecker.cs

@@ -55,7 +55,6 @@ public async Task<int> CheckWithLocalLinksJson(DiagnosticsCollector collector, s
 		var resolver = new CrossLinkResolver(fetcher);
 		// ReSharper disable once RedundantAssignment
 		var crossLinks = await resolver.FetchLinks();
-
 		if (string.IsNullOrEmpty(repository))
 			throw new ArgumentNullException(nameof(repository));
 		if (string.IsNullOrEmpty(localLinksJson))
@@ -117,6 +116,8 @@ private async Task<int> ValidateCrossLinks(
 					continue;
 
 				var linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{uri.Scheme}/main/links.json";
+				if (crossLinks.LinkIndexEntries.TryGetValue(uri.Scheme, out var linkIndexEntry))
+					linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkIndexEntry.Path}";
 				_ = resolver.TryResolve(s =>
 				{
 					if (s.Contains("is not a valid link in the"))

src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs

@@ -14,7 +14,8 @@ public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfigurat
 {
 	public override async Task<FetchedCrossLinks> Fetch()
 	{
-		var dictionary = new Dictionary<string, LinkReference>();
+		var linkReferences = new Dictionary<string, LinkReference>();
+		var linkIndexEntries = new Dictionary<string, LinkIndexEntry>();
 		var declaredRepositories = new HashSet<string>();
 		var repositories = configuration.ReferenceRepositories.Values.Concat<Repository>([configuration.Narrative]);
 
@@ -25,13 +26,16 @@ public override async Task<FetchedCrossLinks> Fetch()
 			var repositoryName = repository.Name;
 			_ = declaredRepositories.Add(repositoryName);
 			var linkReference = await Fetch(repositoryName);
-			dictionary.Add(repositoryName, linkReference);
+			linkReferences.Add(repositoryName, linkReference);
+			var linkIndexReference = await GetLinkIndexEntry(repositoryName);
+			linkIndexEntries.Add(repositoryName, linkIndexReference);
 		}
 
 		return new FetchedCrossLinks
 		{
 			DeclaredRepositories = declaredRepositories,
-			LinkReferences = dictionary.ToFrozenDictionary(),
+			LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(),
+			LinkReferences = linkReferences.ToFrozenDictionary(),
 			FromConfiguration = true
 		};
 	}

tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs

@@ -47,11 +47,20 @@ public Task<FetchedCrossLinks> FetchLinks()
 		LinkReferences.Add("docs-content", reference);
 		LinkReferences.Add("kibana", reference);
 		DeclaredRepositories.AddRange(["docs-content", "kibana", "elasticsearch"]);
+
+		var indexEntries = LinkReferences.ToDictionary(e => e.Key, e => new LinkIndexEntry
+		{
+			Repository = e.Key,
+			Path = $"elastic/asciidocalypse/{e.Key}/links.json",
+			Branch = "main",
+			ETag = Guid.NewGuid().ToString()
+		});
 		_crossLinks = new FetchedCrossLinks
 		{
 			DeclaredRepositories = DeclaredRepositories,
 			LinkReferences = LinkReferences.ToFrozenDictionary(),
-			FromConfiguration = true
+			FromConfiguration = true,
+			LinkIndexEntries = indexEntries.ToFrozenDictionary()
 		};
 		return Task.FromResult(_crossLinks);
 	}

tests/authoring/Framework/TestCrossLinkResolver.fs

@@ -9,6 +9,7 @@ open System.Collections.Generic
 open System.Collections.Frozen
 open System.Runtime.InteropServices
 open System.Threading.Tasks
+open System.Linq
 open Elastic.Markdown.CrossLinks
 open Elastic.Markdown.IO.Configuration
 open Elastic.Markdown.IO.State
@@ -62,21 +63,41 @@ type TestCrossLinkResolver (config: ConfigurationFile) =
             this.LinkReferences.Add("kibana", reference)
             this.DeclaredRepositories.Add("docs-content") |> ignore;
             this.DeclaredRepositories.Add("kibana") |> ignore;
-            this.DeclaredRepositories.Add("elasticsearch") |> ignore;
+            this.DeclaredRepositories.Add("elasticsearch") |> ignore
+
+            let indexEntries =
+                this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair<string, LinkReference>) -> LinkIndexEntry(
+                    Repository = e.Key,
+                    Path = $"elastic/asciidocalypse/{e.Key}/links.json",
+                    Branch = "main",
+                    ETag = Guid.NewGuid().ToString()
+                 ))
+
             let crossLinks =
                 FetchedCrossLinks(
                     DeclaredRepositories=this.DeclaredRepositories,
                     LinkReferences=this.LinkReferences.ToFrozenDictionary(),
-                    FromConfiguration=true
+                    FromConfiguration=true,
+                    LinkIndexEntries=indexEntries.ToFrozenDictionary()
                 )
             Task.FromResult crossLinks
 
         member this.TryResolve(errorEmitter, warningEmitter, crossLinkUri, [<Out>]resolvedUri : byref<Uri|null>) =
+            let indexEntries =
+                this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair<string, LinkReference>) -> LinkIndexEntry(
+                    Repository = e.Key,
+                    Path = $"elastic/asciidocalypse/{e.Key}/links.json",
+                    Branch = "main",
+                    ETag = Guid.NewGuid().ToString()
+                 ));
+
             let crossLinks =
                 FetchedCrossLinks(
                     DeclaredRepositories=this.DeclaredRepositories,
                     LinkReferences=this.LinkReferences.ToFrozenDictionary(),
-                    FromConfiguration=true
+                    FromConfiguration=true,
+                    LinkIndexEntries=indexEntries.ToFrozenDictionary()
+
                 )
             CrossLinkResolver.TryResolve(errorEmitter, warningEmitter, crossLinks, uriResolver, crossLinkUri, &resolvedUri);