From 4b9f6938d5291ae3919c219356a79f754f59f39c Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 13 Mar 2025 17:18:03 +0100 Subject: [PATCH 1/2] Remove hardcoded links to link references, use advertised path from the registry --- docs/_docset.yml | 1 + docs/testing/cross-links.md | 4 ++- .../ConfigurationCrossLinkFetcher.cs | 10 ++++--- .../CrossLinks/CrossLinkFetcher.cs | 25 ++++++++++++----- .../CrossLinks/CrossLinkResolver.cs | 21 ++++++++------- .../CrossLinks/IUriEnvironmentResolver.cs | 3 ++- .../InboundLinks/LinkIndexCrossLinkFetcher.cs | 9 ++++--- .../InboundLinks/LinkIndexLinkChecker.cs | 3 ++- .../Building/AssemblerCrossLinkFetcher.cs | 10 ++++--- .../TestCrossLinkResolver.cs | 11 +++++++- .../Framework/TestCrossLinkResolver.fs | 27 ++++++++++++++++--- 11 files changed, 92 insertions(+), 32 deletions(-) diff --git a/docs/_docset.yml b/docs/_docset.yml index fed81948b..7a6d6f6d1 100644 --- a/docs/_docset.yml +++ b/docs/_docset.yml @@ -1,6 +1,7 @@ project: 'doc-builder' cross_links: - docs-content + - cloud exclude: - '_*.md' subs: diff --git a/docs/testing/cross-links.md b/docs/testing/cross-links.md index 45e722b6a..d0cbefb86 100644 --- a/docs/testing/cross-links.md +++ b/docs/testing/cross-links.md @@ -2,6 +2,8 @@ [Elasticsearch](docs-content://index.md) +[Cloud](cloud://reference/cloud-enterprise/index.md) + [Kibana][1] -[1]: docs-content://index.md \ No newline at end of file +[1]: docs-content://index.md diff --git a/src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs b/src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs index 9b2928cd3..f01f06048 100644 --- a/src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs +++ b/src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs @@ -13,7 +13,8 @@ public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILog { public override async Task Fetch() { - var dictionary = new Dictionary(); + var linkReferences = new Dictionary(); + var linkIndexEntries = new Dictionary(); var declaredRepositories = new HashSet(); foreach (var repository in configuration.CrossLinkRepositories) { @@ -21,7 +22,9 @@ public override async Task Fetch() try { var linkReference = await Fetch(repository); - dictionary.Add(repository, linkReference); + linkReferences.Add(repository, linkReference); + var linkIndexReference = await GetLinkIndexEntry(repository); + linkIndexEntries.Add(repository, linkIndexReference); } catch when (repository == "docs-content") { @@ -36,7 +39,8 @@ public override async Task Fetch() return new FetchedCrossLinks { DeclaredRepositories = declaredRepositories, - LinkReferences = dictionary.ToFrozenDictionary(), + LinkReferences = linkReferences.ToFrozenDictionary(), + LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(), FromConfiguration = true }; } diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs index 822493549..0b855055b 100644 --- a/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs @@ -18,11 +18,14 @@ public record FetchedCrossLinks public required bool FromConfiguration { get; init; } + public required FrozenDictionary LinkIndexEntries { get; init; } + public static FetchedCrossLinks Empty { get; } = new() { DeclaredRepositories = [], LinkReferences = new Dictionary().ToFrozenDictionary(), - FromConfiguration = false + FromConfiguration = false, + LinkIndexEntries = new Dictionary().ToFrozenDictionary() }; } @@ -41,7 +44,7 @@ protected async Task FetchLinkIndex() { if (_linkIndex is not null) { - _logger.LogInformation("Using cached link index"); + _logger.LogTrace("Using cached link index"); return _linkIndex; } var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json"; @@ -51,16 +54,26 @@ protected async Task FetchLinkIndex() return _linkIndex; } + protected async Task GetLinkIndexEntry(string repository) + { + var linkIndex = await FetchLinkIndex(); + if (linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) + return repositoryLinks.First().Value; + throw new Exception($"Repository {repository} not found in link index"); + } + protected async Task Fetch(string repository) { var linkIndex = await FetchLinkIndex(); if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) throw new Exception($"Repository {repository} not found in link index"); - if (!repositoryLinks.TryGetValue("main", out var linkIndexEntry)) - throw new Exception($"Repository {repository} not found in link index"); + if (repositoryLinks.TryGetValue("main", out var linkIndexEntry)) + return await FetchLinkIndexEntry(repository, linkIndexEntry); + if (repositoryLinks.TryGetValue("master", out linkIndexEntry)) + return await FetchLinkIndexEntry(repository, linkIndexEntry); + throw new Exception($"Repository {repository} not found in link index"); - return await FetchLinkIndexEntry(repository, linkIndexEntry); } protected async Task FetchLinkIndexEntry(string repository, LinkIndexEntry linkIndexEntry) @@ -69,7 +82,7 @@ protected async Task FetchLinkIndexEntry(string repository, LinkI if (linkReference is not null) return linkReference; - var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{repository}/main/links.json"; + var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkIndexEntry.Path}"; _logger.LogInformation("Fetching links.json for '{Repository}': {Url}", repository, url); var json = await _client.GetStringAsync(url); linkReference = Deserialize(json); diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs index e559dc5be..701385d90 100644 --- a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs @@ -79,7 +79,7 @@ public static bool TryResolve( resolvedUri = null; var lookup = fetchedCrossLinks.LinkReferences; if (crossLinkUri.Scheme != "asciidocalypse" && lookup.TryGetValue(crossLinkUri.Scheme, out var linkReference)) - return TryFullyValidate(errorEmitter, uriResolver, linkReference, crossLinkUri, out resolvedUri); + return TryFullyValidate(errorEmitter, uriResolver, fetchedCrossLinks, linkReference, crossLinkUri, out resolvedUri); // TODO this is temporary while we wait for all links.json to be published // Here we just silently rewrite the cross_link to the url @@ -103,20 +103,19 @@ public static bool TryResolve( return true; } - private static bool TryFullyValidate( - Action errorEmitter, + private static bool TryFullyValidate(Action errorEmitter, IUriEnvironmentResolver uriResolver, + FetchedCrossLinks fetchedCrossLinks, LinkReference linkReference, Uri crossLinkUri, - [NotNullWhen(true)] out Uri? resolvedUri - ) + [NotNullWhen(true)] out Uri? resolvedUri) { resolvedUri = null; var lookupPath = (crossLinkUri.Host + '/' + crossLinkUri.AbsolutePath.TrimStart('/')).Trim('/'); if (string.IsNullOrEmpty(lookupPath) && crossLinkUri.Host.EndsWith(".md")) lookupPath = crossLinkUri.Host; - if (!LookupLink(errorEmitter, linkReference, crossLinkUri, ref lookupPath, out var link, out var lookupFragment)) + if (!LookupLink(errorEmitter, fetchedCrossLinks, linkReference, crossLinkUri, ref lookupPath, out var link, out var lookupFragment)) return false; var path = ToTargetUrlPath(lookupPath); @@ -142,14 +141,13 @@ private static bool TryFullyValidate( return true; } - private static bool LookupLink( - Action errorEmitter, + private static bool LookupLink(Action errorEmitter, + FetchedCrossLinks crossLinks, LinkReference linkReference, Uri crossLinkUri, ref string lookupPath, [NotNullWhen(true)] out LinkMetadata? link, - [NotNullWhen(true)] out string? lookupFragment - ) + [NotNullWhen(true)] out string? lookupFragment) { lookupFragment = null; @@ -171,6 +169,9 @@ private static bool LookupLink( } var linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{crossLinkUri.Scheme}/main/links.json"; + if (crossLinks.LinkIndexEntries.TryGetValue(crossLinkUri.Scheme, out var linkIndexEntry)) + linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkIndexEntry.Path}"; + errorEmitter($"'{lookupPath}' is not a valid link in the '{crossLinkUri.Scheme}' cross link index: {linksJson}"); return false; } diff --git a/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs b/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs index f6fd1d228..4bad70808 100644 --- a/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs +++ b/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs @@ -21,11 +21,12 @@ public Uri Resolve(Uri crossLinkUri, string path) /// Hardcoding these for now, we'll have an index.json pointing to all links.json files /// at some point from which we can query the branch soon. - private static string GetBranch(Uri crossLinkUri) + public static string GetBranch(Uri crossLinkUri) { var branch = crossLinkUri.Scheme switch { "docs-content" => "main", + "cloud" => "master", _ => "main" }; return branch; diff --git a/src/Elastic.Markdown/InboundLinks/LinkIndexCrossLinkFetcher.cs b/src/Elastic.Markdown/InboundLinks/LinkIndexCrossLinkFetcher.cs index 886e2561c..bcfda1d0a 100644 --- a/src/Elastic.Markdown/InboundLinks/LinkIndexCrossLinkFetcher.cs +++ b/src/Elastic.Markdown/InboundLinks/LinkIndexCrossLinkFetcher.cs @@ -13,21 +13,24 @@ public class LinksIndexCrossLinkFetcher(ILoggerFactory logger) : CrossLinkFetche { public override async Task Fetch() { - var dictionary = new Dictionary(); + var linkReferences = new Dictionary(); + var linkEntries = new Dictionary(); var declaredRepositories = new HashSet(); var linkIndex = await FetchLinkIndex(); foreach (var (repository, value) in linkIndex.Repositories) { var linkIndexEntry = value.First().Value; + linkEntries.Add(repository, linkIndexEntry); var linkReference = await FetchLinkIndexEntry(repository, linkIndexEntry); - dictionary.Add(repository, linkReference); + linkReferences.Add(repository, linkReference); _ = declaredRepositories.Add(repository); } return new FetchedCrossLinks { DeclaredRepositories = declaredRepositories, - LinkReferences = dictionary.ToFrozenDictionary(), + LinkReferences = linkReferences.ToFrozenDictionary(), + LinkIndexEntries = linkEntries.ToFrozenDictionary(), FromConfiguration = false }; } diff --git a/src/Elastic.Markdown/InboundLinks/LinkIndexLinkChecker.cs b/src/Elastic.Markdown/InboundLinks/LinkIndexLinkChecker.cs index 151b36382..fcf079948 100644 --- a/src/Elastic.Markdown/InboundLinks/LinkIndexLinkChecker.cs +++ b/src/Elastic.Markdown/InboundLinks/LinkIndexLinkChecker.cs @@ -55,7 +55,6 @@ public async Task CheckWithLocalLinksJson(DiagnosticsCollector collector, s var resolver = new CrossLinkResolver(fetcher); // ReSharper disable once RedundantAssignment var crossLinks = await resolver.FetchLinks(); - if (string.IsNullOrEmpty(repository)) throw new ArgumentNullException(nameof(repository)); if (string.IsNullOrEmpty(localLinksJson)) @@ -117,6 +116,8 @@ private async Task ValidateCrossLinks( continue; var linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{uri.Scheme}/main/links.json"; + if (crossLinks.LinkIndexEntries.TryGetValue(uri.Scheme, out var linkIndexEntry)) + linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkIndexEntry.Path}"; _ = resolver.TryResolve(s => { if (s.Contains("is not a valid link in the")) diff --git a/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs b/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs index c5c67df93..b5e30c720 100644 --- a/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs +++ b/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs @@ -14,7 +14,8 @@ public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfigurat { public override async Task Fetch() { - var dictionary = new Dictionary(); + var linkReferences = new Dictionary(); + var linkIndexEntries = new Dictionary(); var declaredRepositories = new HashSet(); var repositories = configuration.ReferenceRepositories.Values.Concat([configuration.Narrative]); @@ -25,13 +26,16 @@ public override async Task Fetch() var repositoryName = repository.Name; _ = declaredRepositories.Add(repositoryName); var linkReference = await Fetch(repositoryName); - dictionary.Add(repositoryName, linkReference); + linkReferences.Add(repositoryName, linkReference); + var linkIndexReference = await GetLinkIndexEntry(repositoryName); + linkIndexEntries.Add(repositoryName, linkIndexReference); } return new FetchedCrossLinks { DeclaredRepositories = declaredRepositories, - LinkReferences = dictionary.ToFrozenDictionary(), + LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(), + LinkReferences = linkReferences.ToFrozenDictionary(), FromConfiguration = true }; } diff --git a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs index e1e9e326b..9f9cc894b 100644 --- a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs +++ b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs @@ -47,11 +47,20 @@ public Task FetchLinks() LinkReferences.Add("docs-content", reference); LinkReferences.Add("kibana", reference); DeclaredRepositories.AddRange(["docs-content", "kibana", "elasticsearch"]); + + var indexEntries = LinkReferences.ToDictionary(e => e.Key, e => new LinkIndexEntry + { + Repository = e.Key, + Path = $"elastic/asciidocalypse/{e.Key}/links.json", + Branch = "main", + ETag = Guid.NewGuid().ToString() + }); _crossLinks = new FetchedCrossLinks { DeclaredRepositories = DeclaredRepositories, LinkReferences = LinkReferences.ToFrozenDictionary(), - FromConfiguration = true + FromConfiguration = true, + LinkIndexEntries = indexEntries.ToFrozenDictionary() }; return Task.FromResult(_crossLinks); } diff --git a/tests/authoring/Framework/TestCrossLinkResolver.fs b/tests/authoring/Framework/TestCrossLinkResolver.fs index 9989cdcac..f43202ad6 100644 --- a/tests/authoring/Framework/TestCrossLinkResolver.fs +++ b/tests/authoring/Framework/TestCrossLinkResolver.fs @@ -9,6 +9,7 @@ open System.Collections.Generic open System.Collections.Frozen open System.Runtime.InteropServices open System.Threading.Tasks +open System.Linq open Elastic.Markdown.CrossLinks open Elastic.Markdown.IO.Configuration open Elastic.Markdown.IO.State @@ -62,21 +63,41 @@ type TestCrossLinkResolver (config: ConfigurationFile) = this.LinkReferences.Add("kibana", reference) this.DeclaredRepositories.Add("docs-content") |> ignore; this.DeclaredRepositories.Add("kibana") |> ignore; - this.DeclaredRepositories.Add("elasticsearch") |> ignore; + this.DeclaredRepositories.Add("elasticsearch") |> ignore + + let indexEntries = + this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkIndexEntry( + Repository = e.Key, + Path = $"elastic/asciidocalypse/{e.Key}/links.json", + Branch = "main", + ETag = Guid.NewGuid().ToString() + )) + let crossLinks = FetchedCrossLinks( DeclaredRepositories=this.DeclaredRepositories, LinkReferences=this.LinkReferences.ToFrozenDictionary(), - FromConfiguration=true + FromConfiguration=true, + LinkIndexEntries=indexEntries.ToFrozenDictionary() ) Task.FromResult crossLinks member this.TryResolve(errorEmitter, warningEmitter, crossLinkUri, []resolvedUri : byref) = + let indexEntries = + this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkIndexEntry( + Repository = e.Key, + Path = $"elastic/asciidocalypse/{e.Key}/links.json", + Branch = "main", + ETag = Guid.NewGuid().ToString() + )); + let crossLinks = FetchedCrossLinks( DeclaredRepositories=this.DeclaredRepositories, LinkReferences=this.LinkReferences.ToFrozenDictionary(), - FromConfiguration=true + FromConfiguration=true, + LinkIndexEntries=indexEntries.ToFrozenDictionary() + ) CrossLinkResolver.TryResolve(errorEmitter, warningEmitter, crossLinks, uriResolver, crossLinkUri, &resolvedUri); From 66e08951e95c23e191cc0400422e09bdc391c38f Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 13 Mar 2025 17:33:35 +0100 Subject: [PATCH 2/2] remove cross link tests from docs --- docs/_docset.yml | 1 - docs/testing/cross-links.md | 2 -- src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/_docset.yml b/docs/_docset.yml index 7a6d6f6d1..fed81948b 100644 --- a/docs/_docset.yml +++ b/docs/_docset.yml @@ -1,7 +1,6 @@ project: 'doc-builder' cross_links: - docs-content - - cloud exclude: - '_*.md' subs: diff --git a/docs/testing/cross-links.md b/docs/testing/cross-links.md index d0cbefb86..4f0863d4c 100644 --- a/docs/testing/cross-links.md +++ b/docs/testing/cross-links.md @@ -2,8 +2,6 @@ [Elasticsearch](docs-content://index.md) -[Cloud](cloud://reference/cloud-enterprise/index.md) - [Kibana][1] [1]: docs-content://index.md diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs index 0b855055b..05cf6d661 100644 --- a/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs @@ -92,7 +92,7 @@ protected async Task FetchLinkIndexEntry(string repository, LinkI private void WriteLinksJsonCachedFile(string repository, LinkIndexEntry linkIndexEntry, string json) { - var cachedFileName = $"links-elastic-{repository}-main-{linkIndexEntry.ETag}.json"; + var cachedFileName = $"links-elastic-{repository}-{linkIndexEntry.Branch}-{linkIndexEntry.ETag}.json"; var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "links", cachedFileName); if (File.Exists(cachedPath)) return;