Add inbound links validation

.editorconfig

@@ -227,6 +227,7 @@ dotnet_diagnostic.IDE0057.severity = none
 dotnet_diagnostic.IDE0051.severity = suggestion
 dotnet_diagnostic.IDE0059.severity = suggestion
 
+
 [DocumentationWebHost.cs]
 dotnet_diagnostic.IL3050.severity = none
 dotnet_diagnostic.IL2026.severity = none

src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs → Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs

@@ -8,7 +8,7 @@
 using Spectre.Console;
 using Diagnostic = Elastic.Markdown.Diagnostics.Diagnostic;
 
-namespace Documentation.Builder.Diagnostics.Console;
+namespace Elastic.Documentation.Tooling.Diagnostics.Console;
 
 public class ConsoleDiagnosticsCollector(ILoggerFactory loggerFactory, ICoreService? githubActions = null)
 	: DiagnosticsCollector([new Log(loggerFactory.CreateLogger<Log>()), new GithubAnnotationOutput(githubActions)]

src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs → Elastic.Documentation.Tooling/Diagnostics/Console/ErrataFileSourceRepository.cs

@@ -10,7 +10,7 @@
 using Spectre.Console;
 using Diagnostic = Elastic.Markdown.Diagnostics.Diagnostic;
 
-namespace Documentation.Builder.Diagnostics.Console;
+namespace Elastic.Documentation.Tooling.Diagnostics.Console;
 
 public class ErrataFileSourceRepository : ISourceRepository
 {

src/docs-builder/Diagnostics/Console/GithubAnnotationOutput.cs → Elastic.Documentation.Tooling/Diagnostics/Console/GithubAnnotationOutput.cs

@@ -6,7 +6,7 @@
 using Actions.Core.Services;
 using Elastic.Markdown.Diagnostics;
 
-namespace Documentation.Builder.Diagnostics.Console;
+namespace Elastic.Documentation.Tooling.Diagnostics.Console;
 
 public class GithubAnnotationOutput(ICoreService? githubActions) : IDiagnosticsOutput
 {

src/docs-builder/Diagnostics/Log.cs → Elastic.Documentation.Tooling/Diagnostics/Log.cs

@@ -5,7 +5,7 @@
 using Elastic.Markdown.Diagnostics;
 using Microsoft.Extensions.Logging;
 
-namespace Documentation.Builder.Diagnostics;
+namespace Elastic.Documentation.Tooling.Diagnostics;
 
 // named Log for terseness on console output
 public class Log(ILogger logger) : IDiagnosticsOutput

Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj

@@ -12,6 +12,11 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2"/>
     <PackageReference Include="Github.Actions.Core" Version="9.0.0"/>
     <PackageReference Include="Crayon" Version="2.0.69"/>
+    <PackageReference Include="Errata" Version="0.13.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\src\Elastic.Markdown\Elastic.Markdown.csproj" />
   </ItemGroup>
 
 </Project>

actions/validate-inbound-local/action.yml

@@ -0,0 +1,10 @@
+name: 'Validate Inbound Links'
+description: 'Validates all published cross links from all known repositories against local links.json'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Validate Inbound Links
+      uses: elastic/docs-builder/actions/assembler@main
+      with:
+        command: "link validate-inbound-local"

docs-builder.sln

@@ -55,6 +55,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler", "assembler", "{
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Tooling", "Elastic.Documentation.Tooling\Elastic.Documentation.Tooling.csproj", "{4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "validate-inbound-local", "validate-inbound-local", "{6E2ED6CC-AFC1-4E58-965D-6AEC500EBB46}"
+	ProjectSection(SolutionItems) = preProject
+		actions\validate-inbound-local\action.yml = actions\validate-inbound-local\action.yml
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -113,5 +118,6 @@ Global
 		{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{CFEE9FAD-9E0C-4C0E-A0C2-B97D594C14B5} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
+		{6E2ED6CC-AFC1-4E58-965D-6AEC500EBB46} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 	EndGlobalSection
 EndGlobal

src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs

@@ -0,0 +1,44 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Collections.Frozen;
+using Elastic.Markdown.IO.Configuration;
+using Elastic.Markdown.IO.State;
+using Microsoft.Extensions.Logging;
+
+namespace Elastic.Markdown.CrossLinks;
+
+public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILoggerFactory logger) : CrossLinkFetcher(logger)
+{
+	public override async Task<FetchedCrossLinks> Fetch()
+	{
+		var dictionary = new Dictionary<string, LinkReference>();
+		var declaredRepositories = new HashSet<string>();
+		foreach (var repository in configuration.CrossLinkRepositories)
+		{
+			_ = declaredRepositories.Add(repository);
+			try
+			{
+				var linkReference = await Fetch(repository);
+				dictionary.Add(repository, linkReference);
+			}
+			catch when (repository == "docs-content")
+			{
+				throw;
+			}
+			catch when (repository != "docs-content")
+			{
+				// TODO: ignored for now while we wait for all links.json files to populate
+			}
+		}
+
+		return new FetchedCrossLinks
+		{
+			DeclaredRepositories = declaredRepositories,
+			LinkReferences = dictionary.ToFrozenDictionary()
+		};
+	}
+
+
+}

src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs

@@ -0,0 +1,121 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Collections.Frozen;
+using System.Text.Json;
+using Elastic.Markdown.IO;
+using Elastic.Markdown.IO.State;
+using Microsoft.Extensions.Logging;
+
+namespace Elastic.Markdown.CrossLinks;
+
+public record FetchedCrossLinks
+{
+	public required FrozenDictionary<string, LinkReference> LinkReferences { get; init; }
+	public required HashSet<string> DeclaredRepositories { get; init; }
+
+	public static FetchedCrossLinks Empty { get; } = new()
+	{
+		DeclaredRepositories = [],
+		LinkReferences = new Dictionary<string, LinkReference>().ToFrozenDictionary()
+	};
+}
+
+public abstract class CrossLinkFetcher(ILoggerFactory logger) : IDisposable
+{
+	private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkFetcher));
+	private readonly HttpClient _client = new();
+	private LinkIndex? _linkIndex;
+
+	public static LinkReference Deserialize(string json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
+
+	public abstract Task<FetchedCrossLinks> Fetch();
+
+	protected async Task<LinkIndex> FetchLinkIndex()
+	{
+		if (_linkIndex is not null)
+		{
+			_logger.LogInformation("Using cached link index");
+			return _linkIndex;
+		}
+		var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json";
+		_logger.LogInformation("Fetching {Url}", url);
+		var json = await _client.GetStringAsync(url);
+		_linkIndex = LinkIndex.Deserialize(json);
+		return _linkIndex;
+	}
+
+	protected async Task<LinkReference> Fetch(string repository)
+	{
+		var linkIndex = await FetchLinkIndex();
+		if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
+			throw new Exception($"Repository {repository} not found in link index");
+
+		if (!repositoryLinks.TryGetValue("main", out var linkIndexEntry))
+			throw new Exception($"Repository {repository} not found in link index");
+
+		return await FetchLinkIndexEntry(repository, linkIndexEntry);
+	}
+
+	protected async Task<LinkReference> FetchLinkIndexEntry(string repository, LinkIndexEntry linkIndexEntry)
+	{
+		var linkReference = await TryGetCachedLinkReference(repository, linkIndexEntry);
+		if (linkReference is not null)
+			return linkReference;
+
+		var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{repository}/main/links.json";
+		_logger.LogInformation("Fetching links.json for '{Repository}': {Url}", repository, url);
+		var json = await _client.GetStringAsync(url);
+		linkReference = Deserialize(json);
+		WriteLinksJsonCachedFile(repository, linkIndexEntry, json);
+		return linkReference;
+	}
+
+	private void WriteLinksJsonCachedFile(string repository, LinkIndexEntry linkIndexEntry, string json)
+	{
+		var cachedFileName = $"links-elastic-{repository}-main-{linkIndexEntry.ETag}.json";
+		var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "links", cachedFileName);
+		if (File.Exists(cachedPath))
+			return;
+		try
+		{
+			_ = Directory.CreateDirectory(Path.GetDirectoryName(cachedPath)!);
+			File.WriteAllText(cachedPath, json);
+		}
+		catch (Exception e)
+		{
+			_logger.LogError(e, "Failed to write cached link reference {CachedPath}", cachedPath);
+		}
+	}
+
+	private async Task<LinkReference?> TryGetCachedLinkReference(string repository, LinkIndexEntry linkIndexEntry)
+	{
+		var cachedFileName = $"links-elastic-{repository}-main-{linkIndexEntry.ETag}.json";
+		var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "links", cachedFileName);
+		if (File.Exists(cachedPath))
+		{
+			try
+			{
+				var json = await File.ReadAllTextAsync(cachedPath);
+				var linkReference = Deserialize(json);
+				return linkReference;
+			}
+			catch (Exception e)
+			{
+				_logger.LogError(e, "Failed to read cached link reference {CachedPath}", cachedPath);
+				return null;
+			}
+		}
+		return null;
+
+	}
+
+	public void Dispose()
+	{
+		_client.Dispose();
+		logger.Dispose();
+		GC.SuppressFinalize(this);
+	}
+}

src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs

@@ -2,13 +2,10 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-using System.Collections.Frozen;
 using System.Diagnostics.CodeAnalysis;
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using Elastic.Markdown.IO.Configuration;
 using Elastic.Markdown.IO.State;
-using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown.CrossLinks;
 
@@ -36,61 +33,34 @@ public record LinkIndexEntry
 
 public interface ICrossLinkResolver
 {
-	Task FetchLinks();
+	Task<FetchedCrossLinks> FetchLinks();
 	bool TryResolve(Action<string> errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri);
 }
 
-public class CrossLinkResolver(ConfigurationFile configuration, ILoggerFactory logger) : ICrossLinkResolver
+public class CrossLinkResolver(CrossLinkFetcher fetcher) : ICrossLinkResolver
 {
-	private readonly string[] _links = configuration.CrossLinkRepositories;
-	private FrozenDictionary<string, LinkReference> _linkReferences = new Dictionary<string, LinkReference>().ToFrozenDictionary();
-	private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkResolver));
-	private readonly HashSet<string> _declaredRepositories = [];
+	private FetchedCrossLinks _linkReferences = FetchedCrossLinks.Empty;
 
-	public static LinkReference Deserialize(string json) =>
-		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
-
-	public async Task FetchLinks()
+	public async Task<FetchedCrossLinks> FetchLinks()
 	{
-		using var client = new HttpClient();
-		var dictionary = new Dictionary<string, LinkReference>();
-		foreach (var link in _links)
-		{
-			_ = _declaredRepositories.Add(link);
-			try
-			{
-				var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{link}/main/links.json";
-				_logger.LogInformation("Fetching {Url}", url);
-				var json = await client.GetStringAsync(url);
-				var linkReference = Deserialize(json);
-				dictionary.Add(link, linkReference);
-			}
-			catch when (link == "docs-content")
-			{
-				throw;
-			}
-			catch when (link != "docs-content")
-			{
-				// TODO: ignored for now while we wait for all links.json files to populate
-			}
-		}
-
-		_linkReferences = dictionary.ToFrozenDictionary();
+		_linkReferences = await fetcher.Fetch();
+		return _linkReferences;
 	}
 
 	public bool TryResolve(Action<string> errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) =>
-		TryResolve(errorEmitter, _declaredRepositories, _linkReferences, crossLinkUri, out resolvedUri);
+		TryResolve(errorEmitter, _linkReferences, crossLinkUri, out resolvedUri);
 
-	private static Uri BaseUri { get; } = new Uri("https://docs-v3-preview.elastic.dev");
+	private static Uri BaseUri { get; } = new("https://docs-v3-preview.elastic.dev");
 
 	public static bool TryResolve(
 		Action<string> errorEmitter,
-		HashSet<string> declaredRepositories,
-		IDictionary<string, LinkReference> lookup,
+		FetchedCrossLinks fetchedCrossLinks,
 		Uri crossLinkUri,
 		[NotNullWhen(true)] out Uri? resolvedUri
 	)
 	{
+		var lookup = fetchedCrossLinks.LinkReferences;
+		var declaredRepositories = fetchedCrossLinks.DeclaredRepositories;
 		resolvedUri = null;
 		if (crossLinkUri.Scheme == "docs-content")
 		{
@@ -180,6 +150,7 @@ private static bool LookupLink(
 
 			return ResolveLinkRedirect(targets, errorEmitter, linkReference, crossLinkUri, ref lookupPath, out link, ref lookupFragment);
 		}
+
 		if (linkReference.Links.TryGetValue(lookupPath, out link))
 		{
 			lookupFragment = crossLinkUri.Fragment;

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -71,7 +71,7 @@ public async Task GenerateAll(Cancel ctx)
 			return;
 
 		_logger.LogInformation($"Fetching external links");
-		await Resolver.FetchLinks();
+		_ = await Resolver.FetchLinks();
 
 		await ResolveDirectoryTree(ctx);
 

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -40,7 +40,8 @@ public DocumentationSet(BuildContext context, ILoggerFactory logger, ICrossLinkR
 		SourcePath = context.SourcePath;
 		OutputPath = context.OutputPath;
 		RelativeSourcePath = Path.GetRelativePath(Paths.Root.FullName, SourcePath.FullName);
-		LinkResolver = linkResolver ?? new CrossLinkResolver(context.Configuration, logger);
+		LinkResolver =
+			linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, logger));
 		Configuration = context.Configuration;
 
 		MarkdownParser = new MarkdownParser(SourcePath, context, GetMarkdownFile, context.Configuration, LinkResolver);