diff --git a/.editorconfig b/.editorconfig index 18ee93c55..0e45b376c 100644 --- a/.editorconfig +++ b/.editorconfig @@ -227,6 +227,7 @@ dotnet_diagnostic.IDE0057.severity = none dotnet_diagnostic.IDE0051.severity = suggestion dotnet_diagnostic.IDE0059.severity = suggestion + [DocumentationWebHost.cs] dotnet_diagnostic.IL3050.severity = none dotnet_diagnostic.IL2026.severity = none diff --git a/src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs b/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs similarity index 95% rename from src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs rename to Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs index f4747454c..3ca857500 100644 --- a/src/docs-builder/Diagnostics/Console/ConsoleDiagnosticsCollector.cs +++ b/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs @@ -8,7 +8,7 @@ using Spectre.Console; using Diagnostic = Elastic.Markdown.Diagnostics.Diagnostic; -namespace Documentation.Builder.Diagnostics.Console; +namespace Elastic.Documentation.Tooling.Diagnostics.Console; public class ConsoleDiagnosticsCollector(ILoggerFactory loggerFactory, ICoreService? githubActions = null) : DiagnosticsCollector([new Log(loggerFactory.CreateLogger()), new GithubAnnotationOutput(githubActions)] diff --git a/src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs b/Elastic.Documentation.Tooling/Diagnostics/Console/ErrataFileSourceRepository.cs similarity index 97% rename from src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs rename to Elastic.Documentation.Tooling/Diagnostics/Console/ErrataFileSourceRepository.cs index 07121aec9..391a4259e 100644 --- a/src/docs-builder/Diagnostics/Console/ErrataFileSourceRepository.cs +++ b/Elastic.Documentation.Tooling/Diagnostics/Console/ErrataFileSourceRepository.cs @@ -10,7 +10,7 @@ using Spectre.Console; using Diagnostic = Elastic.Markdown.Diagnostics.Diagnostic; -namespace Documentation.Builder.Diagnostics.Console; +namespace Elastic.Documentation.Tooling.Diagnostics.Console; public class ErrataFileSourceRepository : ISourceRepository { diff --git a/src/docs-builder/Diagnostics/Console/GithubAnnotationOutput.cs b/Elastic.Documentation.Tooling/Diagnostics/Console/GithubAnnotationOutput.cs similarity index 94% rename from src/docs-builder/Diagnostics/Console/GithubAnnotationOutput.cs rename to Elastic.Documentation.Tooling/Diagnostics/Console/GithubAnnotationOutput.cs index 067e3d67b..635fbde91 100644 --- a/src/docs-builder/Diagnostics/Console/GithubAnnotationOutput.cs +++ b/Elastic.Documentation.Tooling/Diagnostics/Console/GithubAnnotationOutput.cs @@ -6,7 +6,7 @@ using Actions.Core.Services; using Elastic.Markdown.Diagnostics; -namespace Documentation.Builder.Diagnostics.Console; +namespace Elastic.Documentation.Tooling.Diagnostics.Console; public class GithubAnnotationOutput(ICoreService? githubActions) : IDiagnosticsOutput { diff --git a/src/docs-builder/Diagnostics/Log.cs b/Elastic.Documentation.Tooling/Diagnostics/Log.cs similarity index 92% rename from src/docs-builder/Diagnostics/Log.cs rename to Elastic.Documentation.Tooling/Diagnostics/Log.cs index 8148f3c28..d4ccd5589 100644 --- a/src/docs-builder/Diagnostics/Log.cs +++ b/Elastic.Documentation.Tooling/Diagnostics/Log.cs @@ -5,7 +5,7 @@ using Elastic.Markdown.Diagnostics; using Microsoft.Extensions.Logging; -namespace Documentation.Builder.Diagnostics; +namespace Elastic.Documentation.Tooling.Diagnostics; // named Log for terseness on console output public class Log(ILogger logger) : IDiagnosticsOutput diff --git a/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj b/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj index 6287544a0..ca15acba2 100644 --- a/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj +++ b/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj @@ -12,6 +12,11 @@ + + + + + diff --git a/actions/validate-inbound-local/action.yml b/actions/validate-inbound-local/action.yml new file mode 100644 index 000000000..6ae079bc3 --- /dev/null +++ b/actions/validate-inbound-local/action.yml @@ -0,0 +1,10 @@ +name: 'Validate Inbound Links' +description: 'Validates all published cross links from all known repositories against local links.json' + +runs: + using: "composite" + steps: + - name: Validate Inbound Links + uses: elastic/docs-builder/actions/assembler@main + with: + command: "link validate-inbound-local" \ No newline at end of file diff --git a/docs-builder.sln b/docs-builder.sln index e04bb687d..f5da68c18 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -55,6 +55,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler", "assembler", "{ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Tooling", "Elastic.Documentation.Tooling\Elastic.Documentation.Tooling.csproj", "{4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "validate-inbound-local", "validate-inbound-local", "{6E2ED6CC-AFC1-4E58-965D-6AEC500EBB46}" + ProjectSection(SolutionItems) = preProject + actions\validate-inbound-local\action.yml = actions\validate-inbound-local\action.yml + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -113,5 +118,6 @@ Global {7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {CFEE9FAD-9E0C-4C0E-A0C2-B97D594C14B5} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {4CCE599A-B9FE-4DF2-8763-34CF0A99D4AA} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} + {6E2ED6CC-AFC1-4E58-965D-6AEC500EBB46} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} EndGlobalSection EndGlobal diff --git a/src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs b/src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs new file mode 100644 index 000000000..63323dd4d --- /dev/null +++ b/src/Elastic.Markdown/CrossLinks/ConfigurationCrossLinkFetcher.cs @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using Elastic.Markdown.IO.Configuration; +using Elastic.Markdown.IO.State; +using Microsoft.Extensions.Logging; + +namespace Elastic.Markdown.CrossLinks; + +public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILoggerFactory logger) : CrossLinkFetcher(logger) +{ + public override async Task Fetch() + { + var dictionary = new Dictionary(); + var declaredRepositories = new HashSet(); + foreach (var repository in configuration.CrossLinkRepositories) + { + _ = declaredRepositories.Add(repository); + try + { + var linkReference = await Fetch(repository); + dictionary.Add(repository, linkReference); + } + catch when (repository == "docs-content") + { + throw; + } + catch when (repository != "docs-content") + { + // TODO: ignored for now while we wait for all links.json files to populate + } + } + + return new FetchedCrossLinks + { + DeclaredRepositories = declaredRepositories, + LinkReferences = dictionary.ToFrozenDictionary() + }; + } + + +} diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs new file mode 100644 index 000000000..1606f9689 --- /dev/null +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkFetcher.cs @@ -0,0 +1,121 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using System.Text.Json; +using Elastic.Markdown.IO; +using Elastic.Markdown.IO.State; +using Microsoft.Extensions.Logging; + +namespace Elastic.Markdown.CrossLinks; + +public record FetchedCrossLinks +{ + public required FrozenDictionary LinkReferences { get; init; } + public required HashSet DeclaredRepositories { get; init; } + + public static FetchedCrossLinks Empty { get; } = new() + { + DeclaredRepositories = [], + LinkReferences = new Dictionary().ToFrozenDictionary() + }; +} + +public abstract class CrossLinkFetcher(ILoggerFactory logger) : IDisposable +{ + private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkFetcher)); + private readonly HttpClient _client = new(); + private LinkIndex? _linkIndex; + + public static LinkReference Deserialize(string json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; + + public abstract Task Fetch(); + + protected async Task FetchLinkIndex() + { + if (_linkIndex is not null) + { + _logger.LogInformation("Using cached link index"); + return _linkIndex; + } + var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json"; + _logger.LogInformation("Fetching {Url}", url); + var json = await _client.GetStringAsync(url); + _linkIndex = LinkIndex.Deserialize(json); + return _linkIndex; + } + + protected async Task Fetch(string repository) + { + var linkIndex = await FetchLinkIndex(); + if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) + throw new Exception($"Repository {repository} not found in link index"); + + if (!repositoryLinks.TryGetValue("main", out var linkIndexEntry)) + throw new Exception($"Repository {repository} not found in link index"); + + return await FetchLinkIndexEntry(repository, linkIndexEntry); + } + + protected async Task FetchLinkIndexEntry(string repository, LinkIndexEntry linkIndexEntry) + { + var linkReference = await TryGetCachedLinkReference(repository, linkIndexEntry); + if (linkReference is not null) + return linkReference; + + var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{repository}/main/links.json"; + _logger.LogInformation("Fetching links.json for '{Repository}': {Url}", repository, url); + var json = await _client.GetStringAsync(url); + linkReference = Deserialize(json); + WriteLinksJsonCachedFile(repository, linkIndexEntry, json); + return linkReference; + } + + private void WriteLinksJsonCachedFile(string repository, LinkIndexEntry linkIndexEntry, string json) + { + var cachedFileName = $"links-elastic-{repository}-main-{linkIndexEntry.ETag}.json"; + var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "links", cachedFileName); + if (File.Exists(cachedPath)) + return; + try + { + _ = Directory.CreateDirectory(Path.GetDirectoryName(cachedPath)!); + File.WriteAllText(cachedPath, json); + } + catch (Exception e) + { + _logger.LogError(e, "Failed to write cached link reference {CachedPath}", cachedPath); + } + } + + private async Task TryGetCachedLinkReference(string repository, LinkIndexEntry linkIndexEntry) + { + var cachedFileName = $"links-elastic-{repository}-main-{linkIndexEntry.ETag}.json"; + var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "links", cachedFileName); + if (File.Exists(cachedPath)) + { + try + { + var json = await File.ReadAllTextAsync(cachedPath); + var linkReference = Deserialize(json); + return linkReference; + } + catch (Exception e) + { + _logger.LogError(e, "Failed to read cached link reference {CachedPath}", cachedPath); + return null; + } + } + return null; + + } + + public void Dispose() + { + _client.Dispose(); + logger.Dispose(); + GC.SuppressFinalize(this); + } +} diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs index dbdf61b29..66ec9dfeb 100644 --- a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs @@ -2,13 +2,10 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using System.Collections.Frozen; using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Serialization; -using Elastic.Markdown.IO.Configuration; using Elastic.Markdown.IO.State; -using Microsoft.Extensions.Logging; namespace Elastic.Markdown.CrossLinks; @@ -36,61 +33,34 @@ public record LinkIndexEntry public interface ICrossLinkResolver { - Task FetchLinks(); + Task FetchLinks(); bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri); } -public class CrossLinkResolver(ConfigurationFile configuration, ILoggerFactory logger) : ICrossLinkResolver +public class CrossLinkResolver(CrossLinkFetcher fetcher) : ICrossLinkResolver { - private readonly string[] _links = configuration.CrossLinkRepositories; - private FrozenDictionary _linkReferences = new Dictionary().ToFrozenDictionary(); - private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkResolver)); - private readonly HashSet _declaredRepositories = []; + private FetchedCrossLinks _linkReferences = FetchedCrossLinks.Empty; - public static LinkReference Deserialize(string json) => - JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; - - public async Task FetchLinks() + public async Task FetchLinks() { - using var client = new HttpClient(); - var dictionary = new Dictionary(); - foreach (var link in _links) - { - _ = _declaredRepositories.Add(link); - try - { - var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{link}/main/links.json"; - _logger.LogInformation("Fetching {Url}", url); - var json = await client.GetStringAsync(url); - var linkReference = Deserialize(json); - dictionary.Add(link, linkReference); - } - catch when (link == "docs-content") - { - throw; - } - catch when (link != "docs-content") - { - // TODO: ignored for now while we wait for all links.json files to populate - } - } - - _linkReferences = dictionary.ToFrozenDictionary(); + _linkReferences = await fetcher.Fetch(); + return _linkReferences; } public bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) => - TryResolve(errorEmitter, _declaredRepositories, _linkReferences, crossLinkUri, out resolvedUri); + TryResolve(errorEmitter, _linkReferences, crossLinkUri, out resolvedUri); - private static Uri BaseUri { get; } = new Uri("https://docs-v3-preview.elastic.dev"); + private static Uri BaseUri { get; } = new("https://docs-v3-preview.elastic.dev"); public static bool TryResolve( Action errorEmitter, - HashSet declaredRepositories, - IDictionary lookup, + FetchedCrossLinks fetchedCrossLinks, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri ) { + var lookup = fetchedCrossLinks.LinkReferences; + var declaredRepositories = fetchedCrossLinks.DeclaredRepositories; resolvedUri = null; if (crossLinkUri.Scheme == "docs-content") { @@ -180,6 +150,7 @@ private static bool LookupLink( return ResolveLinkRedirect(targets, errorEmitter, linkReference, crossLinkUri, ref lookupPath, out link, ref lookupFragment); } + if (linkReference.Links.TryGetValue(lookupPath, out link)) { lookupFragment = crossLinkUri.Fragment; diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 46fe8516f..d0e5a5d6d 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -71,7 +71,7 @@ public async Task GenerateAll(Cancel ctx) return; _logger.LogInformation($"Fetching external links"); - await Resolver.FetchLinks(); + _ = await Resolver.FetchLinks(); await ResolveDirectoryTree(ctx); diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 4ecbca534..e5227c934 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -40,7 +40,8 @@ public DocumentationSet(BuildContext context, ILoggerFactory logger, ICrossLinkR SourcePath = context.SourcePath; OutputPath = context.OutputPath; RelativeSourcePath = Path.GetRelativePath(Paths.Root.FullName, SourcePath.FullName); - LinkResolver = linkResolver ?? new CrossLinkResolver(context.Configuration, logger); + LinkResolver = + linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, logger)); Configuration = context.Configuration; MarkdownParser = new MarkdownParser(SourcePath, context, GetMarkdownFile, context.Configuration, LinkResolver); diff --git a/src/Elastic.Markdown/IO/Paths.cs b/src/Elastic.Markdown/IO/Paths.cs index 8307df6be..2ebf62780 100644 --- a/src/Elastic.Markdown/IO/Paths.cs +++ b/src/Elastic.Markdown/IO/Paths.cs @@ -5,6 +5,10 @@ namespace Elastic.Markdown.IO; public static class Paths { + public static readonly DirectoryInfo Root = RootDirectoryInfo(); + + public static readonly DirectoryInfo ApplicationData = GetApplicationFolder(); + private static DirectoryInfo RootDirectoryInfo() { var directory = new DirectoryInfo(Directory.GetCurrentDirectory()); @@ -14,8 +18,6 @@ private static DirectoryInfo RootDirectoryInfo() return directory ?? new DirectoryInfo(Directory.GetCurrentDirectory()); } - public static readonly DirectoryInfo Root = RootDirectoryInfo(); - /// Used in debug to locate static folder so we can change js/css files while the server is running public static DirectoryInfo? GetSolutionDirectory() { @@ -24,4 +26,15 @@ private static DirectoryInfo RootDirectoryInfo() directory = directory.Parent; return directory; } + + // ~/Library/Application\ Support/ on osx + // XDG_DATA_HOME or home/.local/share on linux + // %LOCAL_APPLICATION_DATA% windows + private static DirectoryInfo GetApplicationFolder() + { + var localPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); + var elasticPath = Path.Combine(localPath, "elastic", "docs-builder"); + return new DirectoryInfo(elasticPath); + } + } diff --git a/src/Elastic.Markdown/IO/State/LinkReference.cs b/src/Elastic.Markdown/IO/State/LinkReference.cs index a26a898bc..f4d895a14 100644 --- a/src/Elastic.Markdown/IO/State/LinkReference.cs +++ b/src/Elastic.Markdown/IO/State/LinkReference.cs @@ -61,6 +61,9 @@ public record LinkReference public static string SerializeRedirects(Dictionary? redirects) => JsonSerializer.Serialize(redirects, SourceGenerationContext.Default.DictionaryStringLinkRedirect); + public static LinkReference Deserialize(string json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; + public static LinkReference Create(DocumentationSet set) { var redirects = set.Configuration.Redirects; diff --git a/src/docs-assembler/Cli/LinkCommands.cs b/src/docs-assembler/Cli/LinkCommands.cs index 5bca2c38c..cf7ac67fd 100644 --- a/src/docs-assembler/Cli/LinkCommands.cs +++ b/src/docs-assembler/Cli/LinkCommands.cs @@ -2,16 +2,20 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.IO.Abstractions; using System.Text; +using Actions.Core.Services; using Amazon.S3; using Amazon.S3.Model; using ConsoleAppFramework; +using Documentation.Assembler.Links; using Elastic.Markdown.CrossLinks; +using Elastic.Markdown.IO.Discovery; using Microsoft.Extensions.Logging; namespace Documentation.Assembler.Cli; -internal sealed class LinkCommands(ILoggerFactory logger) +internal sealed class LinkCommands(ILoggerFactory logger, ICoreService githubActionsService) { private void AssignOutputLogger() { @@ -22,6 +26,35 @@ private void AssignOutputLogger() #pragma warning restore CA2254 } + /// + /// Validate all published cross_links in all published links.json files. + /// + /// + [Command("validate-inbound-all")] + public async Task ValidateAllInboundLinks(Cancel ctx = default) + { + AssignOutputLogger(); + return await new LinkIndexLinkChecker(logger).CheckAll(githubActionsService, ctx); + } + + /// + /// Create an index.json file from all discovered links.json files in our S3 bucket + /// + /// + /// + /// + [Command("validate-inbound-local")] + public async Task ValidateLocalInboundLinks(string? repository = null, string? file = null, Cancel ctx = default) + { + AssignOutputLogger(); + file ??= ".artifacts/docs/html/links.json"; + repository ??= GitCheckoutInformation.Create(new FileSystem()).RepositoryName; + if (repository == null) + throw new Exception("Unable to determine repository name"); + + return await new LinkIndexLinkChecker(logger).CheckWithLocalLinksJson(githubActionsService, repository, file, ctx); + } + /// /// Create an index.json file from all discovered links.json files in our S3 bucket /// diff --git a/src/docs-assembler/Links/LinkIndexCrossLinkFetcher.cs b/src/docs-assembler/Links/LinkIndexCrossLinkFetcher.cs new file mode 100644 index 000000000..ae02140c3 --- /dev/null +++ b/src/docs-assembler/Links/LinkIndexCrossLinkFetcher.cs @@ -0,0 +1,33 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using Elastic.Markdown.CrossLinks; +using Elastic.Markdown.IO.State; +using Microsoft.Extensions.Logging; + +namespace Documentation.Assembler.Links; + +public class LinksIndexCrossLinkFetcher(ILoggerFactory logger) : CrossLinkFetcher(logger) +{ + public override async Task Fetch() + { + var dictionary = new Dictionary(); + var declaredRepositories = new HashSet(); + var linkIndex = await FetchLinkIndex(); + foreach (var (repository, value) in linkIndex.Repositories) + { + var linkIndexEntry = value.First().Value; + var linkReference = await FetchLinkIndexEntry(repository, linkIndexEntry); + dictionary.Add(repository, linkReference); + _ = declaredRepositories.Add(repository); + } + + return new FetchedCrossLinks + { + DeclaredRepositories = declaredRepositories, + LinkReferences = dictionary.ToFrozenDictionary() + }; + } +} diff --git a/src/docs-assembler/Links/LinkIndexLinkChecker.cs b/src/docs-assembler/Links/LinkIndexLinkChecker.cs new file mode 100644 index 000000000..d940f8538 --- /dev/null +++ b/src/docs-assembler/Links/LinkIndexLinkChecker.cs @@ -0,0 +1,110 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using Actions.Core.Services; +using Elastic.Documentation.Tooling.Diagnostics.Console; +using Elastic.Markdown.CrossLinks; +using Elastic.Markdown.IO; +using Elastic.Markdown.IO.State; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Documentation.Assembler.Links; + +public class LinkIndexLinkChecker(ILoggerFactory logger) +{ + private readonly ILogger _logger = logger.CreateLogger(); + + public async Task CheckAll(ICoreService githubActionsService, Cancel ctx) + { + var fetcher = new LinksIndexCrossLinkFetcher(logger); + var resolver = new CrossLinkResolver(fetcher); + //todo add ctx + var crossLinks = await resolver.FetchLinks(); + + return await ValidateCrossLinks(githubActionsService, crossLinks, resolver, null, ctx); + } + + public async Task CheckWithLocalLinksJson( + ICoreService githubActionsService, + string repository, + string localLinksJson, + Cancel ctx + ) + { + var fetcher = new LinksIndexCrossLinkFetcher(logger); + var resolver = new CrossLinkResolver(fetcher); + var crossLinks = await resolver.FetchLinks(); + + if (string.IsNullOrEmpty(repository)) + throw new ArgumentNullException(nameof(repository)); + if (string.IsNullOrEmpty(localLinksJson)) + throw new ArgumentNullException(nameof(repository)); + + _logger.LogInformation("Checking '{Repository}' with local '{LocalLinksJson}'", repository, localLinksJson); + + if (!Path.IsPathRooted(localLinksJson)) + localLinksJson = Path.Combine(Paths.Root.FullName, localLinksJson); + + var dictionary = crossLinks.LinkReferences.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + try + { + var json = await File.ReadAllTextAsync(localLinksJson, ctx); + var localLinkReference = LinkReference.Deserialize(json); + + dictionary[repository] = localLinkReference; + } + catch (Exception e) + { + _logger.LogError(e, "Failed to read {LocalLinksJson}", localLinksJson); + throw; + } + crossLinks = crossLinks with { LinkReferences = dictionary.ToFrozenDictionary() }; + + _logger.LogInformation("Validating all cross links to {Repository}:// from all repositories published to link-index.json", repository); + + return await ValidateCrossLinks(githubActionsService, crossLinks, resolver, [repository], ctx); + } + + private async Task ValidateCrossLinks( + ICoreService githubActionsService, + FetchedCrossLinks crossLinks, + CrossLinkResolver resolver, + string[]? repositoryFilter, + Cancel ctx) + { + var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService); + _ = collector.StartAsync(ctx); + foreach (var (repository, linkReference) in crossLinks.LinkReferences) + { + _logger.LogInformation("Validating {Repository}", repository); + foreach (var crossLink in linkReference.CrossLinks) + { + // if we are filtering we only want errors from inbound links to a certain + // repository + var uri = new Uri(crossLink); + if (repositoryFilter != null && uri.Scheme != repository) + continue; + + _ = resolver.TryResolve(s => + { + if (s.Contains("is not a valid link in the")) + { + var error = $"'elastic/{repository}' links to unknown file: " + s; + error = error.Replace("is not a valid link in the", "in the"); + collector.EmitError(repository, error); + return; + } + + collector.EmitError(repository, s); + + }, uri, out _); + } + } + collector.Channel.TryComplete(); + await collector.StopAsync(ctx); + return collector.Errors + collector.Warnings; + } +} diff --git a/src/docs-builder/Cli/CheckForUpdatesFilter.cs b/src/docs-builder/Cli/CheckForUpdatesFilter.cs index aaf5728bd..45ba4a2bd 100644 --- a/src/docs-builder/Cli/CheckForUpdatesFilter.cs +++ b/src/docs-builder/Cli/CheckForUpdatesFilter.cs @@ -5,22 +5,13 @@ using System.Reflection; using ConsoleAppFramework; using Elastic.Markdown.Helpers; +using Elastic.Markdown.IO; namespace Documentation.Builder.Cli; -internal sealed class CheckForUpdatesFilter : ConsoleAppFilter +internal sealed class CheckForUpdatesFilter(ConsoleAppFilter next) : ConsoleAppFilter(next) { - private readonly FileInfo _stateFile; - - public CheckForUpdatesFilter(ConsoleAppFilter next) : base(next) - { - // ~/Library/Application\ Support/ on osx - // XDG_DATA_HOME or home/.local/share on linux - // %LOCAL_APPLICATION_DATA% windows - var localPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); - var elasticPath = Path.Combine(localPath, "elastic"); - _stateFile = new FileInfo(Path.Combine(elasticPath, "docs-build-check.state")); - } + private readonly FileInfo _stateFile = new(Path.Combine(Paths.ApplicationData.FullName, "docs-build-check.state")); public override async Task InvokeAsync(ConsoleAppContext context, Cancel ctx) { diff --git a/src/docs-builder/Cli/Commands.cs b/src/docs-builder/Cli/Commands.cs index f29a22a55..6e1b29bb7 100644 --- a/src/docs-builder/Cli/Commands.cs +++ b/src/docs-builder/Cli/Commands.cs @@ -4,8 +4,8 @@ using System.IO.Abstractions; using Actions.Core.Services; using ConsoleAppFramework; -using Documentation.Builder.Diagnostics.Console; using Documentation.Builder.Http; +using Elastic.Documentation.Tooling.Diagnostics.Console; using Elastic.Documentation.Tooling.Filters; using Elastic.Markdown; using Elastic.Markdown.IO; diff --git a/src/docs-builder/Diagnostics/LiveMode/LiveModeDiagnosticsCollector.cs b/src/docs-builder/Diagnostics/LiveMode/LiveModeDiagnosticsCollector.cs index 63e5ba851..66f39cadf 100644 --- a/src/docs-builder/Diagnostics/LiveMode/LiveModeDiagnosticsCollector.cs +++ b/src/docs-builder/Diagnostics/LiveMode/LiveModeDiagnosticsCollector.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using Elastic.Documentation.Tooling.Diagnostics; using Elastic.Markdown.Diagnostics; using Microsoft.Extensions.Logging; using Diagnostic = Elastic.Markdown.Diagnostics.Diagnostic; diff --git a/src/docs-builder/docs-builder.csproj b/src/docs-builder/docs-builder.csproj index dde666ae1..59eabd4fd 100644 --- a/src/docs-builder/docs-builder.csproj +++ b/src/docs-builder/docs-builder.csproj @@ -23,7 +23,6 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - diff --git a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs index 1781901dd..87d80c6bb 100644 --- a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs +++ b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.Collections.Frozen; using System.Diagnostics.CodeAnalysis; using Elastic.Markdown.CrossLinks; using Elastic.Markdown.IO.State; @@ -11,10 +12,11 @@ namespace Elastic.Markdown.Tests; public class TestCrossLinkResolver : ICrossLinkResolver { + private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty; private Dictionary LinkReferences { get; } = []; private HashSet DeclaredRepositories { get; } = []; - public Task FetchLinks() + public Task FetchLinks() { // language=json var json = """ @@ -40,13 +42,18 @@ public Task FetchLinks() } } """; - var reference = CrossLinkResolver.Deserialize(json); + var reference = CrossLinkFetcher.Deserialize(json); LinkReferences.Add("docs-content", reference); LinkReferences.Add("kibana", reference); DeclaredRepositories.AddRange(["docs-content", "kibana", "elasticsearch"]); - return Task.CompletedTask; + _crossLinks = new FetchedCrossLinks + { + DeclaredRepositories = DeclaredRepositories, + LinkReferences = LinkReferences.ToFrozenDictionary() + }; + return Task.FromResult(_crossLinks); } public bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) => - CrossLinkResolver.TryResolve(errorEmitter, DeclaredRepositories, LinkReferences, crossLinkUri, out resolvedUri); + CrossLinkResolver.TryResolve(errorEmitter, _crossLinks, crossLinkUri, out resolvedUri); } diff --git a/tests/authoring/Framework/TestCrossLinkResolver.fs b/tests/authoring/Framework/TestCrossLinkResolver.fs index 8d2cf3ccc..e673466eb 100644 --- a/tests/authoring/Framework/TestCrossLinkResolver.fs +++ b/tests/authoring/Framework/TestCrossLinkResolver.fs @@ -6,6 +6,7 @@ namespace authoring open System open System.Collections.Generic +open System.Collections.Frozen open System.Runtime.InteropServices open System.Threading.Tasks open Elastic.Markdown.CrossLinks @@ -55,15 +56,25 @@ type TestCrossLinkResolver (config: ConfigurationFile) = } } """ - let reference = CrossLinkResolver.Deserialize json + let reference = CrossLinkFetcher.Deserialize json this.LinkReferences.Add("docs-content", reference) this.LinkReferences.Add("kibana", reference) this.DeclaredRepositories.Add("docs-content") |> ignore; this.DeclaredRepositories.Add("kibana") |> ignore; this.DeclaredRepositories.Add("elasticsearch") |> ignore; - Task.CompletedTask + let crossLinks = + FetchedCrossLinks( + DeclaredRepositories=this.DeclaredRepositories, + LinkReferences=this.LinkReferences.ToFrozenDictionary() + ) + Task.FromResult crossLinks member this.TryResolve(errorEmitter, crossLinkUri, []resolvedUri : byref) = - CrossLinkResolver.TryResolve(errorEmitter, this.DeclaredRepositories, this.LinkReferences, crossLinkUri, &resolvedUri); + let crossLinks = + FetchedCrossLinks( + DeclaredRepositories=this.DeclaredRepositories, + LinkReferences=this.LinkReferences.ToFrozenDictionary() + ) + CrossLinkResolver.TryResolve(errorEmitter, crossLinks, crossLinkUri, &resolvedUri);