diff --git a/docs/docset.yml b/docs/docset.yml index 28f960baf..51f0fea74 100644 --- a/docs/docset.yml +++ b/docs/docset.yml @@ -1,4 +1,6 @@ project: 'doc-builder' +cross_links: + - docs-content # docs-builder will warn for links to external hosts not declared here external_hosts: - slack.com diff --git a/docs/testing/cross-links.md b/docs/testing/cross-links.md index 590adb5e0..45e722b6a 100644 --- a/docs/testing/cross-links.md +++ b/docs/testing/cross-links.md @@ -1,7 +1,7 @@ # Cross Links -[Elasticsearch](elasticsearch://index.md) +[Elasticsearch](docs-content://index.md) [Kibana][1] -[1]: kibana://index.md \ No newline at end of file +[1]: docs-content://index.md \ No newline at end of file diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs new file mode 100644 index 000000000..fd0c37b60 --- /dev/null +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs @@ -0,0 +1,93 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using Elastic.Markdown.IO.Configuration; +using Elastic.Markdown.IO.State; +using Microsoft.Extensions.Logging; + +namespace Elastic.Markdown.CrossLinks; + +public interface ICrossLinkResolver +{ + Task FetchLinks(); + bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri); +} + +public class CrossLinkResolver(ConfigurationFile configuration, ILoggerFactory logger) : ICrossLinkResolver +{ + private readonly string[] _links = configuration.CrossLinkRepositories; + private FrozenDictionary _linkReferences = new Dictionary().ToFrozenDictionary(); + private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkResolver)); + + public static LinkReference Deserialize(string json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; + + public async Task FetchLinks() + { + using var client = new HttpClient(); + var dictionary = new Dictionary(); + foreach (var link in _links) + { + var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{link}/main/links.json"; + _logger.LogInformation($"Fetching {url}"); + var json = await client.GetStringAsync(url); + var linkReference = Deserialize(json); + dictionary.Add(link, linkReference); + } + _linkReferences = dictionary.ToFrozenDictionary(); + } + + public bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) => + TryResolve(errorEmitter, _linkReferences, crossLinkUri, out resolvedUri); + + private static Uri BaseUri { get; } = new Uri("https://docs-v3-preview.elastic.dev"); + + public static bool TryResolve(Action errorEmitter, IDictionary lookup, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) + { + resolvedUri = null; + if (!lookup.TryGetValue(crossLinkUri.Scheme, out var linkReference)) + { + errorEmitter($"'{crossLinkUri.Scheme}' is not declared as valid cross link repository in docset.yml under cross_links"); + return false; + } + var lookupPath = crossLinkUri.AbsolutePath.TrimStart('/'); + if (string.IsNullOrEmpty(lookupPath) && crossLinkUri.Host.EndsWith(".md")) + lookupPath = crossLinkUri.Host; + + if (!linkReference.Links.TryGetValue(lookupPath, out var link)) + { + errorEmitter($"'{lookupPath}' is not a valid link in the '{crossLinkUri.Scheme}' cross link repository."); + return false; + } + + //https://docs-v3-preview.elastic.dev/elastic/docs-content/tree/main/cloud-account/change-your-password + var path = lookupPath.Replace(".md", ""); + if (path.EndsWith("/index")) + path = path.Substring(0, path.Length - 6); + if (path == "index") + path = string.Empty; + + if (!string.IsNullOrEmpty(crossLinkUri.Fragment)) + { + if (link.Anchors is null) + { + errorEmitter($"'{lookupPath}' does not have any anchors so linking to '{crossLinkUri.Fragment}' is impossible."); + return false; + } + + if (!link.Anchors.Contains(crossLinkUri.Fragment.TrimStart('#'))) + { + errorEmitter($"'{lookupPath}' has no anchor named: '{crossLinkUri.Fragment}'."); + return false; + } + path += crossLinkUri.Fragment; + } + + resolvedUri = new Uri(BaseUri, $"elastic/{crossLinkUri.Scheme}/tree/main/{path}"); + return true; + } +} diff --git a/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs b/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs index c3b7ac3ff..d89089d6b 100644 --- a/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs +++ b/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs @@ -7,6 +7,7 @@ using Elastic.Markdown.Myst.Directives; using Markdig.Helpers; using Markdig.Parsers; +using Markdig.Syntax.Inlines; namespace Elastic.Markdown.Diagnostics; @@ -131,4 +132,50 @@ public static void EmitWarning(this IBlockExtension block, string message) }; block.Build.Collector.Channel.Write(d); } + + + public static void EmitError(this InlineProcessor processor, LinkInline inline, string message) + { + var url = inline.Url; + var line = inline.Line + 1; + var column = inline.Column; + var length = url?.Length ?? 1; + + var context = processor.GetContext(); + if (context.SkipValidation) + return; + var d = new Diagnostic + { + Severity = Severity.Error, + File = processor.GetContext().Path.FullName, + Column = column, + Line = line, + Message = message, + Length = length + }; + context.Build.Collector.Channel.Write(d); + } + + + public static void EmitWarning(this InlineProcessor processor, LinkInline inline, string message) + { + var url = inline.Url; + var line = inline.Line + 1; + var column = inline.Column; + var length = url?.Length ?? 1; + + var context = processor.GetContext(); + if (context.SkipValidation) + return; + var d = new Diagnostic + { + Severity = Severity.Warning, + File = processor.GetContext().Path.FullName, + Column = column, + Line = line, + Message = message, + Length = length + }; + context.Build.Collector.Channel.Write(d); + } } diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 728021163..ad1c56cb8 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -1,9 +1,11 @@ // Licensed to Elasticsearch B.V under one or more agreements. // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information + using System.IO.Abstractions; using System.Reflection; using System.Text.Json; +using Elastic.Markdown.CrossLinks; using Elastic.Markdown.IO; using Elastic.Markdown.IO.State; using Elastic.Markdown.Slices; @@ -20,6 +22,7 @@ public class DocumentationGenerator public DocumentationSet DocumentationSet { get; } public BuildContext Context { get; } + public ICrossLinkResolver Resolver { get; } public DocumentationGenerator( DocumentationSet docSet, @@ -32,6 +35,7 @@ ILoggerFactory logger DocumentationSet = docSet; Context = docSet.Context; + Resolver = docSet.LinkResolver; HtmlWriter = new HtmlWriter(DocumentationSet, _writeFileSystem); _logger.LogInformation($"Created documentation set for: {DocumentationSet.Name}"); @@ -66,6 +70,9 @@ public async Task GenerateAll(Cancel ctx) if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges)) return; + _logger.LogInformation($"Fetching external links"); + await Resolver.FetchLinks(); + await ResolveDirectoryTree(ctx); await ProcessDocumentationFiles(offendingFiles, outputSeenChanges, ctx); @@ -77,6 +84,7 @@ public async Task GenerateAll(Cancel ctx) _logger.LogInformation($"Generating documentation compilation state"); await GenerateDocumentationState(ctx); + _logger.LogInformation($"Generating links.json"); await GenerateLinkReference(ctx); diff --git a/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs b/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs index cbb0a4f40..6e8f24186 100644 --- a/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs +++ b/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs @@ -19,6 +19,8 @@ public record ConfigurationFile : DocumentationFile public string? Project { get; } public Glob[] Exclude { get; } = []; + public string[] CrossLinkRepositories { get; } = []; + public IReadOnlyCollection TableOfContents { get; } = []; public HashSet Files { get; } = new(StringComparer.OrdinalIgnoreCase); @@ -72,6 +74,9 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon .Select(Glob.Parse) .ToArray(); break; + case "cross_links": + CrossLinkRepositories = ReadStringArray(entry).ToArray(); + break; case "subs": _substitutions = ReadDictionary(entry); break; diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 1403bed0b..135d66064 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -4,10 +4,12 @@ using System.Collections.Frozen; using System.IO.Abstractions; +using Elastic.Markdown.CrossLinks; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO.Configuration; using Elastic.Markdown.IO.Navigation; using Elastic.Markdown.Myst; +using Microsoft.Extensions.Logging; namespace Elastic.Markdown.IO; @@ -29,15 +31,18 @@ public class DocumentationSet public MarkdownParser MarkdownParser { get; } - public DocumentationSet(BuildContext context) + public ICrossLinkResolver LinkResolver { get; } + + public DocumentationSet(BuildContext context, ILoggerFactory logger, ICrossLinkResolver? linkResolver = null) { Context = context; SourcePath = context.SourcePath; OutputPath = context.OutputPath; RelativeSourcePath = Path.GetRelativePath(Paths.Root.FullName, SourcePath.FullName); Configuration = new ConfigurationFile(context.ConfigurationPath, SourcePath, context); + LinkResolver = linkResolver ?? new CrossLinkResolver(Configuration, logger); - MarkdownParser = new MarkdownParser(SourcePath, context, GetMarkdownFile, Configuration); + MarkdownParser = new MarkdownParser(SourcePath, context, GetMarkdownFile, Configuration, LinkResolver); Name = SourcePath.FullName; OutputStateFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, ".doc.state")); diff --git a/src/Elastic.Markdown/IO/State/LinkReference.cs b/src/Elastic.Markdown/IO/State/LinkReference.cs index 896fed0c3..d0ebb6978 100644 --- a/src/Elastic.Markdown/IO/State/LinkReference.cs +++ b/src/Elastic.Markdown/IO/State/LinkReference.cs @@ -11,11 +11,11 @@ public record LinkMetadata { [JsonPropertyName("anchors")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public required string[]? Anchors { get; init; } = []; + public string[]? Anchors { get; init; } = []; [JsonPropertyName("hidden")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public required bool Hidden { get; init; } + public bool Hidden { get; init; } } public record LinkReference @@ -26,7 +26,7 @@ public record LinkReference [JsonPropertyName("url_path_prefix")] public required string? UrlPathPrefix { get; init; } - /// Mapping of relative filepath and all the page's anchors for deeplinks + /// Mapping of relative filepath and all the page's anchors for deep links [JsonPropertyName("links")] public required Dictionary Links { get; init; } = []; diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs index b5f907722..9284b5711 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs @@ -225,8 +225,9 @@ private void WriteIncludeBlock(HtmlRenderer renderer, IncludeBlock block) if (!block.Found || block.IncludePath is null) return; - var parser = new MarkdownParser(block.DocumentationSourcePath, block.Build, block.GetDocumentationFile, - block.Configuration); + var parser = new MarkdownParser( + block.DocumentationSourcePath, block.Build, block.GetDocumentationFile, + block.Configuration, block.LinksResolver); var file = block.FileSystem.FileInfo.New(block.IncludePath); var document = parser.ParseAsync(file, block.FrontMatter, default).GetAwaiter().GetResult(); var html = document.ToHtml(MarkdownParser.Pipeline); @@ -240,7 +241,10 @@ private void WriteSettingsBlock(HtmlRenderer renderer, SettingsBlock block) if (!block.Found || block.IncludePath is null) return; - var parser = new MarkdownParser(block.DocumentationSourcePath, block.Build, block.GetDocumentationFile, block.Configuration); + var parser = new MarkdownParser( + block.DocumentationSourcePath, block.Build, block.GetDocumentationFile, block.Configuration + , block.LinksResolver + ); var file = block.FileSystem.FileInfo.New(block.IncludePath); diff --git a/src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs b/src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs index b05f0609d..30564430a 100644 --- a/src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information using System.IO.Abstractions; +using Elastic.Markdown.CrossLinks; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; using Elastic.Markdown.IO.Configuration; @@ -25,6 +26,8 @@ public class IncludeBlock(DirectiveBlockParser parser, ParserContext context) : public ConfigurationFile Configuration { get; } = context.Configuration; + public ICrossLinkResolver LinksResolver { get; } = context.LinksResolver; + public IFileSystem FileSystem { get; } = context.Build.ReadFileSystem; public IDirectoryInfo DocumentationSourcePath { get; } = context.Parser.SourcePath; @@ -40,7 +43,6 @@ public class IncludeBlock(DirectiveBlockParser parser, ParserContext context) : public string? Caption { get; private set; } public string? Label { get; private set; } - //TODO add all options from //https://mystmd.org/guide/directives#directive-include public override void FinalizeAndValidate(ParserContext context) diff --git a/src/Elastic.Markdown/Myst/Directives/SettingsBlock.cs b/src/Elastic.Markdown/Myst/Directives/SettingsBlock.cs index 5bc5577e0..e659b323e 100644 --- a/src/Elastic.Markdown/Myst/Directives/SettingsBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/SettingsBlock.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information using System.IO.Abstractions; +using Elastic.Markdown.CrossLinks; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; using Elastic.Markdown.IO.Configuration; @@ -17,6 +18,8 @@ public class SettingsBlock(DirectiveBlockParser parser, ParserContext context) : public ConfigurationFile Configuration { get; } = context.Configuration; + public ICrossLinkResolver LinksResolver { get; } = context.LinksResolver; + public IFileSystem FileSystem { get; } = context.Build.ReadFileSystem; public IFileInfo IncludeFrom { get; } = context.Path; diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index 85aeb70cd..b75885d89 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; using System.IO.Abstractions; using System.Text.RegularExpressions; using Elastic.Markdown.Diagnostics; @@ -37,16 +38,14 @@ public void Setup(MarkdownPipeline pipeline, IMarkdownRenderer renderer) { } internal partial class LinkRegexExtensions { - [GeneratedRegex(@"\s\=(?\d+%?)(?:x(?\d+%?))?$", RegexOptions.IgnoreCase, "en-US")] public static partial Regex MatchTitleStylingInstructions(); - } public class DiagnosticLinkInlineParser : LinkInlineParser { // See https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml for a list of URI schemes - private static readonly ImmutableHashSet ExcludedSchemes = ["http", "https", "tel", "jdbc"]; + private static readonly ImmutableHashSet ExcludedSchemes = ["http", "https", "tel", "jdbc", "mailto"]; public override bool Match(InlineProcessor processor, ref StringSlice slice) { @@ -59,15 +58,15 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice) if (IsInCommentBlock(link) || context.SkipValidation) return match; - ValidateAndProcessLink(processor, link, context); + ValidateAndProcessLink(link, processor, context); - ParseStylingInstructions(processor, link, context); + ParseStylingInstructions(link); return match; } - private void ParseStylingInstructions(InlineProcessor processor, LinkInline link, ParserContext context) + private void ParseStylingInstructions(LinkInline link) { if (string.IsNullOrWhiteSpace(link.Title) || link.Title.IndexOf('=') < 0) return; @@ -95,48 +94,47 @@ private void ParseStylingInstructions(InlineProcessor processor, LinkInline link private static bool IsInCommentBlock(LinkInline link) => link.Parent?.ParentBlock is CommentBlock; - private void ValidateAndProcessLink(InlineProcessor processor, LinkInline link, ParserContext context) + private void ValidateAndProcessLink(LinkInline link, InlineProcessor processor, ParserContext context) { var url = link.Url; - var line = link.Line + 1; - var column = link.Column; - var length = url?.Length ?? 1; - if (!ValidateBasicUrl(processor, url, line, column, length)) + if (!ValidateBasicUrl(link, processor, url)) return; var uri = Uri.TryCreate(url, UriKind.Absolute, out var u) ? u : null; if (IsCrossLink(uri)) { - ProcessCrossLink(link, context, line, column, length); + ProcessCrossLink(link, processor, context, uri); return; } - if (ValidateExternalUri(processor, uri, context, line, column, length)) + if (ValidateExternalUri(link, processor, context, uri)) return; - ProcessInternalLink(processor, link, context, line, column, length); + ProcessInternalLink(link, processor, context); } - private bool ValidateBasicUrl(InlineProcessor processor, string? url, int line, int column, int length) + private bool ValidateBasicUrl(LinkInline link, InlineProcessor processor, string? url) { if (string.IsNullOrEmpty(url)) { - processor.EmitWarning(line, column, length, "Found empty url"); + processor.EmitWarning(link, "Found empty url"); return false; } + if (url.Contains("{{") || url.Contains("}}")) { - processor.EmitWarning(line, column, length, + processor.EmitWarning(link, "The url contains a template expression. Please do not use template expressions in links. " + "See https://github.com/elastic/docs-builder/issues/182 for further information."); return false; } + return true; } - private bool ValidateExternalUri(InlineProcessor processor, Uri? uri, ParserContext context, int line, int column, int length) + private bool ValidateExternalUri(LinkInline link, InlineProcessor processor, ParserContext context, Uri? uri) { if (uri == null) return false; @@ -148,32 +146,33 @@ private bool ValidateExternalUri(InlineProcessor processor, Uri? uri, ParserCont if (!context.Configuration.ExternalLinkHosts.Contains(baseDomain)) { processor.EmitWarning( - line, - column, - length, + link, $"External URI '{uri}' is not allowed. Add '{baseDomain}' to the " + $"'external_hosts' list in the configuration file '{context.Configuration.SourceFile}' " + "to allow links to this domain." ); } + return true; } - private static void ProcessCrossLink(LinkInline link, ParserContext context, int line, int column, int length) + private static void ProcessCrossLink(LinkInline link, InlineProcessor processor, ParserContext context, Uri uri) { var url = link.Url; if (url != null) context.Build.Collector.EmitCrossLink(url); - // TODO: The link is not rendered correctly yet, will be fixed in a follow-up + + if (context.LinksResolver.TryResolve(s => processor.EmitError(link, s), uri, out var resolvedUri)) + link.Url = resolvedUri.ToString(); } - private static void ProcessInternalLink(InlineProcessor processor, LinkInline link, ParserContext context, int line, int column, int length) + private static void ProcessInternalLink(LinkInline link, InlineProcessor processor, ParserContext context) { var (url, anchor) = SplitUrlAndAnchor(link.Url ?? string.Empty); var includeFrom = GetIncludeFromPath(url, context); var file = ResolveFile(context, url); - ValidateInternalUrl(processor, url, includeFrom, line, column, length, context); - ProcessLinkText(processor, link, context, url, anchor, line, column, length, file); + ValidateInternalUrl(processor, url, includeFrom, link, context); + ProcessLinkText(processor, link, context, url, anchor, file); UpdateLinkUrl(link, url, context, anchor, file); } @@ -188,17 +187,17 @@ private static string GetIncludeFromPath(string url, ParserContext context) => ? context.Parser.SourcePath.FullName : context.Path.Directory!.FullName; - private static void ValidateInternalUrl(InlineProcessor processor, string url, string includeFrom, int line, int column, int length, ParserContext context) + private static void ValidateInternalUrl(InlineProcessor processor, string url, string includeFrom, LinkInline link, ParserContext context) { if (string.IsNullOrWhiteSpace(url)) return; var pathOnDisk = Path.Combine(includeFrom, url.TrimStart('/')); if (!context.Build.ReadFileSystem.File.Exists(pathOnDisk)) - processor.EmitError(line, column, length, $"`{url}` does not exist. resolved to `{pathOnDisk}"); + processor.EmitError(link, $"`{url}` does not exist. resolved to `{pathOnDisk}"); } - private static void ProcessLinkText(InlineProcessor processor, LinkInline link, ParserContext context, string url, string? anchor, int line, int column, int length, IFileInfo file) + private static void ProcessLinkText(InlineProcessor processor, LinkInline link, ParserContext context, string url, string? anchor, IFileInfo file) { if (link.FirstChild != null && string.IsNullOrEmpty(anchor)) return; @@ -207,7 +206,7 @@ private static void ProcessLinkText(InlineProcessor processor, LinkInline link, if (markdown == null) { - processor.EmitWarning(line, column, length, + processor.EmitWarning(link, $"'{url}' could not be resolved to a markdown file while creating an auto text link, '{file.FullName}' does not exist."); return; } @@ -216,7 +215,7 @@ private static void ProcessLinkText(InlineProcessor processor, LinkInline link, if (!string.IsNullOrEmpty(anchor)) { - ValidateAnchor(processor, markdown, anchor, line, column, length); + ValidateAnchor(processor, markdown, anchor, link); if (link.FirstChild == null && markdown.TableOfContents.TryGetValue(anchor, out var heading)) title += " > " + heading.Heading; } @@ -232,10 +231,10 @@ private static IFileInfo ResolveFile(ParserContext context, string url) => ? context.Build.ReadFileSystem.FileInfo.New(Path.Combine(context.Build.SourcePath.FullName, url.TrimStart('/'))) : context.Build.ReadFileSystem.FileInfo.New(Path.Combine(context.Path.Directory!.FullName, url)); - private static void ValidateAnchor(InlineProcessor processor, MarkdownFile markdown, string anchor, int line, int column, int length) + private static void ValidateAnchor(InlineProcessor processor, MarkdownFile markdown, string anchor, LinkInline link) { if (!markdown.Anchors.Contains(anchor)) - processor.EmitError(line, column, length, $"`{anchor}` does not exist in {markdown.FileName}."); + processor.EmitError(link, $"`{anchor}` does not exist in {markdown.FileName}."); } private static void UpdateLinkUrl(LinkInline link, string url, ParserContext context, string? anchor, IFileInfo file) @@ -264,9 +263,9 @@ private static string GetRootRelativePath(ParserContext context, IFileInfo file) return file.FullName.Replace(docsetDirectory!.FullName, string.Empty); } - private static bool IsCrossLink(Uri? uri) => + private static bool IsCrossLink([NotNullWhen(true)] Uri? uri) => uri != null // This means it's not a local && !ExcludedSchemes.Contains(uri.Scheme) && !uri.IsFile - && Path.GetExtension(uri.OriginalString) == ".md"; + && !string.IsNullOrEmpty(uri.Scheme); } diff --git a/src/Elastic.Markdown/Myst/MarkdownParser.cs b/src/Elastic.Markdown/Myst/MarkdownParser.cs index 59dc8cc0e..a2e124cfe 100644 --- a/src/Elastic.Markdown/Myst/MarkdownParser.cs +++ b/src/Elastic.Markdown/Myst/MarkdownParser.cs @@ -4,8 +4,10 @@ using System.IO.Abstractions; using Cysharp.IO; +using Elastic.Markdown.CrossLinks; using Elastic.Markdown.IO; using Elastic.Markdown.IO.Configuration; +using Elastic.Markdown.IO.State; using Elastic.Markdown.Myst.CodeBlocks; using Elastic.Markdown.Myst.Comments; using Elastic.Markdown.Myst.Directives; @@ -23,12 +25,16 @@ public class MarkdownParser( IDirectoryInfo sourcePath, BuildContext context, Func? getDocumentationFile, - ConfigurationFile configuration) + ConfigurationFile configuration, + ICrossLinkResolver linksResolver + ) { public IDirectoryInfo SourcePath { get; } = sourcePath; private BuildContext Context { get; } = context; + private ICrossLinkResolver LinksResolver { get; } = linksResolver; + // ReSharper disable once InconsistentNaming private static MarkdownPipeline? _minimalPipeline; public static MarkdownPipeline MinimalPipeline @@ -86,7 +92,7 @@ public static MarkdownPipeline Pipeline public Task MinimalParseAsync(IFileInfo path, Cancel ctx) { - var context = new ParserContext(this, path, null, Context, Configuration) + var context = new ParserContext(this, path, null, Context, Configuration, LinksResolver) { SkipValidation = true, GetDocumentationFile = getDocumentationFile @@ -96,7 +102,7 @@ public Task MinimalParseAsync(IFileInfo path, Cancel ctx) public Task ParseAsync(IFileInfo path, YamlFrontMatter? matter, Cancel ctx) { - var context = new ParserContext(this, path, matter, Context, Configuration) + var context = new ParserContext(this, path, matter, Context, Configuration, LinksResolver) { GetDocumentationFile = getDocumentationFile }; @@ -127,7 +133,7 @@ private async Task ParseAsync( public MarkdownDocument Parse(string yaml, IFileInfo parent, YamlFrontMatter? matter) { - var context = new ParserContext(this, parent, matter, Context, Configuration) + var context = new ParserContext(this, parent, matter, Context, Configuration, LinksResolver) { GetDocumentationFile = getDocumentationFile }; diff --git a/src/Elastic.Markdown/Myst/ParserContext.cs b/src/Elastic.Markdown/Myst/ParserContext.cs index 01d3ddefb..313c64aa0 100644 --- a/src/Elastic.Markdown/Myst/ParserContext.cs +++ b/src/Elastic.Markdown/Myst/ParserContext.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions; +using Elastic.Markdown.CrossLinks; using Elastic.Markdown.Diagnostics; using Elastic.Markdown.IO; using Elastic.Markdown.IO.Configuration; @@ -14,12 +15,6 @@ namespace Elastic.Markdown.Myst; public static class ParserContextExtensions { - public static BuildContext GetBuildContext(this InlineProcessor processor) => - processor.GetContext().Build; - - public static BuildContext GetBuildContext(this BlockProcessor processor) => - processor.GetContext().Build; - public static ParserContext GetContext(this InlineProcessor processor) => processor.Context as ParserContext ?? throw new InvalidOperationException($"Provided context is not a {nameof(ParserContext)}"); @@ -36,13 +31,15 @@ public ParserContext( IFileInfo path, YamlFrontMatter? frontMatter, BuildContext context, - ConfigurationFile configuration) + ConfigurationFile configuration, + ICrossLinkResolver linksResolver) { Parser = markdownParser; Path = path; FrontMatter = frontMatter; Build = context; Configuration = configuration; + LinksResolver = linksResolver; if (frontMatter?.Properties is not { Count: > 0 }) Substitutions = configuration.Substitutions; @@ -70,6 +67,7 @@ public ParserContext( } public ConfigurationFile Configuration { get; } + public ICrossLinkResolver LinksResolver { get; } public MarkdownParser Parser { get; } public IFileInfo Path { get; } public YamlFrontMatter? FrontMatter { get; } diff --git a/src/Elastic.Markdown/Slices/HtmlWriter.cs b/src/Elastic.Markdown/Slices/HtmlWriter.cs index 5332e8835..d4b10fa39 100644 --- a/src/Elastic.Markdown/Slices/HtmlWriter.cs +++ b/src/Elastic.Markdown/Slices/HtmlWriter.cs @@ -83,9 +83,7 @@ public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, Cancel var rendered = await RenderLayout(markdown, ctx); string path; if (outputFile.Name == "index.md") - { path = Path.ChangeExtension(outputFile.FullName, ".html"); - } else { var dir = outputFile.Directory is null diff --git a/src/docs-builder/Cli/Commands.cs b/src/docs-builder/Cli/Commands.cs index 541104466..5fdb6b3ad 100644 --- a/src/docs-builder/Cli/Commands.cs +++ b/src/docs-builder/Cli/Commands.cs @@ -75,7 +75,7 @@ public async Task Generate( Collector = new ConsoleDiagnosticsCollector(logger, githubActionsService), AllowIndexing = allowIndexing != null }; - var set = new DocumentationSet(context); + var set = new DocumentationSet(context, logger); var generator = new DocumentationGenerator(set, logger); await generator.GenerateAll(ctx); @@ -136,7 +136,7 @@ public async Task Move( { Collector = new ConsoleDiagnosticsCollector(logger, null), }; - var set = new DocumentationSet(context); + var set = new DocumentationSet(context, logger); var moveCommand = new Move(fileSystem, fileSystem, set, logger); return await moveCommand.Execute(source, target, dryRun ?? false, ctx); diff --git a/src/docs-builder/Http/ReloadableGeneratorState.cs b/src/docs-builder/Http/ReloadableGeneratorState.cs index 38de0b06b..0ae1cdb8c 100644 --- a/src/docs-builder/Http/ReloadableGeneratorState.cs +++ b/src/docs-builder/Http/ReloadableGeneratorState.cs @@ -20,14 +20,14 @@ ILoggerFactory logger private IDirectoryInfo? SourcePath { get; } = sourcePath; private IDirectoryInfo? OutputPath { get; } = outputPath; - private DocumentationGenerator _generator = new(new DocumentationSet(context), logger); + private DocumentationGenerator _generator = new(new DocumentationSet(context, logger), logger); public DocumentationGenerator Generator => _generator; public async Task ReloadAsync(Cancel ctx) { SourcePath?.Refresh(); OutputPath?.Refresh(); - var docSet = new DocumentationSet(context); + var docSet = new DocumentationSet(context, logger); var generator = new DocumentationGenerator(docSet, logger); await generator.ResolveDirectoryTree(ctx); Interlocked.Exchange(ref _generator, generator); diff --git a/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs b/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs index 2a040a0e5..59ec4c8c3 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs @@ -73,7 +73,8 @@ protected DirectiveTest(ITestOutputHelper output, [LanguageInjection("markdown") { Collector = Collector }; - Set = new DocumentationSet(context); + var linkResolver = new TestCrossLinkResolver(); + Set = new DocumentationSet(context, logger, linkResolver); File = Set.GetMarkdownFile(FileSystem.FileInfo.New("docs/index.md")) ?? throw new NullReferenceException(); Html = default!; //assigned later Document = default!; @@ -85,6 +86,7 @@ public virtual async ValueTask InitializeAsync() { _ = Collector.StartAsync(default); + await Set.LinkResolver.FetchLinks(); Document = await File.ParseFullAsync(default); var html = File.CreateHtml(Document).AsSpan(); var find = ""; diff --git a/tests/Elastic.Markdown.Tests/DocSet/NavigationTestsBase.cs b/tests/Elastic.Markdown.Tests/DocSet/NavigationTestsBase.cs index 9cb5802c0..6439a53a8 100644 --- a/tests/Elastic.Markdown.Tests/DocSet/NavigationTestsBase.cs +++ b/tests/Elastic.Markdown.Tests/DocSet/NavigationTestsBase.cs @@ -30,7 +30,8 @@ protected NavigationTestsBase(ITestOutputHelper output) Collector = collector }; - Set = new DocumentationSet(context); + var linkResolver = new TestCrossLinkResolver(); + Set = new DocumentationSet(context, LoggerFactory, linkResolver); Set.Files.Should().HaveCountGreaterThan(10); Generator = new DocumentationGenerator(Set, LoggerFactory); diff --git a/tests/Elastic.Markdown.Tests/Inline/InlineLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/InlineLinkTests.cs index 37729e2ba..c93f0019f 100644 --- a/tests/Elastic.Markdown.Tests/Inline/InlineLinkTests.cs +++ b/tests/Elastic.Markdown.Tests/Inline/InlineLinkTests.cs @@ -130,9 +130,7 @@ public class CrossLinkReferenceTest(ITestOutputHelper output) : LinkTestBase(out public void GeneratesHtml() => // language=html Html.Should().Contain( - // TODO: The link is not rendered correctly yet, will be fixed in a follow-up - - """

test

""" + """

test

""" ); [Fact] @@ -157,8 +155,7 @@ Go to [test](kibana://index.md) public void GeneratesHtml() => // language=html Html.Should().Contain( - // TODO: The link is not rendered correctly yet, will be fixed in a follow-up - """

Go to test

""" + """

Go to test

""" ); [Fact] diff --git a/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs b/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs index b95dc4267..737692f5a 100644 --- a/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs +++ b/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs @@ -114,7 +114,8 @@ protected InlineTest( Collector = Collector, UrlPathPrefix = "/docs" }; - Set = new DocumentationSet(context); + var linkResolver = new TestCrossLinkResolver(); + Set = new DocumentationSet(context, logger, linkResolver); File = Set.GetMarkdownFile(FileSystem.FileInfo.New("docs/index.md")) ?? throw new NullReferenceException(); Html = default!; //assigned later Document = default!; @@ -127,6 +128,7 @@ public virtual async ValueTask InitializeAsync() _ = Collector.StartAsync(default); await Set.ResolveDirectoryTree(default); + await Set.LinkResolver.FetchLinks(); Document = await File.ParseFullAsync(default); var html = File.CreateHtml(Document).AsSpan(); diff --git a/tests/Elastic.Markdown.Tests/MockFileSystemExtensions.cs b/tests/Elastic.Markdown.Tests/MockFileSystemExtensions.cs index 0ddca7448..cef4161bc 100644 --- a/tests/Elastic.Markdown.Tests/MockFileSystemExtensions.cs +++ b/tests/Elastic.Markdown.Tests/MockFileSystemExtensions.cs @@ -13,6 +13,9 @@ public static void GenerateDocSetYaml(this MockFileSystem fileSystem, IDirectory { // language=yaml var yaml = new StringWriter(); + yaml.WriteLine("cross_links:"); + yaml.WriteLine(" - docs-content"); + yaml.WriteLine(" - kibana"); yaml.WriteLine("toc:"); var markdownFiles = fileSystem.Directory .EnumerateFiles(root.FullName, "*.md", SearchOption.AllDirectories); @@ -31,5 +34,4 @@ public static void GenerateDocSetYaml(this MockFileSystem fileSystem, IDirectory fileSystem.AddFile(Path.Combine(root.FullName, "docset.yml"), new MockFileData(yaml.ToString())); } - } diff --git a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs index 7ff84650a..c4b29a125 100644 --- a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs +++ b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs @@ -27,7 +27,8 @@ public async Task CreatesDefaultOutputDirectory() { Collector = new DiagnosticsCollector([]) }; - var set = new DocumentationSet(context); + var linkResolver = new TestCrossLinkResolver(); + var set = new DocumentationSet(context, logger, linkResolver); var generator = new DocumentationGenerator(set, logger); await generator.GenerateAll(TestContext.Current.CancellationToken); diff --git a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs new file mode 100644 index 000000000..5e118ea39 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs @@ -0,0 +1,48 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics.CodeAnalysis; +using Elastic.Markdown.CrossLinks; +using Elastic.Markdown.IO.State; + +namespace Elastic.Markdown.Tests; + +public class TestCrossLinkResolver : ICrossLinkResolver +{ + public Dictionary LinkReferences { get; } = new(); + + public Task FetchLinks() + { + var json = """ + { + "origin": { + "branch": "main", + "remote": " https://github.com/elastic/docs-conten", + "ref": "76aac68d066e2af935c38bca8ce04d3ee67a8dd9" + }, + "url_path_prefix": "/elastic/docs-content/tree/main", + "cross_links": [], + "links": { + "index.md": {}, + "get-started/index.md": { + "anchors": [ + "elasticsearch-intro-elastic-stack", + "elasticsearch-intro-use-cases" + ] + }, + "solutions/observability/apps/apm-server-binary.md": { + "anchors": [ "apm-deb" ] + } + } + } + """; + var reference = CrossLinkResolver.Deserialize(json); + LinkReferences.Add("docs-content", reference); + LinkReferences.Add("kibana", reference); + return Task.CompletedTask; + } + + public bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) => + CrossLinkResolver.TryResolve(errorEmitter, LinkReferences, crossLinkUri, out resolvedUri); +} diff --git a/tests/authoring/Container/DefinitionLists.fs b/tests/authoring/Container/DefinitionLists.fs index 3076fe459..c03c06a9a 100644 --- a/tests/authoring/Container/DefinitionLists.fs +++ b/tests/authoring/Container/DefinitionLists.fs @@ -12,8 +12,9 @@ type ``simple multiline definition with markup`` () = static let markdown = Setup.Markdown """ This is my `definition` : And this is the definition **body** + Which may contain multiple lines -""" + """ [] let ``validate HTML`` () = @@ -21,8 +22,8 @@ This is my `definition`
This is my definition
-

And this is the definition body
- Which may contain multiple lines

+

And this is the definition body

+

Which may contain multiple lines

""" diff --git a/tests/authoring/Framework/Setup.fs b/tests/authoring/Framework/Setup.fs index 5e0ecb1f8..c103b052d 100644 --- a/tests/authoring/Framework/Setup.fs +++ b/tests/authoring/Framework/Setup.fs @@ -11,6 +11,7 @@ open System.IO open System.IO.Abstractions.TestingHelpers open System.Threading.Tasks open Elastic.Markdown +open Elastic.Markdown.CrossLinks open Elastic.Markdown.IO open JetBrains.Annotations open Xunit @@ -39,6 +40,8 @@ type Setup = ) = let root = fileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/")); let yaml = new StringWriter(); + yaml.WriteLine("cross_links:"); + yaml.WriteLine(" - docs-content"); yaml.WriteLine("toc:"); let markdownFiles = fileSystem.Directory.EnumerateFiles(root.FullName, "*.md", SearchOption.AllDirectories) markdownFiles @@ -74,8 +77,10 @@ type Setup = let collector = TestDiagnosticsCollector(); let context = BuildContext(fileSystem, Collector=collector) - let set = DocumentationSet(context); - let generator = DocumentationGenerator(set, new TestLoggerFactory()) + let logger = new TestLoggerFactory() + let linkResolver = TestCrossLinkResolver() + let set = DocumentationSet(context, logger, linkResolver); + let generator = DocumentationGenerator(set, logger) let markdownFiles = files diff --git a/tests/authoring/Framework/TestCrossLinkResolver.fs b/tests/authoring/Framework/TestCrossLinkResolver.fs new file mode 100644 index 000000000..2d6927bd5 --- /dev/null +++ b/tests/authoring/Framework/TestCrossLinkResolver.fs @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace authoring + +open System +open System.Collections.Generic +open System.Runtime.InteropServices +open System.Threading.Tasks +open Elastic.Markdown.CrossLinks +open Elastic.Markdown.IO.State + +type TestCrossLinkResolver () = + + let references = Dictionary() + member this.LinkReferences = references + + interface ICrossLinkResolver with + member this.FetchLinks() = + // language=json + let json = """{ + "origin": { + "branch": "main", + "remote": " https://github.com/elastic/docs-conten", + "ref": "76aac68d066e2af935c38bca8ce04d3ee67a8dd9" + }, + "url_path_prefix": "/elastic/docs-content/tree/main", + "cross_links": [], + "links": { + "index.md": {}, + "get-started/index.md": { + "anchors": [ + "elasticsearch-intro-elastic-stack", + "elasticsearch-intro-use-cases" + ] + }, + "solutions/observability/apps/apm-server-binary.md": { + "anchors": [ "apm-deb" ] + } + } +} +""" + let reference = CrossLinkResolver.Deserialize json + this.LinkReferences.Add("docs-content", reference) + this.LinkReferences.Add("kibana", reference) + Task.CompletedTask + + member this.TryResolve(errorEmitter, crossLinkUri, []resolvedUri : byref) = + CrossLinkResolver.TryResolve(errorEmitter, this.LinkReferences, crossLinkUri, &resolvedUri); + + diff --git a/tests/authoring/Inline/CrossLinks.fs b/tests/authoring/Inline/CrossLinks.fs new file mode 100644 index 000000000..8c8d934f8 --- /dev/null +++ b/tests/authoring/Inline/CrossLinks.fs @@ -0,0 +1,80 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +module ``inline elements``.``cross links`` + +open Xunit +open authoring + +type ``cross-link makes it into html`` () = + + static let markdown = Setup.Markdown """ +[APM Server binary](docs-content:/solutions/observability/apps/apm-server-binary.md) +""" + + [] + let ``validate HTML`` () = + markdown |> convertsToHtml """ +

+ APM Server binary + +

+ """ + + [] + let ``has no errors`` () = markdown |> hasNoErrors + + [] + let ``has no warning`` () = markdown |> hasNoWarnings + +type ``error when using wrong scheme`` () = + + static let markdown = Setup.Markdown """ +[APM Server binary](docs-x:/solutions/observability/apps/apm-server-binary.md) +""" + + [] + let ``error on bad scheme`` () = + markdown + |> hasError "'docs-x' is not declared as valid cross link repository in docset.yml under cross_links" + + [] + let ``has no warning`` () = markdown |> hasNoWarnings + +type ``error when bad anchor is used`` () = + + static let markdown = Setup.Markdown """ +[APM Server binary](docs-content:/solutions/observability/apps/apm-server-binary.md#apm-deb-x) +""" + + [] + let ``error when linking to unknown anchor`` () = + markdown + |> hasError "'solutions/observability/apps/apm-server-binary.md' has no anchor named: '#apm-deb-x" + + [] + let ``has no warning`` () = markdown |> hasNoWarnings + +type ``link to valid anchor`` () = + + static let markdown = Setup.Markdown """ +[APM Server binary](docs-content:/solutions/observability/apps/apm-server-binary.md#apm-deb) +""" + + [] + let ``validate HTML`` () = + markdown |> convertsToHtml """ +

+ APM Server binary + +

+ """ + + [] + let ``has no errors`` () = markdown |> hasNoErrors + + [] + let ``has no warning`` () = markdown |> hasNoWarnings diff --git a/tests/authoring/authoring.fsproj b/tests/authoring/authoring.fsproj index fd6524003..6cfe75fcf 100644 --- a/tests/authoring/authoring.fsproj +++ b/tests/authoring/authoring.fsproj @@ -32,6 +32,7 @@ + @@ -48,6 +49,7 @@ +