Support anchors and table of contents from included files

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -9,12 +9,19 @@
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.State;
 using Elastic.Markdown.Slices;
+using Markdig.Syntax;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown;
 
+public interface IConversionCollector
+{
+	void Collect(MarkdownFile file, MarkdownDocument document, string html);
+}
+
 public class DocumentationGenerator
 {
+	private readonly IConversionCollector? _conversionCollector;
 	private readonly IFileSystem _readFileSystem;
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
@@ -26,9 +33,11 @@ public class DocumentationGenerator
 
 	public DocumentationGenerator(
 		DocumentationSet docSet,
-		ILoggerFactory logger
+		ILoggerFactory logger,
+		IConversionCollector? conversionCollector = null
 	)
 	{
+		_conversionCollector = conversionCollector;
 		_readFileSystem = docSet.Context.ReadFileSystem;
 		_writeFileSystem = docSet.Context.WriteFileSystem;
 		_logger = logger.CreateLogger(nameof(DocumentationGenerator));
@@ -161,7 +170,7 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
 		var outputFile = OutputFile(file.RelativePath);
 		if (file is MarkdownFile markdown)
-			await HtmlWriter.WriteAsync(outputFile, markdown, token);
+			await HtmlWriter.WriteAsync(outputFile, markdown, _conversionCollector, token);
 		else
 		{
 			if (outputFile.Directory is { Exists: false })

src/Elastic.Markdown/IO/DocumentationFile.cs

@@ -2,6 +2,9 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
+using Elastic.Markdown.Myst;
+using Elastic.Markdown.Myst.FrontMatter;
+using Elastic.Markdown.Slices;
 
 namespace Elastic.Markdown.IO;
 
@@ -22,4 +25,32 @@ public record ExcludedFile(IFileInfo SourceFile, IDirectoryInfo RootPath)
 	: DocumentationFile(SourceFile, RootPath);
 
 public record SnippetFile(IFileInfo SourceFile, IDirectoryInfo RootPath)
-	: DocumentationFile(SourceFile, RootPath);
+	: DocumentationFile(SourceFile, RootPath)
+{
+	private SnippetAnchors? Anchors { get; set; }
+	private bool _parsed;
+
+	public SnippetAnchors? GetAnchors(
+		DocumentationSet set,
+		MarkdownParser parser,
+		YamlFrontMatter? frontMatter
+	)
+	{
+		if (_parsed)
+			return Anchors;
+		if (!SourceFile.Exists)
+		{
+			_parsed = true;
+			return null;
+		}
+
+		var document = parser.MinimalParseAsync(SourceFile, default).GetAwaiter().GetResult();
+		var toc = MarkdownFile.GetAnchors(set, parser, frontMatter, document, new Dictionary<string, string>(), out var anchors);
+		Anchors = new SnippetAnchors(anchors, toc);
+		_parsed = true;
+		return Anchors;
+	}
+}
+
+public record SnippetAnchors(string[] Anchors, IReadOnlyCollection<PageTocItem> TableOfContentItems);
+

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -187,16 +187,16 @@ private DocumentationFile CreateMarkDownFile(IFileInfo file, BuildContext contex
 		if (Configuration.Exclude.Any(g => g.IsMatch(relativePath)))
 			return new ExcludedFile(file, SourcePath);
 
-		if (Configuration.Files.Contains(relativePath))
-			return new MarkdownFile(file, SourcePath, MarkdownParser, context);
-
-		if (Configuration.Globs.Any(g => g.IsMatch(relativePath)))
-			return new MarkdownFile(file, SourcePath, MarkdownParser, context);
-
 		// we ignore files in folders that start with an underscore
 		if (relativePath.Contains("_snippets"))
 			return new SnippetFile(file, SourcePath);
 
+		if (Configuration.Files.Contains(relativePath))
+			return new MarkdownFile(file, SourcePath, MarkdownParser, context, this);
+
+		if (Configuration.Globs.Any(g => g.IsMatch(relativePath)))
+			return new MarkdownFile(file, SourcePath, MarkdownParser, context, this);
+
 		// we ignore files in folders that start with an underscore
 		if (relativePath.IndexOf("/_", StringComparison.Ordinal) > 0 || relativePath.StartsWith('_'))
 			return new ExcludedFile(file, SourcePath);

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -21,7 +21,15 @@ public record MarkdownFile : DocumentationFile
 {
 	private string? _navigationTitle;
 
-	public MarkdownFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParser parser, BuildContext context)
+	private readonly DocumentationSet _set;
+
+	public MarkdownFile(
+		IFileInfo sourceFile,
+		IDirectoryInfo rootPath,
+		MarkdownParser parser,
+		BuildContext context,
+		DocumentationSet set
+	)
 		: base(sourceFile, rootPath)
 	{
 		FileName = sourceFile.Name;
@@ -30,6 +38,7 @@ public MarkdownFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParse
 		MarkdownParser = parser;
 		Collector = context.Collector;
 		_configurationFile = context.Configuration.SourceFile;
+		_set = set;
 	}
 
 	public string Id { get; } = Guid.NewGuid().ToString("N")[..8];
@@ -191,36 +200,78 @@ private void ReadDocumentInstructions(MarkdownDocument document)
 		else if (Title.AsSpan().ReplaceSubstitutions(subs, out var replacement))
 			Title = replacement;
 
-		var contents = document
+		if (RelativePath.Contains("esql-functions-operators"))
+		{
+
+		}
+		var toc = GetAnchors(_set, MarkdownParser, YamlFrontMatter, document, subs, out var anchors);
+
+		_tableOfContent.Clear();
+		foreach (var t in toc)
+			_tableOfContent[t.Slug] = t;
+
+
+		foreach (var label in anchors)
+			_ = _anchors.Add(label);
+
+		_instructionsParsed = true;
+	}
+
+	public static List<PageTocItem> GetAnchors(
+		DocumentationSet set,
+		MarkdownParser parser,
+		YamlFrontMatter? frontMatter,
+		MarkdownDocument document,
+		IReadOnlyDictionary<string, string> subs,
+		out string[] anchors)
+	{
+		var includeBlocks = document.Descendants<IncludeBlock>().ToArray();
+		var includes = includeBlocks
+			.Where(i => i.Found)
+			.Select(i =>
+			{
+				var path = i.IncludePathFromSourceDirectory;
+				if (path is null
+					|| !set.FlatMappedFiles.TryGetValue(path, out var file)
+					|| file is not SnippetFile snippet)
+					return null;
+
+				return snippet.GetAnchors(set, parser, frontMatter);
+			})
+			.Where(i => i is not null)
+			.ToArray();
+
+		var includedTocs = includes.SelectMany(i => i!.TableOfContentItems).ToArray();
+		var toc = document
 			.Descendants<HeadingBlock>()
 			.Where(block => block is { Level: >= 2 })
 			.Select(h => (h.GetData("header") as string, h.GetData("anchor") as string, h.Level))
 			.Select(h =>
 			{
 				var header = h.Item1!.StripMarkdown();
-				if (header.AsSpan().ReplaceSubstitutions(subs, out var replacement))
-					header = replacement;
 				return new PageTocItem { Heading = header, Slug = (h.Item2 ?? header).Slugify(), Level = h.Level };
 			})
+			.Concat(includedTocs)
+			.Select(toc => subs.Count == 0
+				? toc
+				: toc.Heading.AsSpan().ReplaceSubstitutions(subs, out var r)
+					? toc with { Heading = r }
+					: toc)
 			.ToList();
 
-		_tableOfContent.Clear();
-		foreach (var t in contents)
-			_tableOfContent[t.Slug] = t;
-
-		var anchors = document.Descendants<DirectiveBlock>()
-			.Select(b => b.CrossReferenceName)
-			.Where(l => !string.IsNullOrWhiteSpace(l))
-			.Select(s => s.Slugify())
-			.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
-			.Concat(_tableOfContent.Values.Select(t => t.Slug))
-			.Where(anchor => !string.IsNullOrEmpty(anchor))
-			.ToArray();
-
-		foreach (var label in anchors)
-			_ = _anchors.Add(label);
-
-		_instructionsParsed = true;
+		var includedAnchors = includes.SelectMany(i => i!.Anchors).ToArray();
+		anchors =
+		[
+			..document.Descendants<DirectiveBlock>()
+				.Select(b => b.CrossReferenceName)
+				.Where(l => !string.IsNullOrWhiteSpace(l))
+				.Select(s => s.Slugify())
+				.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
+				.Concat(toc.Select(t => t.Slug))
+				.Where(anchor => !string.IsNullOrEmpty(anchor))
+				.Concat(includedAnchors)
+		];
+		return toc;
 	}
 
 	private YamlFrontMatter ProcessYamlFrontMatter(MarkdownDocument document)

src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs

@@ -231,8 +231,6 @@ private static void WriteIncludeBlock(HtmlRenderer renderer, IncludeBlock block)
 		var document = parser.ParseAsync(file, block.FrontMatter, default).GetAwaiter().GetResult();
 		var html = document.ToHtml(MarkdownParser.Pipeline);
 		_ = renderer.Write(html);
-		//var slice = Include.Create(new IncludeViewModel { Html = html });
-		//RenderRazorSlice(slice, renderer, block);
 	}
 
 	private static void WriteSettingsBlock(HtmlRenderer renderer, SettingsBlock block)

src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs

@@ -35,6 +35,7 @@ public class IncludeBlock(DirectiveBlockParser parser, ParserContext context) :
 	public YamlFrontMatter? FrontMatter { get; } = context.FrontMatter;
 
 	public string? IncludePath { get; private set; }
+	public string? IncludePathFromSourceDirectory { get; private set; }
 
 	public bool Found { get; private set; }
 
@@ -69,6 +70,7 @@ private void ExtractInclusionPath(ParserContext context)
 			includeFrom = DocumentationSourcePath.FullName;
 
 		IncludePath = Path.Combine(includeFrom, includePath.TrimStart('/'));
+		IncludePathFromSourceDirectory = Path.GetRelativePath(DocumentationSourcePath.FullName, IncludePath);
 		if (FileSystem.File.Exists(IncludePath))
 			Found = true;
 		else

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -190,7 +190,7 @@ private static void ValidateInternalUrl(InlineProcessor processor, string url, s
 		if (string.IsNullOrWhiteSpace(url))
 			return;
 
-		var pathOnDisk = Path.Combine(includeFrom, url.TrimStart('/'));
+		var pathOnDisk = Path.GetFullPath(Path.Combine(includeFrom, url.TrimStart('/')));
 		if (!context.Build.ReadFileSystem.File.Exists(pathOnDisk))
 			processor.EmitError(link, $"`{url}` does not exist. resolved to `{pathOnDisk}");
 	}

src/Elastic.Markdown/Slices/HtmlWriter.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
 using Elastic.Markdown.IO;
+using Markdig.Syntax;
 using RazorSlices;
 
 namespace Elastic.Markdown.Slices;
@@ -26,6 +27,11 @@ private async Task<string> RenderNavigation(MarkdownFile markdown, Cancel ctx =
 	public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = default)
 	{
 		var document = await markdown.ParseFullAsync(ctx);
+		return await RenderLayout(markdown, document, ctx);
+	}
+
+	public async Task<string> RenderLayout(MarkdownFile markdown, MarkdownDocument document, Cancel ctx = default)
+	{
 		var html = MarkdownFile.CreateHtml(document);
 		await DocumentationSet.Tree.Resolve(ctx);
 		_renderedNavigation ??= await RenderNavigation(markdown, ctx);
@@ -57,7 +63,7 @@ public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = defau
 		return await slice.RenderAsync(cancellationToken: ctx);
 	}
 
-	public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, Cancel ctx = default)
+	public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, IConversionCollector? collector, Cancel ctx = default)
 	{
 		if (outputFile.Directory is { Exists: false })
 			outputFile.Directory.Create();
@@ -79,7 +85,9 @@ public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, Cancel
 				: Path.Combine(dir, "index.html");
 		}
 
-		var rendered = await RenderLayout(markdown, ctx);
+		var document = await markdown.ParseFullAsync(ctx);
+		var rendered = await RenderLayout(markdown, document, ctx);
+		collector?.Collect(markdown, document, rendered);
 		await writeFileSystem.File.WriteAllTextAsync(path, rendered, ctx);
 	}
 

src/Elastic.Markdown/Slices/_ViewModels.cs

@@ -69,7 +69,7 @@ public string Link(string path)
 	}
 }
 
-public class PageTocItem
+public record PageTocItem
 {
 	public required string Heading { get; init; }
 	public required string Slug { get; init; }