From 5e48683f00db3caa6cf9deb0bfdbcd6b83459530 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Mon, 17 Mar 2025 16:27:35 +0100 Subject: [PATCH 01/13] Add global navigation and improve diagnostic handling This commit introduces a GlobalNavigation feature, parsing the new `navigation.yml` file to handle global navigation references. It also replaces `BuildContext` with `DiagnosticsCollector` for diagnostics, ensuring clearer separation of concerns. Additionally, it includes new tests and updates project files for better testing and navigation handling. --- docs-builder.sln | 7 + .../ProcessorDiagnosticExtensions.cs | 22 + .../IO/Configuration/ConfigurationFile.cs | 4 +- .../IO/Configuration/RedirectFile.cs | 2 +- .../TableOfContentsConfiguration.cs | 2 +- .../IO/Configuration/YamlStreamReader.cs | 14 +- src/docs-assembler/AssembleContext.cs | 9 +- .../Building/AssemblerBuilder.cs | 1 + src/docs-assembler/Cli/RepositoryCommands.cs | 2 + .../Configuration/AssemblyConfiguration.cs | 9 +- .../Navigation/GlobalNavigation.cs | 123 +++++ src/docs-assembler/YamlStaticContext.cs | 15 + src/docs-assembler/docs-assembler.csproj | 1 + src/docs-assembler/navigation.yml | 447 ++++++++++++++++++ .../GlobalNavigationTests.cs | 24 + .../docs-assembler.Tests.csproj | 37 ++ .../FunctionTest.cs | 18 + .../docs-assembler.Tests.Tests.csproj | 18 + 18 files changed, 735 insertions(+), 20 deletions(-) create mode 100644 src/docs-assembler/Navigation/GlobalNavigation.cs create mode 100644 src/docs-assembler/YamlStaticContext.cs create mode 100644 src/docs-assembler/navigation.yml create mode 100644 tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs create mode 100644 tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj create mode 100644 tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/FunctionTest.cs create mode 100644 tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/docs-assembler.Tests.Tests.csproj diff --git a/docs-builder.sln b/docs-builder.sln index d8d67fc4c..f300cc7c0 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -73,6 +73,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "update-reference-index", "u EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-lambda-index-publisher", "src\docs-lambda-index-publisher\docs-lambda-index-publisher.csproj", "{C559D52D-100B-4B2B-BE87-2344D835761D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler.Tests", "tests\docs-assembler.Tests\src\docs-assembler.Tests\docs-assembler.Tests.csproj", "{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -122,6 +124,10 @@ Global {C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.Build.0 = Debug|Any CPU {C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.ActiveCfg = Release|Any CPU {C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.Build.0 = Release|Any CPU + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} @@ -138,5 +144,6 @@ Global {6554F917-73CE-4B3D-9101-F28EAA762C6B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {9FEC15F6-13F8-40B1-A66A-EB054E49E680} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {C559D52D-100B-4B2B-BE87-2344D835761D} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} + {CDC0ECF4-6597-4FBA-8D25-5C244F0877E3} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5} EndGlobalSection EndGlobal diff --git a/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs b/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs index e337a8262..441fc462b 100644 --- a/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs +++ b/src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs @@ -98,6 +98,28 @@ public static void EmitWarning(this BuildContext context, IFileInfo file, string context.Collector.Channel.Write(d); } + public static void EmitError(this DiagnosticsCollector collector, IFileInfo file, string message, Exception? e = null) + { + var d = new Diagnostic + { + Severity = Severity.Error, + File = file.FullName, + Message = message + (e != null ? Environment.NewLine + e : string.Empty), + }; + collector.Channel.Write(d); + } + + public static void EmitWarning(this DiagnosticsCollector collector, IFileInfo file, string message) + { + var d = new Diagnostic + { + Severity = Severity.Warning, + File = file.FullName, + Message = message, + }; + collector.Channel.Write(d); + } + public static void EmitError(this IBlockExtension block, string message, Exception? e = null) { if (block.SkipValidation) diff --git a/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs b/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs index d6f9b79e0..824fb25cb 100644 --- a/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs +++ b/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs @@ -61,7 +61,7 @@ public ConfigurationFile(BuildContext context) var redirectFile = new RedirectFile(redirectFileInfo, _context); Redirects = redirectFile.Redirects; - var reader = new YamlStreamReader(sourceFile, _context); + var reader = new YamlStreamReader(sourceFile, _context.Collector); try { foreach (var entry in reader.Read()) @@ -104,7 +104,7 @@ public ConfigurationFile(BuildContext context) } //we read it twice to ensure we read 'toc' last - reader = new YamlStreamReader(sourceFile, _context); + reader = new YamlStreamReader(sourceFile, _context.Collector); foreach (var entry in reader.Read()) { switch (entry.Key) diff --git a/src/Elastic.Markdown/IO/Configuration/RedirectFile.cs b/src/Elastic.Markdown/IO/Configuration/RedirectFile.cs index 1b84024a1..9c24d2066 100644 --- a/src/Elastic.Markdown/IO/Configuration/RedirectFile.cs +++ b/src/Elastic.Markdown/IO/Configuration/RedirectFile.cs @@ -22,7 +22,7 @@ public RedirectFile(IFileInfo source, BuildContext context) if (!source.Exists) return; - var reader = new YamlStreamReader(Source, Context); + var reader = new YamlStreamReader(Source, Context.Collector); try { foreach (var entry in reader.Read()) diff --git a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs index 732b95400..92bd00d6e 100644 --- a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs +++ b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs @@ -229,7 +229,7 @@ public IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyVa if (!found) return null; - var tocYamlReader = new YamlStreamReader(source, _context); + var tocYamlReader = new YamlStreamReader(source, _context.Collector); foreach (var kv in tocYamlReader.Read()) { switch (kv.Key) diff --git a/src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs b/src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs index 502bfbcdd..c85d5836e 100644 --- a/src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs +++ b/src/Elastic.Markdown/IO/Configuration/YamlStreamReader.cs @@ -16,10 +16,10 @@ public record YamlToplevelKey public required KeyValuePair Entry { get; init; } } -public class YamlStreamReader(IFileInfo source, BuildContext context) +public class YamlStreamReader(IFileInfo source, DiagnosticsCollector collector) { - public IFileInfo Source { get; init; } = source; - public BuildContext Context { get; init; } = context; + private IFileInfo Source { get; init; } = source; + private DiagnosticsCollector Collector { get; init; } = collector; public IEnumerable Read() { @@ -30,7 +30,7 @@ public IEnumerable Read() if (yaml.Documents.Count == 0) { - Context.EmitWarning(Source, "empty redirect file"); + Collector.EmitWarning(Source, "empty redirect file"); yield break; } // Examine the stream @@ -171,7 +171,7 @@ public void EmitWarning(string message, YamlNode? node) => EmitWarning(message, node?.Start, node?.End, (node as YamlScalarNode)?.Value?.Length); public void EmitError(string message, Exception e) => - Context.Collector.EmitError(Source.FullName, message, e); + Collector.EmitError(Source.FullName, message, e); private void EmitError(string message, Mark? start = null, Mark? end = null, int? length = null) { @@ -185,7 +185,7 @@ private void EmitError(string message, Mark? start = null, Mark? end = null, int Column = start.HasValue ? (int)start.Value.Column : null, Length = length }; - Context.Collector.Channel.Write(d); + Collector.Channel.Write(d); } public void EmitWarning(string message, Mark? start = null, Mark? end = null, int? length = null) { @@ -199,6 +199,6 @@ public void EmitWarning(string message, Mark? start = null, Mark? end = null, in Column = start.HasValue ? (int)start.Value.Column : null, Length = length }; - Context.Collector.Channel.Write(d); + Collector.Channel.Write(d); } } diff --git a/src/docs-assembler/AssembleContext.cs b/src/docs-assembler/AssembleContext.cs index a262bf5c2..70ab0765d 100644 --- a/src/docs-assembler/AssembleContext.cs +++ b/src/docs-assembler/AssembleContext.cs @@ -21,6 +21,8 @@ public class AssembleContext public IFileInfo ConfigurationPath { get; } + public IFileInfo NavigationPath { get; } + public IDirectoryInfo CheckoutDirectory { get; set; } public IDirectoryInfo OutputDirectory { get; set; } @@ -47,9 +49,14 @@ public AssembleContext( // This will live in docs-content soon if (!ReadFileSystem.File.Exists(configPath)) ExtractAssemblerConfiguration(configPath); - ConfigurationPath = ReadFileSystem.FileInfo.New(configPath); Configuration = AssemblyConfiguration.Deserialize(ReadFileSystem.File.ReadAllText(ConfigurationPath.FullName)); + + var navigationPath = Path.Combine(Paths.Root.FullName, "src", "docs-assembler", "navigation.yml"); + if (!ReadFileSystem.File.Exists(navigationPath)) + ExtractAssemblerConfiguration(navigationPath); + NavigationPath = ReadFileSystem.FileInfo.New(navigationPath); + CheckoutDirectory = ReadFileSystem.DirectoryInfo.New(checkoutDirectory ?? ".artifacts/checkouts"); OutputDirectory = ReadFileSystem.DirectoryInfo.New(output ?? ".artifacts/assembly"); } diff --git a/src/docs-assembler/Building/AssemblerBuilder.cs b/src/docs-assembler/Building/AssemblerBuilder.cs index 367ea0e83..6e71037f8 100644 --- a/src/docs-assembler/Building/AssemblerBuilder.cs +++ b/src/docs-assembler/Building/AssemblerBuilder.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using Documentation.Assembler.Configuration; +using Documentation.Assembler.Navigation; using Documentation.Assembler.Sourcing; using Elastic.Markdown; using Elastic.Markdown.CrossLinks; diff --git a/src/docs-assembler/Cli/RepositoryCommands.cs b/src/docs-assembler/Cli/RepositoryCommands.cs index 59d6c3e86..70381c93c 100644 --- a/src/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/docs-assembler/Cli/RepositoryCommands.cs @@ -7,6 +7,7 @@ using Actions.Core.Services; using ConsoleAppFramework; using Documentation.Assembler.Building; +using Documentation.Assembler.Navigation; using Documentation.Assembler.Sourcing; using Elastic.Documentation.Tooling.Diagnostics.Console; using Microsoft.Extensions.Logging; @@ -79,6 +80,7 @@ public async Task BuildAll( if (checkouts.Length == 0) throw new Exception("No checkouts found"); + var globalNavigation = GlobalNavigation.Deserialize(assembleContext, checkouts); var builder = new AssemblerBuilder(logger, assembleContext); await builder.BuildAllAsync(checkouts, env, ctx); diff --git a/src/docs-assembler/Configuration/AssemblyConfiguration.cs b/src/docs-assembler/Configuration/AssemblyConfiguration.cs index 73c39734b..e394aee68 100644 --- a/src/docs-assembler/Configuration/AssemblyConfiguration.cs +++ b/src/docs-assembler/Configuration/AssemblyConfiguration.cs @@ -6,20 +6,13 @@ namespace Documentation.Assembler.Configuration; -[YamlStaticContext] -[YamlSerializable(typeof(AssemblyConfiguration))] -[YamlSerializable(typeof(Repository))] -[YamlSerializable(typeof(NarrativeRepository))] -[YamlSerializable(typeof(PublishEnvironment))] -public partial class YamlStaticContext; - public record AssemblyConfiguration { public static AssemblyConfiguration Deserialize(string yaml) { var input = new StringReader(yaml); - var deserializer = new StaticDeserializerBuilder(new YamlStaticContext()) + var deserializer = new StaticDeserializerBuilder(new Assembler.YamlStaticContext()) .IgnoreUnmatchedProperties() .Build(); diff --git a/src/docs-assembler/Navigation/GlobalNavigation.cs b/src/docs-assembler/Navigation/GlobalNavigation.cs new file mode 100644 index 000000000..759cc5462 --- /dev/null +++ b/src/docs-assembler/Navigation/GlobalNavigation.cs @@ -0,0 +1,123 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using DotNet.Globbing; +using Elastic.Markdown.IO.Configuration; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; + +namespace Documentation.Assembler.Navigation; + +public record TableOfContentsReference +{ + public required string Source { get; init; } + public required string? PathPrefix { get; init; } + public required IReadOnlyCollection Children { get; init; } +} + +public record GlobalNavigation +{ + public IReadOnlyCollection References { get; init; } = []; + + public static GlobalNavigation Deserialize(AssembleContext context) + { + var globalConfig = new GlobalNavigation(); + var reader = new YamlStreamReader(context.NavigationPath, context.Collector); + try + { + foreach (var entry in reader.Read()) + { + switch (entry.Key) + { + case "toc": + globalConfig = globalConfig with + { + References = ReadChildren(reader, entry.Entry, new Queue()) + }; + break; + } + } + } + catch (Exception e) + { + reader.EmitError("Could not load docset.yml", e); + throw; + } + + return globalConfig; + } + + private static IReadOnlyCollection ReadChildren( + YamlStreamReader reader, + KeyValuePair entry, + Queue parents + ) + { + var entries = new List(); + if (entry.Value is not YamlSequenceNode sequence) + { + if (entry.Key is YamlScalarNode scalarKey) + { + var key = scalarKey.Value; + reader.EmitWarning($"'{key}' is not an array"); + } + else + reader.EmitWarning($"'{entry.Key}' is not an array"); + + return entries; + } + + foreach (var tocEntry in sequence.Children.OfType()) + { + var child = ReadChild(reader, tocEntry, parents); + if (child is not null) + entries.Add(child); + } + + //TableOfContents = entries; + return entries; + } + + private static TableOfContentsReference? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry, Queue parents) + { + string? source = null; + string? pathPrefix = null; + IReadOnlyCollection? children = null; + foreach (var entry in tocEntry.Children) + { + var key = ((YamlScalarNode)entry.Key).Value; + switch (key) + { + case "toc": + source = reader.ReadString(entry); + break; + case "path_prefix": + pathPrefix = reader.ReadString(entry); + break; + case "children": + if (source is null && pathPrefix is null) + { + reader.EmitWarning("toc entry has no toc or path_prefix defined"); + continue; + } + var path = source ?? pathPrefix; + parents.Enqueue(path!); + children = ReadChildren(reader, entry, parents); + break; + } + } + + if (source is not null) + { + return new TableOfContentsReference + { + Source = source, + Children = children ?? [], + PathPrefix = pathPrefix + }; + } + + return null; + } +} diff --git a/src/docs-assembler/YamlStaticContext.cs b/src/docs-assembler/YamlStaticContext.cs new file mode 100644 index 000000000..fd57edc10 --- /dev/null +++ b/src/docs-assembler/YamlStaticContext.cs @@ -0,0 +1,15 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Documentation.Assembler.Configuration; +using YamlDotNet.Serialization; + +namespace Documentation.Assembler; + +[YamlStaticContext] +[YamlSerializable(typeof(AssemblyConfiguration))] +[YamlSerializable(typeof(Repository))] +[YamlSerializable(typeof(NarrativeRepository))] +[YamlSerializable(typeof(PublishEnvironment))] +public partial class YamlStaticContext; diff --git a/src/docs-assembler/docs-assembler.csproj b/src/docs-assembler/docs-assembler.csproj index 733f18395..ec1ec216d 100644 --- a/src/docs-assembler/docs-assembler.csproj +++ b/src/docs-assembler/docs-assembler.csproj @@ -31,5 +31,6 @@ + diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml new file mode 100644 index 000000000..a01ca9f5a --- /dev/null +++ b/src/docs-assembler/navigation.yml @@ -0,0 +1,447 @@ +############################################# +# KEY FOR REFERENCE SECTION +# ✅ = toc should be ready to go! +# 🔜 = ready in a PR/branch, +# but needs to be approved and merged +# 📝 = no PR started yet +############################################# + +toc: + ############# + # NARRATIVE # + ############# + - toc: get-started + - toc: solutions + - toc: manage-data + - toc: explore-analyze + - toc: deploy-manage + - toc: cloud-account + - toc: troubleshoot + + ########## + # EXTEND # + ########## + # I didn't touch this section (yet?) + - toc: extend + children: + - toc: kibana://extend + - toc: logstash://extend + - toc: elasticsearch://extend + #- toc: beats://extend + - toc: integrations://extend + + ################# + # RELEASE NOTES # + ################# + # I didn't touch this section (yet?) + - toc: release-notes + children: + - toc: elasticsearch://release-notes + - toc: docs-content://release-notes/apm + + ############# + # REFERENCE # + ############# + - toc: reference + children: + # Security + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/security/toc.yml + - toc: docs-content://reference/security + path_prefix: reference/security + # Children include: Endpoint command reference, Elastic Defend, + # Fields and object schemas + children: + # 📝 TO DO: I don't remember the repo name + # 📝 TO DO: toc.yml needs to be created + - toc: that-other-sec-repo://reference + path_prefix: reference/security/prebuilt-rules + # Children include the entire AsciiDoc book + + # Observability + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/observability/toc.yml + - toc: docs-content://reference/observability + path_prefix: reference/observability + # Children include: Fields and object schemas, Elastic Entity Model, + # Infrastructure app fields + + # Search + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/search/toc.yml + - toc: docs-content://reference/search + path_prefix: reference/search + children: + # Search UI + # ✅ https://github.com/elastic/search-ui/blob/main/docs/reference/toc.yml + - toc: search-ui://reference + path_prefix: reference/search-ui + # Children include the entire AsciiDoc book + + # Elasticsearch and index management + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/elasticsearch/toc.yml + - toc: docs-content://reference/elasticsearch + children: + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/elasticsearch/toc.yml + - toc: elasticsearch://reference/elasticsearch + path_prefix: reference/elasticsearch + # Children include: Configuration, JVM settings, Roles, + # Elasticsearch privileges, Index settings, Index lifecycle actions, + # REST APIs, Mapping, Elasticsearch audit events, Command line tools + + # Curator + # ✅ https://github.com/elastic/curator/blob/master/docs/reference/toc.yml + - toc: curator://reference + path_prefix: reference/elasticsearch/curator + # Children include the entire AsciiDoc book + + # Clients + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/elasticsearch-clients/toc.yml + - toc: docs-content://reference/elasticsearch/clients + path_prefix: reference/elasticsearch/clients + children: + + # Eland + # ✅ https://github.com/elastic/eland/blob/main/docs/reference/toc.yml + - toc: eland/docs/reference + path_prefix: reference/elasticsearch/clients/eland + # Children include the entire AsciiDoc book + + # Go + # ✅ https://github.com/elastic/go-elasticsearch/blob/main/docs/reference/toc.yml + - toc: go-elasticsearch://reference + path_prefix: reference/elasticsearch/clients/go + # Children include the entire AsciiDoc book + + # Java + # ✅ https://github.com/elastic/elasticsearch-java/blob/main/docs/reference/toc.yml + - toc: elasticsearch-java://reference + path_prefix: reference/elasticsearch/clients/java + # Children include the entire AsciiDoc book + + # JavaScript + # ✅ https://github.com/elastic/elasticsearch-js/blob/main/docs/reference/toc.yml + - toc: elasticsearch-js://reference + path_prefix: reference/elasticsearch/clients/js + # Children include the entire AsciiDoc book + + # .NET + # ✅ https://github.com/elastic/elasticsearch-net/blob/main/docs/reference/toc.yml + - toc: elasticsearch-net://reference + path_prefix: reference/elasticsearch/clients/net + # Children include the entire AsciiDoc book + + # PHP + # ✅ https://github.com/elastic/elasticsearch-php/blob/main/docs/reference/toc.yml + - toc: elasticsearch-php://reference + path_prefix: reference/elasticsearch/clients/php + # Children include the entire AsciiDoc book + + # Python + # ✅ https://github.com/elastic/elasticsearch-py/blob/main/docs/reference/toc.yml + - toc: elasticsearch-py://reference + path_prefix: reference/elasticsearch/clients/py + # Children include the entire AsciiDoc book + + # Ruby + # ✅ https://github.com/elastic/elasticsearch-ruby/blob/main/docs/reference/toc.yml + - toc: elasticsearch-ruby://reference + path_prefix: reference/elasticsearch/clients/ruby + # Children include the entire AsciiDoc book + + # Rust + # ✅ https://github.com/elastic/elasticsearch-rs/blob/main/docs/reference/toc.yml + - toc: elasticsearch-rs://reference + path_prefix: reference/elasticsearch/clients/rs + # Children include the entire AsciiDoc book + + # Community-contributed clients + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/community-contributed/toc.yml + - toc: elasticsearch://reference/community-contributed + path_prefix: reference/elasticsearch/clients/community-contributed + + # Ingestion tools + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/ingestion-tools/toc.yml + - toc: docs-content://reference/ingestion-tools + path_prefix: reference/ingestion-tools + children: + + # Fleet and Elastic Agent + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/fleet/toc.yml + - toc: docs-content://reference/fleet + path_prefix: reference/fleet + # Children include the entire AsciiDoc book + + # APM + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/apm/toc.yml + - toc: docs-content://reference/apm + path_prefix: reference/apm + # Children include: APM settings, APM settings for Elastic Cloud, + # APM settings for Elastic Cloud Enterprise + children: + # APM Attacher for Kubernetes + # ✅ https://github.com/elastic/apm-k8s-attacher/blob/main/docs/reference/toc.yml + - toc: apm-k8s-attacher://reference + path_prefix: reference/apm/k8s-attacher + # Children include the entire AsciiDoc book + + # APM Architecture for AWS Lambda + # ✅ https://github.com/elastic/apm-aws-lambda/blob/main/docs/reference/toc.yml + - toc: apm-aws-lambda://reference + path_prefix: reference/apm/aws-lambda + # Children include the entire AsciiDoc book + + # APM agents + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/apm-agents/toc.yml + - toc: docs-content://reference/ingestion-tools/apm/agents + path_prefix: reference/apm/agents + children: + # APM Android agent + # ✅ https://github.com/elastic/apm-agent-android/blob/main/docs/reference/toc.yml + - toc: apm-agent-android://reference + path_prefix: reference/apm/agents/android + # Children include the entire AsciiDoc book + + # APM .NET agent + # ✅ https://github.com/elastic/apm-agent-dotnet/blob/main/docs/reference/toc.yml + - toc: apm-agent-dotnet://reference + path_prefix: reference/apm/agents/dotnet + # Children include the entire AsciiDoc book + + # APM Go agent + # ✅ https://github.com/elastic/apm-agent-go/blob/main/docs/reference/toc.yml + - toc: apm-agent-go://reference + path_prefix: reference/apm/agents/go + # Children include the entire AsciiDoc book + + # APM iOS agent + # ✅ https://github.com/elastic/apm-agent-ios/blob/main/docs/reference/toc.yml + - toc: apm-agent-ios://reference + path_prefix: reference/apm/agents/ios + # Children include the entire AsciiDoc book + + # APM Java agent + # ✅ https://github.com/elastic/apm-agent-java/blob/main/docs/reference/toc.yml + - toc: apm-agent-java://reference + path_prefix: reference/apm/agents/java + # Children include the entire AsciiDoc book + + # APM Node.js agent + # 🔜 https://github.com/colleenmcginnis/apm-agent-nodejs/blob/migrate-docs/docs/reference/toc.yml + - toc: apm-agent-nodejs://reference + path_prefix: reference/apm/agents/nodejs + # Children include the entire AsciiDoc book + + # APM PHP agent + # 🔜 https://github.com/colleenmcginnis/apm-agent-php/blob/migrate-docs/docs/reference/toc.yml + - toc: apm-agent-php://reference + path_prefix: reference/apm/agents/php + # Children include the entire AsciiDoc book + + # APM Python agent + # 🔜 https://github.com/colleenmcginnis/apm-agent-python/blob/migrate-docs/docs/reference/toc.yml + - toc: apm-agent-python://reference + path_prefix: reference/apm/agents/python + # Children include the entire AsciiDoc book + + # APM Ruby agent + # 🔜 https://github.com/colleenmcginnis/apm-agent-ruby/blob/migrate-docs/docs/reference/toc.yml + - toc: apm-agent-ruby://reference + path_prefix: reference/apm/agents/ruby + # Children include the entire AsciiDoc book + + # APM RUM JavaScript agent + # ✅ https://github.com/elastic/apm-agent-rum-js/blob/main/docs/reference/toc.yml + - toc: apm-agent-rum-js://reference + path_prefix: reference/apm/agents/rum-js + # Children include the entire AsciiDoc book + + # Beats + # 🔜 https://github.com/colleenmcginnis/beats/blob/migrate-docs/docs/reference/toc.yml + - toc: beats://reference + path_prefix: reference/beats + # Children include all entire AsciiDoc books: Beats, Auditbeat, + # Filebeat, Heartbeat, Metricbeat, Packetbeat, Winlogbeat, + # Elastic logging plugin for Docker + + # Processor reference + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/enrich-processor/toc.yml + - toc: elasticsearch://reference/enrich-processor + path_prefix: reference/enrich-processor + + # Logstash + # ✅ https://github.com/elastic/logstash/blob/main/docs/reference/toc.yml + - toc: logstash://reference + path_prefix: reference/logstash + # Children include the entire AsciiDoc book + + # Logstash plugins + # 📝 TO DO: Migrate all files and create toc.yml + - toc: logstash-docs://reference + path_prefix: reference/logstash/plugins + # Children include the entire AsciiDoc book + + # Elastic Serverless Forwarder for AWS + # ✅ https://github.com/elastic/elastic-serverless-forwarder/blob/main/docs/reference/toc.yml + - toc: elastic-serverless-forwarder://reference + path_prefix: reference/elastic-serverless-forwarder + # Children include the entire AsciiDoc book + + # Search connectors + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/search-connectors/toc.yml + - toc: elasticsearch://reference/search-connectors + path_prefix: reference/search-connectors + + # Elasticsearch Hadoop + # ✅ https://github.com/elastic/elasticsearch-hadoop/blob/main/docs/reference/toc.yml + - toc: elasticsearch-hadoop://reference + path_prefix: reference/elasticsearch-hadoop + # Children include the entire AsciiDoc book + + # Elastic Integrations + # 📝 TO DO: Waiting on integration devs to support 9.0 in all integrations + - toc: integration-docs://reference + path_prefix: reference/integrations + # Children include the entire AsciiDoc book + + # Kibana + # ✅ https://github.com/elastic/kibana/blob/main/docs/reference/toc.yml + - toc: kibana://reference + path_prefix: reference/kibana + # Children include the entire AsciiDoc book + # (minus pages moved to docs-content) + + # Elasticsearch plugins + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/elasticsearch-plugins/toc.yml + - toc: elasticsearch://reference/elasticsearch-plugins + path_prefix: reference/elasticsearch-plugins + + # Query languages + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/query-languages/toc.yml + - toc: elasticsearch://reference/query-languages + path_prefix: reference/query-languages + + # Scripting languages + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/scripting-languages/toc.yml + - toc: elasticsearch://reference/scripting-languages + path_prefix: reference/scripting-languages + + # ECS + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/ecs/toc.yml + - toc: docs-content://reference/ecs + children: + # ECS reference + # ✅ https://github.com/elastic/ecs/blob/main/docs/reference/toc.yml + - toc: ecs://reference + path_prefix: reference/ecs + # Children include the entire AsciiDoc book + + # ECS logging libraries + # ✅ https://github.com/elastic/ecs-logging/blob/main/docs/reference/toc.yml + - toc: ecs-logging://reference + path_prefix: reference/ecs/logging + children: + # ECS Logging .NET + # ✅ https://github.com/elastic/ecs-dotnet/blob/main/docs/reference/toc.yml + - toc: ecs-dotnet://reference + path_prefix: reference/ecs/logging/dotnet + # Children include the entire AsciiDoc book + + # ECS Logging Go (Logrus) + # ✅ https://github.com/elastic/ecs-logging-go-logrus/blob/main/docs/reference/toc.yml + - toc: ecs-logging-go-logrus://reference + path_prefix: reference/ecs/logging/go-logrus + # Children include the entire AsciiDoc book + + # ECS Logging Go (Zap) + # ✅ https://github.com/elastic/ecs-logging-go-zap/blob/main/docs/reference/toc.yml + - toc: ecs-logging-go-zap://reference + path_prefix: reference/ecs/logging/go-zap + # Children include the entire AsciiDoc book + + # ECS Logging Go (Zerolog) + # ✅ https://github.com/elastic/ecs-logging-go-zerolog/blob/main/docs/reference/toc.yml + - toc: ecs-logging-go-zerolog://reference + path_prefix: reference/ecs/logging/go-zerolog + # Children include the entire AsciiDoc book + + # ECS Logging Java + # ✅ https://github.com/elastic/ecs-logging-java/blob/main/docs/reference/toc.yml + - toc: ecs-logging-java://reference + path_prefix: reference/ecs/logging/java + # Children include the entire AsciiDoc book + + # ECS Logging Node.js + # ✅ https://github.com/elastic/ecs-logging-nodejs/blob/main/docs/reference/toc.yml + - toc: ecs-logging-nodejs://reference + path_prefix: reference/ecs/logging/nodejs + # Children include the entire AsciiDoc book + + # ECS Logging PHP + # ✅ https://github.com/elastic/ecs-logging-php/blob/main/docs/reference/toc.yml + - toc: ecs-logging-php://reference + path_prefix: reference/ecs/logging/php + # Children include the entire AsciiDoc book + + # ECS Logging Python + # ✅ https://github.com/elastic/ecs-logging-python/blob/main/docs/reference/toc.yml + - toc: ecs-logging-python://reference + path_prefix: reference/ecs/logging/python + # Children include the entire AsciiDoc book + + # ECS Logging Ruby + # ✅ https://github.com/elastic/ecs-logging-ruby/blob/main/docs/reference/toc.yml + - toc: ecs-logging-ruby://reference + path_prefix: reference/ecs/logging/ruby + # Children include the entire AsciiDoc book + + # Data analysis + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/data-analysis/toc.yml + - toc: docs-content://reference/data-analysis + path_prefix: reference/data-analysis + # Children include: Supplied configurations, Function reference, + # Metrics reference, Canvas function reference + children: + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/text-analysis/toc.yml + - toc: elasticsearch://reference/text-analysis + path_prefix: reference/text-analysis + # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down + # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/aggregations/toc.yml + - toc: elasticsearch://reference/aggregations + path_prefix: reference/aggregations + + # Cloud + # 📝 TO DO: toc.yml needs to be created with one file / no children + - toc: docs-content://reference/cloud + path_prefix: reference/cloud + children: + # Elastic Cloud Enterprise + # Elastic Cloud Hosted + # ✅ https://github.com/elastic/cloud/blob/master/docs/reference/toc.yml + - toc: cloud://reference + path_prefix: reference/cloud + # Children include the entire AsciiDoc book + # (minus pages moved to docs-content) + + # Elastic Cloud on Kubernetes + # ✅ https://github.com/elastic/cloud-on-k8s/blob/main/docs/reference/toc.yml + - toc: cloud-on-k8s://reference + path_prefix: reference/cloud-on-k8s + # Children include the entire AsciiDoc book + # (minus pages moved to docs-content) + + # Elastic cloud control (ECCTL) + # ✅ https://github.com/elastic/ecctl/blob/master/docs/reference/toc.yml + - toc: ecctl://reference + path_prefix: reference/ecctl + # Children include the entire AsciiDoc book + # (minus pages moved to docs-content) + + # Glossary + # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/glossary/toc.yml + - toc: docs-content://reference/glossary \ No newline at end of file diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs new file mode 100644 index 000000000..67313bbf1 --- /dev/null +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs @@ -0,0 +1,24 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions; +using Documentation.Assembler.Navigation; +using Elastic.Markdown.Diagnostics; +using FluentAssertions; + +namespace Documentation.Assembler.Tests; + +public class GlobalNavigationTests +{ + [Fact] + public async Task ParsesGlobalNavigation() + { + await using var collector = new DiagnosticsCollector([]); + _ = collector.StartAsync(TestContext.Current.CancellationToken); + + var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null); + var globalNavigation = GlobalNavigation.Deserialize(assembleContext); + globalNavigation.References.Should().NotBeNull().And.NotBeEmpty(); + } +} diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj b/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj new file mode 100644 index 000000000..8adb49ea6 --- /dev/null +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj @@ -0,0 +1,37 @@ + + + + + net9.0 + enable + enable + + false + true + Exe + + false + true + Documentation.Assembler.Tests + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/FunctionTest.cs b/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/FunctionTest.cs new file mode 100644 index 000000000..28d9969bf --- /dev/null +++ b/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/FunctionTest.cs @@ -0,0 +1,18 @@ +using Xunit; +using Amazon.Lambda.Core; +using Amazon.Lambda.TestUtilities; + +namespace docs_assembler.Tests.Tests; + +public class FunctionTest +{ + [Fact] + public void TestToUpperFunction() + { + // Invoke the lambda function and confirm the string was upper cased. + var context = new TestLambdaContext(); + var upperCase = Function.FunctionHandler("hello world", context); + + Assert.Equal("HELLO WORLD", upperCase); + } +} \ No newline at end of file diff --git a/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/docs-assembler.Tests.Tests.csproj b/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/docs-assembler.Tests.Tests.csproj new file mode 100644 index 000000000..78a156450 --- /dev/null +++ b/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/docs-assembler.Tests.Tests.csproj @@ -0,0 +1,18 @@ + + + net8.0 + enable + enable + true + + + + + + + + + + + + \ No newline at end of file From 75471f96bd99e0f3f1e27d5b6f12e3a1f2caa721 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 18 Mar 2025 15:50:19 +0100 Subject: [PATCH 02/13] Use navigation.yml to control url path prefixes for resolving --- .../CrossLinks/CrossLinkResolver.cs | 2 +- .../CrossLinks/IUriEnvironmentResolver.cs | 2 +- .../Building/AssemblerBuilder.cs | 4 +- .../Building/PublishEnvironmentUriResolver.cs | 31 ++-- src/docs-assembler/Cli/RepositoryCommands.cs | 6 +- .../Navigation/GlobalNavigation.cs | 148 ++++++----------- .../Navigation/GlobalNavigationFile.cs | 154 ++++++++++++++++++ src/docs-assembler/assembler.yml | 2 +- .../TestCrossLinkResolver.cs | 2 +- .../Framework/TestCrossLinkResolver.fs | 2 +- .../GlobalNavigationTests.cs | 56 ++++++- 11 files changed, 283 insertions(+), 126 deletions(-) create mode 100644 src/docs-assembler/Navigation/GlobalNavigationFile.cs diff --git a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs index 701385d90..8eb158704 100644 --- a/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs +++ b/src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs @@ -45,7 +45,7 @@ public interface ICrossLinkResolver public class CrossLinkResolver(CrossLinkFetcher fetcher, IUriEnvironmentResolver? uriResolver = null) : ICrossLinkResolver { private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty; - private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new PreviewEnvironmentUriResolver(); + private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new IsolatedBuildEnvironmentUriResolver(); public async Task FetchLinks() { diff --git a/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs b/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs index 4bad70808..d95e0deaa 100644 --- a/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs +++ b/src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs @@ -9,7 +9,7 @@ public interface IUriEnvironmentResolver Uri Resolve(Uri crossLinkUri, string path); } -public class PreviewEnvironmentUriResolver : IUriEnvironmentResolver +public class IsolatedBuildEnvironmentUriResolver : IUriEnvironmentResolver { private static Uri BaseUri { get; } = new("https://docs-v3-preview.elastic.dev"); diff --git a/src/docs-assembler/Building/AssemblerBuilder.cs b/src/docs-assembler/Building/AssemblerBuilder.cs index 6e71037f8..8d312b40f 100644 --- a/src/docs-assembler/Building/AssemblerBuilder.cs +++ b/src/docs-assembler/Building/AssemblerBuilder.cs @@ -12,14 +12,14 @@ namespace Documentation.Assembler.Building; -public class AssemblerBuilder(ILoggerFactory logger, AssembleContext context) +public class AssemblerBuilder(ILoggerFactory logger, AssembleContext context, GlobalNavigation globalNavigation) { private readonly ILogger _logger = logger.CreateLogger(); public async Task BuildAllAsync(IReadOnlyCollection checkouts, PublishEnvironment environment, Cancel ctx) { var crossLinkFetcher = new AssemblerCrossLinkFetcher(logger, context.Configuration); - var uriResolver = new PublishEnvironmentUriResolver(context.Configuration, environment); + var uriResolver = new PublishEnvironmentUriResolver(globalNavigation, environment); var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, uriResolver); foreach (var checkout in checkouts) diff --git a/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs b/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs index 4d0595842..75d9f6043 100644 --- a/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs +++ b/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs @@ -4,50 +4,45 @@ using System.Collections.Frozen; using Documentation.Assembler.Configuration; +using Documentation.Assembler.Navigation; using Elastic.Markdown.CrossLinks; namespace Documentation.Assembler.Building; public class PublishEnvironmentUriResolver : IUriEnvironmentResolver { + private readonly GlobalNavigation _globalNavigation; private Uri BaseUri { get; } private PublishEnvironment PublishEnvironment { get; } - private PreviewEnvironmentUriResolver PreviewResolver { get; } + private IsolatedBuildEnvironmentUriResolver IsolatedBuildResolver { get; } - private FrozenDictionary AllRepositories { get; } - - public PublishEnvironmentUriResolver(AssemblyConfiguration configuration, PublishEnvironment environment) + public PublishEnvironmentUriResolver(GlobalNavigation globalNavigation, PublishEnvironment environment) { + _globalNavigation = globalNavigation; if (!Uri.TryCreate(environment.Uri, UriKind.Absolute, out var uri)) throw new Exception($"Could not parse uri {environment.Uri} in environment {environment}"); BaseUri = uri; PublishEnvironment = environment; - PreviewResolver = new PreviewEnvironmentUriResolver(); - AllRepositories = configuration.ReferenceRepositories.Values.Concat([configuration.Narrative]) - .ToFrozenDictionary(e => e.Name, e => e); - RepositoryLookup = AllRepositories.GetAlternateLookup>(); + IsolatedBuildResolver = new IsolatedBuildEnvironmentUriResolver(); } - private FrozenDictionary.AlternateLookup> RepositoryLookup { get; } - public Uri Resolve(Uri crossLinkUri, string path) { + // TODO Maybe not needed if (PublishEnvironment.Name == "preview") - return PreviewResolver.Resolve(crossLinkUri, path); + return IsolatedBuildResolver.Resolve(crossLinkUri, path); - var repositoryPath = crossLinkUri.Scheme; - if (RepositoryLookup.TryGetValue(crossLinkUri.Scheme, out var repository)) - repositoryPath = repository.PathPrefix; + var subPath = _globalNavigation.GetSubPath(crossLinkUri, ref path); - var fullPath = (PublishEnvironment.PathPrefix, repositoryPath) switch + var fullPath = (PublishEnvironment.PathPrefix, subPath) switch { (null or "", null or "") => path, - (null or "", var p) => $"{p}/{path}", - (var p, null or "") => $"{p}/{path}", - var (p, pp) => $"{p}/{pp}/{path}" + (null or "", var p) => $"{p}/{path.TrimStart('/')}", + (var p, null or "") => $"{p}/{path.TrimStart('/')}", + var (p, pp) => $"{p}/{pp}/{path.TrimStart('/')}" }; return new Uri(BaseUri, fullPath); diff --git a/src/docs-assembler/Cli/RepositoryCommands.cs b/src/docs-assembler/Cli/RepositoryCommands.cs index 70381c93c..853efe736 100644 --- a/src/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/docs-assembler/Cli/RepositoryCommands.cs @@ -80,8 +80,10 @@ public async Task BuildAll( if (checkouts.Length == 0) throw new Exception("No checkouts found"); - var globalNavigation = GlobalNavigation.Deserialize(assembleContext, checkouts); - var builder = new AssemblerBuilder(logger, assembleContext); + var navigationFile = GlobalNavigationFile.Deserialize(assembleContext); + var globalNavigation = new GlobalNavigation(assembleContext, navigationFile, checkouts); + + var builder = new AssemblerBuilder(logger, assembleContext, globalNavigation); await builder.BuildAllAsync(checkouts, env, ctx); if (strict ?? false) diff --git a/src/docs-assembler/Navigation/GlobalNavigation.cs b/src/docs-assembler/Navigation/GlobalNavigation.cs index 759cc5462..816eb04e2 100644 --- a/src/docs-assembler/Navigation/GlobalNavigation.cs +++ b/src/docs-assembler/Navigation/GlobalNavigation.cs @@ -2,122 +2,76 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using DotNet.Globbing; -using Elastic.Markdown.IO.Configuration; -using YamlDotNet.RepresentationModel; -using YamlDotNet.Serialization; +using System.Collections.Frozen; +using System.Collections.Immutable; +using Documentation.Assembler.Configuration; +using Documentation.Assembler.Sourcing; +using Elastic.Markdown.Diagnostics; namespace Documentation.Assembler.Navigation; -public record TableOfContentsReference -{ - public required string Source { get; init; } - public required string? PathPrefix { get; init; } - public required IReadOnlyCollection Children { get; init; } -} - public record GlobalNavigation { - public IReadOnlyCollection References { get; init; } = []; + private readonly AssembleContext _context; + private readonly FrozenDictionary.AlternateLookup> _checkoutsLookup; + private readonly FrozenDictionary.AlternateLookup> _repoConfigLookup; + private readonly FrozenDictionary.AlternateLookup> _tocLookup; - public static GlobalNavigation Deserialize(AssembleContext context) - { - var globalConfig = new GlobalNavigation(); - var reader = new YamlStreamReader(context.NavigationPath, context.Collector); - try - { - foreach (var entry in reader.Read()) - { - switch (entry.Key) - { - case "toc": - globalConfig = globalConfig with - { - References = ReadChildren(reader, entry.Entry, new Queue()) - }; - break; - } - } - } - catch (Exception e) - { - reader.EmitError("Could not load docset.yml", e); - throw; - } + private FrozenDictionary ConfiguredRepositories { get; } + private FrozenDictionary IndexedTableOfContents { get; } - return globalConfig; - } + public GlobalNavigationFile NavigationConfiguration { get; init; } - private static IReadOnlyCollection ReadChildren( - YamlStreamReader reader, - KeyValuePair entry, - Queue parents - ) - { - var entries = new List(); - if (entry.Value is not YamlSequenceNode sequence) - { - if (entry.Key is YamlScalarNode scalarKey) - { - var key = scalarKey.Value; - reader.EmitWarning($"'{key}' is not an array"); - } - else - reader.EmitWarning($"'{entry.Key}' is not an array"); + private FrozenDictionary Checkouts { get; init; } - return entries; - } + private ImmutableSortedSet TableOfContentsPrefixes { get; } - foreach (var tocEntry in sequence.Children.OfType()) - { - var child = ReadChild(reader, tocEntry, parents); - if (child is not null) - entries.Add(child); - } + public GlobalNavigation(AssembleContext context, GlobalNavigationFile navigationConfiguration, Checkout[] checkouts) + { + _context = context; + NavigationConfiguration = navigationConfiguration; + Checkouts = checkouts.ToDictionary(c => c.Repository.Name, c => c).ToFrozenDictionary(); + _checkoutsLookup = Checkouts.GetAlternateLookup>(); + + var configuration = context.Configuration; + ConfiguredRepositories = configuration.ReferenceRepositories.Values.Concat([configuration.Narrative]) + .ToFrozenDictionary(e => e.Name, e => e); + _repoConfigLookup = ConfiguredRepositories.GetAlternateLookup>(); - //TableOfContents = entries; - return entries; + IndexedTableOfContents = navigationConfiguration.IndexedTableOfContents; + _tocLookup = IndexedTableOfContents.GetAlternateLookup>(); + TableOfContentsPrefixes = navigationConfiguration.IndexedTableOfContents.Keys.OrderByDescending(k => k.Length).ToImmutableSortedSet(); } - private static TableOfContentsReference? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry, Queue parents) + public string GetSubPath(Uri crossLinkUri, ref string path) { - string? source = null; - string? pathPrefix = null; - IReadOnlyCollection? children = null; - foreach (var entry in tocEntry.Children) + if (!_checkoutsLookup.TryGetValue(crossLinkUri.Scheme, out var checkout)) { - var key = ((YamlScalarNode)entry.Key).Value; - switch (key) - { - case "toc": - source = reader.ReadString(entry); - break; - case "path_prefix": - pathPrefix = reader.ReadString(entry); - break; - case "children": - if (source is null && pathPrefix is null) - { - reader.EmitWarning("toc entry has no toc or path_prefix defined"); - continue; - } - var path = source ?? pathPrefix; - parents.Enqueue(path!); - children = ReadChildren(reader, entry, parents); - break; - } + _context.Collector.EmitError(_context.NavigationPath, $"Unable to find checkout for repository: {crossLinkUri.Scheme}"); + // fallback to repository path prefix + if (_repoConfigLookup.TryGetValue(crossLinkUri.Scheme, out var repository)) + return repository.PathPrefix ?? $"reference/{repository.Name}"; + return $"reference/{crossLinkUri.Scheme}"; } + var lookup = crossLinkUri.ToString().AsSpan(); + if (lookup.EndsWith(".md", StringComparison.Ordinal)) + lookup = lookup[..^3]; - if (source is not null) + string? match = null; + foreach (var prefix in TableOfContentsPrefixes) + { + if (!lookup.StartsWith(prefix, StringComparison.Ordinal)) + continue; + match = prefix; + break; + } + if (match is null || !_tocLookup.TryGetValue(match, out var toc)) { - return new TableOfContentsReference - { - Source = source, - Children = children ?? [], - PathPrefix = pathPrefix - }; + _context.Collector.EmitError(_context.NavigationPath, $"Unable to find defined toc for url: {crossLinkUri}"); + return $"reference/{crossLinkUri.Scheme}"; } + path = path.AsSpan().TrimStart(toc.SourcePrefix).ToString(); - return null; + return toc.PathPrefix; } } diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs new file mode 100644 index 000000000..8b3466768 --- /dev/null +++ b/src/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -0,0 +1,154 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Frozen; +using Documentation.Assembler.Configuration; +using Elastic.Markdown.IO.Configuration; +using YamlDotNet.RepresentationModel; + +namespace Documentation.Assembler.Navigation; + +public record TableOfContentsReference +{ + public required Uri Source { get; init; } + public required string SourcePrefix { get; init; } + public required string PathPrefix { get; init; } + public required IReadOnlyCollection Children { get; init; } +} + +public record GlobalNavigationFile +{ + public IReadOnlyCollection TableOfContents { get; init; } = []; + + public FrozenDictionary IndexedTableOfContents { get; init; } = + new Dictionary().ToFrozenDictionary(); + + public static GlobalNavigationFile Deserialize(AssembleContext context) + { + var globalConfig = new GlobalNavigationFile(); + var reader = new YamlStreamReader(context.NavigationPath, context.Collector); + try + { + foreach (var entry in reader.Read()) + { + switch (entry.Key) + { + case "toc": + var toc = ReadChildren(reader, entry.Entry); + var indexed = toc + .SelectMany(YieldAll) + .ToDictionary(t => t.Source.ToString(), t => t) + .ToFrozenDictionary(); + globalConfig = globalConfig with + { + TableOfContents = toc, + IndexedTableOfContents = indexed + }; + break; + } + } + } + catch (Exception e) + { + reader.EmitError("Could not load docset.yml", e); + throw; + } + + return globalConfig; + } + + private static IEnumerable YieldAll(TableOfContentsReference toc) + { + foreach (var tocEntry in toc.Children) + { + yield return tocEntry; + foreach (var child in YieldAll(tocEntry)) + yield return child; + } + } + + private static IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyValuePair entry) + { + var entries = new List(); + if (entry.Value is not YamlSequenceNode sequence) + { + if (entry.Key is YamlScalarNode scalarKey) + { + var key = scalarKey.Value; + reader.EmitWarning($"'{key}' is not an array"); + } + else + reader.EmitWarning($"'{entry.Key}' is not an array"); + + return entries; + } + + foreach (var tocEntry in sequence.Children.OfType()) + { + var child = ReadChild(reader, tocEntry); + if (child is not null) + entries.Add(child); + } + + //TableOfContents = entries; + return entries; + } + + private static TableOfContentsReference? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry) + { + string? source = null; + string? pathPrefix = null; + IReadOnlyCollection? children = null; + Uri? sourceUri = null; + foreach (var entry in tocEntry.Children) + { + var key = ((YamlScalarNode)entry.Key).Value; + switch (key) + { + case "toc": + source = reader.ReadString(entry); + if (source.AsSpan().IndexOf("://") == -1) + source = $"{NarrativeRepository.RepositoryName}://{source}"; + break; + case "path_prefix": + pathPrefix = reader.ReadString(entry); + break; + case "children": + if (source is null && pathPrefix is null) + { + reader.EmitWarning("toc entry has no toc or path_prefix defined"); + continue; + } + if (string.IsNullOrEmpty(pathPrefix) && !Uri.TryCreate(source, UriKind.Absolute, out sourceUri)) + { + reader.EmitWarning($"Source toc entry is not a valid uri: {source}"); + return null; + } + children = ReadChildren(reader, entry); + break; + } + } + + if (source is null) + return null; + + if (!Uri.TryCreate(source, UriKind.Absolute, out sourceUri)) + { + reader.EmitWarning($"Source toc entry is not a valid uri: {source}"); + return null; + } + + var sourcePrefix = $"{sourceUri.Host}/{sourceUri.AbsolutePath.TrimStart('/')}"; + pathPrefix ??= sourcePrefix; + + return new TableOfContentsReference + { + Source = sourceUri, + SourcePrefix = sourcePrefix, + Children = children ?? [], + PathPrefix = pathPrefix + }; + + } +} diff --git a/src/docs-assembler/assembler.yml b/src/docs-assembler/assembler.yml index f65c5df7e..99784ebd3 100644 --- a/src/docs-assembler/assembler.yml +++ b/src/docs-assembler/assembler.yml @@ -1,6 +1,6 @@ environments: production: - uri: https://elastic.co + uri: https://www.elastic.co path_prefix: docs allow_indexing: false staging: diff --git a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs index 9f9cc894b..39b99ca04 100644 --- a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs +++ b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs @@ -12,7 +12,7 @@ namespace Elastic.Markdown.Tests; public class TestCrossLinkResolver : ICrossLinkResolver { - private readonly IUriEnvironmentResolver _uriResolver = new PreviewEnvironmentUriResolver(); + private readonly IUriEnvironmentResolver _uriResolver = new IsolatedBuildEnvironmentUriResolver(); private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty; private Dictionary LinkReferences { get; } = []; private HashSet DeclaredRepositories { get; } = []; diff --git a/tests/authoring/Framework/TestCrossLinkResolver.fs b/tests/authoring/Framework/TestCrossLinkResolver.fs index f43202ad6..0e668343a 100644 --- a/tests/authoring/Framework/TestCrossLinkResolver.fs +++ b/tests/authoring/Framework/TestCrossLinkResolver.fs @@ -18,7 +18,7 @@ type TestCrossLinkResolver (config: ConfigurationFile) = let references = Dictionary() let declared = HashSet() - let uriResolver = PreviewEnvironmentUriResolver() + let uriResolver = IsolatedBuildEnvironmentUriResolver() member this.LinkReferences = references member this.DeclaredRepositories = declared diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs index 67313bbf1..ff6b4f120 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs @@ -3,7 +3,11 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; +using Documentation.Assembler.Building; +using Documentation.Assembler.Configuration; using Documentation.Assembler.Navigation; +using Documentation.Assembler.Sourcing; using Elastic.Markdown.Diagnostics; using FluentAssertions; @@ -11,6 +15,7 @@ namespace Documentation.Assembler.Tests; public class GlobalNavigationTests { + [Fact] public async Task ParsesGlobalNavigation() { @@ -18,7 +23,54 @@ public async Task ParsesGlobalNavigation() _ = collector.StartAsync(TestContext.Current.CancellationToken); var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null); - var globalNavigation = GlobalNavigation.Deserialize(assembleContext); - globalNavigation.References.Should().NotBeNull().And.NotBeEmpty(); + var globalNavigation = GlobalNavigationFile.Deserialize(assembleContext); + globalNavigation.TableOfContents.Should().NotBeNull().And.NotBeEmpty(); + } + + [Fact] + public async Task UriResolving() + { + await using var collector = new DiagnosticsCollector([]); + _ = collector.StartAsync(TestContext.Current.CancellationToken); + + var fs = new FileSystem(); + var assembleContext = new AssembleContext(collector, fs, fs, null, null); + var globalNavigationFile = GlobalNavigationFile.Deserialize(assembleContext); + globalNavigationFile.TableOfContents.Should().NotBeNull().And.NotBeEmpty(); + var globalNavigation = new GlobalNavigation(assembleContext, globalNavigationFile, [ + new Checkout + { + Repository = new Repository { Name = "docs-content", Origin = "elastic/docs-content"}, + HeadReference = Guid.NewGuid().ToString(), + Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "docs-content")) + }, + new Checkout + { + Repository = new Repository { Name = "curator", Origin = "elastic/curator"}, + HeadReference = Guid.NewGuid().ToString(), + Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "curator")) + }, + new Checkout + { + Repository = new Repository { Name = "elasticsearch-net", Origin = "elastic/elasticsearch-net"}, + HeadReference = Guid.NewGuid().ToString(), + Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "elasticsearch-net")) + } + ]); + + var env = assembleContext.Configuration.Environments["production"]; + var uriResolver = new PublishEnvironmentUriResolver(globalNavigation, env); + + // docs-content://reference/apm/something.md - url hasn't changed + var resolvedURi = uriResolver.Resolve(new Uri("docs-content://reference/apm/something.md"), "/reference/apm/something"); + resolvedURi.Should().Be("https://www.elastic.co/docs/reference/apm/something"); + + + resolvedURi = uriResolver.Resolve(new Uri("curator://reference/a/file.md"), "/reference/a/file"); + resolvedURi.Should().Be("https://www.elastic.co/docs/reference/elasticsearch/curator/a/file"); + + //path_prefix: reference/elasticsearch/clients/net + resolvedURi = uriResolver.Resolve(new Uri("elasticsearch-net://reference/b/file.md"), "/reference/b/file"); + resolvedURi.Should().Be("https://www.elastic.co/docs/reference/elasticsearch/clients/net/b/file"); } } From 8c76c0a018af5581375644442f1bca4b06a78ba9 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 18 Mar 2025 16:35:13 +0100 Subject: [PATCH 03/13] path_prefix is mandatory except for top level docs-content references --- src/docs-assembler/Navigation/GlobalNavigationFile.cs | 11 ++++++++--- src/docs-assembler/navigation.yml | 7 ++++++- .../src/docs-assembler.Tests/GlobalNavigationTests.cs | 10 +++++++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs index 8b3466768..2a757ca02 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -100,7 +100,7 @@ private static IReadOnlyCollection ReadChildren(YamlSt string? source = null; string? pathPrefix = null; IReadOnlyCollection? children = null; - Uri? sourceUri = null; + Uri? sourceUri; foreach (var entry in tocEntry.Children) { var key = ((YamlScalarNode)entry.Key).Value; @@ -109,7 +109,10 @@ private static IReadOnlyCollection ReadChildren(YamlSt case "toc": source = reader.ReadString(entry); if (source.AsSpan().IndexOf("://") == -1) + { source = $"{NarrativeRepository.RepositoryName}://{source}"; + pathPrefix = source; + } break; case "path_prefix": pathPrefix = reader.ReadString(entry); @@ -135,11 +138,13 @@ private static IReadOnlyCollection ReadChildren(YamlSt if (!Uri.TryCreate(source, UriKind.Absolute, out sourceUri)) { - reader.EmitWarning($"Source toc entry is not a valid uri: {source}"); + reader.EmitError($"Source toc entry is not a valid uri: {source}", tocEntry); return null; } - var sourcePrefix = $"{sourceUri.Host}/{sourceUri.AbsolutePath.TrimStart('/')}"; + if (string.IsNullOrEmpty(pathPrefix)) + reader.EmitError($"Path prefix is not defined for: {source}, falling back to {sourcePrefix} which may be incorrect", tocEntry); + pathPrefix ??= sourcePrefix; return new TableOfContentsReference diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml index a01ca9f5a..b0efe5bc5 100644 --- a/src/docs-assembler/navigation.yml +++ b/src/docs-assembler/navigation.yml @@ -25,10 +25,14 @@ toc: - toc: extend children: - toc: kibana://extend + path_prefix: extend/kibana - toc: logstash://extend + path_prefix: extend/logstash - toc: elasticsearch://extend - #- toc: beats://extend + path_prefix: extend/elasticsearch - toc: integrations://extend + path_prefix: extend/integrations + #- toc: beats://extend ################# # RELEASE NOTES # @@ -78,6 +82,7 @@ toc: # Elasticsearch and index management # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/elasticsearch/toc.yml - toc: docs-content://reference/elasticsearch + path_prefix: reference/elasticsearch children: # 📝 TO DO: Directory depth in elasticsearch repo: would require toc 2 levels down # 🔜 https://github.com/colleenmcginnis/elasticsearch/blob/docs-assembler-prep/docs/reference/elasticsearch/toc.yml diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs index ff6b4f120..f4f05d0e7 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs @@ -55,6 +55,12 @@ public async Task UriResolving() Repository = new Repository { Name = "elasticsearch-net", Origin = "elastic/elasticsearch-net"}, HeadReference = Guid.NewGuid().ToString(), Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "elasticsearch-net")) + }, + new Checkout + { + Repository = new Repository { Name = "elasticsearch", Origin = "elastic/elasticsearch"}, + HeadReference = Guid.NewGuid().ToString(), + Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "elasticsearch")) } ]); @@ -69,8 +75,10 @@ public async Task UriResolving() resolvedURi = uriResolver.Resolve(new Uri("curator://reference/a/file.md"), "/reference/a/file"); resolvedURi.Should().Be("https://www.elastic.co/docs/reference/elasticsearch/curator/a/file"); - //path_prefix: reference/elasticsearch/clients/net resolvedURi = uriResolver.Resolve(new Uri("elasticsearch-net://reference/b/file.md"), "/reference/b/file"); resolvedURi.Should().Be("https://www.elastic.co/docs/reference/elasticsearch/clients/net/b/file"); + + resolvedURi = uriResolver.Resolve(new Uri("elasticsearch://extend/c/file.md"), "/extend/c/file"); + resolvedURi.Should().Be("https://www.elastic.co/docs/extend/elasticsearch/c/file"); } } From 1c76027dc783f25279c36ab28e235e34e5fc4124 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 18 Mar 2025 18:27:37 +0100 Subject: [PATCH 04/13] stage commit --- src/Elastic.Markdown/BuildContext.cs | 9 ---- .../DocumentationGenerator.cs | 23 +++++++++-- src/Elastic.Markdown/IO/DocumentationSet.cs | 3 +- src/Elastic.Markdown/Slices/HtmlWriter.cs | 1 - src/Elastic.Markdown/Slices/Index.cshtml | 1 - src/Elastic.Markdown/Slices/_ViewModels.cs | 4 +- .../Building/AssemblerBuilder.cs | 29 ++++++------- .../Building/AssemblerCrossLinkFetcher.cs | 8 ++-- .../Configuration/Repository.cs | 9 ---- .../Navigation/GlobalNavigation.cs | 34 +++++++++++---- .../Navigation/GlobalNavigationFile.cs | 12 +----- .../Sourcing/RepositorySourcesFetcher.cs | 7 ---- src/docs-assembler/assembler.yml | 3 ++ src/docs-assembler/navigation.yml | 6 ++- src/docs-builder/Cli/Commands.cs | 3 +- .../OutputDirectoryTests.cs | 1 + tests/authoring/Framework/Setup.fs | 2 +- tests/authoring/Framework/TestValues.fs | 1 + .../GlobalNavigationTests.cs | 41 +++++++------------ 19 files changed, 96 insertions(+), 101 deletions(-) diff --git a/src/Elastic.Markdown/BuildContext.cs b/src/Elastic.Markdown/BuildContext.cs index 7edcb02a8..e5f8a2c77 100644 --- a/src/Elastic.Markdown/BuildContext.cs +++ b/src/Elastic.Markdown/BuildContext.cs @@ -39,15 +39,6 @@ public string? UrlPathPrefix init => _urlPathPrefix = value; } - private readonly string? _staticUrlPathPrefix; - public string? StaticUrlPathPrefix - { - get => !string.IsNullOrWhiteSpace(_staticUrlPathPrefix) - ? $"/{_staticUrlPathPrefix.Trim('/')}" - : UrlPathPrefix; - init => _staticUrlPathPrefix = value; - } - public BuildContext(IFileSystem fileSystem) : this(new DiagnosticsCollector([]), fileSystem, fileSystem, null, null) { } diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 0a5dfacf2..e2d7aaf7d 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -20,8 +20,14 @@ public interface IConversionCollector void Collect(MarkdownFile file, MarkdownDocument document, string html); } +public interface IDocumentationFileOutputProvider +{ + IFileInfo OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath); +} + public class DocumentationGenerator { + private readonly IDocumentationFileOutputProvider? _documentationFileOutputProvider; private readonly ILogger _logger; private readonly IFileSystem _writeFileSystem; private readonly IDocumentationFileExporter _documentationFileExporter; @@ -34,10 +40,12 @@ public class DocumentationGenerator public DocumentationGenerator( DocumentationSet docSet, ILoggerFactory logger, + IDocumentationFileOutputProvider? documentationFileOutputProvider = null, IDocumentationFileExporter? documentationExporter = null, IConversionCollector? conversionCollector = null ) { + _documentationFileOutputProvider = documentationFileOutputProvider; _writeFileSystem = docSet.Build.WriteFileSystem; _logger = logger.CreateLogger(nameof(DocumentationGenerator)); @@ -89,17 +97,19 @@ public async Task GenerateAll(Cancel ctx) await ExtractEmbeddedStaticResources(ctx); - _logger.LogInformation($"Completing diagnostics channel"); - Context.Collector.Channel.TryComplete(); - _logger.LogInformation($"Generating documentation compilation state"); await GenerateDocumentationState(ctx); _logger.LogInformation($"Generating links.json"); await GenerateLinkReference(ctx); + } + public async Task StopDiagnosticCollection(Cancel ctx) + { _logger.LogInformation($"Completing diagnostics channel"); + Context.Collector.Channel.TryComplete(); + _logger.LogInformation($"Completing diagnostics channel"); await Context.Collector.StopAsync(ctx); _logger.LogInformation($"Completed diagnostics channel"); @@ -173,7 +183,12 @@ private async Task ProcessFile(HashSet offendingFiles, DocumentationFile private IFileInfo OutputFile(string relativePath) { var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputDirectory.FullName, relativePath)); - return outputFile; + if (relativePath.StartsWith("_static")) + return outputFile; + + return _documentationFileOutputProvider is not null + ? _documentationFileOutputProvider.OutputFile(DocumentationSet, outputFile, relativePath) + : outputFile; } private bool CompilationNotNeeded(GenerationState? generationState, out HashSet offendingFiles, diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 156998c38..6c24a054a 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -10,6 +10,7 @@ using Elastic.Markdown.Extensions; using Elastic.Markdown.Extensions.DetectionRules; using Elastic.Markdown.IO.Configuration; +using Elastic.Markdown.IO.Discovery; using Elastic.Markdown.IO.Navigation; using Elastic.Markdown.Myst; using Microsoft.Extensions.Logging; @@ -81,7 +82,7 @@ public DocumentationSet(BuildContext build, ILoggerFactory logger, ICrossLinkRes }; MarkdownParser = new MarkdownParser(build, resolver); - Name = SourceDirectory.FullName; + Name = Build.Git.RepositoryName ?? "unavailable"; OutputStateFile = OutputDirectory.FileSystem.FileInfo.New(Path.Combine(OutputDirectory.FullName, ".doc.state")); LinkReferenceFile = OutputDirectory.FileSystem.FileInfo.New(Path.Combine(OutputDirectory.FullName, "links.json")); diff --git a/src/Elastic.Markdown/Slices/HtmlWriter.cs b/src/Elastic.Markdown/Slices/HtmlWriter.cs index edf61aeb0..b960617ca 100644 --- a/src/Elastic.Markdown/Slices/HtmlWriter.cs +++ b/src/Elastic.Markdown/Slices/HtmlWriter.cs @@ -98,7 +98,6 @@ public async Task RenderLayout(MarkdownFile markdown, MarkdownDocument d TopLevelNavigationItems = [.. topLevelNavigationItems], NavigationHtml = navigationHtml, UrlPathPrefix = markdown.UrlPathPrefix, - StaticUrlPathPrefix = DocumentationSet.Build.StaticUrlPathPrefix, Applies = markdown.YamlFrontMatter?.AppliesTo, GithubEditUrl = editUrl, AllowIndexing = DocumentationSet.Build.AllowIndexing && !markdown.Hidden, diff --git a/src/Elastic.Markdown/Slices/Index.cshtml b/src/Elastic.Markdown/Slices/Index.cshtml index e4b84c90a..ec51e5529 100644 --- a/src/Elastic.Markdown/Slices/Index.cshtml +++ b/src/Elastic.Markdown/Slices/Index.cshtml @@ -13,7 +13,6 @@ NavigationHtml = Model.NavigationHtml, TopLevelNavigationItems = Model.TopLevelNavigationItems, UrlPathPrefix = Model.UrlPathPrefix, - StaticUrlPathPrefix = Model.StaticUrlPathPrefix, GithubEditUrl = Model.GithubEditUrl, AllowIndexing = Model.AllowIndexing, Features = Model.Features diff --git a/src/Elastic.Markdown/Slices/_ViewModels.cs b/src/Elastic.Markdown/Slices/_ViewModels.cs index 503855b82..6fffe5a16 100644 --- a/src/Elastic.Markdown/Slices/_ViewModels.cs +++ b/src/Elastic.Markdown/Slices/_ViewModels.cs @@ -23,7 +23,6 @@ public class IndexViewModel public required string NavigationHtml { get; init; } public required string? UrlPathPrefix { get; init; } - public required string? StaticUrlPathPrefix { get; init; } public required string? GithubEditUrl { get; init; } public required ApplicableTo? Applies { get; init; } public required bool AllowIndexing { get; init; } @@ -45,7 +44,6 @@ public class LayoutViewModel public required MarkdownFile? Previous { get; init; } public required MarkdownFile? Next { get; init; } public required string NavigationHtml { get; init; } - public required string? StaticUrlPathPrefix { get; init; } public required string? UrlPathPrefix { get; init; } public required string? GithubEditUrl { get; init; } public required bool AllowIndexing { get; init; } @@ -69,7 +67,7 @@ public MarkdownFile[] Parents public string Static(string path) { path = $"_static/{path.TrimStart('/')}"; - return $"{StaticUrlPathPrefix}/{path}"; + return $"{UrlPathPrefix}/{path}"; } public string Link(string path) diff --git a/src/docs-assembler/Building/AssemblerBuilder.cs b/src/docs-assembler/Building/AssemblerBuilder.cs index 8d312b40f..0dd681c36 100644 --- a/src/docs-assembler/Building/AssemblerBuilder.cs +++ b/src/docs-assembler/Building/AssemblerBuilder.cs @@ -14,8 +14,6 @@ namespace Documentation.Assembler.Building; public class AssemblerBuilder(ILoggerFactory logger, AssembleContext context, GlobalNavigation globalNavigation) { - private readonly ILogger _logger = logger.CreateLogger(); - public async Task BuildAllAsync(IReadOnlyCollection checkouts, PublishEnvironment environment, Cancel ctx) { var crossLinkFetcher = new AssemblerCrossLinkFetcher(logger, context.Configuration); @@ -24,14 +22,19 @@ public async Task BuildAllAsync(IReadOnlyCollection checkouts, Publish foreach (var checkout in checkouts) { + if (checkout.Repository.Skip) + { + context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Skipping {checkout.Repository.Origin} as its not yet been migrated to V3"); + continue; + } + try { await BuildAsync(checkout, environment, crossLinkResolver, ctx); } catch (Exception e) when (e.Message.Contains("Can not locate docset.yml file in")) { - // TODO: we should only ignore this temporarily while migration is ongoing - _logger.LogWarning("Skipping {Checkout} as its not yet been migrated to V3", checkout.Directory.FullName); + context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Skipping {checkout.Repository.Origin} as its not yet been migrated to V3"); } catch (Exception e) { @@ -39,31 +42,25 @@ public async Task BuildAllAsync(IReadOnlyCollection checkouts, Publish throw; } } + + context.Collector.Channel.TryComplete(); + await context.Collector.StopAsync(ctx); } private async Task BuildAsync(Checkout checkout, PublishEnvironment environment, CrossLinkResolver crossLinkResolver, Cancel ctx) { var path = checkout.Directory.FullName; - var localPathPrefix = checkout.Repository.PathPrefix; - var pathPrefix = (environment.PathPrefix, localPathPrefix) switch - { - (null or "", null or "") => null, - (null or "", _) => localPathPrefix, - (_, null or "") => environment.PathPrefix, - var (globalPrefix, docsetPrefix) => $"{globalPrefix}/{docsetPrefix}" - }; - var output = localPathPrefix != null ? Path.Combine(context.OutputDirectory.FullName, localPathPrefix) : context.OutputDirectory.FullName; + var output = environment.PathPrefix != null ? Path.Combine(context.OutputDirectory.FullName, environment.PathPrefix) : context.OutputDirectory.FullName; var buildContext = new BuildContext(context.Collector, context.ReadFileSystem, context.WriteFileSystem, path, output) { - StaticUrlPathPrefix = environment.PathPrefix, - UrlPathPrefix = pathPrefix, + UrlPathPrefix = environment.PathPrefix, Force = true, AllowIndexing = environment.AllowIndexing }; var set = new DocumentationSet(buildContext, logger, crossLinkResolver); - var generator = new DocumentationGenerator(set, logger); + var generator = new DocumentationGenerator(set, logger, globalNavigation); await generator.GenerateAll(ctx); } } diff --git a/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs b/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs index b5e30c720..4f9ac338f 100644 --- a/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs +++ b/src/docs-assembler/Building/AssemblerCrossLinkFetcher.cs @@ -21,10 +21,12 @@ public override async Task Fetch() foreach (var repository in repositories) { - if (repository.Skip) - continue; var repositoryName = repository.Name; _ = declaredRepositories.Add(repositoryName); + + if (repository.Skip) + continue; + var linkReference = await Fetch(repositoryName); linkReferences.Add(repositoryName, linkReference); var linkIndexReference = await GetLinkIndexEntry(repositoryName); @@ -36,7 +38,7 @@ public override async Task Fetch() DeclaredRepositories = declaredRepositories, LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(), LinkReferences = linkReferences.ToFrozenDictionary(), - FromConfiguration = true + FromConfiguration = false }; } } diff --git a/src/docs-assembler/Configuration/Repository.cs b/src/docs-assembler/Configuration/Repository.cs index 68e9aaa5b..46ad8462f 100644 --- a/src/docs-assembler/Configuration/Repository.cs +++ b/src/docs-assembler/Configuration/Repository.cs @@ -10,7 +10,6 @@ public record NarrativeRepository : Repository { public static string RepositoryName { get; } = "docs-content"; public override string Name { get; set; } = RepositoryName; - public override string? PathPrefix { get; set; } } public record Repository @@ -27,14 +26,6 @@ public record Repository [YamlMember(Alias = "checkout_strategy")] public string CheckoutStrategy { get; set; } = "partial"; - private string? _pathPrefix; - [YamlMember(Alias = "path_prefix")] - public virtual string? PathPrefix - { - get => _pathPrefix ?? $"reference/{Name}"; - set => _pathPrefix = value; - } - [YamlMember(Alias = "skip")] public bool Skip { get; set; } } diff --git a/src/docs-assembler/Navigation/GlobalNavigation.cs b/src/docs-assembler/Navigation/GlobalNavigation.cs index 816eb04e2..317bd20ee 100644 --- a/src/docs-assembler/Navigation/GlobalNavigation.cs +++ b/src/docs-assembler/Navigation/GlobalNavigation.cs @@ -4,13 +4,16 @@ using System.Collections.Frozen; using System.Collections.Immutable; +using System.IO.Abstractions; using Documentation.Assembler.Configuration; using Documentation.Assembler.Sourcing; +using Elastic.Markdown; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.IO; namespace Documentation.Assembler.Navigation; -public record GlobalNavigation +public record GlobalNavigation : IDocumentationFileOutputProvider { private readonly AssembleContext _context; private readonly FrozenDictionary.AlternateLookup> _checkoutsLookup; @@ -45,13 +48,13 @@ public GlobalNavigation(AssembleContext context, GlobalNavigationFile navigation public string GetSubPath(Uri crossLinkUri, ref string path) { - if (!_checkoutsLookup.TryGetValue(crossLinkUri.Scheme, out var checkout)) + if (!_checkoutsLookup.TryGetValue(crossLinkUri.Scheme, out _)) { - _context.Collector.EmitError(_context.NavigationPath, $"Unable to find checkout for repository: {crossLinkUri.Scheme}"); - // fallback to repository path prefix - if (_repoConfigLookup.TryGetValue(crossLinkUri.Scheme, out var repository)) - return repository.PathPrefix ?? $"reference/{repository.Name}"; - return $"reference/{crossLinkUri.Scheme}"; + _context.Collector.EmitError(_context.ConfigurationPath, + !_repoConfigLookup.TryGetValue(crossLinkUri.Scheme, out _) + ? $"Repository: '{crossLinkUri.Scheme}' is not defined in assembler.yml" + : $"Unable to find checkout for repository: {crossLinkUri.Scheme}" + ); } var lookup = crossLinkUri.ToString().AsSpan(); if (lookup.EndsWith(".md", StringComparison.Ordinal)) @@ -74,4 +77,21 @@ public string GetSubPath(Uri crossLinkUri, ref string path) return toc.PathPrefix; } + + public IFileInfo OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath) + { + if (relativePath.StartsWith("_static/", StringComparison.Ordinal)) + return defaultOutputFile; + + var outputDirectory = documentationSet.OutputDirectory; + var match = TableOfContentsPrefixes.FirstOrDefault(prefix => relativePath.StartsWith(prefix, StringComparison.Ordinal)); + if (match is null || !_tocLookup.TryGetValue(match, out var toc)) + { + _context.Collector.EmitError(_context.NavigationPath, $"Unable to find defined toc for output file: {documentationSet.Name}://{relativePath}"); + return defaultOutputFile; + } + var fs = defaultOutputFile.FileSystem; + var path = fs.Path.Combine(outputDirectory.FullName, toc.PathPrefix, relativePath); + return fs.FileInfo.New(path); + } } diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs index 2a757ca02..728ec6098 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -60,9 +60,9 @@ public static GlobalNavigationFile Deserialize(AssembleContext context) private static IEnumerable YieldAll(TableOfContentsReference toc) { + yield return toc; foreach (var tocEntry in toc.Children) { - yield return tocEntry; foreach (var child in YieldAll(tocEntry)) yield return child; } @@ -90,8 +90,6 @@ private static IReadOnlyCollection ReadChildren(YamlSt if (child is not null) entries.Add(child); } - - //TableOfContents = entries; return entries; } @@ -100,7 +98,6 @@ private static IReadOnlyCollection ReadChildren(YamlSt string? source = null; string? pathPrefix = null; IReadOnlyCollection? children = null; - Uri? sourceUri; foreach (var entry in tocEntry.Children) { var key = ((YamlScalarNode)entry.Key).Value; @@ -123,11 +120,6 @@ private static IReadOnlyCollection ReadChildren(YamlSt reader.EmitWarning("toc entry has no toc or path_prefix defined"); continue; } - if (string.IsNullOrEmpty(pathPrefix) && !Uri.TryCreate(source, UriKind.Absolute, out sourceUri)) - { - reader.EmitWarning($"Source toc entry is not a valid uri: {source}"); - return null; - } children = ReadChildren(reader, entry); break; } @@ -136,7 +128,7 @@ private static IReadOnlyCollection ReadChildren(YamlSt if (source is null) return null; - if (!Uri.TryCreate(source, UriKind.Absolute, out sourceUri)) + if (!Uri.TryCreate(source, UriKind.Absolute, out var sourceUri)) { reader.EmitError($"Source toc entry is not a valid uri: {source}", tocEntry); return null; diff --git a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index 80aa4b515..a4aa96cc3 100644 --- a/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -44,13 +44,6 @@ public async Task> AcquireAllLatest(Cancel ctx = d var dict = new ConcurrentDictionary(); var checkouts = new ConcurrentBag(); - if (context.OutputDirectory.Exists) - { - _logger.LogInformation("Cleaning output directory: {OutputDirectory}", context.OutputDirectory.FullName); - context.OutputDirectory.Delete(true); - } - - _logger.LogInformation("Cloning narrative content: {Repository}", NarrativeRepository.RepositoryName); var checkout = CloneOrUpdateRepository(Configuration.Narrative, NarrativeRepository.RepositoryName, dict); checkouts.Add(checkout); diff --git a/src/docs-assembler/assembler.yml b/src/docs-assembler/assembler.yml index 99784ebd3..453c11925 100644 --- a/src/docs-assembler/assembler.yml +++ b/src/docs-assembler/assembler.yml @@ -15,6 +15,9 @@ environments: narrative: checkout_strategy: full references: + #TODO TEMPORARY REMOVE + asciidocalypse: + skip: true apm-agent-android: apm-agent-dotnet: apm-agent-go: diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml index b0efe5bc5..2f3bcd213 100644 --- a/src/docs-assembler/navigation.yml +++ b/src/docs-assembler/navigation.yml @@ -41,7 +41,9 @@ toc: - toc: release-notes children: - toc: elasticsearch://release-notes + path_prefix: release-notes/elasticsearch - toc: docs-content://release-notes/apm + path_prefix: release-notes/apm ############# # REFERENCE # @@ -338,6 +340,7 @@ toc: # ECS # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/ecs/toc.yml - toc: docs-content://reference/ecs + path_prefix: reference/ecs children: # ECS reference # ✅ https://github.com/elastic/ecs/blob/main/docs/reference/toc.yml @@ -449,4 +452,5 @@ toc: # Glossary # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/glossary/toc.yml - - toc: docs-content://reference/glossary \ No newline at end of file + - toc: docs-content://reference/glossary + path_prefix: reference/glossary \ No newline at end of file diff --git a/src/docs-builder/Cli/Commands.cs b/src/docs-builder/Cli/Commands.cs index c7e87c5a7..0f5b03bdd 100644 --- a/src/docs-builder/Cli/Commands.cs +++ b/src/docs-builder/Cli/Commands.cs @@ -133,8 +133,9 @@ public async Task Generate( metadataOnly ??= metaValue; var exporter = metadataOnly.HasValue && metadataOnly.Value ? new NoopDocumentationFileExporter() : null; - var generator = new DocumentationGenerator(set, logger, exporter); + var generator = new DocumentationGenerator(set, logger, null, exporter); await generator.GenerateAll(ctx); + await generator.StopDiagnosticCollection(ctx); if (runningOnCi) await githubActionsService.SetOutputAsync("landing-page-path", set.MarkdownFiles.First().Value.Url); if (bool.TryParse(githubActionsService.GetInput("strict"), out var strictValue) && strictValue) diff --git a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs index e11aadfcf..0f6decbb0 100644 --- a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs +++ b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs @@ -27,6 +27,7 @@ public async Task CreatesDefaultOutputDirectory() var generator = new DocumentationGenerator(set, logger); await generator.GenerateAll(TestContext.Current.CancellationToken); + await generator.StopDiagnosticCollection(TestContext.Current.CancellationToken); fileSystem.Directory.Exists(".artifacts").Should().BeTrue(); diff --git a/tests/authoring/Framework/Setup.fs b/tests/authoring/Framework/Setup.fs index 7c179b915..36ea48380 100644 --- a/tests/authoring/Framework/Setup.fs +++ b/tests/authoring/Framework/Setup.fs @@ -102,7 +102,7 @@ type Setup = let conversionCollector = TestConversionCollector() let linkResolver = TestCrossLinkResolver(context.Configuration) let set = DocumentationSet(context, logger, linkResolver); - let generator = DocumentationGenerator(set, logger, null, conversionCollector) + let generator = DocumentationGenerator(set, logger, null, null, conversionCollector) let context = { Collector = collector diff --git a/tests/authoring/Framework/TestValues.fs b/tests/authoring/Framework/TestValues.fs index 7dc33ad07..c68c6c6b2 100644 --- a/tests/authoring/Framework/TestValues.fs +++ b/tests/authoring/Framework/TestValues.fs @@ -99,6 +99,7 @@ and MarkdownTestContext = member this.Bootstrap () = backgroundTask { let! ctx = Async.CancellationToken do! this.Generator.GenerateAll(ctx) + do! this.Generator.StopDiagnosticCollection(ctx) let results = this.ConversionCollector.Results diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs index f4f05d0e7..094d15ce4 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs @@ -15,7 +15,6 @@ namespace Documentation.Assembler.Tests; public class GlobalNavigationTests { - [Fact] public async Task ParsesGlobalNavigation() { @@ -25,8 +24,19 @@ public async Task ParsesGlobalNavigation() var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null); var globalNavigation = GlobalNavigationFile.Deserialize(assembleContext); globalNavigation.TableOfContents.Should().NotBeNull().And.NotBeEmpty(); + var docsContentKeys = globalNavigation.IndexedTableOfContents.Keys + .Where(k => k.StartsWith("docs-content://", StringComparison.Ordinal)).ToArray(); + docsContentKeys.Should().Contain("docs-content://solutions/"); } + public static Checkout CreateCheckout(IFileSystem fs, string name) => + new() + { + Repository = new Repository { Name = name, Origin = $"elastic/{name}" }, + HeadReference = Guid.NewGuid().ToString(), + Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", name)) + }; + [Fact] public async Task UriResolving() { @@ -37,32 +47,9 @@ public async Task UriResolving() var assembleContext = new AssembleContext(collector, fs, fs, null, null); var globalNavigationFile = GlobalNavigationFile.Deserialize(assembleContext); globalNavigationFile.TableOfContents.Should().NotBeNull().And.NotBeEmpty(); - var globalNavigation = new GlobalNavigation(assembleContext, globalNavigationFile, [ - new Checkout - { - Repository = new Repository { Name = "docs-content", Origin = "elastic/docs-content"}, - HeadReference = Guid.NewGuid().ToString(), - Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "docs-content")) - }, - new Checkout - { - Repository = new Repository { Name = "curator", Origin = "elastic/curator"}, - HeadReference = Guid.NewGuid().ToString(), - Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "curator")) - }, - new Checkout - { - Repository = new Repository { Name = "elasticsearch-net", Origin = "elastic/elasticsearch-net"}, - HeadReference = Guid.NewGuid().ToString(), - Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "elasticsearch-net")) - }, - new Checkout - { - Repository = new Repository { Name = "elasticsearch", Origin = "elastic/elasticsearch"}, - HeadReference = Guid.NewGuid().ToString(), - Directory = fs.DirectoryInfo.New(fs.Path.Combine(".artifacts", "checkouts", "elasticsearch")) - } - ]); + string[] repos = ["docs-content", "curator", "elasticsearch-net", "elasticsearch"]; + var checkouts = repos.Select(r => CreateCheckout(fs, r)).ToArray(); + var globalNavigation = new GlobalNavigation(assembleContext, globalNavigationFile, checkouts); var env = assembleContext.Configuration.Environments["production"]; var uriResolver = new PublishEnvironmentUriResolver(globalNavigation, env); From b627f974f86ae56f0dc1295d5259fb76ec5bde7b Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 18 Mar 2025 20:08:30 +0100 Subject: [PATCH 05/13] stage commit --- .../DocumentationGenerator.cs | 9 ++-- .../IO/Discovery/GitCheckoutInformation.cs | 14 ++++-- .../Navigation/GlobalNavigation.cs | 27 +++++++++-- .../Navigation/GlobalNavigationFile.cs | 47 +++++++++++++------ src/docs-assembler/navigation.yml | 14 ++++-- 5 files changed, 78 insertions(+), 33 deletions(-) diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 6b3e62f43..ef817dc35 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -22,7 +22,7 @@ public interface IConversionCollector public interface IDocumentationFileOutputProvider { - IFileInfo OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath); + IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath); } public class DocumentationGenerator @@ -180,6 +180,8 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx) var path = a.Replace("Elastic.Markdown.", "").Replace("_static.", $"_static{Path.DirectorySeparatorChar}"); var outputFile = OutputFile(path); + if (outputFile is null) + continue; await _documentationFileExporter.CopyEmbeddedResource(outputFile, resourceStream, ctx); _logger.LogDebug("Copied static embedded resource {Path}", path); } @@ -197,10 +199,11 @@ private async Task ProcessFile(HashSet offendingFiles, DocumentationFile _logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName); var outputFile = OutputFile(file.RelativePath); - await _documentationFileExporter.ProcessFile(file, outputFile, token); + if (outputFile is not null) + await _documentationFileExporter.ProcessFile(file, outputFile, token); } - private IFileInfo OutputFile(string relativePath) + private IFileInfo? OutputFile(string relativePath) { var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputDirectory.FullName, relativePath)); if (relativePath.StartsWith("_static")) diff --git a/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs b/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs index 0d6996a57..45a23140a 100644 --- a/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs +++ b/src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs @@ -46,21 +46,25 @@ public static GitCheckoutInformation Create(IDirectoryInfo source, IFileSystem f } var fakeRef = Guid.NewGuid().ToString()[..16]; - var gitConfig = Git(source, ".git/config"); + var gitConfig = Git(source, Path.Combine(".git", "config")); if (!gitConfig.Exists) { - logger?.LogInformation("Git checkout information not available."); - return Unavailable; + gitConfig = Git(source, Path.Combine("..", ".git", "config")); + if (!gitConfig.Exists) + { + logger?.LogInformation("Git checkout information not available."); + return Unavailable; + } } - var head = Read(source, ".git/HEAD") ?? fakeRef; + var head = Read(source, Path.Combine(".git", "HEAD")) ?? fakeRef; var gitRef = head; var branch = head.Replace("refs/heads/", string.Empty); //not detached HEAD if (head.StartsWith("ref:")) { head = head.Replace("ref: ", string.Empty); - gitRef = Read(source, ".git/" + head) ?? fakeRef; + gitRef = Read(source, Path.Combine(".git", head)) ?? fakeRef; branch = branch.Replace("ref: ", string.Empty); } else diff --git a/src/docs-assembler/Navigation/GlobalNavigation.cs b/src/docs-assembler/Navigation/GlobalNavigation.cs index 317bd20ee..c09e5f757 100644 --- a/src/docs-assembler/Navigation/GlobalNavigation.cs +++ b/src/docs-assembler/Navigation/GlobalNavigation.cs @@ -78,19 +78,36 @@ public string GetSubPath(Uri crossLinkUri, ref string path) return toc.PathPrefix; } - public IFileInfo OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath) + public IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath) { if (relativePath.StartsWith("_static/", StringComparison.Ordinal)) return defaultOutputFile; var outputDirectory = documentationSet.OutputDirectory; - var match = TableOfContentsPrefixes.FirstOrDefault(prefix => relativePath.StartsWith(prefix, StringComparison.Ordinal)); + var fs = defaultOutputFile.FileSystem; + var relativePathSpan = relativePath.AsSpan(); + + var lookup = $"{documentationSet.Name}://{relativePath}"; + var match = TableOfContentsPrefixes.FirstOrDefault(prefix => lookup.StartsWith(prefix, StringComparison.Ordinal)); if (match is null || !_tocLookup.TryGetValue(match, out var toc)) { - _context.Collector.EmitError(_context.NavigationPath, $"Unable to find defined toc for output file: {documentationSet.Name}://{relativePath}"); - return defaultOutputFile; + if (relativePath.StartsWith("raw-migrated-files/", StringComparison.Ordinal)) + return null; + if (relativePath.StartsWith("images/", StringComparison.Ordinal)) + return null; + if (relativePath.StartsWith("examples/", StringComparison.Ordinal)) + return null; + if (relativePath.StartsWith("docset.yml", StringComparison.Ordinal)) + return null; + if (relativePath.StartsWith("doc_examples", StringComparison.Ordinal)) + return null; + if (relativePath.EndsWith(".asciidoc", StringComparison.Ordinal)) + return null; + + var fallBack = fs.Path.Combine(outputDirectory.FullName, "_failed", documentationSet.Name, relativePath); + _context.Collector.EmitError(_context.NavigationPath, $"No toc for output path: '{lookup}' falling back to: '{fallBack}'"); + return fs.FileInfo.New(fallBack); } - var fs = defaultOutputFile.FileSystem; var path = fs.Path.Combine(outputDirectory.FullName, toc.PathPrefix, relativePath); return fs.FileInfo.New(path); } diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs index 728ec6098..e5059299a 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -35,7 +35,7 @@ public static GlobalNavigationFile Deserialize(AssembleContext context) switch (entry.Key) { case "toc": - var toc = ReadChildren(reader, entry.Entry); + var toc = ReadChildren(reader, entry.Entry, null); var indexed = toc .SelectMany(YieldAll) .ToDictionary(t => t.Source.ToString(), t => t) @@ -68,33 +68,34 @@ private static IEnumerable YieldAll(TableOfContentsRef } } - private static IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyValuePair entry) + private static IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyValuePair entry, string? parent) { var entries = new List(); - if (entry.Value is not YamlSequenceNode sequence) + if (entry.Key is not YamlScalarNode { Value: { } key } scalarKey) { - if (entry.Key is YamlScalarNode scalarKey) - { - var key = scalarKey.Value; - reader.EmitWarning($"'{key}' is not an array"); - } - else - reader.EmitWarning($"'{entry.Key}' is not an array"); + reader.EmitWarning($"key '{entry.Key}' is not string"); + return []; + } - return entries; + if (entry.Value is not YamlSequenceNode sequence) + { + reader.EmitWarning($"'{scalarKey.Value}' is not an array"); + return []; } foreach (var tocEntry in sequence.Children.OfType()) { - var child = ReadChild(reader, tocEntry); + var child = ReadChild(reader, tocEntry, parent); if (child is not null) entries.Add(child); } + return entries; } - private static TableOfContentsReference? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry) + private static TableOfContentsReference? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry, string? parent) { + string? repository = null; string? source = null; string? pathPrefix = null; IReadOnlyCollection? children = null; @@ -107,9 +108,14 @@ private static IReadOnlyCollection ReadChildren(YamlSt source = reader.ReadString(entry); if (source.AsSpan().IndexOf("://") == -1) { + parent = source; source = $"{NarrativeRepository.RepositoryName}://{source}"; pathPrefix = source; } + + break; + case "repo": + repository = reader.ReadString(entry); break; case "path_prefix": pathPrefix = reader.ReadString(entry); @@ -120,11 +126,22 @@ private static IReadOnlyCollection ReadChildren(YamlSt reader.EmitWarning("toc entry has no toc or path_prefix defined"); continue; } - children = ReadChildren(reader, entry); + + children = ReadChildren(reader, entry, parent); break; } } + if (repository is not null) + { + if (source is not null) + reader.EmitError($"toc config defines 'repo' can not be combined with 'toc': {source}", tocEntry); + if (children is not null) + reader.EmitError($"toc config defines 'repo' can not be combined with 'children'", tocEntry); + pathPrefix = string.Join("/", [parent, repository]); + source = $"{repository}://{parent}"; + } + if (source is null) return null; @@ -133,6 +150,7 @@ private static IReadOnlyCollection ReadChildren(YamlSt reader.EmitError($"Source toc entry is not a valid uri: {source}", tocEntry); return null; } + var sourcePrefix = $"{sourceUri.Host}/{sourceUri.AbsolutePath.TrimStart('/')}"; if (string.IsNullOrEmpty(pathPrefix)) reader.EmitError($"Path prefix is not defined for: {source}, falling back to {sourcePrefix} which may be incorrect", tocEntry); @@ -146,6 +164,5 @@ private static IReadOnlyCollection ReadChildren(YamlSt Children = children ?? [], PathPrefix = pathPrefix }; - } } diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml index 2f3bcd213..f33d7329d 100644 --- a/src/docs-assembler/navigation.yml +++ b/src/docs-assembler/navigation.yml @@ -40,10 +40,14 @@ toc: # I didn't touch this section (yet?) - toc: release-notes children: - - toc: elasticsearch://release-notes - path_prefix: release-notes/elasticsearch - - toc: docs-content://release-notes/apm - path_prefix: release-notes/apm + # repo is short for + # - toc: :// + # path_prefix: / + - repo: elasticsearch + - repo: apm + - repo: apm-agent-android + - repo: apm-agent-go + - repo: apm-agent-dotnet ############# # REFERENCE # @@ -108,7 +112,7 @@ toc: # Eland # ✅ https://github.com/elastic/eland/blob/main/docs/reference/toc.yml - - toc: eland/docs/reference + - toc: eland://reference path_prefix: reference/elasticsearch/clients/eland # Children include the entire AsciiDoc book From 73ba35f55640c1d794c04426e9a84a3cafce9767 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 18 Mar 2025 22:38:35 +0100 Subject: [PATCH 06/13] File copy now in a decent place --- .../Console/ConsoleDiagnosticsCollector.cs | 2 +- src/Elastic.Markdown/BuildContext.cs | 2 + .../Diagnostics/DiagnosticsChannel.cs | 10 ++-- .../DocumentationGenerator.cs | 8 ++- src/Elastic.Markdown/IO/DocumentationSet.cs | 4 +- .../Building/AssemblerBuilder.cs | 11 ++-- src/docs-assembler/Cli/RepositoryCommands.cs | 6 ++- .../Navigation/GlobalNavigation.cs | 29 +++++++++-- .../Navigation/GlobalNavigationFile.cs | 2 +- src/docs-assembler/navigation.yml | 52 +++++++++++++++++-- 10 files changed, 105 insertions(+), 21 deletions(-) diff --git a/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs b/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs index d40e285b8..5aa5ce497 100644 --- a/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs +++ b/src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs @@ -24,7 +24,7 @@ protected override void HandleItem(Diagnostic diagnostic) _errors.Add(diagnostic); else if (diagnostic.Severity == Severity.Warning) _warnings.Add(diagnostic); - else + else if (!NoHints) _hints.Add(diagnostic); } diff --git a/src/Elastic.Markdown/BuildContext.cs b/src/Elastic.Markdown/BuildContext.cs index 503cc9c40..8eb18e002 100644 --- a/src/Elastic.Markdown/BuildContext.cs +++ b/src/Elastic.Markdown/BuildContext.cs @@ -29,6 +29,8 @@ public record BuildContext public bool Force { get; init; } + public bool SkipMetadata { get; init; } + // This property is used to determine if the site should be indexed by search engines public bool AllowIndexing { get; init; } diff --git a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs index 7a67e643c..c04101d1f 100644 --- a/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs +++ b/src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs @@ -85,10 +85,12 @@ public class DiagnosticsCollector(IReadOnlyCollection output public HashSet OffendingFiles { get; } = []; - public HashSet InUseSubstitutionKeys { get; } = []; + public ConcurrentDictionary InUseSubstitutionKeys { get; } = []; public ConcurrentBag CrossLinks { get; } = []; + public bool NoHints { get; init; } + public Task StartAsync(Cancel cancellationToken) { if (_started is not null) @@ -117,6 +119,8 @@ void Drain() { while (Channel.Reader.TryRead(out var item)) { + if (item.Severity == Severity.Hint && NoHints) + continue; IncrementSeverityCount(item); HandleItem(item); _ = OffendingFiles.Add(item.File); @@ -132,7 +136,7 @@ private void IncrementSeverityCount(Diagnostic item) _ = Interlocked.Increment(ref _errors); else if (item.Severity == Severity.Warning) _ = Interlocked.Increment(ref _warnings); - else if (item.Severity == Severity.Hint) + else if (item.Severity == Severity.Hint && !NoHints) _ = Interlocked.Increment(ref _hints); } @@ -175,5 +179,5 @@ public async ValueTask DisposeAsync() } public void CollectUsedSubstitutionKey(ReadOnlySpan key) => - _ = InUseSubstitutionKeys.Add(key.ToString()); + _ = InUseSubstitutionKeys.TryAdd(key.ToString(), true); } diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index ef817dc35..fa2b5edd9 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -82,7 +82,7 @@ public async Task ResolveDirectoryTree(Cancel ctx) public async Task GenerateAll(Cancel ctx) { var generationState = GetPreviousGenerationState(); - if (Context.Force || generationState == null) + if (!Context.SkipMetadata && (Context.Force || generationState == null)) DocumentationSet.ClearOutputDirectory(); if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges)) @@ -99,6 +99,9 @@ public async Task GenerateAll(Cancel ctx) await ExtractEmbeddedStaticResources(ctx); + if (Context.SkipMetadata) + return; + _logger.LogInformation($"Generating documentation compilation state"); await GenerateDocumentationState(ctx); @@ -150,7 +153,8 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) => private void HintUnusedSubstitutionKeys() { var definedKeys = new HashSet(Context.Configuration.Substitutions.Keys.ToArray()); - var keysNotInUse = definedKeys.Except(Context.Collector.InUseSubstitutionKeys).ToArray(); + var inUse = new HashSet(Context.Collector.InUseSubstitutionKeys.Keys); + var keysNotInUse = definedKeys.Except(inUse).ToArray(); // If we have less than 20 unused keys emit them separately // Otherwise emit one hint with all of them for brevity if (keysNotInUse.Length >= 20) diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 6c24a054a..fd0af16ff 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -91,7 +91,7 @@ public DocumentationSet(BuildContext build, ILoggerFactory logger, ICrossLinkRes .SelectMany(extension => extension.ScanDocumentationFiles(ScanDocumentationFiles, DefaultFileHandling)) .ToArray(); - Files = files.Concat(additionalSources).ToArray(); + Files = files.Concat(additionalSources).Where(f => f is not ExcludedFile).ToArray(); LastWrite = Files.Max(f => f.SourceFile.LastWriteTimeUtc); @@ -154,7 +154,7 @@ private DocumentationFile DefaultFileHandling(IFileInfo file, IDirectoryInfo sou if (documentationFile is not null) return documentationFile; } - return new StaticFile(file, sourceDirectory); + return new ExcludedFile(file, sourceDirectory); } private void ValidateRedirectsExists() diff --git a/src/docs-assembler/Building/AssemblerBuilder.cs b/src/docs-assembler/Building/AssemblerBuilder.cs index 0dd681c36..71eec9a75 100644 --- a/src/docs-assembler/Building/AssemblerBuilder.cs +++ b/src/docs-assembler/Building/AssemblerBuilder.cs @@ -20,11 +20,15 @@ public async Task BuildAllAsync(IReadOnlyCollection checkouts, Publish var uriResolver = new PublishEnvironmentUriResolver(globalNavigation, environment); var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, uriResolver); + if (context.OutputDirectory.Exists) + context.OutputDirectory.Delete(true); + context.OutputDirectory.Create(); + foreach (var checkout in checkouts) { if (checkout.Repository.Skip) { - context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Skipping {checkout.Repository.Origin} as its not yet been migrated to V3"); + context.Collector.EmitWarning(context.ConfigurationPath.FullName, $"Skipping {checkout.Repository.Origin} as its marked as skip in configuration"); continue; } @@ -55,8 +59,9 @@ private async Task BuildAsync(Checkout checkout, PublishEnvironment environment, var buildContext = new BuildContext(context.Collector, context.ReadFileSystem, context.WriteFileSystem, path, output) { UrlPathPrefix = environment.PathPrefix, - Force = true, - AllowIndexing = environment.AllowIndexing + Force = false, + AllowIndexing = environment.AllowIndexing, + SkipMetadata = true }; var set = new DocumentationSet(buildContext, logger, crossLinkResolver); diff --git a/src/docs-assembler/Cli/RepositoryCommands.cs b/src/docs-assembler/Cli/RepositoryCommands.cs index 853efe736..1579e8a8b 100644 --- a/src/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/docs-assembler/Cli/RepositoryCommands.cs @@ -63,7 +63,11 @@ public async Task BuildAll( var githubEnvironmentInput = githubActionsService.GetInput("environment"); environment ??= !string.IsNullOrEmpty(githubEnvironmentInput) ? githubEnvironmentInput : "dev"; - await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService); + await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService) + { + NoHints = true + }; + _ = collector.StartAsync(ctx); var assembleContext = new AssembleContext(collector, new FileSystem(), new FileSystem(), null, null) diff --git a/src/docs-assembler/Navigation/GlobalNavigation.cs b/src/docs-assembler/Navigation/GlobalNavigation.cs index c09e5f757..32b9b475d 100644 --- a/src/docs-assembler/Navigation/GlobalNavigation.cs +++ b/src/docs-assembler/Navigation/GlobalNavigation.cs @@ -70,7 +70,9 @@ public string GetSubPath(Uri crossLinkUri, ref string path) } if (match is null || !_tocLookup.TryGetValue(match, out var toc)) { - _context.Collector.EmitError(_context.NavigationPath, $"Unable to find defined toc for url: {crossLinkUri}"); + //TODO remove + if (crossLinkUri.Scheme != "asciidocalypse") + _context.Collector.EmitError(_context.NavigationPath, $"Unable to find defined toc for url: {crossLinkUri}"); return $"reference/{crossLinkUri.Scheme}"; } path = path.AsSpan().TrimStart(toc.SourcePrefix).ToString(); @@ -85,10 +87,22 @@ public string GetSubPath(Uri crossLinkUri, ref string path) var outputDirectory = documentationSet.OutputDirectory; var fs = defaultOutputFile.FileSystem; - var relativePathSpan = relativePath.AsSpan(); - var lookup = $"{documentationSet.Name}://{relativePath}"; - var match = TableOfContentsPrefixes.FirstOrDefault(prefix => lookup.StartsWith(prefix, StringComparison.Ordinal)); + var lookup = $"{documentationSet.Name}://{relativePath}".AsSpan(); + if (lookup.StartsWith("docs-content://serverless/", StringComparison.Ordinal)) + return null; + if (lookup.StartsWith("eland://sphinx/", StringComparison.Ordinal)) + return null; + + + string? match = null; + foreach (var prefix in TableOfContentsPrefixes) + { + if (!lookup.StartsWith(prefix, StringComparison.Ordinal)) + continue; + match = prefix; + break; + } if (match is null || !_tocLookup.TryGetValue(match, out var toc)) { if (relativePath.StartsWith("raw-migrated-files/", StringComparison.Ordinal)) @@ -108,7 +122,12 @@ public string GetSubPath(Uri crossLinkUri, ref string path) _context.Collector.EmitError(_context.NavigationPath, $"No toc for output path: '{lookup}' falling back to: '{fallBack}'"); return fs.FileInfo.New(fallBack); } - var path = fs.Path.Combine(outputDirectory.FullName, toc.PathPrefix, relativePath); + + var newPath = relativePath.AsSpan().TrimStart(toc.SourcePrefix.TrimEnd('/')).TrimStart('/').ToString(); + var path = fs.Path.Combine(outputDirectory.FullName, toc.PathPrefix, newPath); + if (path.Contains("deploy-manage")) + { + } return fs.FileInfo.New(path); } } diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs index e5059299a..7c5dd315e 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -109,8 +109,8 @@ private static IReadOnlyCollection ReadChildren(YamlSt if (source.AsSpan().IndexOf("://") == -1) { parent = source; - source = $"{NarrativeRepository.RepositoryName}://{source}"; pathPrefix = source; + source = $"{NarrativeRepository.RepositoryName}://{source}"; } break; diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml index f33d7329d..99c472f54 100644 --- a/src/docs-assembler/navigation.yml +++ b/src/docs-assembler/navigation.yml @@ -43,11 +43,54 @@ toc: # repo is short for # - toc: :// # path_prefix: / - - repo: elasticsearch - - repo: apm + - repo: asciidocalypse - repo: apm-agent-android - - repo: apm-agent-go - repo: apm-agent-dotnet + - repo: apm-agent-go + - repo: apm-agent-ios + - repo: apm-agent-java + - repo: apm-agent-nodejs + - repo: apm-agent-php + - repo: apm-agent-python + - repo: apm-agent-ruby + - repo: apm-agent-rum-js + - repo: apm-aws-lambda + - repo: apm-k8s-attacher + - repo: beats + - repo: cloud-on-k8s + - repo: cloud + - repo: curator + - repo: ecctl + - repo: ecs-dotnet + - repo: ecs-logging-go-logrus + - repo: ecs-logging-go-zap + - repo: ecs-logging-go-zerolog + - repo: ecs-logging-java + - repo: ecs-logging-nodejs + - repo: ecs-logging-php + - repo: ecs-logging-python + - repo: ecs-logging-ruby + - repo: ecs-logging + - repo: ecs + - repo: eland + - repo: elastic-serverless-forwarder + - repo: elasticsearch-hadoop + - repo: elasticsearch-java + - repo: elasticsearch-js + - repo: elasticsearch-net + - repo: elasticsearch-php + - repo: elasticsearch-py + - repo: elasticsearch-rs + - repo: elasticsearch-ruby + - repo: elasticsearch + - repo: go-elasticsearch + - repo: integrations + - repo: kibana + - repo: logstash-docs + - repo: logstash + - repo: search-ui + - repo: integration-docs + - repo: security-docs ############# # REFERENCE # @@ -175,6 +218,9 @@ toc: - toc: docs-content://reference/ingestion-tools path_prefix: reference/ingestion-tools children: + # Elasticsearch (TODO discuss with Colleen) + - toc: elasticsearch://reference/ingestion-tools/enrich-processor + path_prefix: reference/elasticsearch/enrich-processor # Fleet and Elastic Agent # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/fleet/toc.yml From 5f1760a73000ec0a38e29fd1ddeaf002590f49b0 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 18 Mar 2025 23:07:18 +0100 Subject: [PATCH 07/13] temporary home for in flux paths --- src/docs-assembler/navigation.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml index 99c472f54..e1731048f 100644 --- a/src/docs-assembler/navigation.yml +++ b/src/docs-assembler/navigation.yml @@ -218,9 +218,17 @@ toc: - toc: docs-content://reference/ingestion-tools path_prefix: reference/ingestion-tools children: - # Elasticsearch (TODO discuss with Colleen) + # I don't know (TODO discuss with Colleen) - toc: elasticsearch://reference/ingestion-tools/enrich-processor path_prefix: reference/elasticsearch/enrich-processor + - toc: elasticsearch://reference/ingestion-tools/search-connectors + path_prefix: reference/elasticsearch/search-connectors + - toc: elasticsearch://reference/data-analysis + path_prefix: reference/elasticsearch/data-analysis + - toc: security-docs://reference/prebuilt-rules + path_prefix: reference/prebuilt-rules + - toc: elasticsearch://reference + path_prefix: reference/elasticsearch # Fleet and Elastic Agent # 🔜 https://github.com/elastic/docs-content/blob/prepare-for-assembler/reference/fleet/toc.yml From 8ad455e25989d7ac33fb75b5fafa4ce83deee5ac Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 18 Mar 2025 23:10:39 +0100 Subject: [PATCH 08/13] fix failing test --- tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs index 0f6decbb0..8d00c2464 100644 --- a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs +++ b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs @@ -15,7 +15,13 @@ public async Task CreatesDefaultOutputDirectory() var logger = new TestLoggerFactory(output); var fileSystem = new MockFileSystem(new Dictionary { - { "docs/docset.yml", new MockFileData("") }, + { "docs/docset.yml", + //language=yaml + new MockFileData(""" +project: test +toc: +- file: index.md +""") }, { "docs/index.md", new MockFileData("test") } }, new MockFileSystemOptions { From 9bf69dba18f4e70732c1fde5ff8911832d98e63c Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Tue, 18 Mar 2025 23:12:21 +0100 Subject: [PATCH 09/13] remove not referenced test project --- .../docs-assembler.Tests.Tests/FunctionTest.cs | 18 ------------------ .../docs-assembler.Tests.Tests.csproj | 18 ------------------ 2 files changed, 36 deletions(-) delete mode 100644 tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/FunctionTest.cs delete mode 100644 tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/docs-assembler.Tests.Tests.csproj diff --git a/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/FunctionTest.cs b/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/FunctionTest.cs deleted file mode 100644 index 28d9969bf..000000000 --- a/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/FunctionTest.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Xunit; -using Amazon.Lambda.Core; -using Amazon.Lambda.TestUtilities; - -namespace docs_assembler.Tests.Tests; - -public class FunctionTest -{ - [Fact] - public void TestToUpperFunction() - { - // Invoke the lambda function and confirm the string was upper cased. - var context = new TestLambdaContext(); - var upperCase = Function.FunctionHandler("hello world", context); - - Assert.Equal("HELLO WORLD", upperCase); - } -} \ No newline at end of file diff --git a/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/docs-assembler.Tests.Tests.csproj b/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/docs-assembler.Tests.Tests.csproj deleted file mode 100644 index 78a156450..000000000 --- a/tests/docs-assembler.Tests/test/docs-assembler.Tests.Tests/docs-assembler.Tests.Tests.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net8.0 - enable - enable - true - - - - - - - - - - - - \ No newline at end of file From aa9a3e86db5b954b5c0bda7bc4d5c97f568be53d Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 19 Mar 2025 07:14:15 +0100 Subject: [PATCH 10/13] Restrict docset.yml configs that define toc.yml sections to ONLY link to sub toc.yml files (#767) * Restrict docset.yml configs that define toc.yml sections to ONLY link to sub toc.yml files * relax check for narrative docs too, plays by different rules * dotnet format * reference Project directly --- docs/_docset.yml | 3 +++ .../IO/Configuration/ConfigurationFile.cs | 21 +++++++++++++++---- .../IO/Configuration/ITocItem.cs | 2 ++ .../TableOfContentsConfiguration.cs | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/docs/_docset.yml b/docs/_docset.yml index 8292d5b3e..5f0cdbef8 100644 --- a/docs/_docset.yml +++ b/docs/_docset.yml @@ -1,5 +1,8 @@ project: 'doc-builder' max_toc_depth: 2 +# indicates this documentation set is not linkable by assembler. +# relaxes a few restrictions around toc building and file placement +dev_docs: true cross_links: - docs-content exclude: diff --git a/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs b/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs index 824fb25cb..d9ffcf9ae 100644 --- a/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs +++ b/src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs @@ -44,6 +44,15 @@ public record ConfigurationFile : DocumentationFile private FeatureFlags? _featureFlags; public FeatureFlags Features => _featureFlags ??= new FeatureFlags(_features); + /// This is a documentation set that is not linked to by assembler. + /// Setting this to true relaxes a few restrictions such as mixing toc references with file and folder reference + public bool DevelopmentDocs { get; } + + // TODO ensure project key is `docs-content` + private bool IsNarrativeDocs => + Project is not null + && Project.Equals("Elastic documentation", StringComparison.OrdinalIgnoreCase); + public ConfigurationFile(BuildContext context) : base(context.ConfigurationPath, context.DocumentationSourceDirectory) { @@ -74,6 +83,9 @@ public ConfigurationFile(BuildContext context) case "max_toc_depth": MaxTocDepth = int.TryParse(reader.ReadString(entry.Entry), out var maxTocDepth) ? maxTocDepth : 1; break; + case "dev_docs": + DevelopmentDocs = bool.TryParse(reader.ReadString(entry.Entry), out var devDocs) && devDocs; + break; case "exclude": var excludes = YamlStreamReader.ReadStringArray(entry.Entry); Exclude = [.. excludes.Where(s => !string.IsNullOrEmpty(s)).Select(Glob.Parse)]; @@ -111,8 +123,11 @@ public ConfigurationFile(BuildContext context) { case "toc": var toc = new TableOfContentsConfiguration(this, _context, 0, ""); - var entries = toc.ReadChildren(reader, entry.Entry); - TableOfContents = entries; + var children = toc.ReadChildren(reader, entry.Entry); + var tocEntries = children.OfType().ToArray(); + if (!DevelopmentDocs && !IsNarrativeDocs && tocEntries.Length > 0 && children.Count != tocEntries.Length) + reader.EmitError("toc links to other toc sections it may only contain other toc references", entry.Key); + TableOfContents = children; Files = toc.Files; //side-effect ripe for refactor break; } @@ -142,6 +157,4 @@ private IReadOnlyCollection InstantiateExtensions() return list.AsReadOnly(); } - - } diff --git a/src/Elastic.Markdown/IO/Configuration/ITocItem.cs b/src/Elastic.Markdown/IO/Configuration/ITocItem.cs index bd9c69c16..6d09e492d 100644 --- a/src/Elastic.Markdown/IO/Configuration/ITocItem.cs +++ b/src/Elastic.Markdown/IO/Configuration/ITocItem.cs @@ -9,3 +9,5 @@ public interface ITocItem; public record FileReference(string Path, bool Found, bool Hidden, IReadOnlyCollection Children) : ITocItem; public record FolderReference(string Path, bool Found, bool InNav, IReadOnlyCollection Children) : ITocItem; + +public record TocReference(string Path, bool Found, bool InNav, IReadOnlyCollection Children) : FolderReference(Path, Found, InNav, Children); diff --git a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs index 92bd00d6e..3f30775fb 100644 --- a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs +++ b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs @@ -112,7 +112,7 @@ public IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyVa foreach (var f in toc.Files) _ = Files.Add(f); - return [new FolderReference($"{parentPath}".TrimStart(Path.DirectorySeparatorChar), folderFound, inNav, toc.TableOfContents)]; + return [new TocReference($"{parentPath}".TrimStart(Path.DirectorySeparatorChar), folderFound, inNav, toc.TableOfContents)]; } if (file is not null) From 807e87f63d58acc7f6f7c8ca4b2b4bd1d26699d1 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Wed, 19 Mar 2025 09:53:57 +0100 Subject: [PATCH 11/13] Fix layout and horizontal scrollable tables (#772) * Fix layout and horizontal scrollable tables mend * Use spacing var * Add custom scrollbar * tweaks --- .../Assets/markdown/table.css | 18 +++++++++- src/Elastic.Markdown/Helpers/Htmx.cs | 2 +- .../Slices/Layout/_PagesNav.cshtml | 2 +- .../Slices/Layout/_TableOfContents.cshtml | 2 +- .../Slices/Layout/_TocTree.cshtml | 2 +- src/Elastic.Markdown/Slices/_Layout.cshtml | 36 +++++++++---------- .../Inline/AnchorLinkTests.cs | 10 +++--- .../Inline/DirectiveBlockLinkTests.cs | 2 +- .../Inline/InlineAnchorTests.cs | 2 +- .../Inline/InlineLinkTests.cs | 12 +++---- 10 files changed, 51 insertions(+), 37 deletions(-) diff --git a/src/Elastic.Markdown/Assets/markdown/table.css b/src/Elastic.Markdown/Assets/markdown/table.css index c36715131..b657c6715 100644 --- a/src/Elastic.Markdown/Assets/markdown/table.css +++ b/src/Elastic.Markdown/Assets/markdown/table.css @@ -1,8 +1,21 @@ + @layer components { + .table-wrapper { - @apply max-w-full overflow-x-scroll my-4; + @apply w-full overflow-x-scroll my-4; + + &::-webkit-scrollbar { + @apply h-2 bg-grey-10 border-b-1 border-grey-20; + } + &::-webkit-scrollbar-thumb { + @apply bg-grey-80 rounded-full; + } + &::-webkit-scrollbar-thumb:hover { + @apply bg-grey-100; + } + table { @apply w-full border-collapse border-1 border-grey-20; } @@ -13,6 +26,9 @@ thead { @apply font-sans font-semibold text-left align-top border-b-1 border-grey-20 bg-grey-10; + :empty { + display: none; + } } tbody { diff --git a/src/Elastic.Markdown/Helpers/Htmx.cs b/src/Elastic.Markdown/Helpers/Htmx.cs index 15fe388ef..c4d4bd377 100644 --- a/src/Elastic.Markdown/Helpers/Htmx.cs +++ b/src/Elastic.Markdown/Helpers/Htmx.cs @@ -14,7 +14,7 @@ public static string GetHxSelectOob(FeatureFlags features, string? pathPrefix, s if (features.IsPrimaryNavEnabled && currentUrl == pathPrefix + "/") return "#main-container,#primary-nav,#secondary-nav"; - var selectTargets = "#primary-nav,#secondary-nav,#content-container"; + var selectTargets = "#primary-nav,#secondary-nav,#content-container,#toc-nav"; if (!HasSameTopLevelGroup(pathPrefix, currentUrl, targetUrl) && features.IsPrimaryNavEnabled) selectTargets += ",#pages-nav"; return selectTargets; diff --git a/src/Elastic.Markdown/Slices/Layout/_PagesNav.cshtml b/src/Elastic.Markdown/Slices/Layout/_PagesNav.cshtml index 0d266954c..caa5cfef0 100644 --- a/src/Elastic.Markdown/Slices/Layout/_PagesNav.cshtml +++ b/src/Elastic.Markdown/Slices/Layout/_PagesNav.cshtml @@ -1,5 +1,5 @@ @inherits RazorSlice -