From 1ac2f302aca59fd67d6a21eece5fa6ba15235f1e Mon Sep 17 00:00:00 2001 From: Colleen McGinnis Date: Wed, 26 Mar 2025 10:35:43 -0500 Subject: [PATCH 1/5] add detection rules --- src/docs-assembler/assembler.yml | 5 ++--- src/docs-assembler/navigation.yml | 9 +++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/docs-assembler/assembler.yml b/src/docs-assembler/assembler.yml index ce60ea571..43834d35e 100644 --- a/src/docs-assembler/assembler.yml +++ b/src/docs-assembler/assembler.yml @@ -43,6 +43,8 @@ references: current: master curator: current: master + detection-rules: + checkout_strategy: full ecctl: current: master ecs-dotnet: @@ -76,6 +78,3 @@ references: logstash: search-ui: integration-docs: - # security-docs: - # # wait for move to https://github.com/elastic/detection-rules/pull/4507/files - # skip: true diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml index 27a077458..f974e02b8 100644 --- a/src/docs-assembler/navigation.yml +++ b/src/docs-assembler/navigation.yml @@ -140,12 +140,9 @@ toc: path_prefix: reference/security # Children include: Endpoint command reference, Elastic Defend, # Fields and object schemas - #children: - # 📝 TO DO: Update when rules are moved to elastic/detection-rules - # This makes sense for now. I'm not sure when these files will be - # moved to https://github.com/elastic/detection-rules. - # - toc: security-docs://reference/prebuilt-rules - # path_prefix: reference/security/prebuilt-rules + children: + - repo: detection-rules + # path_prefix: reference/security/prebuilt-rules # Observability # ✅ https://github.com/elastic/docs-content/blob/main/reference/observability/toc.yml From 6310bb25121fde7d3d7d574368a763664e19a246 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 26 Mar 2025 21:10:16 +0100 Subject: [PATCH 2/5] Add initial support for detection-rules in docs-assembler --- .../TableOfContentsConfiguration.cs | 5 +-- src/docs-assembler/AssembleSources.cs | 3 +- .../Building/PublishEnvironmentUriResolver.cs | 6 +++- .../Navigation/GlobalNavigationFile.cs | 3 +- .../Navigation/GlobalNavigationHtmlWriter.cs | 32 +++++++++++++++++-- .../GlobalNavigationPathProvider.cs | 27 +++++++++++++--- src/docs-assembler/navigation.yml | 4 +-- .../GlobalNavigationTests.cs | 20 ++++++++++-- 8 files changed, 83 insertions(+), 17 deletions(-) diff --git a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs index 312a42e69..0988ae8eb 100644 --- a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs +++ b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs @@ -20,9 +20,10 @@ public static class ContentSourceMoniker public static string CreateString(string repo, string? path) { - if (string.IsNullOrWhiteSpace(path)) + path = path?.Replace("\\", "/").Trim('/'); + if (string.IsNullOrWhiteSpace(path) || path == ".") return $"{repo}://"; - return $"{repo}://{path.Replace("\\", "/").Trim('/')}/"; + return $"{repo}://{path}/"; } } diff --git a/src/docs-assembler/AssembleSources.cs b/src/docs-assembler/AssembleSources.cs index 709276e39..b172bcd90 100644 --- a/src/docs-assembler/AssembleSources.cs +++ b/src/docs-assembler/AssembleSources.cs @@ -199,7 +199,8 @@ static void ReadBlock( if (source is null) return; - if (!Uri.TryCreate(source.TrimEnd('/') + '/', UriKind.Absolute, out var sourceUri)) + source = source.EndsWith("://") ? source : source.TrimEnd('/') + "/"; + if (!Uri.TryCreate(source, UriKind.Absolute, out var sourceUri)) { reader.EmitError($"Source toc entry is not a valid uri: {source}", tocEntry); return; diff --git a/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs b/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs index d1269491f..c504bfdff 100644 --- a/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs +++ b/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs @@ -27,7 +27,11 @@ public PublishEnvironmentUriResolver(FrozenDictionary t TableOfContentsPrefixes = [..topLevelMappings .Values - .Select(v => v.Source.ToString()) + .Select(p => + { + var source = p.Source.ToString(); + return source.EndsWith(":///") ? source[..^1] : source; + }) .OrderByDescending(v => v.Length) ]; diff --git a/src/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/docs-assembler/Navigation/GlobalNavigationFile.cs index e85cb87d2..1719885d7 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -226,7 +226,8 @@ private IReadOnlyCollection ReadChildren(string key, YamlStreamRea if (source is null) return pathPrefix; - if (!Uri.TryCreate(source.TrimEnd('/') + "/", UriKind.Absolute, out sourceUri)) + source = source.EndsWith("://") ? source : source.TrimEnd('/') + "/"; + if (!Uri.TryCreate(source, UriKind.Absolute, out sourceUri)) { reader.EmitError($"Source toc entry is not a valid uri: {source}", tocEntry); return pathPrefix; diff --git a/src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs b/src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs index 818b5515a..0c41159cd 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs @@ -31,15 +31,41 @@ public class GlobalNavigationHtmlWriter( { assembleContext.Collector.EmitWarning(assembleContext.NavigationPath.FullName, $"Could not find a toc tree for {topLevelUri.TopLevelSource}"); return (tree, tree.Source); - } + return (topLevel, topLevelUri.TopLevelSource); } + public static TableOfContentsTree? GetCurrentTree(INavigation navigation) + { + if (navigation is TableOfContentsTree tree) + return tree; + if (navigation is not DocumentationGroup group) + return null; + if (group.NavigationRoot is TableOfContentsTree root) + return root; + var i = 0; + while (group.Parent != null) + { + group = group.Parent; + if (group.Parent is TableOfContentsTree groupTree) + return groupTree; + if (group.NavigationRoot is TableOfContentsTree treeRoot) + return treeRoot; + i++; + if (i > 100) + throw new InvalidOperationException("Could not find parent"); + } + + return null; + } + public async Task RenderNavigation(INavigation currentRootNavigation, Cancel ctx = default) { - if (currentRootNavigation is not TableOfContentsTree tree) - throw new InvalidOperationException($"Expected a {nameof(DocumentationGroup)}"); + var tree = GetCurrentTree(currentRootNavigation) + ?? throw new InvalidOperationException( + $"Expected a {nameof(TableOfContentsTree)} but got {currentRootNavigation.GetType().Name} for {nameof(currentRootNavigation)}" + ); if (Phantoms.Contains(tree.Source)) return string.Empty; diff --git a/src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs b/src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs index 82558024f..d391c6eb1 100644 --- a/src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs +++ b/src/docs-assembler/Navigation/GlobalNavigationPathProvider.cs @@ -6,6 +6,7 @@ using System.IO.Abstractions; using Elastic.Markdown; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.Extensions.DetectionRules; using Elastic.Markdown.IO; using Elastic.Markdown.IO.Configuration; @@ -16,7 +17,7 @@ public record GlobalNavigationPathProvider : IDocumentationFileOutputProvider private readonly AssembleSources _assembleSources; private readonly AssembleContext _context; - private ImmutableSortedSet TableOfContentsPrefixes { get; } + public ImmutableSortedSet TableOfContentsPrefixes { get; } private ImmutableSortedSet PhantomPrefixes { get; } public GlobalNavigationPathProvider(GlobalNavigationFile navigationFile, AssembleSources assembleSources, AssembleContext context) @@ -26,12 +27,20 @@ public GlobalNavigationPathProvider(GlobalNavigationFile navigationFile, Assembl TableOfContentsPrefixes = [..assembleSources.TocTopLevelMappings .Values - .Select(v => v.Source.ToString()) + .Select(p => + { + var source = p.Source.ToString(); + return source.EndsWith(":///") ? source[..^1] : source; + }) .OrderByDescending(v => v.Length) ]; PhantomPrefixes = [..navigationFile.Phantoms - .Select(p => p.Source.ToString()) + .Select(p => + { + var source = p.Source.ToString(); + return source.EndsWith(":///") ? source[..^1] : source; + }) .OrderByDescending(v => v.Length) .ToArray() ]; @@ -39,13 +48,23 @@ public GlobalNavigationPathProvider(GlobalNavigationFile navigationFile, Assembl public IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath) { + if (relativePath.StartsWith("_static/", StringComparison.Ordinal)) return defaultOutputFile; + + + var repositoryName = documentationSet.Build.Git.RepositoryName; var outputDirectory = documentationSet.OutputDirectory; var fs = defaultOutputFile.FileSystem; - var repositoryName = documentationSet.Build.Git.RepositoryName; + if (repositoryName == "detection-rules") + { + var output = DetectionRuleFile.OutputPath(defaultOutputFile, documentationSet.Build); + var md = fs.FileInfo.New(Path.ChangeExtension(output.FullName, "md")); + relativePath = Path.GetRelativePath(documentationSet.OutputDirectory.FullName, md.FullName); + } + var l = ContentSourceMoniker.CreateString(repositoryName, relativePath).TrimEnd('/'); var lookup = l.AsSpan(); diff --git a/src/docs-assembler/navigation.yml b/src/docs-assembler/navigation.yml index f974e02b8..90c040dd2 100644 --- a/src/docs-assembler/navigation.yml +++ b/src/docs-assembler/navigation.yml @@ -141,8 +141,8 @@ toc: # Children include: Endpoint command reference, Elastic Defend, # Fields and object schemas children: - - repo: detection-rules - # path_prefix: reference/security/prebuilt-rules + - toc: detection-rules:// + path_prefix: reference/security/prebuilt-rules # Observability # ✅ https://github.com/elastic/docs-content/blob/main/reference/observability/toc.yml diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs index 31d1f03b0..2b91bfa12 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs @@ -73,6 +73,21 @@ public async Task ReadAllPathPrefixes() pathPrefixes.Should().Contain(new Uri("eland://reference/elasticsearch/clients/eland/")); } + [Fact] + public async Task PathProvider() + { + Assert.SkipUnless(HasCheckouts(), $"Requires local checkout folder: {CheckoutDirectory.FullName}"); + + var assembleSources = await Setup(); + + var navigationFile = new GlobalNavigationFile(Context, assembleSources); + var pathProvider = new GlobalNavigationPathProvider(navigationFile, assembleSources, Context); + + assembleSources.TocTopLevelMappings.Should().NotBeEmpty().And.ContainKey(new Uri("detection-rules://")); + pathProvider.TableOfContentsPrefixes.Should().Contain("detection-rules://"); + } + + [Fact] public async Task ParsesReferences() { @@ -89,6 +104,8 @@ public async Task ParsesReferences() assembleSources.TocTopLevelMappings.Should().NotBeEmpty().And.ContainKey(expectedRoot); assembleSources.TocTopLevelMappings[sut].ParentSource.Should().Be(expectedParent); + assembleSources.TocTopLevelMappings.Should().NotBeEmpty().And.ContainKey(new Uri("detection-rules://")); + var navigationFile = new GlobalNavigationFile(Context, assembleSources); var referenceToc = navigationFile.TableOfContents.FirstOrDefault(t => t.Source == expectedRoot); referenceToc.Should().NotBeNull(); @@ -110,9 +127,6 @@ public async Task ParsesReferences() var referenceNav = navigation.NavigationLookup[expectedRoot]; navigation.NavigationItems.Should().HaveSameCount(navigation.NavigationLookup); - var referenceOrder = referenceNav.Group.NavigationItems.OfType() - .Last().Source.Should().Be(new Uri("docs-content://reference/glossary/")); - referenceNav.Should().NotBeNull(); referenceNav.NavigationLookup.Should().NotContainKey(clients); referenceNav.Group.NavigationItems.OfType() From 17b1b5edc191d5a601bc13d126d29fc382a355c9 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 28 Mar 2025 12:25:45 +0100 Subject: [PATCH 3/5] embedd detection rules overview page with assembler appropiate urls --- .../DetectionRules/DetectionRuleFile.cs | 8 +++++- .../DetectionRulesDocsBuilderExtension.cs | 6 +++++ .../DiagnosticLinkInlineParser.cs | 25 ++++++++++++++++--- .../Building/PublishEnvironmentUriResolver.cs | 5 ++++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs index 82ce13379..fed529906 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs @@ -18,6 +18,10 @@ public DetectionRuleOverviewFile(IFileInfo sourceFile, IDirectoryInfo rootPath, public RuleReference[] Rules { get; set; } = []; + private Dictionary Files { get; } = []; + + public void AddDetectionRuleFile(DetectionRuleFile df, RuleReference ruleReference) => Files[ruleReference.Path] = df; + protected override Task GetMinimalParseDocumentAsync(Cancel ctx) { Title = "Detection Rules Overview"; @@ -57,9 +61,10 @@ private string GetMarkdown() """; foreach (var r in group.OrderBy(r => r.Rule.Name)) { + var url = Files[r.Path].Url; markdown += $""" -[{r.Rule.Name}]({r.Path})
+[{r.Rule.Name}](!{url})
"""; } @@ -69,6 +74,7 @@ private string GetMarkdown() return markdown; } + } public record DetectionRuleFile : MarkdownFile diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs index 4c8638fc5..90630d093 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs @@ -32,17 +32,23 @@ public void CreateNavigationItem( } + private DetectionRuleOverviewFile? _overviewFile; public void Visit(DocumentationFile file, ITocItem tocItem) { // TODO the parsing of rules should not happen at ITocItem reading time. // ensure the file has an instance of the rule the reference parsed. if (file is DetectionRuleFile df && tocItem is RuleReference r) + { df.Rule = r.Rule; + _overviewFile?.AddDetectionRuleFile(df, r); + + } if (file is DetectionRuleOverviewFile of && tocItem is RuleOverviewReference or) { var rules = or.Children.OfType().ToArray(); of.Rules = rules; + _overviewFile = of; } } diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index 9ebba741d..29362dfc9 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -189,11 +189,30 @@ private static void ProcessCrossLink(LinkInline link, InlineProcessor processor, private static void ProcessInternalLink(LinkInline link, InlineProcessor processor, ParserContext context) { + if (link.Url != null && link.Url.StartsWith('!')) + { + // [](!/already/resolved/url) internal syntax to allow markdown embedding already resolved links + var verbatimUrl = link.Url[1..]; + link.Url = verbatimUrl; + var md = ResolveFile(context, verbatimUrl); + _ = SetLinkData(link, processor, context, md, verbatimUrl); + return; + } + var (url, anchor) = SplitUrlAndAnchor(link.Url ?? string.Empty); var includeFrom = GetIncludeFromPath(url, context); var file = ResolveFile(context, url); ValidateInternalUrl(processor, url, includeFrom, link, context); + var linkMarkdown = SetLinkData(link, processor, context, file, url); + + ProcessLinkText(processor, link, linkMarkdown, anchor, url, file); + UpdateLinkUrl(link, url, context, anchor); + } + + private static MarkdownFile? SetLinkData(LinkInline link, InlineProcessor processor, ParserContext context, + IFileInfo file, string url) + { if (context.DocumentationFileLookup(context.MarkdownSourcePath) is MarkdownFile currentMarkdown) { link.SetData(nameof(currentMarkdown.NavigationRoot), currentMarkdown.NavigationRoot); @@ -210,15 +229,13 @@ private static void ProcessInternalLink(LinkInline link, InlineProcessor process var linkMarkdown = context.DocumentationFileLookup(file) as MarkdownFile; if (linkMarkdown is not null) link.SetData($"Target{nameof(currentMarkdown.NavigationRoot)}", linkMarkdown.NavigationRoot); - - ProcessLinkText(processor, link, linkMarkdown, anchor, url, file); - UpdateLinkUrl(link, url, context, anchor); + return linkMarkdown; } private static (string url, string? anchor) SplitUrlAndAnchor(string fullUrl) { var parts = fullUrl.Split('#'); - return (parts[0], parts.Length > 1 ? parts[1].Trim() : null); + return (parts[0].TrimStart('!'), parts.Length > 1 ? parts[1].Trim() : null); } private static string GetIncludeFromPath(string url, ParserContext context) => diff --git a/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs b/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs index 109b3c286..874933a7e 100644 --- a/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs +++ b/src/docs-assembler/Building/PublishEnvironmentUriResolver.cs @@ -40,6 +40,11 @@ public PublishEnvironmentUriResolver(FrozenDictionary t public Uri Resolve(Uri crossLinkUri, string path) { + if (crossLinkUri.Scheme == "detection-rules") + { + + } + var subPath = GetSubPathPrefix(crossLinkUri, ref path); var fullPath = (PublishEnvironment.PathPrefix, subPath) switch From 04a55dd4b03465444b6870e6da960d11cb86e9f8 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 28 Mar 2025 12:43:45 +0100 Subject: [PATCH 4/5] detect edgecase of documentation set having only one documentationgroup with one index in its navigation to dedup --- .../Extensions/DetectionRules/DetectionRulesReference.cs | 2 +- src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs index 723ca4616..a9db2a53f 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs @@ -13,7 +13,7 @@ public record RuleOverviewReference( IReadOnlyCollection Children, IReadOnlyCollection DetectionRuleFolders ) - : FileReference(TableOfContentsScope, Path, Found, true, Children); + : FileReference(TableOfContentsScope, Path, Found, false, Children); public record RuleReference( ITableOfContentsScope TableOfContentsScope, diff --git a/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs b/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs index bbccd5b70..a283a075c 100644 --- a/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs +++ b/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs @@ -88,6 +88,11 @@ public TableOfContentsTree( Source = source; TreeCollector.Collect(source, this); DocumentationSet = documentationSet; + + //edge case if tree only holds a single group ensure we collapse it down to the root (this) + if (NavigationItems.Count == 1 && NavigationItems.First() is GroupNavigationItem { Group.NavigationItems.Count: 0 }) + NavigationItems = []; + } internal TableOfContentsTree( From 3b1674997920ec23dbc73d570f7be85fe0378423 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 28 Mar 2025 12:57:43 +0100 Subject: [PATCH 5/5] Render groups with no navigation items as filereferences --- .../Slices/Layout/_TocTreeNav.cshtml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Elastic.Markdown/Slices/Layout/_TocTreeNav.cshtml b/src/Elastic.Markdown/Slices/Layout/_TocTreeNav.cshtml index 839190453..82ea6080d 100644 --- a/src/Elastic.Markdown/Slices/Layout/_TocTreeNav.cshtml +++ b/src/Elastic.Markdown/Slices/Layout/_TocTreeNav.cshtml @@ -21,6 +21,20 @@ } + else if (item is GroupNavigationItem { Group: { NavigationItems.Count: 0, Index: not null } } group) + { + var f = group.Group.Index; +
  • + + @f.NavigationTitle + +
    • + } else if (item is GroupNavigationItem folder) { var g = folder.Group;