From 32c6148793dafe84378c7cb324ea73479cad075f Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 19 Mar 2025 21:09:17 +0100 Subject: [PATCH 1/3] Updates to DetectionRulesExtension - Allow multiple rule folders to be configured - Skip `_deprecated/*` rules --- .../DetectionRulesDocsBuilderExtension.cs | 42 ++++++++++--------- .../DetectionRules/DetectionRulesReference.cs | 2 - .../TableOfContentsConfiguration.cs | 17 ++++---- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs index 35e26379c..c8cd31dd0 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs @@ -12,7 +12,8 @@ namespace Elastic.Markdown.Extensions.DetectionRules; public class DetectionRulesDocsBuilderExtension(BuildContext build) : IDocsBuilderExtension { private BuildContext Build { get; } = build; - public bool InjectsIntoNavigation(ITocItem tocItem) => tocItem is RulesFolderReference; + + public bool InjectsIntoNavigation(ITocItem tocItem) => false; public void CreateNavigationItem( DocumentationGroup? parent, @@ -25,14 +26,6 @@ public void CreateNavigationItem( ref int fileIndex, int index) { - var detectionRulesFolder = (RulesFolderReference)tocItem; - var children = detectionRulesFolder.Children; - var group = new DocumentationGroup(Build, lookups with { TableOfContents = children }, ref fileIndex, depth + 1, inNav) - { - Parent = parent - }; - groups.Add(group); - navigationItems.Add(new GroupNavigation(index, depth, group)); } public void Visit(DocumentationFile file, ITocItem tocItem) @@ -75,13 +68,25 @@ Func defaultFileHandling }).ToArray(); } - public IReadOnlyCollection CreateTableOfContentItems( - string parentPath, - string detectionRules, - HashSet files - ) + public IReadOnlyCollection CreateTableOfContentItems(ConfigurationFile configuration, string parentPath, + string[] detectionRuleFolders, + HashSet files) + { + var tocItems = new List(); + foreach (var detectionRuleFolder in detectionRuleFolders) + { + var children = ReadDetectionRuleFolder(configuration, parentPath, files, detectionRuleFolder); + tocItems.AddRange(children); + } + return tocItems + .OrderBy(d => d is RuleReference r ? r.Rule.Name : null, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private IReadOnlyCollection ReadDetectionRuleFolder(ConfigurationFile configuration, string parentPath, HashSet files, + string detectionRuleFolder) { - var detectionRulesFolder = Path.Combine(parentPath, detectionRules).TrimStart(Path.DirectorySeparatorChar); + var detectionRulesFolder = Path.Combine(parentPath, detectionRuleFolder).TrimStart(Path.DirectorySeparatorChar); var fs = Build.ReadFileSystem; var sourceDirectory = Build.DocumentationSourceDirectory; var path = fs.DirectoryInfo.New(fs.Path.GetFullPath(fs.Path.Combine(sourceDirectory.FullName, detectionRulesFolder))); @@ -91,22 +96,21 @@ HashSet files .Where(f => !f.Directory!.Attributes.HasFlag(FileAttributes.Hidden) && !f.Directory!.Attributes.HasFlag(FileAttributes.System)) .Where(f => f.Extension is ".md" or ".toml") .Where(f => f.Name != "README.md") + .Where(f => !f.FullName.Contains("_deprecated")) .Select(f => { var relativePath = Path.GetRelativePath(sourceDirectory.FullName, f.FullName); if (f.Extension == ".toml") { var rule = DetectionRule.From(f); - return new RuleReference(Build.Configuration, relativePath, detectionRules, true, [], rule); + return new RuleReference(configuration, relativePath, detectionRuleFolder, true, [], rule); } _ = files.Add(relativePath); - return new FileReference(Build.Configuration, relativePath, true, false, []); + return new FileReference(configuration, relativePath, true, false, []); }) - .OrderBy(d => d is RuleReference r ? r.Rule.Name : null, StringComparer.OrdinalIgnoreCase) .ToArray(); - //return [new RulesFolderReference(detectionRulesFolder, true, children)]; return children; } } diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs index 58cfc4e41..c502c6ec3 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs @@ -6,8 +6,6 @@ namespace Elastic.Markdown.Extensions.DetectionRules; -public record RulesFolderReference(ITableOfContentsScope TableOfContentsScope, string Path, bool Found, IReadOnlyCollection Children) : ITocItem; - public record RuleReference( ITableOfContentsScope TableOfContentsScope, string Path, diff --git a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs index e5a8c6857..ff72100dc 100644 --- a/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs +++ b/src/Elastic.Markdown/IO/Configuration/TableOfContentsConfiguration.cs @@ -75,7 +75,7 @@ public IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyVa { string? file = null; string? folder = null; - string? detectionRules = null; + string[]? detectionRules = null; TableOfContentsConfiguration? toc = null; var fileFound = false; var folderFound = false; @@ -140,7 +140,7 @@ public IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyVa else { var extension = _configuration.EnabledExtensions.OfType().First(); - children = extension.CreateTableOfContentItems(parentPath, detectionRules, Files); + children = extension.CreateTableOfContentItems(_configuration, parentPath, detectionRules, Files); } } return [new FileReference(this, $"{parentPath}{Path.DirectorySeparatorChar}{file}".TrimStart(Path.DirectorySeparatorChar), fileFound, hiddenFile, children ?? [])]; @@ -173,20 +173,23 @@ public IReadOnlyCollection ReadChildren(YamlStreamReader reader, KeyVa return folder; } - private string? ReadDetectionRules(YamlStreamReader reader, KeyValuePair entry, string parentPath, out bool found) + private string[]? ReadDetectionRules(YamlStreamReader reader, KeyValuePair entry, string parentPath, out bool found) { found = false; - var folder = reader.ReadString(entry); - if (folder is not null) + var folders = YamlStreamReader.ReadStringArray(entry); + foreach (var folder in folders) { + if (string.IsNullOrWhiteSpace(folder)) + continue; + var path = Path.Combine(_rootPath.FullName, parentPath.TrimStart(Path.DirectorySeparatorChar), folder); if (!_context.ReadFileSystem.DirectoryInfo.New(path).Exists) reader.EmitError($"Directory '{path}' does not exist", entry.Key); else found = true; - } - return folder; + } + return folders.Length == 0 ? null : folders; } private string? ReadFile(YamlStreamReader reader, KeyValuePair entry, string parentPath, out bool found) From 22b97ae53cf6cd6dfc68c4be18d56ca60a93b019 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 19 Mar 2025 22:31:12 +0100 Subject: [PATCH 2/3] Add support for displaying threat information --- .../DetectionRules/DetectionRule.cs | 132 +++++++++++++++++- .../DetectionRules/DetectionRuleFile.cs | 103 ++++++++++++-- .../DetectionRulesDocsBuilderExtension.cs | 1 + 3 files changed, 216 insertions(+), 20 deletions(-) diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs index 43edbd5df..a3528654f 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs @@ -9,6 +9,32 @@ namespace Elastic.Markdown.Extensions.DetectionRules; +public record DetectionRuleThreat +{ + public required string Framework { get; init; } + public required DetectionRuleTechnique[] Techniques { get; init; } = []; + public required DetectionRuleTactic Tactic { get; init; } +} + +public record DetectionRuleTactic +{ + public required string Id { get; init; } + public required string Name { get; init; } + public required string Reference { get; init; } +} + +public record DetectionRuleSubTechnique +{ + public required string Id { get; init; } + public required string Name { get; init; } + public required string Reference { get; init; } +} + +public record DetectionRuleTechnique : DetectionRuleSubTechnique +{ + public required DetectionRuleSubTechnique[] SubTechniques { get; init; } = []; +} + public record DetectionRule { public required string Name { get; init; } @@ -19,6 +45,8 @@ public record DetectionRule public required string? Query { get; init; } + public required string? Setup { get; init; } + public required string[]? Tags { get; init; } public required string Severity { get; init; } @@ -39,6 +67,8 @@ public record DetectionRule public required string[]? References { get; init; } public required string Version { get; init; } + public required DetectionRuleThreat[] Threats { get; init; } = []; + public static DetectionRule From(IFileInfo source) { TomlDocument model; @@ -51,12 +81,15 @@ public static DetectionRule From(IFileInfo source) { throw new Exception($"Could not parse toml in: {source.FullName}", e); } + if (!model.TryGetValue("metadata", out var node) || node is not TomlTable metadata) throw new Exception($"Could not find metadata section in {source.FullName}"); if (!model.TryGetValue("rule", out node) || node is not TomlTable rule) throw new Exception($"Could not find rule section in {source.FullName}"); + var threats = GetThreats(rule); + return new DetectionRule { Authors = TryGetStringArray(rule, "author"), @@ -64,19 +97,108 @@ public static DetectionRule From(IFileInfo source) Type = rule.GetString("type"), Language = TryGetString(rule, "language"), License = rule.GetString("license"), - RiskScore = TryRead(rule, "risk_score"), + RiskScore = TryGetInt(rule, "risk_score") ?? 0, RuleId = rule.GetString("rule_id"), Severity = rule.GetString("severity"), Tags = TryGetStringArray(rule, "tags"), Indices = TryGetStringArray(rule, "index"), References = TryGetStringArray(rule, "references"), IndicesFromDateMath = TryGetString(rule, "from"), + Setup = TryGetString(rule, "setup"), Query = TryGetString(rule, "query"), Note = TryGetString(rule, "note"), Name = rule.GetString("name"), RunsEvery = "?", MaximumAlertsPerExecution = "?", Version = "?", + Threats = threats + }; + } + + private static DetectionRuleThreat[] GetThreats(TomlTable model) + { + if (!model.TryGetValue("threat", out var node) || node is not TomlArray threats) + return []; + + var threatsList = new List(threats.ArrayValues.Count); + foreach (var value in threats) + { + if (value is not TomlTable threatTable) + continue; + + var framework = threatTable.GetString("framework"); + var techniques = ReadTechniques(threatTable); + + var tactic = ReadTactic(threatTable); + var threat = new DetectionRuleThreat + { + Framework = framework, + Techniques = techniques.ToArray(), + Tactic = tactic + }; + threatsList.Add(threat); + } + + return threatsList.ToArray(); + } + + private static IReadOnlyCollection ReadTechniques(TomlTable threatTable) + { + var techniquesArray = threatTable.TryGetValue("technique", out var node) && node is TomlArray ta ? ta : null; + if (techniquesArray is null) + return []; + var techniques = new List(techniquesArray.Count); + foreach (var t in techniquesArray) + { + if (t is not TomlTable techniqueTable) + continue; + var id = techniqueTable.GetString("id"); + var name = techniqueTable.GetString("name"); + var reference = techniqueTable.GetString("reference"); + techniques.Add(new DetectionRuleTechnique + { + Id = id, + Name = name, + Reference = reference, + SubTechniques = ReadSubTechniques(techniqueTable).ToArray() + }); + } + return techniques; + } + private static IReadOnlyCollection ReadSubTechniques(TomlTable techniqueTable) + { + var subArray = techniqueTable.TryGetValue("subtechnique", out var node) && node is TomlArray ta ? ta : null; + if (subArray is null) + return []; + var subTechniques = new List(subArray.Count); + foreach (var t in subArray) + { + if (t is not TomlTable subTechniqueTable) + continue; + var id = subTechniqueTable.GetString("id"); + var name = subTechniqueTable.GetString("name"); + var reference = subTechniqueTable.GetString("reference"); + subTechniques.Add(new DetectionRuleSubTechnique + { + Id = id, + Name = name, + Reference = reference + }); + } + return subTechniques; + } + + private static DetectionRuleTactic ReadTactic(TomlTable threatTable) + { + var tacticTable = threatTable.GetSubTable("tactic"); + var id = tacticTable.GetString("id"); + var name = tacticTable.GetString("name"); + var reference = tacticTable.GetString("reference"); + return new DetectionRuleTactic + { + Id = id, + Name = name, + Reference = reference }; } @@ -86,10 +208,6 @@ public static DetectionRule From(IFileInfo source) private static string? TryGetString(TomlTable table, string key) => table.TryGetValue(key, out var node) && node is TomlString t ? t.Value : null; - private static TTarget? TryRead(TomlTable table, string key) => - table.TryGetValue(key, out var node) && node is TTarget t ? t : default; - - private static TTarget Read(TomlTable table, string key) => - TryRead(table, key) ?? throw new Exception($"Could not find {key} in {table}"); - + private static int? TryGetInt(TomlTable table, string key) => + table.TryGetValue(key, out var node) && node is TomlLong t ? (int)t.Value : null; } diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs index 98b54faa4..1e90e7f2c 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs @@ -27,47 +27,124 @@ DocumentationSet set protected override Task GetMinimalParseDocumentAsync(Cancel ctx) { - var document = MarkdownParser.MinimalParseStringAsync(Rule?.Note ?? string.Empty, SourceFile, null); Title = Rule?.Name; + var markdown = GetMarkdown(); + var document = MarkdownParser.MinimalParseStringAsync(markdown, SourceFile, null); return Task.FromResult(document); } protected override Task GetParseDocumentAsync(Cancel ctx) { - if (Rule == null) - return Task.FromResult(MarkdownParser.ParseStringAsync($"# {Title}", SourceFile, null)); + var markdown = GetMarkdown(); + var document = MarkdownParser.ParseStringAsync(markdown, SourceFile, null); + return Task.FromResult(document); + } + private string GetMarkdown() + { + if (Rule is null) + return $"# {Title}"; // language=markdown - var markdown = $""" + var markdown = +$""" # {Rule.Name} +{Rule.Description} + **Rule type**: {Rule.Type} + **Rule indices**: {RenderArray(Rule.Indices)} + **Rule Severity**: {Rule.Severity} + **Risk Score**: {Rule.RiskScore} + **Runs every**: {Rule.RunsEvery} + **Searches indices from**: `{Rule.IndicesFromDateMath}` + **Maximum alerts per execution**: {Rule.MaximumAlertsPerExecution} -**References**: {RenderArray(Rule.References)} + +**References**: {RenderArray((Rule.References ?? []).Select(r => $"[{r}]({r})").ToArray())} + **Tags**: {RenderArray(Rule.Tags)} + **Version**: {Rule.Version} + **Rule authors**: {RenderArray(Rule.Authors)} + **Rule license**: {Rule.License} +"""; + // language=markdown + if (!string.IsNullOrWhiteSpace(Rule.Setup)) + { + markdown += +$""" -## Investigation guide + {Rule.Setup} +"""; + } -{Rule.Note} + // language=markdown + if (!string.IsNullOrWhiteSpace(Rule.Note)) + { + markdown += +$""" -## Rule Query + ## Investigation guide -```{Rule.Language ?? Rule.Type} -{Rule.Query} -``` + {Rule.Note} """; - var document = MarkdownParser.ParseStringAsync(markdown, SourceFile, null); - return Task.FromResult(document); + } + // language=markdown + if (!string.IsNullOrWhiteSpace(Rule.Query)) + { + markdown += +$""" + + ## Rule Query + + ```{Rule.Language ?? Rule.Type} + {Rule.Query} + ``` + """; + } + + foreach (var threat in Rule.Threats) + { + // language=markdown + markdown += +$""" + +**Framework:** {threat.Framework} + +* Tactic: + * Name: {threat.Tactic.Name} + * Id: {threat.Tactic.Id} + * Reference URL: [{threat.Tactic.Reference}]({threat.Tactic.Reference}) + +"""; + foreach (var technique in threat.Techniques) + { + // language=markdown + markdown += TechniqueMarkdown(technique, "Technique"); + foreach (var subTechnique in technique.SubTechniques) + markdown += TechniqueMarkdown(subTechnique, "Sub Technique"); + } + } + return markdown; } + private static string TechniqueMarkdown(DetectionRuleSubTechnique technique, string header) => +$""" + +* {header}: + * Name: {technique.Name} + * Id: {technique.Id} + * Reference URL: [{technique.Reference}]({technique.Reference}) + +"""; + private static string RenderArray(string[]? values) { if (values == null || values.Length == 0) diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs index c8cd31dd0..d3af44c02 100644 --- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs +++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions; +using Elastic.Markdown.Helpers; using Elastic.Markdown.IO; using Elastic.Markdown.IO.Configuration; using Elastic.Markdown.IO.Navigation; From c2410ffb0b501f8a8c32731a1611f40abdcf610b Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 19 Mar 2025 22:33:32 +0100 Subject: [PATCH 3/3] Fallbacks for `kuery` and `lucene` languages --- .../Myst/CodeBlocks/EnhancedCodeBlockParser.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs index e42b4a676..74f4b7466 100644 --- a/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs +++ b/src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockParser.cs @@ -96,6 +96,9 @@ public override bool Close(BlockProcessor processor, Block block) "console-result" => "json", "terminal" => "bash", "painless" => "java", + //TODO support these natively + "kuery" => "json", + "lucene" => "json", _ => codeBlock.Language }; if (!string.IsNullOrEmpty(codeBlock.Language) && !CodeBlock.Languages.Contains(codeBlock.Language))