From 92eaa5b18ab4d613e1adc39732c491f54682035a Mon Sep 17 00:00:00 2001 From: Andrew Pease <7442091+peasead@users.noreply.github.com> Date: Mon, 26 Apr 2021 07:07:04 -0500 Subject: [PATCH] [New Rule] Threat intel indicator match rule (#1133) --- detection_rules/beats.py | 3 + detection_rules/cli_utils.py | 2 +- detection_rules/ecs.py | 6 +- detection_rules/main.py | 3 +- detection_rules/packaging.py | 2 +- detection_rules/rule.py | 8 +- detection_rules/schemas/definitions.py | 12 +- .../threat_intel_module_match.toml | 147 ++++++++++++++++++ tests/test_all_rules.py | 17 +- 9 files changed, 181 insertions(+), 19 deletions(-) create mode 100644 rules/cross-platform/threat_intel_module_match.toml diff --git a/detection_rules/beats.py b/detection_rules/beats.py index 859ac3f0af6..464e60c8d10 100644 --- a/detection_rules/beats.py +++ b/detection_rules/beats.py @@ -174,6 +174,9 @@ def get_beats_sub_schema(schema: dict, beat: str, module: str, *datasets: str): dataset_dir = module_dir.get("folders", {}).get(dataset, {}) flattened.extend(get_field_schema(dataset_dir, prefix=module + ".", include_common=True)) + # we also need to capture (beta?) fields which are directly within the module _meta.files.fields + flattened.extend(get_field_schema(module_dir, include_common=True)) + return {field["name"]: field for field in sorted(flattened, key=lambda f: f["name"])} diff --git a/detection_rules/cli_utils.py b/detection_rules/cli_utils.py index e59b626cc88..4b7fd610c28 100644 --- a/detection_rules/cli_utils.py +++ b/detection_rules/cli_utils.py @@ -73,7 +73,7 @@ def get_collection(*args, **kwargs): rules = RuleCollection() - if not rule_name or rule_id or rule_files: + if not (rule_name or rule_id or rule_files): client_error('Required: at least one of --rule-id, --rule-file, or --directory') rules.load_files(Path(p) for p in rule_files) diff --git a/detection_rules/ecs.py b/detection_rules/ecs.py index dd4a67b9bd5..fa6a4e7b233 100644 --- a/detection_rules/ecs.py +++ b/detection_rules/ecs.py @@ -9,6 +9,7 @@ import os import shutil import json +from pathlib import Path import requests import eql @@ -99,7 +100,7 @@ def get_max_version(include_master=False): versions = get_schema_map().keys() if include_master and any([v.startswith('master') for v in versions]): - return glob.glob(os.path.join(ECS_SCHEMAS_DIR, 'master*'))[0] + return list(Path(ECS_SCHEMAS_DIR).glob('master*'))[0].name return str(max([Version(v) for v in versions if not v.startswith('master')])) @@ -107,6 +108,9 @@ def get_max_version(include_master=False): @cached def get_schema(version=None, name='ecs_flat'): """Get schema by version.""" + if version == 'master': + version = get_max_version(include_master=True) + return get_schemas()[version or str(get_max_version())][name] diff --git a/detection_rules/main.py b/detection_rules/main.py index 3607ab91fbc..42a9cd2cff5 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -113,7 +113,8 @@ def name_to_filename(name): @root.command('toml-lint') -@click.option('--rule-file', '-f', type=click.Path('r'), help='Optionally specify a specific rule file only') +@click.option('--rule-file', '-f', type=click.Path(), multiple=True, + help='Optionally specify rule files') def toml_lint(rule_file): """Cleanup files with some simple toml formatting.""" if rule_file: diff --git a/detection_rules/packaging.py b/detection_rules/packaging.py index e24f66a7afe..2e61033239c 100644 --- a/detection_rules/packaging.py +++ b/detection_rules/packaging.py @@ -301,7 +301,7 @@ def export(self, outfile, downgrade_version=None, verbose=True, skip_unsupported output_lines = [json.dumps(downgrade_contents_from_rule(r, downgrade_version), sort_keys=True) for r in self.rules] else: - output_lines = [json.dumps(r.contents, sort_keys=True) for r in self.rules] + output_lines = [json.dumps(r.contents.data.to_dict(), sort_keys=True) for r in self.rules] outfile.write_text('\n'.join(output_lines) + '\n') diff --git a/detection_rules/rule.py b/detection_rules/rule.py index 8c3bc4a0223..10d1b39f8ad 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -30,8 +30,8 @@ class RuleMeta(MarshmallowDataclassMixin): deprecation_date: Optional[definitions.Date] # Optional fields - beats_version: Optional[definitions.SemVer] - ecs_versions: Optional[List[definitions.SemVer]] + beats_version: Optional[definitions.BranchVer] + ecs_versions: Optional[List[definitions.BranchVer]] comments: Optional[str] maturity: Optional[definitions.Maturity] os_type_list: Optional[List[definitions.OSType]] @@ -161,8 +161,8 @@ class BaseRuleData(MarshmallowDataclassMixin): severity: definitions.Severity tags: Optional[List[str]] throttle: Optional[str] - timeline_id: Optional[str] - timeline_title: Optional[str] + timeline_id: Optional[definitions.TimelineTemplateId] + timeline_title: Optional[definitions.TimelineTemplateTitle] timestamp_override: Optional[str] to: Optional[str] type: Literal[definitions.RuleType] diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py index e3c82d0550c..51643f2dd92 100644 --- a/detection_rules/schemas/definitions.py +++ b/detection_rules/schemas/definitions.py @@ -5,7 +5,7 @@ """Custom shared definitions for schemas.""" -from typing import Literal +from typing import Literal, Final from marshmallow import validate from marshmallow_dataclass import NewType @@ -35,7 +35,15 @@ OPERATORS = ['equals'] +TIMELINE_TEMPLATES: Final[dict] = { + 'db366523-f1c6-4c1f-8731-6ce5ed9e5717': 'Generic Endpoint Timeline', + '91832785-286d-4ebe-b884-1a208d111a70': 'Generic Network Timeline', + '76e52245-7519-4251-91ab-262fb1a1728c': 'Generic Process Timeline', + '495ad7a7-316e-4544-8a0f-9c098daee76e': 'Generic Threat Match Timeline' +} + +BranchVer = NewType('BranchVer', str, validate=validate.Regexp(BRANCH_PATTERN)) CodeString = NewType("CodeString", str) ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN)) Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN)) @@ -57,4 +65,6 @@ TacticURL = NewType('TacticURL', str, validate=validate.Regexp(TACTIC_URL)) TechniqueURL = NewType('TechniqueURL', str, validate=validate.Regexp(TECHNIQUE_URL)) ThresholdValue = NewType("ThresholdValue", int, validate=validate.Range(min=1)) +TimelineTemplateId = NewType('TimelineTemplateId', str, validate=validate.OneOf(list(TIMELINE_TEMPLATES))) +TimelineTemplateTitle = NewType('TimelineTemplateTitle', str, validate=validate.OneOf(TIMELINE_TEMPLATES.values())) UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN)) diff --git a/rules/cross-platform/threat_intel_module_match.toml b/rules/cross-platform/threat_intel_module_match.toml new file mode 100644 index 00000000000..899b24f6424 --- /dev/null +++ b/rules/cross-platform/threat_intel_module_match.toml @@ -0,0 +1,147 @@ +[metadata] +creation_date = "2021/04/21" +maturity = "production" +updated_date = "2021/04/21" + +[rule] +author = ["Elastic"] +description = """ +This rule is triggered when indicators from the Threat Intel Filebeat module has a match against local file or network observations. +""" +from = "now-10m" +index = ["auditbeat-*", "endgame-*", "filebeat-*", "logs-*", "packetbeat-*", "winlogbeat-*"] +interval = "9m" +language = "kuery" +license = "Elastic License v2" +name = "Threat Intel Filebeat Module Indicator Match" +note = """ +## Triage and Analysis +If an indicator matches a local observation, the following enriched fields will be generated to identify the indicator, field, and type matched. + +- `threatintel.indicator.matched.atomic` - this identifies the atomic indicator that matched the local observation +- `threatintel.indicator.matched.field` - this identifies the indicator field that matched the local observation +- `threatintel.indicator.matched.type` - this identifies the indicator type that matched the local observation +""" +references = [ "https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-threatintel.html"] +risk_score = 99 +rule_id = "dc672cb7-d5df-4d1f-a6d7-0841b1caafb9" +severity = "critical" +tags = ["Elastic", "Windows", "Elastic Endgame", "Network", "Continuous Monitoring", "SecOps", "Monitoring"] +timeline_id = "495ad7a7-316e-4544-8a0f-9c098daee76e" +timeline_title = "Generic Threat Match Timeline" +type = "threat_match" + +threat_index = [ "filebeat-*"] +threat_indicator_path = "" +threat_language = "kuery" + +threat_query = """ +event.module:threatintel and + (threatintel.indicator.file.hash.*:* or threatintel.indicator.file.pe.imphash:* or threatintel.indicator.ip:* or + threatintel.indicator.registry.path:* or threatintel.indicator.url.full:*) +""" + +query = """ +file.hash.*:* or file.pe.imphash:* or source.ip:* or destination.ip:* or url.full:* or registry.path:* +""" + + +[[rule.threat_filters]] +[rule.threat_filters."$state"] +store = "appState" +[rule.threat_filters.meta] +negate = false +disabled = false +type = "phrase" +key = "event.module" +[rule.threat_filters.meta.params] +query = "threatintel" +[rule.threat_filters.query.match_phrase] +"event.module" = "threatintel" + +[[rule.threat_filters]] +[rule.threat_filters."$state"] +store = "appState" +[rule.threat_filters.meta] +negate = false +disabled = false +type = "phrase" +key = "event.category" +[rule.threat_filters.meta.params] +query = "threat" +[rule.threat_filters.query.match_phrase] +"event.category" = "threat" + +[[rule.threat_filters]] +[rule.threat_filters."$state"] +store = "appState" +[rule.threat_filters.meta] +negate = false +disabled = false +type = "phrase" +key = "event.kind" +[rule.threat_filters.meta.params] +query = "enrichment" +[rule.threat_filters.query.match_phrase] +"event.kind" = "enrichment" + +[[rule.threat_filters]] +[rule.threat_filters."$state"] +store = "appState" +[rule.threat_filters.meta] +negate = false +disabled = false +type = "phrase" +key = "event.type" +[rule.threat_filters.meta.params] +query = "indicator" +[rule.threat_filters.query.match_phrase] +"event.type" = "indicator" + +[[rule.threat_mapping]] +[[rule.threat_mapping.entries]] +field = "file.hash.md5" +type = "mapping" +value = "threatintel.indicator.file.hash.md5" + +[[rule.threat_mapping]] +[[rule.threat_mapping.entries]] +field = "file.hash.sha1" +type = "mapping" +value = "threatintel.indicator.file.hash.sha1" + +[[rule.threat_mapping]] +[[rule.threat_mapping.entries]] +field = "file.hash.sha256" +type = "mapping" +value = "threatintel.indicator.file.hash.sha256" + +[[rule.threat_mapping]] +[[rule.threat_mapping.entries]] +field = "file.pe.imphash" +type = "mapping" +value = "threatintel.indicator.file.pe.imphash" + +[[rule.threat_mapping]] +[[rule.threat_mapping.entries]] +field = "source.ip" +type = "mapping" +value = "threatintel.indicator.ip" + +[[rule.threat_mapping]] +[[rule.threat_mapping.entries]] +field = "destination.ip" +type = "mapping" +value = "threatintel.indicator.ip" + +[[rule.threat_mapping]] +[[rule.threat_mapping.entries]] +field = "url.full" +type = "mapping" +value = "threatintel.indicator.url.full" + +[[rule.threat_mapping]] +[[rule.threat_mapping.entries]] +field = "registry.path" +type = "mapping" +value = "threatintel.indicator.registry.path" diff --git a/tests/test_all_rules.py b/tests/test_all_rules.py index 0faadf0828d..da6c3f330a6 100644 --- a/tests/test_all_rules.py +++ b/tests/test_all_rules.py @@ -299,14 +299,10 @@ def test_primary_tactic_as_tag(self): class TestRuleTimelines(BaseRuleTest): """Test timelines in rules are valid.""" - TITLES = { - 'db366523-f1c6-4c1f-8731-6ce5ed9e5717': 'Generic Endpoint Timeline', - '91832785-286d-4ebe-b884-1a208d111a70': 'Generic Network Timeline', - '76e52245-7519-4251-91ab-262fb1a1728c': 'Generic Process Timeline' - } - def test_timeline_has_title(self): """Ensure rules with timelines have a corresponding title.""" + from detection_rules.schemas.definitions import TIMELINE_TEMPLATES + for rule in self.all_rules: timeline_id = rule.contents.data.timeline_id timeline_title = rule.contents.data.timeline_title @@ -317,13 +313,14 @@ def test_timeline_has_title(self): if timeline_id: unknown_id = f'{self.rule_str(rule)} Unknown timeline_id: {timeline_id}.' - unknown_id += f' replace with {", ".join(self.TITLES)} or update this unit test with acceptable ids' - self.assertIn(timeline_id, list(self.TITLES), unknown_id) + unknown_id += f' replace with {", ".join(TIMELINE_TEMPLATES)} ' \ + f'or update this unit test with acceptable ids' + self.assertIn(timeline_id, list(TIMELINE_TEMPLATES), unknown_id) unknown_title = f'{self.rule_str(rule)} unknown timeline_title: {timeline_title}' - unknown_title += f' replace with {", ".join(self.TITLES.values())}' + unknown_title += f' replace with {", ".join(TIMELINE_TEMPLATES.values())}' unknown_title += ' or update this unit test with acceptable titles' - self.assertEqual(timeline_title, self.TITLES[timeline_id], ) + self.assertEqual(timeline_title, TIMELINE_TEMPLATES[timeline_id], unknown_title) class TestRuleFiles(BaseRuleTest):