Merge pull request #2 from elastic/main

[New Rule] Threat intel indicator match rule (elastic#1133)

detection_rules/beats.py

@@ -174,6 +174,9 @@ def get_beats_sub_schema(schema: dict, beat: str, module: str, *datasets: str):
         dataset_dir = module_dir.get("folders", {}).get(dataset, {})
         flattened.extend(get_field_schema(dataset_dir, prefix=module + ".", include_common=True))
 
+    # we also need to capture (beta?) fields which are directly within the module _meta.files.fields
+    flattened.extend(get_field_schema(module_dir, include_common=True))
+
     return {field["name"]: field for field in sorted(flattened, key=lambda f: f["name"])}
 
 

detection_rules/cli_utils.py

@@ -73,7 +73,7 @@ def get_collection(*args, **kwargs):
 
         rules = RuleCollection()
 
-        if not rule_name or rule_id or rule_files:
+        if not (rule_name or rule_id or rule_files):
             client_error('Required: at least one of --rule-id, --rule-file, or --directory')
 
         rules.load_files(Path(p) for p in rule_files)

detection_rules/ecs.py

@@ -9,6 +9,7 @@
 import os
 import shutil
 import json
+from pathlib import Path
 
 import requests
 import eql
@@ -99,14 +100,17 @@ def get_max_version(include_master=False):
     versions = get_schema_map().keys()
 
     if include_master and any([v.startswith('master') for v in versions]):
-        return glob.glob(os.path.join(ECS_SCHEMAS_DIR, 'master*'))[0]
+        return list(Path(ECS_SCHEMAS_DIR).glob('master*'))[0].name
 
     return str(max([Version(v) for v in versions if not v.startswith('master')]))
 
 
 @cached
 def get_schema(version=None, name='ecs_flat'):
     """Get schema by version."""
+    if version == 'master':
+        version = get_max_version(include_master=True)
+
     return get_schemas()[version or str(get_max_version())][name]
 
 

detection_rules/main.py

@@ -113,7 +113,8 @@ def name_to_filename(name):
 
 
 @root.command('toml-lint')
-@click.option('--rule-file', '-f', type=click.Path('r'), help='Optionally specify a specific rule file only')
+@click.option('--rule-file', '-f', type=click.Path(), multiple=True,
+              help='Optionally specify rule files')
 def toml_lint(rule_file):
     """Cleanup files with some simple toml formatting."""
     if rule_file:

detection_rules/packaging.py

@@ -301,7 +301,7 @@ def export(self, outfile, downgrade_version=None, verbose=True, skip_unsupported
                 output_lines = [json.dumps(downgrade_contents_from_rule(r, downgrade_version), sort_keys=True)
                                 for r in self.rules]
         else:
-            output_lines = [json.dumps(r.contents, sort_keys=True) for r in self.rules]
+            output_lines = [json.dumps(r.contents.data.to_dict(), sort_keys=True) for r in self.rules]
 
         outfile.write_text('\n'.join(output_lines) + '\n')
 

detection_rules/rule.py

@@ -30,8 +30,8 @@ class RuleMeta(MarshmallowDataclassMixin):
     deprecation_date: Optional[definitions.Date]
 
     # Optional fields
-    beats_version: Optional[definitions.SemVer]
-    ecs_versions: Optional[List[definitions.SemVer]]
+    beats_version: Optional[definitions.BranchVer]
+    ecs_versions: Optional[List[definitions.BranchVer]]
     comments: Optional[str]
     maturity: Optional[definitions.Maturity]
     os_type_list: Optional[List[definitions.OSType]]
@@ -161,8 +161,8 @@ class BaseRuleData(MarshmallowDataclassMixin):
     severity: definitions.Severity
     tags: Optional[List[str]]
     throttle: Optional[str]
-    timeline_id: Optional[str]
-    timeline_title: Optional[str]
+    timeline_id: Optional[definitions.TimelineTemplateId]
+    timeline_title: Optional[definitions.TimelineTemplateTitle]
     timestamp_override: Optional[str]
     to: Optional[str]
     type: Literal[definitions.RuleType]

detection_rules/schemas/definitions.py

@@ -5,7 +5,7 @@
 
 """Custom shared definitions for schemas."""
 
-from typing import Literal
+from typing import Literal, Final
 
 from marshmallow import validate
 from marshmallow_dataclass import NewType
@@ -35,7 +35,15 @@
 
 OPERATORS = ['equals']
 
+TIMELINE_TEMPLATES: Final[dict] = {
+    'db366523-f1c6-4c1f-8731-6ce5ed9e5717': 'Generic Endpoint Timeline',
+    '91832785-286d-4ebe-b884-1a208d111a70': 'Generic Network Timeline',
+    '76e52245-7519-4251-91ab-262fb1a1728c': 'Generic Process Timeline',
+    '495ad7a7-316e-4544-8a0f-9c098daee76e': 'Generic Threat Match Timeline'
+}
 
+
+BranchVer = NewType('BranchVer', str, validate=validate.Regexp(BRANCH_PATTERN))
 CodeString = NewType("CodeString", str)
 ConditionSemVer = NewType('ConditionSemVer', str, validate=validate.Regexp(CONDITION_VERSION_PATTERN))
 Date = NewType('Date', str, validate=validate.Regexp(DATE_PATTERN))
@@ -57,4 +65,6 @@
 TacticURL = NewType('TacticURL', str, validate=validate.Regexp(TACTIC_URL))
 TechniqueURL = NewType('TechniqueURL', str, validate=validate.Regexp(TECHNIQUE_URL))
 ThresholdValue = NewType("ThresholdValue", int, validate=validate.Range(min=1))
+TimelineTemplateId = NewType('TimelineTemplateId', str, validate=validate.OneOf(list(TIMELINE_TEMPLATES)))
+TimelineTemplateTitle = NewType('TimelineTemplateTitle', str, validate=validate.OneOf(TIMELINE_TEMPLATES.values()))
 UUIDString = NewType('UUIDString', str, validate=validate.Regexp(UUID_PATTERN))

rules/cross-platform/threat_intel_module_match.toml

@@ -0,0 +1,147 @@
+[metadata]
+creation_date = "2021/04/21"
+maturity = "production"
+updated_date = "2021/04/21"
+
+[rule]
+author = ["Elastic"]
+description = """
+This rule is triggered when indicators from the Threat Intel Filebeat module has a match against local file or network observations.
+"""
+from = "now-10m"
+index = ["auditbeat-*", "endgame-*", "filebeat-*", "logs-*", "packetbeat-*", "winlogbeat-*"]
+interval = "9m"
+language = "kuery"
+license = "Elastic License v2"
+name = "Threat Intel Filebeat Module Indicator Match"
+note = """
+## Triage and Analysis
+If an indicator matches a local observation, the following enriched fields will be generated to identify the indicator, field, and type matched.
+
+- `threatintel.indicator.matched.atomic` - this identifies the atomic indicator that matched the local observation
+- `threatintel.indicator.matched.field` - this identifies the indicator field that matched the local observation
+- `threatintel.indicator.matched.type` - this identifies the indicator type that matched the local observation
+"""
+references = [ "https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-threatintel.html"]
+risk_score = 99
+rule_id = "dc672cb7-d5df-4d1f-a6d7-0841b1caafb9"
+severity = "critical"
+tags = ["Elastic", "Windows", "Elastic Endgame", "Network", "Continuous Monitoring", "SecOps", "Monitoring"]
+timeline_id = "495ad7a7-316e-4544-8a0f-9c098daee76e"
+timeline_title = "Generic Threat Match Timeline"
+type = "threat_match"
+
+threat_index = [ "filebeat-*"]
+threat_indicator_path = ""
+threat_language = "kuery"
+
+threat_query = """
+event.module:threatintel and
+  (threatintel.indicator.file.hash.*:* or threatintel.indicator.file.pe.imphash:* or threatintel.indicator.ip:* or
+     threatintel.indicator.registry.path:* or threatintel.indicator.url.full:*)
+"""
+
+query = """
+file.hash.*:* or file.pe.imphash:* or source.ip:* or destination.ip:* or url.full:* or registry.path:*
+"""
+
+
+[[rule.threat_filters]]
+[rule.threat_filters."$state"]
+store = "appState"
+[rule.threat_filters.meta]
+negate = false
+disabled = false
+type = "phrase"
+key = "event.module"
+[rule.threat_filters.meta.params]
+query = "threatintel"
+[rule.threat_filters.query.match_phrase]
+"event.module" = "threatintel"
+
+[[rule.threat_filters]]
+[rule.threat_filters."$state"]
+store = "appState"
+[rule.threat_filters.meta]
+negate = false
+disabled = false
+type = "phrase"
+key = "event.category"
+[rule.threat_filters.meta.params]
+query = "threat"
+[rule.threat_filters.query.match_phrase]
+"event.category" = "threat"
+
+[[rule.threat_filters]]
+[rule.threat_filters."$state"]
+store = "appState"
+[rule.threat_filters.meta]
+negate = false
+disabled = false
+type = "phrase"
+key = "event.kind"
+[rule.threat_filters.meta.params]
+query = "enrichment"
+[rule.threat_filters.query.match_phrase]
+"event.kind" = "enrichment"
+
+[[rule.threat_filters]]
+[rule.threat_filters."$state"]
+store = "appState"
+[rule.threat_filters.meta]
+negate = false
+disabled = false
+type = "phrase"
+key = "event.type"
+[rule.threat_filters.meta.params]
+query = "indicator"
+[rule.threat_filters.query.match_phrase]
+"event.type" = "indicator"
+
+[[rule.threat_mapping]]
+[[rule.threat_mapping.entries]]
+field = "file.hash.md5"
+type = "mapping"
+value = "threatintel.indicator.file.hash.md5"
+
+[[rule.threat_mapping]]
+[[rule.threat_mapping.entries]]
+field = "file.hash.sha1"
+type = "mapping"
+value = "threatintel.indicator.file.hash.sha1"
+
+[[rule.threat_mapping]]
+[[rule.threat_mapping.entries]]
+field = "file.hash.sha256"
+type = "mapping"
+value = "threatintel.indicator.file.hash.sha256"
+
+[[rule.threat_mapping]]
+[[rule.threat_mapping.entries]]
+field = "file.pe.imphash"
+type = "mapping"
+value = "threatintel.indicator.file.pe.imphash"
+
+[[rule.threat_mapping]]
+[[rule.threat_mapping.entries]]
+field = "source.ip"
+type = "mapping"
+value = "threatintel.indicator.ip"
+
+[[rule.threat_mapping]]
+[[rule.threat_mapping.entries]]
+field = "destination.ip"
+type = "mapping"
+value = "threatintel.indicator.ip"
+
+[[rule.threat_mapping]]
+[[rule.threat_mapping.entries]]
+field = "url.full"
+type = "mapping"
+value = "threatintel.indicator.url.full"
+
+[[rule.threat_mapping]]
+[[rule.threat_mapping.entries]]
+field = "registry.path"
+type = "mapping"
+value = "threatintel.indicator.registry.path"

tests/test_all_rules.py

@@ -299,14 +299,10 @@ def test_primary_tactic_as_tag(self):
 class TestRuleTimelines(BaseRuleTest):
     """Test timelines in rules are valid."""
 
-    TITLES = {
-        'db366523-f1c6-4c1f-8731-6ce5ed9e5717': 'Generic Endpoint Timeline',
-        '91832785-286d-4ebe-b884-1a208d111a70': 'Generic Network Timeline',
-        '76e52245-7519-4251-91ab-262fb1a1728c': 'Generic Process Timeline'
-    }
-
     def test_timeline_has_title(self):
         """Ensure rules with timelines have a corresponding title."""
+        from detection_rules.schemas.definitions import TIMELINE_TEMPLATES
+
         for rule in self.all_rules:
             timeline_id = rule.contents.data.timeline_id
             timeline_title = rule.contents.data.timeline_title
@@ -317,13 +313,14 @@ def test_timeline_has_title(self):
 
             if timeline_id:
                 unknown_id = f'{self.rule_str(rule)} Unknown timeline_id: {timeline_id}.'
-                unknown_id += f' replace with {", ".join(self.TITLES)} or update this unit test with acceptable ids'
-                self.assertIn(timeline_id, list(self.TITLES), unknown_id)
+                unknown_id += f' replace with {", ".join(TIMELINE_TEMPLATES)} ' \
+                              f'or update this unit test with acceptable ids'
+                self.assertIn(timeline_id, list(TIMELINE_TEMPLATES), unknown_id)
 
                 unknown_title = f'{self.rule_str(rule)} unknown timeline_title: {timeline_title}'
-                unknown_title += f' replace with {", ".join(self.TITLES.values())}'
+                unknown_title += f' replace with {", ".join(TIMELINE_TEMPLATES.values())}'
                 unknown_title += ' or update this unit test with acceptable titles'
-                self.assertEqual(timeline_title, self.TITLES[timeline_id], )
+                self.assertEqual(timeline_title, TIMELINE_TEMPLATES[timeline_id], unknown_title)
 
 
 class TestRuleFiles(BaseRuleTest):