Fix geomodel noisiness (#1553)

alerts/geomodel/alert.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from enum import Enum
 import math
 from operator import attrgetter
 from typing import List, NamedTuple, Optional
@@ -43,12 +44,31 @@ def to_json(self):
         }
 
 
+class Severity(Enum):
+    '''A representation of the different levels of severity that an alert can
+    be raised to.
+    '''
+
+    STATUS = 'STATUS'
+    INFO = 'INFO'
+    WARNING = 'WARNING'
+    CRITICAL = 'CRITICAL'
+    ERROR = 'ERROR'
+
+
 class Alert(NamedTuple):
     '''A container for the data the alerts output by GeoModel contain.
+
     '''
 
     username: str
     hops: List[Hop]
+    severity: Severity
+    # Because we cannot know ahead of time what factors (see factors.py) will
+    # have been implemented and registered for use, this container should be
+    # thought of as something of a black-box useful only for humans looking
+    # at the alert after it fires.
+    factors: List[dict]
 
 
 def _travel_possible(loc1: Locality, loc2: Locality) -> bool:
@@ -100,7 +120,7 @@ def alert(
     if len(hops) == 0:
         return None
 
-    return Alert(username, hops)
+    return Alert(username, hops, Severity.INFO, [])
 
 
 def summary(alert: Alert) -> str:

alerts/geomodel/config.py

@@ -32,10 +32,25 @@ class Whitelist(NamedTuple):
     cidrs: List[str]
 
 
+class ASNMovement(NamedTuple):
+    '''Configuration for the `asn_movement` factor.
+    '''
+
+    maxmind_db_path: str
+
+
+class Factors(NamedTuple):
+    '''Configuration for factors.
+    '''
+
+    asn_movement: ASNMovement
+
+
 class Config(NamedTuple):
     '''The top-level configuration type.
     '''
 
     localities: Localities
     events: Events
     whitelist: Whitelist
+    factors: Factors

alerts/geomodel/factors.py

@@ -0,0 +1,81 @@
+from typing import Callable, List, NamedTuple
+from functools import reduce
+
+from .alert import Alert, Severity
+
+
+class Enhancement(NamedTuple):
+    '''Information to enhance an `Alert` with, produced by an implementation of
+    a `FactorInterface`.  The `pipe` function handles constructing a modified
+    `Alert` with enhancements applied.
+    '''
+
+    extras: dict
+    severity: Severity
+
+
+# A factor is a sort of plugin intended to enrich a GeoModel alert with extra
+# information that may be useful to incident responders as well as to modify
+# the alert's severity.
+FactorInterface = Callable[[Alert], Enhancement]
+
+
+def pipe(alert: Alert, factors: List[FactorInterface]) -> Alert:
+    '''Run an alert through an ordered pipeline of factors, applying the
+    `Enhancement`s produced by each in turn.
+    '''
+
+    def _apply_enhancement(alert: Alert, enhance: Enhancement) -> Alert:
+        return Alert(
+            username=alert.username,
+            hops=alert.hops,
+            severity=enhance.severity,
+            factors=alert.factors + [enhance.extras])
+
+    return reduce(
+        lambda alrt, fctr: _apply_enhancement(alrt, fctr(alrt)),
+        factors,
+        alert)
+
+
+def asn_movement(db, escalate: Severity) -> FactorInterface:
+    '''Enriches GeoModel alerts with information about the ASNs from which IPs
+    in hops originate.  When movement from one ASN to another is detected, the
+    alert's severity will be raised.
+
+    `maxmind_db_path` is the path to a MaxMind database file containing
+    information about ASNs.
+
+    `escalate` is the severity to (de-)escalate the alert to/from in the case
+    that movement from one ASN to another is detected in the alert.
+    '''
+
+    # Keys in the dictionaries returned by MaxMind.
+    # asn = 'autonomous_system_number'  # currently not used
+    org = 'autonomous_system_organization'
+
+    def factor(alert: Alert) -> Enhancement:
+        ips = [hop.origin.ip for hop in alert.hops]
+        if len(alert.hops) > 0:
+            ips.append(alert.hops[-1].destination.ip)
+
+        # Converting the list of IPs to a set to get the unique items can
+        # result in items being re-arranged.
+        unique_ips = list({ip: True for ip in ips}.keys())
+
+        asn_info = [db.get(ip) for ip in unique_ips]
+        asn_pairs = [
+            (asn_info[i], asn_info[i + 1])
+            for i in range(len(asn_info) - 1)
+        ]
+        asn_hops = [
+            pair
+            for pair in asn_pairs
+            if pair[0][org] != pair[1][org]
+        ]
+
+        return Enhancement(
+            extras={'asn_hops': asn_hops},
+            severity=escalate if len(asn_hops) > 0 else alert.severity)
+
+    return factor

alerts/geomodel_location.json

@@ -13,5 +13,8 @@
   "whitelist": {
     "users": [],
     "cidrs": []
+  },
+  "factors": {
+    "asn_movement": null
   }
 }

alerts/geomodel_location.py

@@ -5,11 +5,11 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 # Copyright (c) 2015 Mozilla Corporation
 
+from datetime import datetime, timedelta
 import json
 import os
 
-from mozdef_util.utilities.toUTC import toUTC
-from datetime import datetime, timedelta
+import maxminddb as mmdb
 
 from lib.alerttask import AlertTask
 from mozdef_util.query_models import\
@@ -18,10 +18,12 @@
     RangeMatch,\
     SubnetMatch,\
     QueryStringMatch as QSMatch
+from mozdef_util.utilities.toUTC import toUTC
 
 import geomodel.alert as alert
 import geomodel.config as config
 import geomodel.execution as execution
+import geomodel.factors as factors
 import geomodel.locality as locality
 
 
@@ -48,6 +50,8 @@ class AlertGeoModel(AlertTask):
     def main(self):
         cfg = self._load_config()
 
+        self.factor_pipeline = self._prepare_factor_pipeline(cfg)
+
         if not self.es.index_exists('localities'):
             settings = {
                 'mappings': {
@@ -145,15 +149,22 @@ def onAggregation(self, agg):
             journal(entry_from_es, cfg.localities.es_index)
 
         if new_alert is not None:
-            summary = alert.summary(new_alert)
+            modded_alert = factors.pipe(new_alert, self.factor_pipeline)
+
+            summary = alert.summary(modded_alert)
 
             alert_dict = self.createAlertDict(
-                summary, 'geomodel', ['geomodel'], events, 'WARNING')
+                summary,
+                'geomodel',
+                ['geomodel'],
+                events,
+                modded_alert.severity.value)
 
             # TODO: When we update to Python 3.7+, change to asdict(alert_produced)
             alert_dict['details'] = {
-                'username': new_alert.username,
+                'username': modded_alert.username,
                 'hops': [hop.to_json() for hop in new_alert.hops],
+                'factors': modded_alert.factors
             }
 
             return alert_dict
@@ -170,4 +181,24 @@ def _load_config(self):
 
             cfg['whitelist'] = config.Whitelist(**cfg['whitelist'])
 
+            asn_mvmt = None
+            if cfg['factors']['asn_movement'] is not None:
+                asn_mvmt = config.ASNMovement(**cfg['factors']['asn_movement'])
+
+            cfg['factors'] = config.Factors(
+                asn_movement=asn_mvmt)
+
             return config.Config(**cfg)
+
+    def _prepare_factor_pipeline(self, cfg):
+        pipeline = []
+
+        if cfg.factors.asn_movement is not None:
+            pipeline.append(
+                factors.asn_movement(
+                    mmdb.open_database(cfg.factors.asn_movement.maxmind_db_path),
+                    alert.Severity.WARNING
+                )
+            )
+
+        return pipeline

requirements.txt

@@ -47,3 +47,4 @@ googleapis-common-protos==1.6.0
 google-cloud-pubsub==1.0.0
 grpc-google-iam-v1==0.12.3
 websocket-client==0.44.0
+maxminddb==1.5.2

tests/alerts/geomodel/test_factors.py

@@ -0,0 +1,90 @@
+from datetime import datetime
+
+import alerts.geomodel.alert as alert
+import alerts.geomodel.factors as factors
+
+
+class MockMMDB:
+    '''Mocks a MaxMind database connection with a dictionary of records mapping
+    IP adresses to dictionaries containing information about ASNs.
+    '''
+
+    def __init__(self, records):
+        self.records = records
+
+    def get(self, ip):
+        return self.records.get(ip)
+
+    def close(self):
+        return
+
+
+def null_origin(ip):
+    return alert.Origin(
+        ip=ip,
+        city='Null',
+        country='NA',
+        latitude=0.0,
+        longitude=0.0,
+        observed=datetime.now(),
+        geopoint='0.0,0.0')
+
+
+# A set of records for a mocked MaxMind database containing information about
+# ASNs used to test the `asn_movement` factor implementation with.
+asn_mvmt_records = {
+    '1.2.3.4': {
+        'autonomous_system_number': 54321,
+        'autonomous_system_organization': 'CLOUDFLARENET'
+    },
+    '4.3.2.1': {
+        'autonomous_system_number': 12345,
+        'autonomous_system_organization': 'MOZILLA_SFO1'
+    },
+    '5.6.7.8': {
+        'autonomous_system_number': 67891,
+        'autonomous_system_organization': 'AMAZONAWSNET'
+    }
+}
+
+
+def test_asn_movement():
+    factor = factors.asn_movement(
+        MockMMDB(asn_mvmt_records),
+        alert.Severity.WARNING)
+
+    test_hops = [
+        alert.Hop(
+            origin=null_origin('1.2.3.4'),
+            destination=null_origin('4.3.2.1')),
+        alert.Hop(
+            origin=null_origin('4.3.2.1'),
+            destination=null_origin('5.6.7.8'))
+    ]
+
+    test_alert = alert.Alert(
+        username='tester',
+        hops=test_hops,
+        severity=alert.Severity.INFO,
+        factors=[])
+
+    pipeline = [factor]
+
+    modified_alert = factors.pipe(test_alert, pipeline)
+
+    assert modified_alert.username == test_alert.username
+    assert modified_alert.severity == alert.Severity.WARNING
+    assert len(modified_alert.factors) == 1
+    assert 'asn_hops' in modified_alert.factors[0]
+    assert len(modified_alert.factors[0]['asn_hops']) == 2
+
+    asn_key = 'autonomous_system_organization'
+    asn1 = modified_alert.factors[0]['asn_hops'][0][0][asn_key]
+    asn2 = modified_alert.factors[0]['asn_hops'][0][1][asn_key]
+    asn3 = modified_alert.factors[0]['asn_hops'][1][0][asn_key]
+    asn4 = modified_alert.factors[0]['asn_hops'][1][1][asn_key]
+
+    assert asn1 == 'CLOUDFLARENET'
+    assert asn2 == 'MOZILLA_SFO1'
+    assert asn3 == 'MOZILLA_SFO1'
+    assert asn4 == 'AMAZONAWSNET'