Enrich GeoModel alerts with information about Tor nodes and VPNs (#1595)

alerts/plugins/geomodel_ipintel_enrichment.json

@@ -0,0 +1,4 @@
+{
+  "intel_file_path": "",
+  "match_tag": "geomodel"
+}

alerts/plugins/geomodel_ipintel_enrichment.py

@@ -0,0 +1,133 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+import json
+import os
+import typing as types
+
+
+CONFIG_FILE = os.path.join(
+    os.path.dirname(__file__),
+    'geomodel_ipintel_enrichment.json')
+
+# TODO: Switch to dataclasses when we move to Python 3.7+
+
+
+class Config(types.NamedTuple):
+    '''Container for the configuration of the plugin.
+
+    `intel_file_path` is a path to an IP reputation JSON file
+    providing information about Tor exit nodes and VPNs.
+
+    `match_tag` is the alert tag to match and run the plugin on.
+    '''
+
+    intel_file_path: str
+    match_tag: str
+
+    def load(file_path: str) -> 'Config':
+        '''Attempt to load a `Config` from a JSON file.
+        '''
+
+        with open(file_path) as cfg_file:
+            return Config(**json.load(cfg_file))
+
+
+class message:
+    '''Alert plugin that handles messages (alerts) tagged as geomodel alerts
+    produced by `geomodel_location.AlertGeoModel`.  This plugin will enrich such
+    alerts with information about Tor exit nodes and/or VPNs identified by any
+    of the IP addresses present in the `details.hops` of the alert.
+
+    Specifically, the alert's `details` dict will have a new field `ipintel`
+    appended containing the contents of an `IPIntel`.
+
+    The alert's summary is also appended with notes about IPs belonging to Tor
+    nodes and VPNs when they are detected.  For example, a summary may take the
+    form
+
+    > user@mozilla.com seen in Dallas,US then Dumbravita,RO
+    > (9293.00 KM in 150.87 minutes); Tor nodes detected: 1.2.3.4, 5.6.7.8
+    > ; VPNs detected: 10.11.12.13
+    '''
+
+    def __init__(self):
+        config = Config.load(CONFIG_FILE)
+
+        self.registration = [config.match_tag]
+
+        with open(config.intel_file_path) as intel_file:
+            self._intel = json.load(intel_file)
+
+    def onMessage(self, message):
+        alert_tags = message.get('tags', [])
+
+        if self.registration[0] in alert_tags:
+            return enrich(message, self._intel)
+
+        return message
+
+
+def enrich(alert, intel):
+    '''Enrich a geomodel alert with intel about IPs that are known Tor exit
+    nodes or members of a VPN.
+    '''
+
+    # The names of classifications present in the intel source that we want to
+    # specifically detect and enrich the alert summary with when detected.
+    tor_class = 'TorNode'
+    vpn_class = 'VPN'
+
+    details = alert.get('details', {})
+
+    hops = details.get('hops', [])
+
+    ips = [
+        hop['origin']['ip']
+        for hop in hops
+    ]
+
+    if len(hops) > 0:
+        ips.append(hops[-1]['destination']['ip'])
+
+    relevant_intel = {
+        ip: intel[ip]
+        for ip in set(ips)
+        if ip in intel
+    }
+
+    ip_intel = []
+
+    for ip, records in relevant_intel.items():
+        ip_intel.extend([
+            {'ip': ip, 'classification': _class, 'threatscore': records[_class]}
+            for _class in records
+        ])
+
+    tor_nodes = [
+        entry['ip']
+        for entry in ip_intel
+        if entry['classification'] == tor_class
+    ]
+
+    vpn_nodes = [
+        entry['ip']
+        for entry in ip_intel
+        if entry['classification'] == vpn_class
+    ]
+
+    if len(tor_nodes) > 0:
+        alert['summary'] += '; Tor nodes detected: {}'.format(
+            ', '.join(tor_nodes))
+
+    if len(vpn_nodes) > 0:
+        alert['summary'] += '; VPNs detected: {}'.format(
+            ', '.join(vpn_nodes))
+
+    details['ipintel'] = ip_intel
+
+    alert['details'] = details
+
+    return alert

cron/update_ipintel.json

@@ -0,0 +1,4 @@
+{
+  "source_url": "",
+  "download_location": ""
+}

cron/update_ipintel.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+import argparse
+import json
+import typing as types
+
+import requests
+
+from mozdef_util.utilities.logger import initLogger, logger
+
+
+# TODO: Move to dataclasses when we switch to Python 3.7+
+
+class Config(types.NamedTuple):
+    '''Container for the configuration data required by the cron task.
+    '''
+
+    source_url: str
+    download_location: str
+
+    def load(path: str) -> 'Config':
+        '''Attempt to load a `Config` from a JSON file.
+        '''
+
+        with open(path) as cfg_file:
+            return Config(**json.load(cfg_file))
+
+
+def download_intel_file(source: str) -> dict:
+    '''Attempt to download the IP Intel file and produce it as JSON.
+    '''
+
+    resp = requests.get(source)
+
+    return resp.json()
+
+
+def main():
+    args_parser = argparse.ArgumentParser(
+        description='Task to update IP Intel JSON')
+    args_parser.add_argument(
+        '-c',
+        '--configfile',
+        help='Path to JSON configuration file to use.')
+
+    args = args_parser.parse_args()
+
+    cfg = Config.load(args.configfile)
+
+    initLogger()
+
+    logger.debug('Downloading IP intel JSON')
+
+    ip_intel_json = download_intel_file(cfg.source_url)
+
+    logger.debug('Writing intel JSON to file')
+
+    with open(cfg.download_location, 'w') as download_location:
+        json.dump(ip_intel_json, download_location)
+
+    logger.debug('Terminating successfully')
+
+
+if __name__ == '__main__':
+    main()

cron/update_ipintel.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2014 Mozilla Corporation
+
+source /opt/mozdef/envs/python/bin/activate
+/opt/mozdef/envs/mozdef/cron/update_ipintel.py -c /opt/mozdef/envs/mozdef/cron/update_ipintel.json

tests/alerts/plugins/test_geomodel_ipintel_enrichment.py

@@ -0,0 +1,81 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# Copyright (c) 2017 Mozilla Corporation
+
+from alerts.plugins.geomodel_ipintel_enrichment import enrich
+
+
+class TestGeoModelEnrichment:
+    def test_enrichment(self):
+        test_alert = {
+            'summary': 'test alert',
+            'details': {
+                'hops': [
+                    {
+                        'origin': {
+                            'ip': '1.2.3.4',
+                        },
+                        'destination': {
+                            'ip': '4.3.2.1',
+                        }
+                    },
+                    {
+                        'origin': {
+                            'ip': '4.3.2.1',
+                        },
+                        'destination': {
+                            'ip': '1.4.2.3',
+                        }
+                    },
+                    {
+                        'origin': {
+                            'ip': '1.4.2.3',
+                        },
+                        'destination': {
+                            'ip': '1.2.3.4',
+                        }
+                    }
+                ]
+            }
+        }
+
+        test_intel = {
+            '1.2.3.4': {
+                'TorNode': 127,
+            },
+            '4.3.2.1': {
+                'Spam': 32,
+                'VPN': 80,
+            }
+        }
+
+        enriched = enrich(test_alert, test_intel)
+
+        # Make sure nothing previously present was changed.
+        assert 'details' in enriched
+        assert 'hops' in enriched['details']
+        assert len(enriched['details']['hops']) == 3
+
+        # Make sure info for the known IPs was added.
+        assert 'ipintel' in enriched['details']
+        assert len(enriched['details']['ipintel']) == 3
+        assert {
+            'ip': '1.2.3.4',
+            'classification': 'TorNode',
+            'threatscore': 127
+        } in enriched['details']['ipintel']
+        assert {
+            'ip': '4.3.2.1',
+            'classification': 'Spam',
+            'threatscore': 32
+        } in enriched['details']['ipintel']
+        assert {
+            'ip': '4.3.2.1',
+            'classification': 'VPN',
+            'threatscore': 80
+        } in enriched['details']['ipintel']
+
+        # Make sure that the alert summary was appended to.
+        assert 'Tor nodes detected: 1.2.3.4' in enriched['summary']
+        assert 'VPNs detected: 4.3.2.1' in enriched['summary']