From 67e2e29fb23e53c73655fe8df9779f7e8cc69796 Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Tue, 30 Oct 2018 15:39:45 +1100 Subject: [PATCH 1/9] Create an alert on attempts to use known exfil domains For example, pastebin.com --- alerts/proxy_drop_exfil_domains.py | 60 ++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 alerts/proxy_drop_exfil_domains.py diff --git a/alerts/proxy_drop_exfil_domains.py b/alerts/proxy_drop_exfil_domains.py new file mode 100644 index 000000000..3f181bb2a --- /dev/null +++ b/alerts/proxy_drop_exfil_domains.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2014 Mozilla Corporation + + +from lib.alerttask import AlertTask +from mozdef_util.query_models import QueryStringMatch, SearchQuery, TermMatch + + +class AlertProxyDropExfilDomain(AlertTask): + def main(self): + self.parse_config('proxy_drop_exfil_domains.conf', ['exfil_domains']) + + search_query = SearchQuery(minutes=20) + + search_query.add_must([ + TermMatch('category', 'squid'), + TermMatch('tags', 'squid'), + TermMatch('details.proxyaction', 'TCP_DENIED/-') + ]) + + # Only notify on certain domains listed in the config + domain_regex = "/({0}).*/".format( + self.config.exfil_daomins.replace(',', '|')) + search_query.add_must([ + QueryStringMatch('details.destination: {}'.format(domain_regex)) + ]) + + self.filtersManual(search_query) + + # Search aggregations on field 'hostname', keep X samples of + # events at most + self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10) + # alert when >= X matching events in an aggregation + # I think it makes sense to alert every time here + self.walkAggregations(threshold=1) + + # Set alert properties + def onAggregation(self, aggreg): + # aggreg['count']: number of items in the aggregation, ex: number of failed login attempts + # aggreg['value']: value of the aggregation field, ex: toto@example.com + # aggreg['events']: list of events in the aggregation + category = 'squid' + tags = ['squid', 'proxy'] + severity = 'WARNING' + + dropped_domains = set() + for event in aggreg['allevents']: + dropped_domains.add(event['_source']['details']['destination']) + + summary = 'Suspicious Proxy DROP event(s) detected from {0} to the following exfil domains: {1}'.format( + aggreg['value'], + ",".join(sorted(dropped_domains)) + ) + + # Create the alert object based on these properties + return self.createAlertDict(summary, category, tags, aggreg['events'], severity) From 3b7b8aee684dd078c04b4a600e8e141fdff115b7 Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Tue, 30 Oct 2018 15:42:26 +1100 Subject: [PATCH 2/9] Create config file for the alert proxy_drop_exfil_domains We would only need to populate this file with additional domains if we would like to extend its capability --- alerts/proxy_drop_exfil_domains.conf | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 alerts/proxy_drop_exfil_domains.conf diff --git a/alerts/proxy_drop_exfil_domains.conf b/alerts/proxy_drop_exfil_domains.conf new file mode 100644 index 000000000..21f4e606f --- /dev/null +++ b/alerts/proxy_drop_exfil_domains.conf @@ -0,0 +1,2 @@ +[options] +exfil_domains = pastebin.com,pasted.co,hastebin.com,dumpz.org,www.pastefs.com,send.firefox.com From bcd79e4d9cc1d08e881c37359c9d3e6ea48fa415 Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Tue, 30 Oct 2018 15:43:59 +1100 Subject: [PATCH 3/9] Create unit test for proxy_drop_exfil_domains alert --- tests/alerts/test_proxy_drop_exfil_domains.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 tests/alerts/test_proxy_drop_exfil_domains.py diff --git a/tests/alerts/test_proxy_drop_exfil_domains.py b/tests/alerts/test_proxy_drop_exfil_domains.py new file mode 100644 index 000000000..09b949172 --- /dev/null +++ b/tests/alerts/test_proxy_drop_exfil_domains.py @@ -0,0 +1,114 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation +from positive_alert_test_case import PositiveAlertTestCase +from negative_alert_test_case import NegativeAlertTestCase +from alert_test_suite import AlertTestSuite + + +class TestAlertProxyDropExfilDomain(AlertTestSuite): + alert_filename = "proxy_drop_executable" + # This event is the default positive event that will cause the + # alert to trigger + default_event = { + "_type": "event", + "_source": { + "category": "squid", + "tags": ["squid"], + "details": { + "sourceipaddress": "1.2.3.4", + "destination": "http://pastebin.com/", + "proxyaction": "TCP_DENIED/-", + } + } + } + + # This event is an alternate destination that we'd want to aggregate + default_event2 = AlertTestSuite.copy(default_event) + default_event2["_source"]["details"]["destination"] = "http://hastebin.com" + + # This event is the default negative event that will not cause the + # alert to trigger + default_negative_event = AlertTestSuite.copy(default_event) + default_negative_event["_source"]["details"]["destination"] = "http://foo.mozilla.com/index.html" + + # This alert is the expected result from running this task + default_alert = { + "category": "squid", + "tags": ['squid', 'proxy'], + "severity": "WARNING", + "summary": 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): http://pastebin.com/', + } + + # This alert is the expected result from this task against multiple matching events + default_alert_aggregated = AlertTestSuite.copy(default_alert) + default_alert_aggregated[ + "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): http://pastebin.com/,http://hastebin.com' + + test_cases = [] + + test_cases.append( + PositiveAlertTestCase( + description="Positive test with default events and default alert expected", + events=AlertTestSuite.create_events(default_event, 1), + expected_alert=default_alert + ) + ) + + test_cases.append( + PositiveAlertTestCase( + description="Positive test with default events and default alert expected - dedup", + events=AlertTestSuite.create_events(default_event, 2), + expected_alert=default_alert + ) + ) + + events1 = AlertTestSuite.create_events(default_event, 1) + events2 = AlertTestSuite.create_events(default_event2, 1) + test_cases.append( + PositiveAlertTestCase( + description="Positive test with default events and default alert expected - different destinations", + events=events1 + events2, + expected_alert=default_alert_aggregated + ) + ) + + test_cases.append( + NegativeAlertTestCase( + description="Negative test with default negative event", + events=AlertTestSuite.create_events(default_negative_event, 1), + ) + ) + + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['category'] = 'bad' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with events with incorrect category", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['tags'] = 'bad tag example' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with events with incorrect tags", + events=events, + ) + ) + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({ + 'minutes': 241}) + event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({ + 'minutes': 241}) + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with old timestamp", + events=events, + ) + ) From 6e9d43983d191b073e7e0460cded6672bd5d400a Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Thu, 1 Nov 2018 15:13:21 +1100 Subject: [PATCH 4/9] Add another domain --- alerts/proxy_drop_exfil_domains.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/proxy_drop_exfil_domains.conf b/alerts/proxy_drop_exfil_domains.conf index 21f4e606f..4e52d556d 100644 --- a/alerts/proxy_drop_exfil_domains.conf +++ b/alerts/proxy_drop_exfil_domains.conf @@ -1,2 +1,2 @@ [options] -exfil_domains = pastebin.com,pasted.co,hastebin.com,dumpz.org,www.pastefs.com,send.firefox.com +exfil_domains = pastebin.com,pasted.co,hastebin.com,dumpz.org,www.pastefs.com,send.firefox.com,gist.github.com From b1ad7b847d4792f75d17dde38e5f1c879ef8fd9f Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Thu, 1 Nov 2018 20:02:14 +1100 Subject: [PATCH 5/9] Update to improve and fix a typo --- alerts/proxy_drop_exfil_domains.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/alerts/proxy_drop_exfil_domains.py b/alerts/proxy_drop_exfil_domains.py index 3f181bb2a..b40cbdbfa 100644 --- a/alerts/proxy_drop_exfil_domains.py +++ b/alerts/proxy_drop_exfil_domains.py @@ -7,10 +7,10 @@ from lib.alerttask import AlertTask -from mozdef_util.query_models import QueryStringMatch, SearchQuery, TermMatch +from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, ExistsMatch, PhraseMatch, WildcardMatch -class AlertProxyDropExfilDomain(AlertTask): +class AlertProxyDropExfilDomains(AlertTask): def main(self): self.parse_config('proxy_drop_exfil_domains.conf', ['exfil_domains']) @@ -19,12 +19,12 @@ def main(self): search_query.add_must([ TermMatch('category', 'squid'), TermMatch('tags', 'squid'), - TermMatch('details.proxyaction', 'TCP_DENIED/-') + TermMatch('details.proxyaction', "TCP_DENIED/-") ]) # Only notify on certain domains listed in the config domain_regex = "/({0}).*/".format( - self.config.exfil_daomins.replace(',', '|')) + self.config.exfil_domains.replace(',', '|')) search_query.add_must([ QueryStringMatch('details.destination: {}'.format(domain_regex)) ]) @@ -47,13 +47,14 @@ def onAggregation(self, aggreg): tags = ['squid', 'proxy'] severity = 'WARNING' - dropped_domains = set() + exfil_domains = set() for event in aggreg['allevents']: - dropped_domains.add(event['_source']['details']['destination']) + domain = event['_source']['details']['destination'].split(':') + exfil_domains.add(domain[0]) - summary = 'Suspicious Proxy DROP event(s) detected from {0} to the following exfil domains: {1}'.format( + summary = 'Suspicious Proxy DROP event(s) detected from {0} to the following exfil domain(s): {1}'.format( aggreg['value'], - ",".join(sorted(dropped_domains)) + ",".join(sorted(exfil_domains)) ) # Create the alert object based on these properties From e40706f7cbd2579d8453fd3d635e75dffefeff27 Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Thu, 1 Nov 2018 20:03:44 +1100 Subject: [PATCH 6/9] Make the tests pass --- tests/alerts/test_proxy_drop_exfil_domains.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/alerts/test_proxy_drop_exfil_domains.py b/tests/alerts/test_proxy_drop_exfil_domains.py index 09b949172..abc734df7 100644 --- a/tests/alerts/test_proxy_drop_exfil_domains.py +++ b/tests/alerts/test_proxy_drop_exfil_domains.py @@ -7,8 +7,10 @@ from alert_test_suite import AlertTestSuite -class TestAlertProxyDropExfilDomain(AlertTestSuite): - alert_filename = "proxy_drop_executable" +class TestProxyDropExfilDomains(AlertTestSuite): + alert_filename = "proxy_drop_exfil_domains" + alert_classname = "AlertProxyDropExfilDomains" + # This event is the default positive event that will cause the # alert to trigger default_event = { @@ -18,7 +20,7 @@ class TestAlertProxyDropExfilDomain(AlertTestSuite): "tags": ["squid"], "details": { "sourceipaddress": "1.2.3.4", - "destination": "http://pastebin.com/", + "destination": "pastebin.com", "proxyaction": "TCP_DENIED/-", } } @@ -26,25 +28,25 @@ class TestAlertProxyDropExfilDomain(AlertTestSuite): # This event is an alternate destination that we'd want to aggregate default_event2 = AlertTestSuite.copy(default_event) - default_event2["_source"]["details"]["destination"] = "http://hastebin.com" + default_event2["_source"]["details"]["destination"] = "hastebin.com" # This event is the default negative event that will not cause the # alert to trigger default_negative_event = AlertTestSuite.copy(default_event) - default_negative_event["_source"]["details"]["destination"] = "http://foo.mozilla.com/index.html" + default_negative_event["_source"]["details"]["destination"] = "foo.mozilla.com" # This alert is the expected result from running this task default_alert = { "category": "squid", "tags": ['squid', 'proxy'], "severity": "WARNING", - "summary": 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): http://pastebin.com/', + "summary": 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): pastebin.com', } # This alert is the expected result from this task against multiple matching events default_alert_aggregated = AlertTestSuite.copy(default_alert) default_alert_aggregated[ - "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): http://pastebin.com/,http://hastebin.com' + "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): hastebin.com,pastebin.com' test_cases = [] From 63edd64b3e14cf7c9ad2c01be26020200a9a212a Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Tue, 6 Nov 2018 12:29:12 +1100 Subject: [PATCH 7/9] Updating the config file To start from a limited number of domains to be on the safe side --- alerts/proxy_drop_exfil_domains.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/proxy_drop_exfil_domains.conf b/alerts/proxy_drop_exfil_domains.conf index 4e52d556d..10c4cb3a8 100644 --- a/alerts/proxy_drop_exfil_domains.conf +++ b/alerts/proxy_drop_exfil_domains.conf @@ -1,2 +1,2 @@ [options] -exfil_domains = pastebin.com,pasted.co,hastebin.com,dumpz.org,www.pastefs.com,send.firefox.com,gist.github.com +exfil_domains = pastebin.com,www.sendspace.com From b157a07b014fc669d56ccc3fd43111d9b5bf65a7 Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Tue, 6 Nov 2018 12:31:34 +1100 Subject: [PATCH 8/9] Updating the unit test file To reflect the configuration change --- tests/alerts/test_proxy_drop_exfil_domains.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/alerts/test_proxy_drop_exfil_domains.py b/tests/alerts/test_proxy_drop_exfil_domains.py index abc734df7..fe12849e1 100644 --- a/tests/alerts/test_proxy_drop_exfil_domains.py +++ b/tests/alerts/test_proxy_drop_exfil_domains.py @@ -28,7 +28,7 @@ class TestProxyDropExfilDomains(AlertTestSuite): # This event is an alternate destination that we'd want to aggregate default_event2 = AlertTestSuite.copy(default_event) - default_event2["_source"]["details"]["destination"] = "hastebin.com" + default_event2["_source"]["details"]["destination"] = "www.sendspace.com" # This event is the default negative event that will not cause the # alert to trigger @@ -46,7 +46,7 @@ class TestProxyDropExfilDomains(AlertTestSuite): # This alert is the expected result from this task against multiple matching events default_alert_aggregated = AlertTestSuite.copy(default_alert) default_alert_aggregated[ - "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): hastebin.com,pastebin.com' + "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): www.sendspace.com,pastebin.com' test_cases = [] From 224c706e660744dc103e16186e9db760eafc1c32 Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Tue, 6 Nov 2018 12:42:50 +1100 Subject: [PATCH 9/9] Updating the unit test again To match the order of domains in the alert vs. summary --- tests/alerts/test_proxy_drop_exfil_domains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/alerts/test_proxy_drop_exfil_domains.py b/tests/alerts/test_proxy_drop_exfil_domains.py index fe12849e1..f06fad046 100644 --- a/tests/alerts/test_proxy_drop_exfil_domains.py +++ b/tests/alerts/test_proxy_drop_exfil_domains.py @@ -46,7 +46,7 @@ class TestProxyDropExfilDomains(AlertTestSuite): # This alert is the expected result from this task against multiple matching events default_alert_aggregated = AlertTestSuite.copy(default_alert) default_alert_aggregated[ - "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): www.sendspace.com,pastebin.com' + "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): pastebin.com,www.sendspace.com' test_cases = []