From e59dc8144b2e11d40520cd5b01fc3a7956c9b47e Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Thu, 11 Oct 2018 16:52:06 -0400 Subject: [PATCH 01/10] Quick initial commit of proxy drop ip destinations --- alerts/proxy_drop_ip.py | 60 ++++++++++++++ tests/alerts/test_proxy_drop_ip.py | 124 +++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 alerts/proxy_drop_ip.py create mode 100644 tests/alerts/test_proxy_drop_ip.py diff --git a/alerts/proxy_drop_ip.py b/alerts/proxy_drop_ip.py new file mode 100644 index 000000000..928ca0a36 --- /dev/null +++ b/alerts/proxy_drop_ip.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2014 Mozilla Corporation + + +from lib.alerttask import AlertTask +from query_models import QueryStringMatch, SearchQuery, TermMatch + + +class AlertProxyDropIP(AlertTask): + def main(self): + self.parse_config('proxy_drop_executable.conf', ['extensions']) + + search_query = SearchQuery(minutes=20) + + search_query.add_must([ + TermMatch('category', 'squid'), + TermMatch('tags', 'squid'), + TermMatch('details.proxyaction', 'TCP_DENIED/-') + ]) + + # Match on 1.1.1.1, http://1.1.1.1, or https://1.1.1.1 + ip_regex = "/^(http:\/\/|https:\/\/)?\d+\.\d+\.\d+\.\d+.*/" + search_query.add_must([ + QueryStringMatch('details.destination: {}'.format(ip_regex)) + ]) + + self.filtersManual(search_query) + + # Search aggregations on field 'hostname', keep X samples of + # events at most + self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10) + # alert when >= X matching events in an aggregation + # I think it makes sense to alert every time here + self.walkAggregations(threshold=1) + + # Set alert properties + def onAggregation(self, aggreg): + # aggreg['count']: number of items in the aggregation, ex: number of failed login attempts + # aggreg['value']: value of the aggregation field, ex: toto@example.com + # aggreg['events']: list of events in the aggregation + category = 'squid' + tags = ['squid', 'proxy'] + severity = 'WARNING' + + dropped_destinations = set() + for event in aggreg['allevents']: + dropped_destinations.add( + event['_source']['details']['destination']) + + summary = 'Suspicious Proxy DROP event(s) detected from {0} to the following IP destination(s): {1}'.format( + aggreg['value'], + ",".join(sorted(dropped_urls)) + ) + + # Create the alert object based on these properties + return self.createAlertDict(summary, category, tags, aggreg['events'], severity) diff --git a/tests/alerts/test_proxy_drop_ip.py b/tests/alerts/test_proxy_drop_ip.py new file mode 100644 index 000000000..fdd6b5819 --- /dev/null +++ b/tests/alerts/test_proxy_drop_ip.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation +from positive_alert_test_case import PositiveAlertTestCase +from negative_alert_test_case import NegativeAlertTestCase +from alert_test_suite import AlertTestSuite + + +class TestAlertProxyDropIP(AlertTestSuite): + alert_filename = "proxy_drop_ip" + # This event is the default positive event that will cause the + # alert to trigger + default_event = { + "_type": "event", + "_source": { + "category": "squid", + "tags": ["squid"], + "details": { + "sourceipaddress": "1.2.3.4", + "destination": "http://1.2.3.5", + "proxyaction": "TCP_DENIED/-", + } + } + } + + # This event is an alternate destination that we'd want to aggregate + default_event2 = AlertTestSuite.copy(default_event) + default_event2["_source"]["details"]["destination"] = "1.2.3.5:1337" + + # This event is the default negative event that will not cause the + # alert to trigger + default_negative_event = AlertTestSuite.copy(default_event) + default_negative_event["_source"]["details"]["destination"] = "http://foo.mozilla.com" + + # This alert is the expected result from running this task + default_alert = { + "category": "squid", + "tags": ['squid', 'proxy'], + "severity": "WARNING", + "summary": 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following executable file destination(s): http://1.2.3.5', + } + + # This alert is the expected result from this task against multiple matching events + default_alert_aggregated = AlertTestSuite.copy(default_alert) + default_alert_aggregated[ + "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following executable file destination(s): http://1.2.3.5,1.2.3.5:1337' + + test_cases = [] + + test_cases.append( + PositiveAlertTestCase( + description="Positive test with default events and default alert expected", + events=AlertTestSuite.create_events(default_event, 1), + expected_alert=default_alert + ) + ) + + test_cases.append( + PositiveAlertTestCase( + description="Positive test with default events and default alert expected - dedup", + events=AlertTestSuite.create_events(default_event, 2), + expected_alert=default_alert + ) + ) + + events1 = AlertTestSuite.create_events(default_event, 1) + events2 = AlertTestSuite.create_events(default_event2, 1) + test_cases.append( + PositiveAlertTestCase( + description="Positive test with default events and default alert expected - different dests", + events=events1 + events2, + expected_alert=default_alert_aggregated + ) + ) + + test_cases.append( + NegativeAlertTestCase( + description="Negative test with default negative event", + events=AlertTestSuite.create_events(default_negative_event, 1), + ) + ) + + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['category'] = 'bad' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with events with incorrect category", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['tags'] = 'bad tag example' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with events with incorrect tags", + events=events, + ) + ) + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({ + 'minutes': 241}) + event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({ + 'minutes': 241}) + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with old timestamp", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['details']['destination'] = 'http://evil.com/evil.pdf' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with events with non blacklisted extension", + events=events, + ) + ) From 301362757d909c0e5a1d05c8fe4d1b287c3e6eb6 Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Fri, 12 Oct 2018 10:03:47 -0400 Subject: [PATCH 02/10] Add examples to querystring class for starting anchor --- .../lib/query_models/test_query_string_match.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/lib/query_models/test_query_string_match.py b/tests/lib/query_models/test_query_string_match.py index ec4f44101..46140e2d1 100644 --- a/tests/lib/query_models/test_query_string_match.py +++ b/tests/lib/query_models/test_query_string_match.py @@ -9,6 +9,7 @@ hostname_test_regex = 'hostname: /(.*\.)*(groupa|groupb)\.(.*\.)*subdomain\.(.*\.)*.*/' filename_matcher = 'summary: /.*\.(exe|sh)/' +ip_matcher = 'destination: /^(http:\/\/|https:\/\/)?\d+\.\d+\.\d+\.\d+.*/' class TestQueryStringMatchPositiveTestSuite(PositiveTestSuite): @@ -43,6 +44,14 @@ def query_tests(self): {'summary': 'test.exe'}, {'summary': 'test.sh'}, ], + + QueryStringMatch(ip_matcher): [ + {'destination': 'http://1.2.3.4/somepath'}, + {'destination': 'https://1.2.3.4/somepath'}, + {'destination': '1.2.3.4:443'}, + {'destination': '1.2.3.4:80'}, + + ], } return tests @@ -84,5 +93,12 @@ def query_tests(self): {'summary': '.exe.test'}, ], + QueryStringMatch(ip_matcher): [ + {'destination': 'http://example.com/somepath'}, + {'destination': 'https://foo.bar.example.com/somepath'}, + {'destination': 'example.com:443'}, + {'destination': 'foo.bar.example.com:80'}, + + ], } return tests From 3c44cd1039b64b0f331c9c54acb6127ca376017c Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Fri, 12 Oct 2018 10:07:15 -0400 Subject: [PATCH 03/10] Remove starting anchor, all lucene is implicitly anchored --- alerts/proxy_drop_ip.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/alerts/proxy_drop_ip.py b/alerts/proxy_drop_ip.py index 928ca0a36..0620f0b1b 100644 --- a/alerts/proxy_drop_ip.py +++ b/alerts/proxy_drop_ip.py @@ -12,8 +12,6 @@ class AlertProxyDropIP(AlertTask): def main(self): - self.parse_config('proxy_drop_executable.conf', ['extensions']) - search_query = SearchQuery(minutes=20) search_query.add_must([ @@ -23,7 +21,7 @@ def main(self): ]) # Match on 1.1.1.1, http://1.1.1.1, or https://1.1.1.1 - ip_regex = "/^(http:\/\/|https:\/\/)?\d+\.\d+\.\d+\.\d+.*/" + ip_regex = "/(http:\/\/|https:\/\/)?\d+\.\d+\.\d+\.\d+.*/" search_query.add_must([ QueryStringMatch('details.destination: {}'.format(ip_regex)) ]) From 05dc85e9b90304214b3e977020607d92f0c760d7 Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Fri, 12 Oct 2018 10:08:03 -0400 Subject: [PATCH 04/10] Remove anchor from query string match test case --- tests/lib/query_models/test_query_string_match.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/query_models/test_query_string_match.py b/tests/lib/query_models/test_query_string_match.py index 46140e2d1..ab5caee22 100644 --- a/tests/lib/query_models/test_query_string_match.py +++ b/tests/lib/query_models/test_query_string_match.py @@ -9,7 +9,7 @@ hostname_test_regex = 'hostname: /(.*\.)*(groupa|groupb)\.(.*\.)*subdomain\.(.*\.)*.*/' filename_matcher = 'summary: /.*\.(exe|sh)/' -ip_matcher = 'destination: /^(http:\/\/|https:\/\/)?\d+\.\d+\.\d+\.\d+.*/' +ip_matcher = 'destination: /(http:\/\/|https:\/\/)?\d+\.\d+\.\d+\.\d+.*/' class TestQueryStringMatchPositiveTestSuite(PositiveTestSuite): From b97603e3366e61cfb29f600ede40d497598ea9d7 Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Fri, 12 Oct 2018 10:27:05 -0400 Subject: [PATCH 05/10] Fix variable copy pasta --- alerts/proxy_drop_ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/proxy_drop_ip.py b/alerts/proxy_drop_ip.py index 0620f0b1b..21e601a89 100644 --- a/alerts/proxy_drop_ip.py +++ b/alerts/proxy_drop_ip.py @@ -51,7 +51,7 @@ def onAggregation(self, aggreg): summary = 'Suspicious Proxy DROP event(s) detected from {0} to the following IP destination(s): {1}'.format( aggreg['value'], - ",".join(sorted(dropped_urls)) + ",".join(sorted(dropped_destinations)) ) # Create the alert object based on these properties From 81bb1f574abd65f1d047c584ad2244c625ea30e0 Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Fri, 12 Oct 2018 12:53:59 -0400 Subject: [PATCH 06/10] Add test cases for query string match to validate IP match pattern --- .../query_models/test_query_string_match.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/lib/query_models/test_query_string_match.py b/tests/lib/query_models/test_query_string_match.py index ab5caee22..6bbaf802c 100644 --- a/tests/lib/query_models/test_query_string_match.py +++ b/tests/lib/query_models/test_query_string_match.py @@ -9,7 +9,9 @@ hostname_test_regex = 'hostname: /(.*\.)*(groupa|groupb)\.(.*\.)*subdomain\.(.*\.)*.*/' filename_matcher = 'summary: /.*\.(exe|sh)/' -ip_matcher = 'destination: /(http:\/\/|https:\/\/)?\d+\.\d+\.\d+\.\d+.*/' + +# Note that this has potential for over-matching on foo.bar.baz.com, which needs further validation in alerts +ip_matcher = 'destination: /.*\..{1,3}\..{1,3}\..{1,3}(:.*|\/.*)/' class TestQueryStringMatchPositiveTestSuite(PositiveTestSuite): @@ -48,10 +50,14 @@ def query_tests(self): QueryStringMatch(ip_matcher): [ {'destination': 'http://1.2.3.4/somepath'}, {'destination': 'https://1.2.3.4/somepath'}, + {'destination': '1.2.3.4/somepath'}, + {'destination': '1.2.3.4/somepath'}, {'destination': '1.2.3.4:443'}, {'destination': '1.2.3.4:80'}, - - ], + # Over-match examples (which need to be validated further in alerts) + {'destination': 'https://foo.bar.baz.com/somepath'}, + {'destination': 'foo.bar.baz.com:80'}, + ] } return tests @@ -94,11 +100,10 @@ def query_tests(self): ], QueryStringMatch(ip_matcher): [ + {'destination': 'https://foo.bar.mozilla.com/somepath'}, + {'destination': 'foo.bar.mozilla.com:80'}, {'destination': 'http://example.com/somepath'}, - {'destination': 'https://foo.bar.example.com/somepath'}, - {'destination': 'example.com:443'}, - {'destination': 'foo.bar.example.com:80'}, - + {'destination': 'example.com:443'} ], } return tests From 6289e52936d3f0f755ac4aef1aeb0afb6fe1a108 Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Fri, 12 Oct 2018 12:54:49 -0400 Subject: [PATCH 07/10] Fix up alert and test expectations to pass --- alerts/proxy_drop_ip.py | 20 ++++++++++++++++---- tests/alerts/test_proxy_drop_ip.py | 6 +++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/alerts/proxy_drop_ip.py b/alerts/proxy_drop_ip.py index 21e601a89..5fd2d2d59 100644 --- a/alerts/proxy_drop_ip.py +++ b/alerts/proxy_drop_ip.py @@ -8,6 +8,7 @@ from lib.alerttask import AlertTask from query_models import QueryStringMatch, SearchQuery, TermMatch +import re class AlertProxyDropIP(AlertTask): @@ -21,7 +22,8 @@ def main(self): ]) # Match on 1.1.1.1, http://1.1.1.1, or https://1.1.1.1 - ip_regex = "/(http:\/\/|https:\/\/)?\d+\.\d+\.\d+\.\d+.*/" + # This will over-match on short 3-char domains like foo.bar.baz.com, but will get weeded out below + ip_regex = '/.*\..{1,3}\..{1,3}\..{1,3}(:.*|\/.*)/' search_query.add_must([ QueryStringMatch('details.destination: {}'.format(ip_regex)) ]) @@ -44,12 +46,22 @@ def onAggregation(self, aggreg): tags = ['squid', 'proxy'] severity = 'WARNING' + # Lucene search has a slight potential for overmatches, so we'd double-check + # with this pattern to ensure it's truely an IP before we add dest to our dropped list + pattern = r'(http:\/\/|https:\/\/|)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}' + dropped_destinations = set() + for event in aggreg['allevents']: - dropped_destinations.add( - event['_source']['details']['destination']) + if re.search(pattern, event['_source']['details']['destination']): + dropped_destinations.add( + event['_source']['details']['destination']) + + # If it's all over-matches, don't throw the alert + if len(dropped_destinations) == 0: + return None - summary = 'Suspicious Proxy DROP event(s) detected from {0} to the following IP destination(s): {1}'.format( + summary = 'Suspicious Proxy DROP event(s) detected from {0} to the following IP-based destination(s): {1}'.format( aggreg['value'], ",".join(sorted(dropped_destinations)) ) diff --git a/tests/alerts/test_proxy_drop_ip.py b/tests/alerts/test_proxy_drop_ip.py index fdd6b5819..678f108b9 100644 --- a/tests/alerts/test_proxy_drop_ip.py +++ b/tests/alerts/test_proxy_drop_ip.py @@ -18,7 +18,7 @@ class TestAlertProxyDropIP(AlertTestSuite): "tags": ["squid"], "details": { "sourceipaddress": "1.2.3.4", - "destination": "http://1.2.3.5", + "destination": "http://1.2.3.5/", "proxyaction": "TCP_DENIED/-", } } @@ -38,13 +38,13 @@ class TestAlertProxyDropIP(AlertTestSuite): "category": "squid", "tags": ['squid', 'proxy'], "severity": "WARNING", - "summary": 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following executable file destination(s): http://1.2.3.5', + "summary": 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following IP-based destination(s): http://1.2.3.5/', } # This alert is the expected result from this task against multiple matching events default_alert_aggregated = AlertTestSuite.copy(default_alert) default_alert_aggregated[ - "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following executable file destination(s): http://1.2.3.5,1.2.3.5:1337' + "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following IP-based destination(s): 1.2.3.5:1337,http://1.2.3.5/' test_cases = [] From 2aa2ad7c1a2258e4904567cf84cab1e2673d820b Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Fri, 12 Oct 2018 13:00:32 -0400 Subject: [PATCH 08/10] Add starting anchor to the validation regex, to prevent matching on in url path links --- alerts/proxy_drop_ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/proxy_drop_ip.py b/alerts/proxy_drop_ip.py index 5fd2d2d59..ce8a83ba2 100644 --- a/alerts/proxy_drop_ip.py +++ b/alerts/proxy_drop_ip.py @@ -48,7 +48,7 @@ def onAggregation(self, aggreg): # Lucene search has a slight potential for overmatches, so we'd double-check # with this pattern to ensure it's truely an IP before we add dest to our dropped list - pattern = r'(http:\/\/|https:\/\/|)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}' + pattern = r'^(http:\/\/|https:\/\/|)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}' dropped_destinations = set() From 9d22548425b6f8681b0f0c2a6c8b1177cf01ed93 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Thu, 25 Oct 2018 18:09:25 -0500 Subject: [PATCH 09/10] Fix import in proxy drop alert --- alerts/proxy_drop_ip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/proxy_drop_ip.py b/alerts/proxy_drop_ip.py index ce8a83ba2..de83c883e 100644 --- a/alerts/proxy_drop_ip.py +++ b/alerts/proxy_drop_ip.py @@ -7,7 +7,7 @@ from lib.alerttask import AlertTask -from query_models import QueryStringMatch, SearchQuery, TermMatch +from mozdef_util.query_models import QueryStringMatch, SearchQuery, TermMatch import re From d75707c9576ab1f69ab669a7ac91577477972757 Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Tue, 30 Oct 2018 10:03:37 -0400 Subject: [PATCH 10/10] Remove boilerplate comments --- alerts/proxy_drop_ip.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/alerts/proxy_drop_ip.py b/alerts/proxy_drop_ip.py index de83c883e..fe343f188 100644 --- a/alerts/proxy_drop_ip.py +++ b/alerts/proxy_drop_ip.py @@ -29,19 +29,11 @@ def main(self): ]) self.filtersManual(search_query) - - # Search aggregations on field 'hostname', keep X samples of - # events at most self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10) - # alert when >= X matching events in an aggregation - # I think it makes sense to alert every time here self.walkAggregations(threshold=1) # Set alert properties def onAggregation(self, aggreg): - # aggreg['count']: number of items in the aggregation, ex: number of failed login attempts - # aggreg['value']: value of the aggregation field, ex: toto@example.com - # aggreg['events']: list of events in the aggregation category = 'squid' tags = ['squid', 'proxy'] severity = 'WARNING' @@ -66,5 +58,4 @@ def onAggregation(self, aggreg): ",".join(sorted(dropped_destinations)) ) - # Create the alert object based on these properties return self.createAlertDict(summary, category, tags, aggreg['events'], severity)