Permalink
Browse files

Merge branch 'master' into remove-cloudy-mozdef-defaults

  • Loading branch information...
gene1wood committed Nov 26, 2018
2 parents 5696674 + 3d05d7b commit 81dec31d3a6a7e6ae2287ccba0522278d30ff896
@@ -22,6 +22,10 @@ The Mozilla Defense Platform (MozDef) seeks to automate the security incident ha

MozDef is in production at Mozilla where we are using it to process over 300 million events per day.

## Give MozDef a Try in AWS:

[![Launch MozDef](docs/source/images/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/mozdef.infosec.allizom.org/cf/mozdef-parent.yml)

## Documentation:

http://mozdef.readthedocs.org/en/latest/
@@ -0,0 +1,43 @@
#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation


from lib.alerttask import AlertTask
from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, ExistsMatch, PhraseMatch, WildcardMatch


class AlertGuardDutyProbe(AlertTask):
def main(self):
# Create a query to look back the last 20 minutes
search_query = SearchQuery(minutes=20)

# Add search terms to our query
search_query.add_must([
TermMatch('source', 'guardduty'),
TermMatch('details.finding.action.actionType', 'PORT_PROBE'),
ExistsMatch('details.sourceipaddress'),
])

self.filtersManual(search_query)
# Search aggregations on field 'sourceipaddress'
# keep X samples of events at most
self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10)
# alert when >= X matching events in an aggregation
self.walkAggregations(threshold=1)

# Set alert properties
def onAggregation(self, aggreg):
# aggreg['count']: number of items in the aggregation, ex: number of failed login attempts
# aggreg['value']: value of the aggregation field, ex: toto@example.com
# aggreg['events']: list of events in the aggregation
category = 'bruteforce'
tags = ['guardduty', 'bruteforce']
severity = 'INFO'
summary = "Guard Duty Port Probe by {}".format(aggreg['value'])

# Create the alert object based on these properties
return self.createAlertDict(summary, category, tags, aggreg['events'], severity)
@@ -64,6 +64,9 @@ Parameters:
MozDefSQSQueueName:
Type: String
Description: The name of the generic SQS queue used to pickup events.
DomainName:
Type: String
Description: The fully qualified domain you'll be hosting MozDef at.
Resources:
MozDefElasticLoadBalancingV2TargetGroup:
Type: AWS::ElasticLoadBalancingV2::TargetGroup
@@ -112,7 +115,10 @@ Resources:
- content: |
OPTIONS_ESSERVERS=${ESURL}
OPTIONS_KIBANAURL=${KibanaURL}
# The OPTIONS_METEOR_KIBANAURL uses the reserved word "relative" which triggers MozDef
# to use relative links to Kibana : https://github.com/mozilla/MozDef/pull/956
OPTIONS_METEOR_KIBANAURL=https://relative:9090/_plugin/kibana/
OPTIONS_METEOR_ROOTURL=https://${DomainName}
# See https://github.com/mozilla-iam/mozilla.oidc.accessproxy/blob/master/README.md#setup
client_id=${OIDCClientId}
client_secret=${OIDCClientSecret}
@@ -129,7 +135,86 @@ Resources:
cookiename=sesmeteor
# Increase the AWS ES total fields limit from 1000 to 4000
OPTIONS_MAPPING_TOTAL_FIELDS_LIMIT=4000
# Set thresholds for attack dataviz lower means more ogres
OPTIONS_IPV4ATTACKERHITCOUNT=5
OPTIONS_IPV4ATTACKERPREFIXLENGTH=24
path: /opt/mozdef/docker/compose/cloudy_mozdef.env
- content: |
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
from celery.schedules import crontab, timedelta
import time
import logging
ALERTS = {
'bruteforce_ssh.AlertBruteforceSsh': {'schedule': crontab(minute='*/1')},
'unauth_ssh.AlertUnauthSSH': {'schedule': crontab(minute='*/1')},
'guard_duty_probe.AlertGuardDutyProbe': {'schedule': crontab(minute='*/1')},
'cloudtrail_logging_disabled.AlertCloudtrailLoggingDisabled': {'schedule': timedelta(minutes=1)},
'cloudtrail_deadman.AlertCloudtrailDeadman': {'schedule': timedelta(hours=1)}
}
ALERT_PLUGINS = [
# 'relative pythonfile name (exclude the .py) - EX: sso_dashboard',
]
RABBITMQ = {
'mqserver': 'rabbitmq',
'mquser': 'guest',
'mqpassword': 'guest',
'mqport': 5672,
'alertexchange': 'alerts',
'alertqueue': 'mozdef.alert'
}
ES = {
'servers': ["${ESURL}"]
}
OPTIONS = {
'defaulttimezone': 'UTC',
}
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'formatters': {
'simple': {
'format': '%(levelname)s %(message)s',
'datefmt': '%y %b %d, %H:%M:%S',
},
'standard': {
'format': '%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d: %(message)s'
}
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'simple'
},
'celery': {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'filename': 'celery.log',
'formatter': 'standard',
'maxBytes': 1024 * 1024 * 100, # 100 mb
},
},
'loggers': {
'celery': {
'handlers': ['celery', 'console'],
'level': 'DEBUG',
},
}
}
logging.Formatter.converter = time.gmtime
path: /opt/mozdef/docker/compose/mozdef_alerts/files/config.py
- content: |
client_id=${OIDCClientId}
client_secret=${OIDCClientSecret}
@@ -39,6 +39,8 @@ Metadata:
default: EC2 SSH Key Name
AMIImageId:
default: EC2 AMI Image ID
DomainName:
default: FQDN to host MozDef at
ACMCertArn:
default: ACM Certificate ARN
OIDCDiscoveryURL:
@@ -66,7 +68,11 @@ Parameters:
AMIImageId:
Type: AWS::EC2::Image::Id
Description: The AMI Image ID to use of the EC2 instance
Default: ami-0c3705bb3b43ad51f
Default: ami-073434079b0366251
DomainName:
Type: String
Description: The fully qualified DNS name you will host CloudyMozDef at.
Default: cloudymozdef.security.allizom.org
ACMCertArn:
Type: String
Description: "The ARN of your pre-issued ACM cert. (Example: arn:aws:acm:us-west-2:123456789012:certificate/abcdef01-2345-6789-abcd-ef0123456789)"
@@ -132,6 +138,7 @@ Resources:
OIDCDiscoveryURL: !Ref OIDCDiscoveryURL
CloudTrailSQSNotificationQueueName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueName
MozDefSQSQueueName: !GetAtt MozDefSQS.Outputs.SQSQueueName
DomainName: !Ref DomainName
Tags:
- Key: application
Value: mozdef
@@ -14,7 +14,10 @@
"instance_type": "t2.large",
"ssh_pty" : "true",
"ssh_username": "ec2-user",
"ami_name": "mozdef_{{timestamp}}"
"ami_name": "mozdef_{{timestamp}}",
"ami_groups": [
"all"
]
}],
"provisioners": [
{ "type": "shell",
@@ -30,7 +33,7 @@
"sudo systemctl enable docker",
"sudo mkdir -p /opt/mozdef/",
"sudo git clone https://github.com/mozilla/MozDef /opt/mozdef",
"cd /opt/mozdef && sudo git checkout guardduty"
"cd /opt/mozdef && sudo git checkout master"
]}
]
}
@@ -153,7 +153,7 @@ def searchMongoAlerts(mozdefdb):
# count by ip
{"$group": {"_id": "$sourceip", "hitcount": {"$sum": 1}}},
# limit to those with X observances
{"$match": {"hitcount": {"$gt": 5}}},
{"$match": {"hitcount": {"$gt": options.ipv4attackerhitcount}}},
# sort
{"$sort": SON([("hitcount", -1), ("_id", -1)])},
# top 10
@@ -166,7 +166,7 @@ def searchMongoAlerts(mozdefdb):
# set CIDR
# todo: lookup ipwhois for asn_cidr value
# potentially with a max mask value (i.e. asn is /8, limit attackers to /24)
ipcidr.prefixlen = 32
ipcidr.prefixlen = options.ipv4attackerprefixlength

# append to or create attacker.
# does this match an existing attacker's indicators
@@ -482,6 +482,10 @@ def initConfig():
# set to either amqp or amqps for ssl
options.mqprotocol = getConfig('mqprotocol', 'amqp', options.configfile)

# Set these settings to change the correlation for attackers
options.ipv4attackerprefixlength = getConfig('ipv4attackerprefixlength', 32, options.configfile)
options.ipv4attackerhitcount = getConfig('ipv4ipv4attackerhitcount', 5, options.configfile)


if __name__ == '__main__':
parser = OptionParser()
@@ -69,6 +69,8 @@ services:
image: mozdef/mozdef_alerts
env_file:
- cloudy_mozdef.env
volumes:
- /opt/mozdef/docker/compose/mozdef_alerts/files/config.py:/opt/mozdef/envs/mozdef/alerts/lib/config.py
restart: always
command: bash -c 'source /opt/mozdef/envs/python/bin/activate && celery -A celeryconfig worker --loglevel=info --beat'
depends_on:
@@ -44,8 +44,9 @@ Requirements:

This test should pass and you will have confirmed you have a working environment.

At this point, begin development and periodically run your unit-tests locally with the following command::
At this point, begin development and periodically run your unit-tests locally with the following commands::

make build-tests
make run-tests TEST_CASE=tests/alerts/[YOUR ALERT TEST FILE].py


@@ -1,11 +1,82 @@
Benchmarking
============
MozDef for AWS
===============

**What is MozDef for AWS**

Cloud based MozDef is an opinionated deployment of the MozDef services created in 2018 to help AWS users
ingest cloudtrail, guardduty, and provide security services.

.. image:: images/cloudformation-launch-stack.png
:target: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/mozdef.infosec.allizom.org/cf/mozdef-parent.yml


Feedback
-----------

MozDef for AWS is new and we'd love your feedback. Try filing GitHub issues here in the repository or connect with us
in the Mozilla Discourse Security Category.

https://discourse.mozilla.org/c/security

You can also take a short survey on MozDef for AWS after you have deployed it.
https://goo.gl/forms/JYjTYDK45d3JdnGd2


Dependencies
--------------

MozDef requires the following:

- A DNS name ( cloudymozdef.security.allizom.org )
- An OIDC Provider with ClientID, ClientSecret, and Discovery URL
- Mozilla Uses Auth0 but you can use any OIDC provider you like: Shibboleth, KeyCloak, AWS Cognito, Okta, Ping (etc)
- An ACM Certificate in the deployment region for your DNS name
- A VPC with three public subnets available.
- It is advised that this VPC be dedicated to MozDef or used solely for security automation.
- An SQS queue recieving GuardDuty events. At the time of writing this is not required but may be required in future.


Supported Regions
------------------

MozDef for AWS is currently only supported in us-west-2 but will onboard additional regions over time.


Architecture
-------------

.. image:: images/MozDefCloudArchitecture.png


Deployment Process
-------------------

1. Launch the one click stack and provide the requisite values.
2. Wait for the stack to complete. You'll see several nested stacks in the Cloudformation console. *Note: This may take a while*
3. Navigate to the URL you set up for MozDef. It should redirect you to the single sign on provider. If successful you'll see the MozDef UI.
4. Try navigating to ElasticSearch https://your_base_url:9090
You should see the following:
::

{
"name" : "SMf4400",
"cluster_name" : "656532927350:mozdef-mozdef-yemjpbnpw8xb",
"cluster_uuid" : "_yBEIsFkQH-nEZfrFgj7mg",
"version" : {
"number" : "5.6.8",
"build_hash" : "688ecce",
"build_date" : "2018-09-11T14:44:40.463Z",
"build_snapshot" : false,
"lucene_version" : "6.6.1"
},
"tagline" : "You Know, for Search"
}

5. Test out Kibana at https://your_base_url:9090/_plugin/kibana/app/kibana#/discover?_g=()


Using MozDef
-------------

Refer back to our other docs on how to use MozDef for general guidance. Cloud specific instructions will evolve here.
If you saw something about MozDef for AWS at re: Invent 2018 and you want to contribute we'd love your PRs.
Binary file not shown.
@@ -31,7 +31,7 @@ def __init__(self):
# AWS guard duty can send IPs in a bunch of places
# Lets pick out some likely targets and format them
# so other mozdef plugins can rely on their location
self.ipaddress_keys =[
self.ipaddress_keys = [
'details.finding.action.networkConnectionAction.remoteIpDetails.ipAddressV4',
'details.finding.action.awsApiCallAction.remoteIpDetails.ipAdrressV4'
]
@@ -43,7 +43,8 @@ def convert_key_date_format(self, needle, haystack):
current_pointer = haystack
for updated_key in num_levels:
if updated_key == num_levels[-1]:
current_pointer[updated_key] = toUTC(current_pointer[updated_key]).isoformat()
current_pointer[updated_key] = toUTC(
current_pointer[updated_key]).isoformat()
return haystack
if updated_key in current_pointer:
current_pointer = current_pointer[updated_key]
@@ -63,12 +64,25 @@ def onMessage(self, message, metadata):
message = self.convert_key_date_format(date_key, message)

# convert the dict to a dot dict for saner deep key/value processing
message=DotDict(message)
message = DotDict(message)
# pull out the likely source IP address
for ipaddress_key in self.ipaddress_keys:
if 'sourceipaddress' not in message['details'].keys():
if key_exists(ipaddress_key,message):
message.details.sourceipaddress = message.get(ipaddress_key)
if key_exists(ipaddress_key, message):
message.details.sourceipaddress = message.get(
ipaddress_key)

# if we still haven't found what we are looking for #U2
# sometimes it's in a list
if 'sourceipaddress' not in message['details'].keys():
if key_exists('details.finding.action.portProbeAction.portProbeDetails', message) \
and isinstance(message.details.finding.action.portProbeAction.portProbeDetails, list):

# inspect the first list entry and see if it contains an IP
portProbeDetails = DotDict(
message.details.finding.action.portProbeAction.portProbeDetails[0])
if key_exists('remoteIpDetails.ipAddressV4', portProbeDetails):
message.details.sourceipaddress = portProbeDetails.remoteIpDetails.ipAddressV4

# recovert the message back to a plain dict
return (dict(message), metadata)
Oops, something went wrong.

0 comments on commit 81dec31

Please sign in to comment.