Skip to content
Permalink
Browse files

Merge pull request #1353 from mozilla/reinforce2019

Reinforce2019 merge
  • Loading branch information...
pwnbus committed Jul 10, 2019
2 parents ca6edb3 + 49317ec commit 693d7ea5cab9a0da6762d671655528bb062c2184
@@ -13,7 +13,7 @@
# XXX TBD this should get wrapped into an object that provides pyconfig
if os.getenv("OPTIONS_MQPROTOCOL", "amqp") == "sqs":
BROKER_URL = "sqs://@"
BROKER_TRANSPORT_OPTIONS = {'region': os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('.')[1]}
BROKER_TRANSPORT_OPTIONS = {'region': os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('.')[1], 'is_secure': True, 'port': 443}
CELERY_RESULT_BACKEND = None
alert_queue_name = os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4]
CELERY_DEFAULT_QUEUE = alert_queue_name
@@ -13,25 +13,24 @@
class AlertCloudtrailExcessiveDescribe(AlertTask):
def main(self):
# Create a query to look back the last 20 minutes
search_query = SearchQuery(minutes=20)
search_query = SearchQuery(minutes=5)

# Add search terms to our query
search_query.add_must([
TermMatch('source', 'cloudtrail'),
TermMatch('details.eventverb', 'Describe'),
ExistsMatch('details.source')
ExistsMatch('details.sourceipv4address')
])

self.filtersManual(search_query)
# We aggregate on details.hostname which is the AWS service name
self.searchEventsAggregated('details.source', samplesLimit=2)
self.walkAggregations(threshold=50)
# We aggregate on details.sourceipv4address which is the AWS service name
self.searchEventsAggregated('details.sourceipv4address', samplesLimit=2)
self.walkAggregations(threshold=5)

def onAggregation(self, aggreg):
category = 'access'
tags = ['cloudtrail']
severity = 'WARNING'
summary = "Excessive Describe calls on {0} ({1})".format(aggreg['value'], aggreg['count'])

summary = "A production service is generating excessive describe calls."
# Create the alert object based on these properties
return self.createAlertDict(summary, category, tags, aggreg['events'], severity)
@@ -7,7 +7,7 @@


from lib.alerttask import AlertTask
from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch
from mozdef_util.query_models import SearchQuery, TermMatch


class AlertCloudtrailPublicBucket(AlertTask):
@@ -16,8 +16,8 @@ def main(self):

search_query.add_must([
TermMatch('source', 'cloudtrail'),
TermMatch('details.eventname', 'PutBucketPolicy'),
ExistsMatch('details.requestparameters.bucketpolicy.statement.principal')
TermMatch('details.eventname', 'CreateBucket'),
TermMatch('details.requestparameters.x-amz-acl', 'public-read-write'),
])

self.filtersManual(search_query)
@@ -27,9 +27,6 @@ def main(self):
# Set alert properties
def onEvent(self, event):
request_parameters = event['_source']['details']['requestparameters']
for statement in request_parameters['bucketpolicy']['statement']:
if statement['principal'] != '*':
return
category = 'access'
tags = ['cloudtrail']
severity = 'INFO'
@@ -159,6 +159,7 @@ def _configureKombu(self):
self.mqConn = kombu.Connection(connString)
if connString.find('sqs') == 0:
self.mqConn.transport_options['region'] = os.getenv('DEFAULT_AWS_REGION', 'us-west-2')
self.mqConn.transport_options['is_secure'] = True
self.alertExchange = kombu.Exchange(
name=RABBITMQ["alertexchange"], type="topic", durable=True
)
@@ -21,8 +21,13 @@ S3_PROD_BUCKET_PATH := mozdef/cf
S3_PROD_BUCKET_URI := s3://$(S3_PROD_BUCKET_NAME)/$(S3_PROD_BUCKET_PATH)
S3_PROD_STACK_URI := https://s3-$(AWS_REGION).amazonaws.com/$(S3_PROD_BUCKET_NAME)/$(S3_PROD_BUCKET_PATH)/

# OIDC_CLIENT_SECRET is set in an environment variable by running "source aws_parameters.sh"
OIDC_CLIENT_SECRET_PARAM_ARG := $(shell test -n "$(OIDC_CLIENT_SECRET)" && echo "ParameterKey=OIDCClientSecret,ParameterValue=$(OIDC_CLIENT_SECRET)")
# OIDC_CLIENT_SECRET and other secrets are set in an environment variable by running "source aws_parameters.sh"
OIDC_CLIENT_SECRET_PARAM_ARG := $(shell test -n "$(OIDC_CLIENT_SECRET)" && echo "ParameterKey=OIDCClientSecret,ParameterValue=$(OIDC_CLIENT_SECRET),UsePreviousValue=false")
ALB_BASIC_AUTH_SECRET_PARAM_ARG := $(shell test -n "$(ALB_BASIC_AUTH_SECRET)" && echo "ParameterKey=ALBBasicAuthSecret,ParameterValue=$(ALB_BASIC_AUTH_SECRET),UsePreviousValue=false")

# Make functions
eq = $(if $(or $(1),$(2)),$(and $(findstring $(1),$(2)),\
$(findstring $(2),$(1))),1)

.PHONY:all
all:
@@ -37,14 +42,22 @@ packer-build-github: ## Build the base AMI with packer
@echo "Branch based build triggered for $(BRANCH)."
ci/pack_and_copy $(BRANCH) $(AMI_MAP_TEMP_FILE)

.PHONY: safety-checks
safety-checks:
@echo "Making sure you have an environment variable OIDC_CLIENT_SECRET set."
@test -n "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -n "$(OIDC_CLIENT_ID)" -o -z "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -z "$(OIDC_CLIENT_ID)"
@echo "Making sure you have either OIDC_CLIENT_ID or ALB_BASIC_AUTH_SECRET set."
# If both are equal then you're either leaking the secret, or, most likely, both are equal to string ""
# which is unsafe (as it would effectively give you a basic auth password of string "")
$(call eq, $(OIDC_CLIENT_ID), $(ALB_BASIC_AUTH_SECRET_PARAM_ARG))

.PHONY: create-dev-stack
create-dev-stack: test ## Create everything you need for a fresh new stack!
create-dev-stack: safety-checks test ## Create everything you need for a fresh new stack!
@export AWS_REGION=$(AWS_REGION)
@echo "Make sure you have an environment variable OIDC_CLIENT_SECRET set."
@test -n "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -n "$(OIDC_CLIENT_ID)" -o -z "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -z "$(OIDC_CLIENT_ID)"
aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_DEV_STACK_URI)mozdef-parent.yml \
--capabilities CAPABILITY_IAM \
--parameters $(OIDC_CLIENT_SECRET_PARAM_ARG) \
$(ALB_BASIC_AUTH_SECRET_PARAM_ARG) \
$(DEV_STACK_PARAMS) \
--output text

@@ -54,12 +67,12 @@ create-dev-s3-bucket:
aws s3api create-bucket --bucket $(S3_DEV_BUCKET_NAME) --acl public-read --create-bucket-configuration LocationConstraint=$(AWS_REGION)

.PHONY: update-dev-stack
update-dev-stack: test ## Updates the nested stack on AWS
update-dev-stack: safety-checks test ## Updates the nested stack on AWS
@export AWS_REGION=$(AWS_REGION)
@test -n "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -n "$(OIDC_CLIENT_ID)" -o -z "$(OIDC_CLIENT_SECRET_PARAM_ARG)" -a -z "$(OIDC_CLIENT_ID)"
aws cloudformation update-stack --stack-name $(STACK_NAME) --template-url $(S3_DEV_STACK_URI)mozdef-parent.yml \
--capabilities CAPABILITY_IAM \
--parameters $(OIDC_CLIENT_SECRET_PARAM_ARG) \
$(ALB_BASIC_AUTH_SECRET_PARAM_ARG) \
$(DEV_STACK_PARAMS) \
--output text

@@ -43,5 +43,10 @@
"ParameterKey": "OIDCClientSecret",
"ParameterValue": "secret-value-goes-here",
"UsePreviousValue": false
},
{
"ParameterKey": "ALBBasicAuthSecret",
"ParameterValue": "secret-value-goes-here",
"UsePreviousValue": false
}
]
@@ -1 +1,2 @@
export OIDC_CLIENT_SECRET=secretgoeshere
export OIDC_CLIENT_SECRET=secretgoeshere
export ALB_BASIC_AUTH_SECRET=secretgoeshere
@@ -2,9 +2,6 @@

set -e # Exit immediately if a command exits with a non-zero status.

echo 'Welcome GitHub webhook to the CodeBuild Job of MozDef.'
echo "It's dangerous to go alone. Take one of these: <%%%%|==========>"

# echo "Begin test of the MozDef codebase."
# export COMPOSE_INTERACTIVE_NO_CLI=1 make tests
# The above does not currently work in a non-interactive TTY.
@@ -24,23 +21,17 @@ echo " Event : ${CODEBUILD_WEBHOOK_EVENT}"
echo " Head Ref : ${CODEBUILD_WEBHOOK_HEAD_REF}"
echo " Trigger : ${CODEBUILD_WEBHOOK_TRIGGER}"

if [[ "branch/master" == "${CODEBUILD_WEBHOOK_TRIGGER}" \
|| "branch/reinforce2019" == "${CODEBUILD_WEBHOOK_TRIGGER}" \
|| "${CODEBUILD_WEBHOOK_TRIGGER}" =~ ^tag\/v[0-9]+\.[0-9]+\.[0-9]+(\-(prod|pre|testing))?$ ]]; then
echo "Codebuild is ubuntu 14.04. Installing packer in order to compensate. Someone should build a CI docker container \;)."
wget -nv https://releases.hashicorp.com/packer/1.3.5/packer_1.3.5_linux_amd64.zip
unzip packer_1.3.5_linux_amd64.zip -d /usr/bin
echo "Building a release. C|_| This may take a bit. Might as well grab a coffee."
BRANCH="`echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f2`"
make build-from-cwd
make hub-login
make BRANCH=${BRANCH} docker-push-tagged
cd cloudy_mozdef
make BRANCH=${BRANCH} packer-build-github
make BRANCH=${BRANCH} publish-versioned-templates
cd ..
echo "End build of the MozDef codebase."
else
echo "Trigger '${CODEBUILD_WEBHOOK_TRIGGER}' was not a commit to branch/master or a conforming git tag name. Skipping build"
fi
echo "Codebuild is ubuntu 14.04. Installing packer in order to compensate. Someone should build a CI docker container \;)."
wget -nv https://releases.hashicorp.com/packer/1.3.5/packer_1.3.5_linux_amd64.zip
unzip packer_1.3.5_linux_amd64.zip -d /usr/bin
echo "Building a release. C|_| This may take a bit. Might as well grab a coffee."
BRANCH="`echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f2`"
make build-from-cwd
make hub-login
make BRANCH=${BRANCH} docker-push-tagged
cd cloudy_mozdef
make BRANCH=${BRANCH} packer-build-github
make BRANCH=${BRANCH} publish-versioned-templates
cd ..
echo "End build of the MozDef codebase."

@@ -40,6 +40,7 @@ Resources:
AlertWritersEnv:
Type: "AWS::Lambda::Function"
Properties:
Description: MozDef alert writer environment used to develop and invoke a MozDef alert
Handler: "lambdalert.handle"
Role:
Fn::GetAtt:
@@ -0,0 +1,148 @@
AWSTemplateFormatVersion: 2010-09-09
Description: CodeBuild CI/CD Job to build on commit
Mappings:
VariableMap:
Variables:
S3BucketToPublishCloudFormationTemplatesTo: public.us-west-2.infosec.mozilla.org
CloudWatchLogGroupName: MozDefCI
CloudWatchLogStreamName: build
Resources:
CodeBuildServiceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service: codebuild.amazonaws.com
Action: sts:AssumeRole
Policies:
- PolicyName: MozDefCodeBuild
PolicyDocument:
Version: 2012-10-17
Statement:
- Sid: ManagePackerEC2Instance
Effect: Allow
Action:
- ec2:DeleteVolume
- ec2:TerminateInstances
- ec2:ModifyInstanceAttribute
- ec2:StopInstances
- ec2:AttachVolume
- ec2:DetachVolume
- ec2:DeleteSnapshot
- ec2:CreateSnapshot
Resource: '*'
Condition:
StringEqualsIfExists:
'ec2:ResourceTag/app': packer-builder-mozdef
- Sid: UploadCloudFormationTemplatesToS3
Effect: Allow
Action:
- s3:PutObject*
- s3:GetObject*
Resource: !Join [ '', [ 'arn:aws:s3::', !FindInMap [ 'VariableMap', 'Variables', 'S3BucketToPublishCloudFormationTemplatesTo' ], '/*' ] ]
- Sid: ListS3BucketContents
Effect: Allow
Action:
- s3:ListBucket*
Resource: !Join [ '', [ 'arn:aws:s3::', !FindInMap [ 'VariableMap', 'Variables', 'S3BucketToPublishCloudFormationTemplatesTo' ] ] ]
- Sid: CreatePackerEC2Instance
Effect: Allow
Action:
- ec2:CreateKeyPair
- ec2:CreateVolume
- ec2:CreateImage
- ec2:CreateSecurityGroup
- ec2:CreateTags
- ec2:ModifyImageAttribute
- ec2:DeregisterImage
- ec2:CopyImage
- ec2:RegisterImage
- ec2:RunInstances
- ec2:DeleteSecurityGroup
- ec2:AuthorizeSecurityGroupIngress
- ec2:DeleteKeyPair
Resource: '*'
- Sid: ReadEC2
Effect: Allow
Action:
- ec2:DescribeInstances
- ec2:DescribeRegions
- ec2:DescribeSnapshots
- ec2:DescribeVolumes
- ec2:DescribeInstanceStatus
- ec2:DescribeTags
- ec2:DescribeSecurityGroups
- ec2:DescribeImages
- ec2:DescribeImageAttribute
- ec2:DescribeSubnets
Resource: '*'
- Sid: ReadSSMParameters
Effect: Allow
Action: ssm:GetParameter
Resource: arn:aws:ssm:*:*:parameter/mozdef/ci/*
# I think these are vestigial, created by the CodeBuild UI.
# Also they're not even the right resource path since they contain "/aws/codebuild/" but the actual LogGroup that CodeBuild writes to doesn't
# e.g. arn:aws:logs:us-west-2:371522382791:log-group:MozDefCI:*
# - Sid: NotSure1
# Effect: Allow
# Action:
# - logs:CreateLogGroup
# - logs:CreateLogStream
# - logs:PutLogEvents
# Resource:
# - !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group:/aws/codebuild/mozdef' ] ]
# - !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group:/aws/codebuild/mozdef:*' ] ]
# - !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group:/aws/codebuild/MozDefCI' ] ]
# - !Join [ ':', [ 'arn:aws:logs', !Ref 'AWS::Region', !Ref 'AWS::AccountId', 'log-group:/aws/codebuild/MozDefCI:*' ] ]
- Sid: NotSure2
Action:
- s3:PutObject
- s3:GetObject
- s3:GetObjectVersion
- s3:GetBucketAcl
- s3:GetBucketLocation
Effect: Allow
Resource:
- arn:aws:s3:::codepipeline-us-west-2-*
CodeBuildProject:
Type: AWS::CodeBuild::Project
Properties:
Name: mozdef
Description: Builds MozDef AMI, dockers containers, and runs test suite. Owner is Andrew Krug.
BadgeEnabled: True
ServiceRole: !GetAtt CodeBuildServiceRole.Arn
Artifacts:
Type: NO_ARTIFACTS
Environment:
Type: LINUX_CONTAINER
ComputeType: BUILD_GENERAL1_MEDIUM
Image: aws/codebuild/docker:18.09.0-1.7.0
Source:
Type: GITHUB
# Auth: # This information is for the AWS CodeBuild console's use only. Your code should not get or set Auth directly.
# SourceIdentifier: # Not sure what this should be yet
BuildSpec: cloudy_mozdef/buildspec.yml
Location: https://github.com/mozilla/MozDef
ReportBuildStatus: True
Triggers:
Webhook: true
FilterGroups:
- - Type: EVENT
Pattern: PUSH
- Type: HEAD_REF # Build on commits to branch reinforce2019
Pattern: '^refs/heads/reinforce2019'
- Type: HEAD_REF # Build on commits to branch master
Pattern: '^refs/heads/master'
- Type: HEAD_REF # Build on tags like v1.2.3 and v1.2.3-testing
Pattern: '^refs/tags\/v[0-9]+\.[0-9]+\.[0-9]+(\-(prod|pre|testing))?$'
Tags:
- Key: app
Value: mozdef
LogsConfig:
CloudWatchLogs:
GroupName: !FindInMap [ 'VariableMap', 'Variables', 'CloudWatchLogGroupName' ]
Status: ENABLED
StreamName: !FindInMap [ 'VariableMap', 'Variables', 'CloudWatchLogStreamName' ]
@@ -28,7 +28,7 @@ Resources:
Type: AWS::ElasticLoadBalancing::LoadBalancer
Properties:
AvailabilityZones:
- us-west-2a
- !Select [ 0, 'Fn::GetAZs': !Ref 'AWS::Region' ]
Listeners:
- LoadBalancerPort: '80'
InstancePort: '80'

0 comments on commit 693d7ea

Please sign in to comment.
You can’t perform that action at this time.