Skip to content
Permalink
Browse files

Move to model where we build once in CodeBuild and fetch images in pa…

…cker
  • Loading branch information...
gene1wood committed Apr 9, 2019
1 parent 3d47a42 commit 7fe0541dcd704a9e557b1d1c5fc87c6f3ca91f74
Showing with 99 additions and 118 deletions.
  1. +7 −1 CHANGELOG
  2. +0 −4 Makefile
  3. +4 −4 cloudy_mozdef/ci/deploy
  4. +57 −60 cloudy_mozdef/packer/packer.json
  5. +31 −49 docs/source/cicd.rst
@@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)

## [Unreleased]

## [v1.38.5] - 2019-04-09
### Changed
- Changed the CI/CD order to now build docker images in CodeBuild, upload them
to DockerHub and then pull them down in the packer instance. Updated docs.

## [v1.38.4] - 2019-04-08
### Fixed
- Docker image tagging for git version tag builds
@@ -64,7 +69,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added checks on sending SQS messages to only accept intra-account messages
- Improved docker performance and disk space requirements

[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.38.4...HEAD
[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.38.5...HEAD
[v1.38.5]: https://github.com/mozilla/MozDef/compare/v1.38.4...v1.38.5
[v1.38.4]: https://github.com/mozilla/MozDef/compare/v1.38.3...v1.38.4
[v1.38.3]: https://github.com/mozilla/MozDef/compare/v1.38.2...v1.38.3
[v1.38.2]: https://github.com/mozilla/MozDef/compare/v1.38.1...v1.38.2
@@ -143,7 +143,3 @@ rebuild: clean build-from-cwd
.PHONY: new-alert
new-alert: ## Create an example alert and working alert unit test
python tests/alert_templater.py

.PHONY: set-version-and-fetch-docker-container
set-version-and-fetch-docker-container: build-from-cwd tag-images # Lock the release of MozDef by pulling the docker containers on AMI build and caching replace all instances of latest in the compose override with the BRANCH
sed -i s/latest/$(BRANCH)/g docker/compose/docker-compose-cloudy-mozdef.yml
@@ -15,20 +15,20 @@ echo "It's dangerous to go alone. Take one of these: <%%%%|==========>"
# Then again we probably do not need to run the test suite here because it has been run three times to get the code here.
# echo "Tests complete.

echo "Processing webhook event for ${CODEBUILD_WEBHOOK_TRIGGER}."
echo "Processing webhook event for '${CODEBUILD_WEBHOOK_TRIGGER}'."

if [[ "branch/master" == "$CODEBUILD_WEBHOOK_TRIGGER" \
|| "$CODEBUILD_WEBHOOK_TRIGGER" =~ ^tag\/v[0-9]+\.[0-9]+\.[0-9]+(\-(prod|pre|testing))?$ ]]; then
echo "Building a release"
echo "C|_| This may take a bit. Might as well grab a coffee."
BRANCH="`echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f2`"
make build-from-cwd
make hub-login
make BRANCH=${BRANCH} docker-push-tagged
cd cloudy_mozdef
BRANCH="`echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f2`"
make BRANCH=${BRANCH} packer-build-github
make BRANCH=${BRANCH} publish-versioned-templates
cd ..
make hub-login
make BRANCH=${BRANCH} docker-push-tagged
fi

echo "End build of the MozDef codebase."
@@ -1,72 +1,69 @@
{
"variables": {
"aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
"aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
"aws_security_token": "{{env `AWS_SESSION_TOKEN`}}"
"aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
"aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}",
"aws_security_token": "{{env `AWS_SESSION_TOKEN`}}"
},
"builders": [{
"type": "amazon-ebs",
"region": "us-west-2",
"access_key": "{{user `aws_access_key`}}",
"secret_key": "{{user `aws_secret_key`}}",
"token": "{{user `aws_security_token`}}",
"source_ami": "ami-0d1000aff9a9bad89",
"instance_type": "t2.large",
"ssh_pty" : "true",
"ssh_username": "ec2-user",
"ami_name": "mozdef_{{timestamp}}",
"launch_block_device_mappings": [
{
"delete_on_termination": true,
"device_name": "/dev/xvda",
"volume_size": 14
"builders": [
{
"type": "amazon-ebs",
"region": "us-west-2",
"access_key": "{{user `aws_access_key`}}",
"secret_key": "{{user `aws_secret_key`}}",
"token": "{{user `aws_security_token`}}",
"source_ami": "ami-0d1000aff9a9bad89",
"instance_type": "t2.large",
"ssh_pty": "true",
"ssh_username": "ec2-user",
"ami_name": "mozdef_{{timestamp}}",
"launch_block_device_mappings": [
{
"delete_on_termination": true,
"device_name": "/dev/xvda",
"volume_size": 14
}
],
"ami_description": "An automated build of MozDef triggered via the makefile.",
"ami_groups": [
"all"
],
"run_tags": {
"app": "packer-builder-mozdef"
},
"run_volume_tags": {
"app": "packer-builder-mozdef"
},
"snapshot_tags": {
"app": "packer-builder-mozdef"
},
"tags": {
"github:Branch": "{{ user `github_branch`}}",
"buildTimestamp": "{{timestamp}}",
"app": "mozdef"
}
],
"ami_description": "An automated build of MozDef triggered via the makefile.",
"ami_groups": [
"all"
],
"run_tags": {
"app": "packer-builder-mozdef"
},
"run_volume_tags": {
"app": "packer-builder-mozdef"
},
"snapshot_tags": {
"app": "packer-builder-mozdef"
},
"tags": {
"github:Branch": "{{ user `github_branch`}}",
"buildTimestamp": "{{timestamp}}",
"app": "mozdef"
}
}],
"provisioners": [
{ "type": "shell",
],
"provisioners": [
{
"type": "shell",
"inline": [
"set -e",
"sudo yum update -y",
"sudo yum makecache fast",
"sudo yum install -y glibc-devel gcc libstdc++ libffi-devel zlib-devel make ",
"sudo yum install -y mysql-devel python python-devel python-pip",
"sudo yum install -y git",
"sudo yum install -y docker",
"sudo yum install -y python3",
"sudo pip install virtualenv ",
"sudo pip install docker-compose",
"sudo yum install -y glibc-devel gcc libstdc++ libffi-devel zlib-devel make mysql-devel python python-devel python-pip git docker python3",
"sudo pip install virtualenv docker-compose",
"sudo systemctl enable docker",
"sudo systemctl start docker",
"sudo mkdir -p /opt/mozdef/",
"sudo mkdir --verbose --parents /opt/mozdef/",
"sudo git clone https://github.com/mozilla/MozDef /opt/mozdef",
"cd /opt/mozdef && sudo git checkout {{ user `github_branch`}}",
"cd /opt/mozdef && sudo git rev-parse HEAD",
"cd /opt/mozdef && sudo touch docker/compose/cloudy_mozdef.env docker/compose/rabbitmq.env docker/compose/cloudy_mozdef_mq_cloudtrail.env docker/compose/cloudy_mozdef_mq_sns_sqs.env docker/compose/cloudy_mozdef_kibana.env",
"cd /opt/mozdef && sudo make BRANCH={{ user `github_branch`}} set-version-and-fetch-docker-container",
"cd /opt/mozdef && sudo docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p mozdef pull",
"rm -rf /home/ec2-user/.ssh/authorized_keys",
"rm -rf /home/ec2-user/.ssh/known_hosts",
"sudo rm -rf /tmp/*",
"sudo rm -rf /home/ec2-user/.bash_history",
"sudo rm -rf /root/.ssh"
]}
]
"cd /opt/mozdef",
"sudo git checkout {{ user `github_branch`}}",
"sudo git rev-parse HEAD",
"sudo touch docker/compose/cloudy_mozdef.env docker/compose/rabbitmq.env docker/compose/cloudy_mozdef_mq_cloudtrail.env docker/compose/cloudy_mozdef_mq_sns_sqs.env docker/compose/cloudy_mozdef_kibana.env",
"sudo sed --in-place s/latest/{{ user `github_branch`}}/g docker/compose/docker-compose-cloudy-mozdef.yml",
"sudo docker-compose --file docker/compose/docker-compose-cloudy-mozdef.yml --project-name mozdef pull",
"sudo rm --recursive --force --verbose /tmp/* /home/ec2-user/.bash_history /root/.ssh /home/ec2-user/.ssh/known_hosts /home/ec2-user/.ssh/authorized_keys"
]
}
]
}
@@ -78,9 +78,9 @@ _________________________________________
The Build Sequence
__________________

* A branch is merged into `master` in the GitHub repo
* GitHub emits a webhook event to AWS CodeBuild indicating that a commit was
pushed to `master`
* A branch is merged into `master` in the GitHub repo or a version git tag is
applied to a commit
* GitHub emits a webhook event to AWS CodeBuild indicating this
* AWS CodeBuild reads the
`buildspec.yml <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/buildspec.yml>`_
file to know what to do
@@ -98,8 +98,28 @@ __________________
target of the `Makefile` which calls `docker-compose build` on the
`docker-compose.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose.yml>`_
file, building the docker images in the AWS CodeBuild environment. These are
built so they can be uploaded later to DockerHub for use by developers and
the community.
built both so they can be consumed later in the build by packer and also
for use by developers and the community.
* `deploy` then calls the
`docker-push-tagged <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L113>`_
make target which calls

* the tag-images_
make target which calls the
`cloudy_mozdef/ci/docker_tag_or_push tag <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
script which applies a docker image tag to the local image that was just
built by AWS CodeBuild.
* the
`hub-tagged <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L116-L117>`_
make target which calls the
`cloudy_mozdef/ci/docker_tag_or_push push <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
script which

* Uploads the local image that was just built by AWS CodeBuild to DockerHub.
If the branch being built is `master` then the image is uploaded both with
a tag of `master` as well as with a tag of `latest`
* If the branch being built is from a version tag (e.g. `v1.2.3`) then the
image is uploaded with only that version tag applied
* The `deploy` script next calls the
`packer-build-github <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/cloudy_mozdef/Makefile#L34-L36>`_
make target in the
@@ -124,31 +144,14 @@ __________________
* Within this ec2 instance, packer `clones the MozDef GitHub repo and checks
out the branch that triggered this build
<https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/cloudy_mozdef/packer/packer.json#L59-L60>`_
* packer calls the `set-version-and-fetch-docker-container <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L148-L149>`_
target of the `Makefile` which

* Calls the
`build-from-cwd <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L78-L79>`_
target of the `Makefile` which calls `docker-compose build` on the
`docker-compose.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose.yml>`_
file, building the docker images in the packer ec2 environment
* Calls the tag-images_
make target which calls the `cloudy_mozdef/ci/docker_tag_or_push tag <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
script.

* This applies a docker image tag to the local image that was just built
by packer. This tag will be referenced by the
`docker-compose-cloudy-mozdef.yml` file when the MozDef ec2 instance is
launched from the AMI.

* Replaces all instances of the word `latest` in the
`docker-compose-cloudy-mozdef.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-cloudy-mozdef.yml>`_
file with either the branch `master` or the version tag (e.g. `v1.2.3`)

* packer replaces all instances of the word `latest` in the
`docker-compose-cloudy-mozdef.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-cloudy-mozdef.yml>`_
file with either the branch `master` or the version tag (e.g. `v1.2.3`)
* packer runs `docker-compose pull` on the
`docker-compose-cloudy-mozdef.yml <https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-cloudy-mozdef.yml>`_
file to pull down any remaining non MozDef container images that weren't
just built in preceding packer steps
file to pull down both the docker images that were just built by AWS
CodeBuild and uploaded to Dockerhub as well as other non MozDef docker
images

* After packer completes executing the steps laid out in `packer.json` inside
the ec2 instance, it generates an AMI from that instance and continues with
@@ -168,26 +171,5 @@ __________________
* uploads the CloudFormation templates to S3 in a directory either called
`master` or the tag version that was built (e.g. `v1.2.3`)

* `deploy` then calls the
`docker-push-tagged <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L113>`_
make target which calls

* the tag-images_
make target which calls the
`cloudy_mozdef/ci/docker_tag_or_push tag <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
script which applies a docker image tag to the local image that was just
built by AWS CodeBuild.
* the
`hub-tagged <https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L116-L117>`_
make target which calls the
`cloudy_mozdef/ci/docker_tag_or_push push <https://github.com/mozilla/MozDef/blob/master/cloudy_mozdef/ci/docker_tag_or_push>`_
script which

* Uploads the local image that was just built by AWS CodeBuild to DockerHub.
If the branch being built is `master` then the image is uploaded both with
a tag of `master` as well as with a tag of `latest`
* If the branch being built is from a version tag (e.g. `v1.2.3`) then the
image is uploaded with only that version tag applied

.. _docker/compose/docker-compose-tests.yml: https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-tests.yml
.. _tag-images: https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L109-L110

0 comments on commit 7fe0541

Please sign in to comment.
You can’t perform that action at this time.