diff --git a/Jenkinsfile b/Jenkinsfile index bd300fb..ce89585 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,6 +11,10 @@ import java.util.Random // Set default variables final default_timeout_minutes = 20 +final codedeploy_target_skip = -1 +// See generally safe key names from https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +final s3_safe_branch_name = env.BRANCH_NAME.replaceAll(/[^0-9a-zA-Z\!\-_\.\*\'\(\)]/ , "_") +def codedeploy_target = codedeploy_target_skip /** Set up CAPTCHA*/ def get_captcha(Long hash_const) { @@ -55,6 +59,25 @@ properties([ defaultValue: false, description: 'Run Packer for this build?' ), + booleanParam( + name: 'Package_CodeDeploy', + defaultValue: false, + description: 'Package CodeDeploy application on this build?' + ), + string( + name: 'Deploy_CodeDeploy', + defaultValue: '', + description: '''Deploy a CodeDeploy archive. + Specify one of the following: + 1. "current" - deploy the CodeDeploy archive from this build + 2. a full S3 URL of a zip file to deploy + 3. an empty string (to skip deployment)''' + ), + booleanParam( + name: 'Skip_Terraform', + defaultValue: false, + description: 'Skip all Terraform steps, including validation and planning? (shortens cycle times when testing other aspects)' + ), booleanParam( name: 'Apply_Terraform', defaultValue: false, @@ -84,7 +107,7 @@ properties([ defaultValue: false, description: """Rotate server instances in Auto Scaling Group? You should do this if you changed ASG size or baked a new AMI. - """ + """ ), booleanParam( name: 'Run_JMeter', @@ -122,11 +145,11 @@ properties([ stage('Preflight') { // Check CAPTCHA - def should_validate_captcha = params.Run_Packer || params.Apply_Terraform || params.Destroy_Terraform || params.Run_JMeter + def should_validate_captcha = params.Run_Packer || params.Apply_Terraform || params.Destroy_Terraform || params.Run_JMeter || params.CodeDeploy_Target if (should_validate_captcha) { if (params.CAPTCHA_Guess == null || params.CAPTCHA_Guess == "") { - throw new Exception("No CAPTCHA guess detected, try again!") + error "No CAPTCHA guess detected, try again!" } def guess = params.CAPTCHA_Guess as Long def hash = params.CAPTCHA_Hash as Long @@ -137,6 +160,28 @@ stage('Preflight') { } else { echo "No CAPTCHA required, continuing" } + + def build_number = env.BUILD_NUMBER as Long + + switch (params.Deploy_CodeDeploy) { + case "": + echo "CodeDeploy deployment target is blank, skipping codedeploy step" + break + case "current": + echo """CodeDeploy: targeting latest build + CodeDeploy: Will use prefix ${s3_safe_branch_name} + """ + codedeploy_target = "current" + break + case ~/^s3:.*/: + echo """CodeDeploy: targeting S3 URL build ${params.Deploy_CodeDeploy}""" + codedeploy_target = params.Deploy_CodeDeploy + break + default: + currentBuild.result = 'ABORTED' + error "CodeDeploy build_number ${build_number} is not understood" + break + } } stage('Checkout') { @@ -153,8 +198,8 @@ stage('Validate') { node { wrap.call({ unstash 'src' - // Validate packer templates, check branch - sh ("./bin/validate.sh") + // Validate packer templates, check branch, lint shell scripts, lint terraform + sh ("SKIP_TERRAFORM=${params.Skip_Terraform} ./bin/validate.sh") }) } } @@ -191,36 +236,40 @@ if (params.Run_Packer) { } } -stage('Build CodeDeploy Archive') { - node { - wrap.call({ - unstash 'src' - sh ("./bin/build-codedeploy.sh") - }) +if (params.Package_CodeDeploy) { + stage('Package CodeDeploy Archive') { + node { + wrap.call({ + unstash 'src' + sh ("./bin/build-codedeploy.sh ${s3_safe_branch_name}") + }) + } } } def terraform_prompt = 'Should we apply the Terraform plan?' -stage('Plan Terraform') { - node { - wrap.call({ - unstash 'src' - def verb = "plan" - if (params.Destroy_Terraform) { - verb += '-destroy'; - terraform_prompt += ' WARNING: will DESTROY resources'; - } - sh (""" - ./bin/terraform.sh ${verb} - """) - }) - stash includes: "**", excludes: ".git/", name: 'plan' +if (! params.Skip_Terraform) { + stage('Plan Terraform') { + node { + wrap.call({ + unstash 'src' + def verb = "plan" + if (params.Destroy_Terraform) { + verb += '-destroy'; + terraform_prompt += ' WARNING: will DESTROY resources'; + } + sh (""" + ./bin/terraform.sh ${verb} + """) + }) + stash includes: "**", excludes: ".git/", name: 'plan' + } } } -if (params.Apply_Terraform || params.Destroy_Terraform) { +if (! params.Skip_Terraform && params.Apply_Terraform || params.Destroy_Terraform) { // See https://support.cloudbees.com/hc/en-us/articles/226554067-Pipeline-How-to-add-an-input-step-with-timeout-that-continues-if-timeout-is-reached-using-a-default-value def userInput = false try { @@ -241,6 +290,29 @@ if (params.Apply_Terraform || params.Destroy_Terraform) { } } +if (params.Rotate_Servers) { + stage('Rotate Servers') { + node { + wrap.call({ + unstash 'src' + sh ("./bin/rotate-asg.sh infra-demo-asg") + }) + } + } +} + +if (params.Deploy_CodeDeploy) { + stage('Deploy') { + node { + wrap.call({ + unstash 'src' + sh ("./bin/deploy-codedeploy.sh ${codedeploy_target} ${s3_safe_branch_name}") + }) + } + } +} + + if (params.Rotate_Servers) { stage('Rotate Servers') { node { diff --git a/README.md b/README.md index 8d432ca..73a18f0 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,10 @@ See the branch [demo-20180926](https://github.com/ModusCreateOrg/devops-infra-de See the branch [demo-20181205](https://github.com/ModusCreateOrg/devops-infra-demo/tree/demo-20181205) for the code for the demo for the [Ansible NYC talk _Ansible Image Bakeries: Best Practices & Pitfalls_](https://www.meetup.com/Ansible-NYC/events/256728741/). Slides from this presentation are on [SlideShare](https://www.slideshare.net/RichardBullingtonMcG/ansible-image-bakeries-best-practices-and-pitfalls). -See the branch [demo-20190130](https://github.com/ModusCreateOrg/devops-infra-demo/tree/demo-20180130) for the code for the demo for the [Big Apple DevOps talk _Monitoring and Alerting as code with Terraform and New Relic_](https://www.meetup.com/Big-Apple-DevOps/events/257744262/). Slides from this presentation are on [Slideshare](https://www.slideshare.net/RichardBullingtonMcG/monitoring-and-alerting-as-code-with-terraform-and-new-relic). +See the branch [demo-20190130](https://github.com/ModusCreateOrg/devops-infra-demo/tree/demo-20190130) for the code for the demo for the [Big Apple DevOps talk _Monitoring and Alerting as code with Terraform and New Relic_](https://www.meetup.com/Big-Apple-DevOps/events/257744262/). Slides from this presentation are on [Slideshare](https://www.slideshare.net/RichardBullingtonMcG/monitoring-and-alerting-as-code-with-terraform-and-new-relic). + +See the branch [demo-20191109](https://github.com/ModusCreateOrg/devops-infra-demo/tree/demo-20191l09) for the code for the demo for the [BSidesCT 2019 talk g Apple DevOps talk _Extensible DevSecOps pipelines with Jenkins, Docker, Terraform, and a kitchen sink full of scanners_](https://bsidesct.org/schedule/). Slides from this presentation are on +[Slideshare](https://www.slideshare.net/RichardBullingtonMcG/extensible-devsecops-pipelines-with-jenkins-docker-terrraform-and-a-kitchen-sink-full-of-scanners) Instructions ------------ diff --git a/ansible/roles/cloudwatch-agent/files/config.json b/ansible/roles/cloudwatch-agent/files/config.json index 219e151..38e2d6f 100755 --- a/ansible/roles/cloudwatch-agent/files/config.json +++ b/ansible/roles/cloudwatch-agent/files/config.json @@ -61,6 +61,11 @@ "file_path": "/var/log/nginx/error.log", "log_group_name": "nginx-error", "log_stream_name": "{instance_id}" + }, + { + "file_path": "/opt/codedeploy-agent/deployment-root/deployment-logs/codedeploy-agent-deployments.log", + "log_group_name": "codedeploy", + "log_stream_name": "{instance_id}" } ] } diff --git a/ansible/roles/scan-openscap/defaults/main.yml b/ansible/roles/scan-openscap/defaults/main.yml index ed0b37a..ac588b1 100644 --- a/ansible/roles/scan-openscap/defaults/main.yml +++ b/ansible/roles/scan-openscap/defaults/main.yml @@ -3,5 +3,6 @@ build_dir: /app/build output_file_html: /app/build/scan-xccdf-results.html output_file_xml: /app/build/scan-xccdf-results.xml -profile: C2S +# Oh no! The old C2S profile is no longer available! +profile: standard xccdf_file: /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml diff --git a/ansible/roles/scan-openscap/tasks/main.yml b/ansible/roles/scan-openscap/tasks/main.yml index f8bf305..a7778fb 100644 --- a/ansible/roles/scan-openscap/tasks/main.yml +++ b/ansible/roles/scan-openscap/tasks/main.yml @@ -18,7 +18,7 @@ cd {{ build_dir }} # This will have a non-zero exit if any of the scans fail, so do not fail immediately on that set +e - oscap xccdf eval --fetch-remote-resources --profile {{ profile }} --results {{ output_file_xml }} {{ xccdf_file }} + oscap xccdf eval --profile {{ profile }} --results {{ output_file_xml }} {{ xccdf_file }} set -e oscap xccdf generate report {{ output_file_xml }} > {{ output_file_html }} args: diff --git a/bin/activate-rvm.sh b/bin/activate-rvm.sh index dc8a93e..467ae11 100755 --- a/bin/activate-rvm.sh +++ b/bin/activate-rvm.sh @@ -2,11 +2,16 @@ # Activate rvm # Source this to activate RVM +# CodeDeploy has no HOME variable defined! +HOME=${HOME:-/centos} RVM_SH=${RVM_SH:-$HOME/.rvm/scripts/rvm} RUBY_VERSION=${RUBY_VERSION:-2.6.3} # rvm hates the bash options -eu +echo -n "Activating RVM. HOME=$HOME id:" +id -a + if [[ ! -f "$RVM_SH" ]]; then echo "Error: $0: RVM_SH $RVM_SH not found" exit 1 diff --git a/bin/build-codedeploy.sh b/bin/build-codedeploy.sh index 126fe6c..8bcbfca 100755 --- a/bin/build-codedeploy.sh +++ b/bin/build-codedeploy.sh @@ -16,20 +16,34 @@ BUILD_DIR="$BASE_DIR/build" ANSIBLE_DIR="$BASE_DIR/ansible" APPLICTION_DIR="$BASE_DIR/application" SRC_DIR="$BASE_DIR/src" +GAUNTLT_DIR="$BASE_DIR/gauntlt" + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." + +# shellcheck disable=SC1090 +. "$DIR/common.sh" +#shellcheck disable=SC1090 +. "$BASE_DIR/env.sh" + GIT_REV="$(git rev-parse --short HEAD)" BUILD_NUMBER=${BUILD_NUMBER:-0} -ARCHIVE="codedeploy-$BUILD_NUMBER-$GIT_REV.zip" +BRANCH_PREFIX=${1:-master} +ARCHIVE="codedeploy-$BRANCH_PREFIX-$BUILD_NUMBER-$GIT_REV.zip" CONTAINERNAME=infra-demo +# Thanks https://stackoverflow.com/questions/33791069/quick-way-to-get-aws-account-number-from-the-cli-tools +AWS_ACCOUNT_ID=$(get_aws_account_id) +BUCKET="codedeploy-$AWS_ACCOUNT_ID" +S3_URL="s3://$BUCKET/$ARCHIVE" echo "GIT_REV=$GIT_REV" +echo "BRANCH_PREFIX=$BRANCH_PREFIX" echo "BUILD_NUMBER=$BUILD_NUMBER" echo "ARCHIVE=$ARCHIVE" +echo "S3_URL=$S3_URL" -# Thanks https://stackoverflow.com/questions/33791069/quick-way-to-get-aws-account-number-from-the-cli-tools -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --output text --query 'Account') -BUCKET="codedeploy-$AWS_ACCOUNT_ID" -S3_URL="s3://$BUCKET/$ARCHIVE" if [[ -d "$BUILD_DIR" ]]; then rm -rf "$BUILD_DIR" @@ -52,8 +66,11 @@ SOURCES="$BASE_DIR/bin $ANSIBLE_DIR $APPLICTION_DIR $SRC_DIR +$GAUNTLT_DIR $BASE_DIR/codedeploy/appspec.yml" +echo "Copying sources into place" for src in $SOURCES; do + echo cp -a "$src" "$BUILD_DIR" cp -a "$src" "$BUILD_DIR" done @@ -64,6 +81,7 @@ done bin \ ansible \ application \ + gauntlt \ src \ venv \ socket diff --git a/bin/codedeploy/AfterInstall.sh b/bin/codedeploy/AfterInstall.sh index 2fcbca4..3b5c57e 100755 --- a/bin/codedeploy/AfterInstall.sh +++ b/bin/codedeploy/AfterInstall.sh @@ -41,3 +41,6 @@ source ${VENV_DIR}/bin/activate set -u newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini" + +echo /app directory: +ls -laZ /app diff --git a/bin/codedeploy/ValidateService.sh b/bin/codedeploy/ValidateService.sh index 61a8c63..6e69712 100755 --- a/bin/codedeploy/ValidateService.sh +++ b/bin/codedeploy/ValidateService.sh @@ -14,6 +14,10 @@ ${DEBUG:-false} && set -vx # and http://wiki.bash-hackers.org/scripting/debuggingtips export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' +GAUNTLT_RESULTS=/app/build/gauntlt-results.html +# TODO: save this to S3 instead +GAUNTLT_RESULTS_SAVE="/home/centos/$DEPLOYMENT_ID-gauntlt-results.html" + check_every() { local delay=${1:-} local host="http://localhost/" @@ -25,4 +29,19 @@ check_every() { done } +echo "Checking web server availability" check_every 2 + +echo "Scanning with openscap and gauntlt" +mkdir -p /app/build /app/ansible/tmp +cat < /dev/null > "$GAUNTLT_RESULTS" +chown -R centos:centos "$GAUNTLT_RESULTS" /app/build /app/ansible/tmp +chmod 755 "$GAUNTLT_RESULTS" /app/build /app/ansible/tmp + +set +e +sudo -u centos HOME=/home/centos /app/bin/ansible.sh scan-openscap.yml scan-gauntlt.yml +RETCODE=$? +set -e +cp "$GAUNTLT_RESULTS" "$GAUNTLT_RESULTS_SAVE" +rm -rf /app/ansible/tmp /app/build +exit "$RETCODE" diff --git a/bin/deploy-codedeploy.sh b/bin/deploy-codedeploy.sh new file mode 100755 index 0000000..fa2c17d --- /dev/null +++ b/bin/deploy-codedeploy.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." + +# shellcheck disable=SC1090 +. "$DIR/common.sh" +#shellcheck disable=SC1090 +. "$BASE_DIR/env.sh" + +BUILD_NUMBER=${BUILD_NUMBER:-0} +BUCKET="codedeploy-$(get_aws_account_id)" +PARAM=${1:-} +BRANCH_PREFIX=${2:-master} +APP_NAME=${3:-tf-infra-demo-app} +DEPLOYMENT_GROUP_NAME=${4:-dev} + +GIT_REV="$(git rev-parse --short HEAD)" +ARCHIVE="codedeploy-$BRANCH_PREFIX-$BUILD_NUMBER-$GIT_REV.zip" +# Thanks https://stackoverflow.com/questions/33791069/quick-way-to-get-aws-account-number-from-the-cli-tools +S3_URL="s3://$BUCKET/$ARCHIVE" + +# Thanks https://stackoverflow.com/questions/33791069/quick-way-to-get-aws-account-number-from-the-cli-tools +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --output text --query 'Account') +BUCKET="codedeploy-$AWS_ACCOUNT_ID" + +case $PARAM in + s3[:]//[a-z0-9]*) + S3_URL="$PARAM" + ARCHIVE=$(cut -d/ -f 3- <<<"$S3_URL") + BUCKET=$(cut -d/ -f 2- <<<"$S3_URL") + ;; + current) + echo "Using current CodeDeploy build: $S3_URL" + ;; + *) + echo "ERROR: Unknown format for $PARAM, exiting" + exit 1 + ;; +esac + + +S3_SHORTHAND="bundleType=zip,bucket=$BUCKET,key=$ARCHIVE" + +echo "BRANCH_PREFIX=$BRANCH_PREFIX" +echo "BUILD_NUMBER=$BUILD_NUMBER" +echo "ARCHIVE=$ARCHIVE" +echo "S3_URL=$S3_URL" +echo "S3_SHORTHAND=$S3_SHORTHAND" + + +DEPLOYMENT_ID=$(aws deploy create-deployment \ + --region "$AWS_DEFAULT_REGION" \ + --output text \ + --query '[deploymentId]' \ + --application-name "$APP_NAME" \ + --deployment-group-name "$DEPLOYMENT_GROUP_NAME" \ + --description "deployment initiated by deploy-codedeploy.sh" \ + --no-ignore-application-stop-failures \ + --s3-location "$S3_SHORTHAND") +echo "CodeDeploy: deployment started $DEPLOYMENT_ID" +echo "CodeDeploy: see https://console.aws.amazon.com/codesuite/codedeploy/deployments/$DEPLOYMENT_ID" +echo "CodeDeploy: waiting for deployment $DEPLOYMENT_ID to complete..." +aws deploy wait deployment-successful --deployment-id "$DEPLOYMENT_ID" diff --git a/bin/validate.sh b/bin/validate.sh index 3ce3976..b6c830d 100755 --- a/bin/validate.sh +++ b/bin/validate.sh @@ -27,20 +27,22 @@ echo "Linting packer files" $DOCKER_PACKER validate app/packer/machines/web-server.json # Ensure that `terraform fmt` comes up clean -echo "Linting terraform files for correctness" -DOCKER_TERRAFORM=$(get_docker_terraform) -init_terraform -$DOCKER_TERRAFORM validate \ - -var 'newrelic_license_key=ZZZZ' \ - -var 'newrelic_api_key=ZZZZ' \ - -var 'newrelic_alert_email=ferd.berferd@example.com' \ -echo "Linting terraform files for formatting" -fmt=$($DOCKER_TERRAFORM fmt) -if [[ -n "$fmt" ]]; then - echo 'ERROR: these files are not formatted correctly. Run "terraform fmt"' - echo "$fmt" - git diff - exit 1 +if [[ "$SKIP_TERRAFORM" == "false" ]]; then + echo "Linting terraform files for correctness" + DOCKER_TERRAFORM=$(get_docker_terraform) + init_terraform + $DOCKER_TERRAFORM validate \ + -var 'newrelic_license_key=ZZZZ' \ + -var 'newrelic_api_key=ZZZZ' \ + -var 'newrelic_alert_email=ferd.berferd@example.com' \ + echo "Linting terraform files for formatting" + fmt=$($DOCKER_TERRAFORM fmt) + if [[ -n "$fmt" ]]; then + echo 'ERROR: these files are not formatted correctly. Run "terraform fmt"' + echo "$fmt" + git diff + exit 1 + fi fi echo "Linting shell scripts" diff --git a/codedeploy/appspec.yml b/codedeploy/appspec.yml index 6c7275f..21be634 100644 --- a/codedeploy/appspec.yml +++ b/codedeploy/appspec.yml @@ -6,6 +6,8 @@ files: destination: /app/application - source: src destination: /app/src + - source: gauntlt + destination: /app/gauntlt - source: bin destination: /app/bin - source: ansible @@ -25,4 +27,4 @@ hooks: timeout: 120 ValidateService: - location: bin/codedeploy/ValidateService.sh - timeout: 60 + timeout: 300 diff --git a/gauntlt/nmap.attack b/gauntlt/nmap.attack index 6467c14..35b9827 100644 --- a/gauntlt/nmap.attack +++ b/gauntlt/nmap.attack @@ -36,12 +36,12 @@ Feature: nmap attacks for localhost and to use this for your tests, change the v Scenario: Output to XML When I launch an "nmap" attack with: """ - nmap -p 22,80,443 -oX foo.xml + nmap -p 22,80,443 -oX /app/build/nmap-results.xml """ - And the file "foo.xml" should contain XML: + And the file "/app/build/nmap-results.xml" should contain XML: | css | | ports port[protocol="tcp"][portid="22"] state[state="open"] | - And the file "foo.xml" should not contain XML: + And the file "/app/build/nmap-results.xml" should not contain XML: | css | | ports port[protocol="tcp"][portid="80"] state[state="open"] | | ports port[protocol="tcp"][portid="443"] state[state="open"] |