bsidesct.org presentation 2019-11-09

Jenkinsfile

@@ -11,6 +11,10 @@ import java.util.Random
 
 // Set default variables
 final default_timeout_minutes = 20
+final codedeploy_target_skip = -1
+// See generally safe key names from https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+final s3_safe_branch_name = env.BRANCH_NAME.replaceAll(/[^0-9a-zA-Z\!\-_\.\*\'\(\)]/ , "_")
+def codedeploy_target = codedeploy_target_skip
 
 /** Set up CAPTCHA*/
 def get_captcha(Long hash_const) {
@@ -55,6 +59,25 @@ properties([
             defaultValue: false,
             description: 'Run Packer for this build?'
         ),
+        booleanParam(
+            name: 'Package_CodeDeploy',
+            defaultValue: false,
+            description: 'Package CodeDeploy application on this build?'
+        ),
+        string(
+            name: 'Deploy_CodeDeploy',
+            defaultValue: '',
+            description: '''Deploy a CodeDeploy archive.
+                            Specify one of the following:
+                            1. "current" - deploy the CodeDeploy archive from this build
+                            2. a full S3 URL of a zip file to deploy
+                            3. an empty string (to skip deployment)'''
+        ),
+        booleanParam(
+            name: 'Skip_Terraform',
+            defaultValue: false,
+            description: 'Skip all Terraform steps, including validation and planning? (shortens cycle times when testing other aspects)'
+        ),
         booleanParam(
             name: 'Apply_Terraform',
             defaultValue: false,
@@ -84,7 +107,7 @@ properties([
             defaultValue: false,
             description: """Rotate server instances in Auto Scaling Group?
                             You should do this if you changed ASG size or baked a new AMI.
-                        """
+                         """
         ),
         booleanParam(
             name: 'Run_JMeter',
@@ -122,11 +145,11 @@ properties([
 stage('Preflight') {
 
     // Check CAPTCHA
-    def should_validate_captcha = params.Run_Packer || params.Apply_Terraform || params.Destroy_Terraform || params.Run_JMeter
+    def should_validate_captcha = params.Run_Packer || params.Apply_Terraform || params.Destroy_Terraform || params.Run_JMeter || params.CodeDeploy_Target
 
     if (should_validate_captcha) {
         if (params.CAPTCHA_Guess == null || params.CAPTCHA_Guess == "") {
-            throw new Exception("No CAPTCHA guess detected, try again!")
+            error "No CAPTCHA guess detected, try again!"
         }
         def guess = params.CAPTCHA_Guess as Long
         def hash = params.CAPTCHA_Hash as Long
@@ -137,6 +160,28 @@ stage('Preflight') {
     } else {
         echo "No CAPTCHA required, continuing"
     }
+
+    def build_number = env.BUILD_NUMBER as Long
+
+    switch (params.Deploy_CodeDeploy) {
+        case "": 
+            echo "CodeDeploy deployment target is blank, skipping codedeploy step"
+            break
+        case "current": 
+            echo """CodeDeploy: targeting latest build
+                    CodeDeploy: Will use prefix ${s3_safe_branch_name}
+                 """
+            codedeploy_target = "current"
+            break
+        case ~/^s3:.*/:
+            echo """CodeDeploy: targeting S3 URL build ${params.Deploy_CodeDeploy}"""
+            codedeploy_target = params.Deploy_CodeDeploy
+            break
+        default:
+            currentBuild.result = 'ABORTED'
+            error "CodeDeploy build_number ${build_number} is not understood"
+            break
+    }
 }
 
 stage('Checkout') {
@@ -153,8 +198,8 @@ stage('Validate') {
     node {
         wrap.call({
             unstash 'src'
-            // Validate packer templates, check branch
-            sh ("./bin/validate.sh")
+            // Validate packer templates, check branch, lint shell scripts, lint terraform
+            sh ("SKIP_TERRAFORM=${params.Skip_Terraform} ./bin/validate.sh")
         })
     }
 }
@@ -191,36 +236,40 @@ if (params.Run_Packer) {
     }
 }
 
-stage('Build CodeDeploy Archive') {
-    node {
-        wrap.call({
-            unstash 'src'
-            sh ("./bin/build-codedeploy.sh")
-        })
+if (params.Package_CodeDeploy) {
+    stage('Package CodeDeploy Archive') {
+        node {
+            wrap.call({
+                unstash 'src'
+                sh ("./bin/build-codedeploy.sh ${s3_safe_branch_name}")
+            })
+        }
     }
 }
 
 def terraform_prompt = 'Should we apply the Terraform plan?'
 
 
-stage('Plan Terraform') {
-    node {
-        wrap.call({
-            unstash 'src'
-            def verb = "plan"
-            if (params.Destroy_Terraform) {
-                verb += '-destroy';
-                terraform_prompt += ' WARNING: will DESTROY resources';
-            }
-            sh ("""
-                ./bin/terraform.sh ${verb}
-                """)
-        })
-        stash includes: "**", excludes: ".git/", name: 'plan'
+if (! params.Skip_Terraform) {
+    stage('Plan Terraform') {
+        node {
+            wrap.call({
+                unstash 'src'
+                def verb = "plan"
+                if (params.Destroy_Terraform) {
+                    verb += '-destroy';
+                    terraform_prompt += ' WARNING: will DESTROY resources';
+                }
+                sh ("""
+                    ./bin/terraform.sh ${verb}
+                    """)
+            })
+            stash includes: "**", excludes: ".git/", name: 'plan'
+        }
     }
 }
 
-if (params.Apply_Terraform || params.Destroy_Terraform) {
+if (! params.Skip_Terraform && params.Apply_Terraform || params.Destroy_Terraform) {
     // See https://support.cloudbees.com/hc/en-us/articles/226554067-Pipeline-How-to-add-an-input-step-with-timeout-that-continues-if-timeout-is-reached-using-a-default-value
     def userInput = false
     try {
@@ -241,6 +290,29 @@ if (params.Apply_Terraform || params.Destroy_Terraform) {
     }
 }
 
+if (params.Rotate_Servers) {
+    stage('Rotate Servers') {
+        node {
+            wrap.call({
+                unstash 'src'
+                sh ("./bin/rotate-asg.sh infra-demo-asg")
+            })
+        }
+    }
+}
+
+if (params.Deploy_CodeDeploy) {
+    stage('Deploy') {
+        node {
+            wrap.call({
+                unstash 'src'
+                sh ("./bin/deploy-codedeploy.sh ${codedeploy_target} ${s3_safe_branch_name}")
+            })
+        }
+    }
+}
+
+
 if (params.Rotate_Servers) {
     stage('Rotate Servers') {
         node {

README.md

@@ -14,7 +14,10 @@ See the branch [demo-20180926](https://github.com/ModusCreateOrg/devops-infra-de
 
 See the branch [demo-20181205](https://github.com/ModusCreateOrg/devops-infra-demo/tree/demo-20181205) for the code for the demo for the [Ansible NYC talk _Ansible Image Bakeries: Best Practices & Pitfalls_](https://www.meetup.com/Ansible-NYC/events/256728741/). Slides from this presentation are on [SlideShare](https://www.slideshare.net/RichardBullingtonMcG/ansible-image-bakeries-best-practices-and-pitfalls).
 
-See the branch [demo-20190130](https://github.com/ModusCreateOrg/devops-infra-demo/tree/demo-20180130) for the code for the demo for the [Big Apple DevOps  talk _Monitoring and Alerting as code with Terraform and New Relic_](https://www.meetup.com/Big-Apple-DevOps/events/257744262/). Slides from this presentation are on [Slideshare](https://www.slideshare.net/RichardBullingtonMcG/monitoring-and-alerting-as-code-with-terraform-and-new-relic).
+See the branch [demo-20190130](https://github.com/ModusCreateOrg/devops-infra-demo/tree/demo-20190130) for the code for the demo for the [Big Apple DevOps  talk _Monitoring and Alerting as code with Terraform and New Relic_](https://www.meetup.com/Big-Apple-DevOps/events/257744262/). Slides from this presentation are on [Slideshare](https://www.slideshare.net/RichardBullingtonMcG/monitoring-and-alerting-as-code-with-terraform-and-new-relic).
+
+See the branch [demo-20191109](https://github.com/ModusCreateOrg/devops-infra-demo/tree/demo-20191l09) for the code for the demo for the [BSidesCT 2019 talk g Apple DevOps  talk _Extensible DevSecOps pipelines with Jenkins, Docker, Terraform, and a kitchen sink full of scanners_](https://bsidesct.org/schedule/). Slides from this presentation are on
+[Slideshare](https://www.slideshare.net/RichardBullingtonMcG/extensible-devsecops-pipelines-with-jenkins-docker-terrraform-and-a-kitchen-sink-full-of-scanners)
 
 Instructions
 ------------

ansible/roles/cloudwatch-agent/files/config.json

@@ -61,6 +61,11 @@
 						"file_path": "/var/log/nginx/error.log",
 						"log_group_name": "nginx-error",
 						"log_stream_name": "{instance_id}"
+					},
+					{
+						"file_path": "/opt/codedeploy-agent/deployment-root/deployment-logs/codedeploy-agent-deployments.log",
+						"log_group_name": "codedeploy",
+						"log_stream_name": "{instance_id}"
 					}
 				]
 			}

ansible/roles/scan-openscap/defaults/main.yml

@@ -3,5 +3,6 @@
 build_dir: /app/build
 output_file_html: /app/build/scan-xccdf-results.html
 output_file_xml: /app/build/scan-xccdf-results.xml
-profile: C2S
+# Oh no! The old C2S profile is no longer available!
+profile: standard
 xccdf_file: /usr/share/xml/scap/ssg/content/ssg-centos7-xccdf.xml

ansible/roles/scan-openscap/tasks/main.yml

@@ -18,7 +18,7 @@
     cd {{ build_dir }}
     # This will have a non-zero exit if any of the scans fail, so do not fail immediately on that
     set +e
-    oscap xccdf eval --fetch-remote-resources --profile {{ profile }} --results {{ output_file_xml }} {{ xccdf_file }}
+    oscap xccdf eval --profile {{ profile }} --results {{ output_file_xml }} {{ xccdf_file }}
     set -e
     oscap xccdf generate report {{ output_file_xml }} > {{ output_file_html }}
   args:

bin/activate-rvm.sh

@@ -2,11 +2,16 @@
 # Activate rvm
 # Source this to activate RVM
 
+# CodeDeploy has no HOME variable defined!
+HOME=${HOME:-/centos}
 RVM_SH=${RVM_SH:-$HOME/.rvm/scripts/rvm}
 RUBY_VERSION=${RUBY_VERSION:-2.6.3}
 
 # rvm hates the bash options -eu
 
+echo -n "Activating RVM. HOME=$HOME id:"
+id -a
+
 if [[ ! -f "$RVM_SH" ]]; then
     echo "Error: $0: RVM_SH $RVM_SH not found"
     exit 1

bin/build-codedeploy.sh

@@ -16,20 +16,34 @@ BUILD_DIR="$BASE_DIR/build"
 ANSIBLE_DIR="$BASE_DIR/ansible"
 APPLICTION_DIR="$BASE_DIR/application"
 SRC_DIR="$BASE_DIR/src"
+GAUNTLT_DIR="$BASE_DIR/gauntlt"
+
+# Credit to http://stackoverflow.com/a/246128/424301
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BASE_DIR="$DIR/.."
+
+# shellcheck disable=SC1090
+. "$DIR/common.sh"
+#shellcheck disable=SC1090
+. "$BASE_DIR/env.sh"
+
 
 GIT_REV="$(git rev-parse --short HEAD)"
 BUILD_NUMBER=${BUILD_NUMBER:-0}
-ARCHIVE="codedeploy-$BUILD_NUMBER-$GIT_REV.zip"
+BRANCH_PREFIX=${1:-master}
+ARCHIVE="codedeploy-$BRANCH_PREFIX-$BUILD_NUMBER-$GIT_REV.zip"
 CONTAINERNAME=infra-demo
+# Thanks https://stackoverflow.com/questions/33791069/quick-way-to-get-aws-account-number-from-the-cli-tools
+AWS_ACCOUNT_ID=$(get_aws_account_id)
+BUCKET="codedeploy-$AWS_ACCOUNT_ID"
+S3_URL="s3://$BUCKET/$ARCHIVE"
 
 echo "GIT_REV=$GIT_REV"
+echo "BRANCH_PREFIX=$BRANCH_PREFIX"
 echo "BUILD_NUMBER=$BUILD_NUMBER"
 echo "ARCHIVE=$ARCHIVE"
+echo "S3_URL=$S3_URL"
 
-# Thanks https://stackoverflow.com/questions/33791069/quick-way-to-get-aws-account-number-from-the-cli-tools
-AWS_ACCOUNT_ID=$(aws sts get-caller-identity --output text --query 'Account')
-BUCKET="codedeploy-$AWS_ACCOUNT_ID"
-S3_URL="s3://$BUCKET/$ARCHIVE"
 
 if [[ -d "$BUILD_DIR" ]]; then
     rm -rf "$BUILD_DIR"
@@ -52,8 +66,11 @@ SOURCES="$BASE_DIR/bin
 $ANSIBLE_DIR
 $APPLICTION_DIR
 $SRC_DIR
+$GAUNTLT_DIR
 $BASE_DIR/codedeploy/appspec.yml"
+echo "Copying sources into place"
 for src in $SOURCES; do
+    echo cp -a "$src" "$BUILD_DIR"
     cp -a "$src" "$BUILD_DIR"
 done
 
@@ -64,6 +81,7 @@ done
         bin \
         ansible \
         application \
+        gauntlt \
         src \
         venv \
         socket

bin/codedeploy/AfterInstall.sh

@@ -41,3 +41,6 @@ source ${VENV_DIR}/bin/activate
 set -u
 newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig"
 sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini"
+
+echo /app directory:
+ls -laZ /app

bin/codedeploy/ValidateService.sh

@@ -14,6 +14,10 @@ ${DEBUG:-false} && set -vx
 # and http://wiki.bash-hackers.org/scripting/debuggingtips
 export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
 
+GAUNTLT_RESULTS=/app/build/gauntlt-results.html
+# TODO: save this to S3 instead
+GAUNTLT_RESULTS_SAVE="/home/centos/$DEPLOYMENT_ID-gauntlt-results.html"
+
 check_every() {
     local delay=${1:-}
     local host="http://localhost/"
@@ -25,4 +29,19 @@ check_every() {
     done
 }
 
+echo "Checking web server availability"
 check_every 2
+
+echo "Scanning with openscap and gauntlt"
+mkdir -p /app/build /app/ansible/tmp
+cat < /dev/null > "$GAUNTLT_RESULTS"
+chown -R centos:centos "$GAUNTLT_RESULTS" /app/build /app/ansible/tmp
+chmod 755 "$GAUNTLT_RESULTS" /app/build /app/ansible/tmp
+
+set +e
+sudo -u centos HOME=/home/centos /app/bin/ansible.sh scan-openscap.yml scan-gauntlt.yml
+RETCODE=$?
+set -e
+cp "$GAUNTLT_RESULTS" "$GAUNTLT_RESULTS_SAVE"
+rm -rf /app/ansible/tmp /app/build
+exit "$RETCODE"