Demo 20190130

.gitignore

@@ -12,4 +12,6 @@
 __pycache__
 build/
 jmeter.log
+scan-xccdf-results.html
+scan-xccdf-results.xml
 venv/

Jenkinsfile

@@ -26,17 +26,22 @@ def get_captcha(Long hash_const) {
 }
 
 def wrap = { fn->
-        ansiColor('xterm') {
-            withCredentials([file(credentialsId: 'terraform-demo.json',
-                variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE')]) {
-                sh ("""
-                    cp env.sh.sample env.sh
-                    rm -rf build
-                    mkdir build
-                """)
+    ansiColor('xterm') {
+        withCredentials(
+            [
+                file(credentialsId: 'terraform-demo.json',
+                     variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'),
+                string(credentialsId: 'newrelic.license.key',
+                       variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'),
+                string(credentialsId: 'newrelic.api.key',
+                       variable: 'NEWRELIC_API_KEY_OVERRIDE'),
+                string(credentialsId: 'newrelic.alert.email',
+                       variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'),
+            ]) {
+                sh ("bin/clean-workspace.sh")
                 fn()
             }
-        }
+    }
 }
 
 final Long XOR_CONST = 3735928559 // 0xdeadbeef
@@ -87,15 +92,12 @@ properties([
             description: "Execute a JMeter load test against the stack"
         ),
         string(
-            name: 'JMETER_threads',
+            name: 'JMETER_num_threads',
             defaultValue: '2',
-            description: """number of jmeter threads. Resulting ASG stable sizes for t2.large instances are:
-            - 2 threads, 3 instances;
-            - 4 threads, 7 instances;
-            """
+            description: "number of jmeter threads."
         ),
         string(
-            name: 'JMETER_ramp_duration',
+            name: 'JMETER_ramp_time',
             defaultValue: '900',
             description: 'period in seconds of ramp-up time.'
         ),
@@ -182,7 +184,7 @@ stage('Build CodeDeploy Archive') {
     node {
         unstash 'src'
         wrap.call({
-            sh ("./codedeploy/bin/build.sh")
+            sh ("./bin/build-codedeploy.sh")
         })
     }
 }
@@ -246,7 +248,7 @@ if (params.Run_JMeter) {
             wrap.call({
                 sh ("""
                     HOST=\$(./bin/terraform.sh output route53-dns)
-                    ./bin/jmeter.sh -Jthreads=${params.JMETER_threads} -Jramp_duration=${params.JMETER_ramp_duration} -Jduration=${params.JMETER_duration} -Jhost=\$HOST
+                    ./bin/jmeter.sh -Jnum_threads=${params.JMETER_num_threads} -Jramp_time=${params.JMETER_ramp_time} -Jduration=${params.JMETER_duration} -Jhost=\$HOST
                     ls -l build
                     """)
                 archiveArtifacts artifacts: 'build/*.jtl, build/*.xml, build/*.csv, build/*.html', fingerprint: true

README.md

@@ -23,6 +23,7 @@ Instructions
 * [Docker](https://docker.com/) (tested with 18.05.0-ce)
 * [Packer](https://www.packer.io/) (tested with 1.0.3)
 * [Terraform](https://www.terraform.io/) (tested with  v0.11.7)
+* [JQ](https://stedolan.github.io/jq/) (tested with 1.3 and 1.5)
 
 Optionally, you can use Vagrant to test ansible playbooks locally and Jenkins to orchestrate creation of AMIs in conjunction with GitHub branches and pull requests.
 
@@ -116,6 +117,8 @@ A JMeter test harness that will allow testing of a the application
 
 A `Jenkinsfile` is provided that will allow Jenkins to execute Packer and Terraform, package a CodeDeploy application, and even run JMeter performance tests. In order for Jenkins to do this, it needs to have AWS credentials set up, preferably through an IAM role, granting full control of EC2 and VPC resources in that account, and write access to the S3 bucket used for storing CodeDeploy applications. Packer needs this in order to create AMIs, key pairs, etc, Terraform needs this to create a VPC and EC2 resources, and CodeDeploy needs this to store the artifact it creates. This could be pared down further through some careful logging and role work.
 
+The Jenkins executor running this job needs to have both a recent Docker and the jq utility (version 1.3 or higher) installed.
+
 The scripts here assume that Jenkins is running on EC2 and uses instance data from the Jenkins executor to infer what VPC and subnet to launch the new EC2 instance into.  The AWS profile IAM user associated with your Jenkins instance or the Jenkins user's AWS credentials should have full control of EC2 in the account you are using.
 
 This script relies on Jenkins having a secret file containing the Google application credentials in JSON with the id "terraform-demo.json". You will need to add that to your Jenkins server's credentials.

ansible/bakery.yml

@@ -3,6 +3,20 @@
 
 # Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for
 # the trick on installing locally using "hosts: 127.0.0.1" and "connection:local"
+- name: Install New Relic Infrastructure
+  hosts: 127.0.0.1
+  connection: local
+  become: yes
+  roles: 
+    - newrelic.newrelic-infra
+  vars:
+    nrinfragent_os_name: CentOS
+    nrinfragent_os_version: 7
+    nrinfragent_config:
+      license_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ
+      log_file: /var/log/newrelic-infra/nr-infra.log
+      log_to_stdout: false
+
 - name: Install Web Application
   hosts: 127.0.0.1
   connection: local
@@ -12,6 +26,7 @@
     - prepare-web-content
     - prepare-codedeploy
 
+
 - name: Harden Server
   hosts: 127.0.0.1
   connection: local

ansible/newrelic-infrastructure.yml

@@ -0,0 +1,21 @@
+---
+# Use ansible to install the newrelic infra agent.
+
+# Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for
+# the trick on installing locally using "hosts: 127.0.0.1" and "connection:local"
+
+- name: Set the newrelic license key
+  hosts: 127.0.0.1
+  connection: local
+  become: yes
+  tasks:
+    - command: bash /app/bin/set-newrelic-license-key.sh
+
+- name: Restart newrelic-infra
+  hosts: 127.0.0.1
+  connection: local
+  become: yes
+  service:
+    name: newerelic-infra
+    state: restarted
+

ansible/requirements.yml

@@ -15,3 +15,6 @@
 # And the ModusCreateOrg fork has been fixed to avoid running stuff from /tmp
 - src: https://github.com/ModusCreateOrg/ansible-aws-codedeploy-agent
 
+# New Relic Infrastructure
+- src: newrelic.newrelic-infra
+

ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2

@@ -3,12 +3,14 @@ venv = /app/venv
 wsgi-file = /app/src/wsgi.py
 chdir = /app/src
 master = 1
-workers = 2
-threads = 8
+workers = 64
+threads = 1
 lazy-apps = 1
 wsgi-env-behaviour = holy
-enable-threads = 1
+enable-threads = 0
 http-auto-chunked = 1
 http-keepalive = 1
 uwsgi-socket = 127.0.0.1:8008
-
+harakiri = 120
+harakiri-verbose
+max-requests = 2048

codedeploy/bin/build.sh → bin/build-codedeploy.sh

@@ -13,11 +13,9 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 BASE_DIR="$DIR/.."
 BUILD_DIR="$BASE_DIR/build"
-ANSIBLE_DIR="$BASE_DIR/../ansible"
-APPLICTION_DIR="$BASE_DIR/../application"
-SRC_DIR="$BASE_DIR/../src"
-VENV_DIR="$BASE_DIR/../venv"
-DOCKER_DIR="$BASE_DIR/.."
+ANSIBLE_DIR="$BASE_DIR/ansible"
+APPLICTION_DIR="$BASE_DIR/application"
+SRC_DIR="$BASE_DIR/src"
 
 GIT_REV="$(git rev-parse --short HEAD)"
 BUILD_NUMBER=${BUILD_NUMBER:-0}
@@ -39,20 +37,22 @@ fi
 mkdir -p "$BUILD_DIR/socket"
 
 echo Build docker container $CONTAINERNAME
-docker build -f=Dockerfile -t "$CONTAINERNAME" "$DOCKER_DIR"
+docker build -f=Dockerfile -t "$CONTAINERNAME" "$BASE_DIR"
 
 echo Create python virtual environment
-docker run --rm -v "$DOCKER_DIR:/src" "$CONTAINERNAME" /bin/bash -c \
-    "mkdir -p /src/venv ; \
-    cp -fa /app/venv/* /src/venv"
+docker run \
+    --rm \
+    -v "$BASE_DIR:/src" \
+    "$CONTAINERNAME" \
+    /bin/bash -c \
+        "mkdir -p /src/build/venv ; \
+        cp -fa /app/venv/* /src/build/venv"
 
 SOURCES="$BASE_DIR/bin
 $ANSIBLE_DIR
 $APPLICTION_DIR
 $SRC_DIR
-$BASE_DIR/appspec.yml
-$BASE_DIR/bin
-$VENV_DIR"
+$BASE_DIR/codedeploy/appspec.yml"
 for src in $SOURCES; do
     cp -a "$src" "$BUILD_DIR"
 done
@@ -70,7 +70,7 @@ done
 )
 
 echo Remove docker generated files
-docker run --rm -v "$DOCKER_DIR:/src" "$CONTAINERNAME" /bin/bash -c \
+docker run --rm -v "$BASE_DIR:/src" "$CONTAINERNAME" /bin/bash -c \
     "rm -rf /src/venv"
 
 cd "$BUILD_DIR"

codedeploy/bin/AfterInstall.sh → bin/clean-workspace.sh

@@ -1,12 +1,8 @@
 #!/usr/bin/env bash
+# clean workspace
 #
-# AfterInstall.sh
-#
-# AWS CodeDeploy After Install hook script
-
 # Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/
 set -euo pipefail
-IFS=$'\n\t'
 
 # Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true"
 ${DEBUG:-false} && set -vx
@@ -17,7 +13,13 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
 # Credit to http://stackoverflow.com/a/246128/424301
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 BASE_DIR="$DIR/.."
-ANSIBLE_DIR="$BASE_DIR/ansible"
+BUILD_DIR="$BASE_DIR/build"
+export BASE_DIR
+
+# shellcheck disable=SC1090
+. "$DIR/common.sh"
 
-# Invoke Ansible for final set up
-ansible-playbook -l localhost "$ANSIBLE_DIR/app-AfterInstall.yml"
+cp "$BASE_DIR/env.sh.sample" "$BASE_DIR/env.sh"
+clean_root_owned_docker_files
+rm -rf "$BUILD_DIR"
+mkdir "$BUILD_DIR"

bin/codedeploy/AfterInstall.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+# AfterInstall.sh
+#
+# AWS CodeDeploy After Install hook script
+
+# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
+IFS=$'\n\t'
+
+# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true"
+${DEBUG:-false} && set -vx
+# Credit to https://stackoverflow.com/a/17805088
+# and http://wiki.bash-hackers.org/scripting/debuggingtips
+export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
+
+# Credit to http://stackoverflow.com/a/246128/424301
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BASE_DIR="$DIR/../.."
+ANSIBLE_DIR="$BASE_DIR/ansible"
+
+# Invoke Ansible for final set up
+ansible-playbook -l localhost "$ANSIBLE_DIR/app-AfterInstall.yml"
+
+# Configure New Relic
+# TODO: move into Ansible playbook app-AfterInstall.yml
+NEWRELIC_CONFIG_DIR=/app
+VENV_DIR=/app/venv
+# Thanks Stack Overflow https://stackoverflow.com/a/9735663/424301
+EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
+EC2_REGION="$(sed 's/[a-z]$//' <<<"$EC2_AVAIL_ZONE")"
+
+NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \
+    --region="$EC2_REGION" \
+    --secret-id newrelic_license \
+    --output text \
+    --query '[SecretString]')
+set +u
+#shellcheck disable=SC1090
+source ${VENV_DIR}/bin/activate
+set -u
+newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig"
+sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini"

codedeploy/bin/ApplicationStart.sh → bin/codedeploy/ApplicationStart.sh

@@ -16,7 +16,7 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
 
 # Credit to http://stackoverflow.com/a/246128/424301
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-BASE_DIR="$DIR/.."
+BASE_DIR="$DIR/../.."
 ANSIBLE_DIR="$BASE_DIR/ansible"
 
 # Invoke Ansible for final set up

bin/set-newrelic-license-key.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Set the newrelic license key fron AWS credentials
+#
+# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
+
+# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true"
+${DEBUG:-false} && set -vx
+# Credit to https://stackoverflow.com/a/17805088
+# and http://wiki.bash-hackers.org/scripting/debuggingtips
+export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
+
+# Credit to http://stackoverflow.com/a/246128/424301
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BASE_DIR="$DIR/.."
+export BASE_DIR
+
+NEWRELIC_CONFIG_FILE="/etc/newrelic-infra.yml"
+# Thanks Stack Overflow https://stackoverflow.com/a/9735663/424301
+EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
+EC2_REGION="$(sed 's/[a-z]$//' <<<"$EC2_AVAIL_ZONE")"
+
+NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \
+    --region="$EC2_REGION" \
+    --secret-id newrelic_license \
+    --output text \
+    --query '[SecretString]')
+
+cp -a "${NEWRELIC_CONFIG_FILE}" "${NEWRELIC_CONFIG_FILE}.orig"
+sed "s/ZZZZ*ZZZZ/${NEWRELIC_LICENSE_KEY}/" "${NEWRELIC_CONFIG_FILE}.orig" > "${NEWRELIC_CONFIG_FILE}"