diff --git a/.gitignore b/.gitignore index ad4f921..8409a05 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ __pycache__ build/ jmeter.log +scan-xccdf-results.html +scan-xccdf-results.xml venv/ diff --git a/Jenkinsfile b/Jenkinsfile index f1533ad..c0f6a61 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,17 +26,22 @@ def get_captcha(Long hash_const) { } def wrap = { fn-> - ansiColor('xterm') { - withCredentials([file(credentialsId: 'terraform-demo.json', - variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE')]) { - sh (""" - cp env.sh.sample env.sh - rm -rf build - mkdir build - """) + ansiColor('xterm') { + withCredentials( + [ + file(credentialsId: 'terraform-demo.json', + variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'), + string(credentialsId: 'newrelic.license.key', + variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.api.key', + variable: 'NEWRELIC_API_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.alert.email', + variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'), + ]) { + sh ("bin/clean-workspace.sh") fn() } - } + } } final Long XOR_CONST = 3735928559 // 0xdeadbeef @@ -87,15 +92,12 @@ properties([ description: "Execute a JMeter load test against the stack" ), string( - name: 'JMETER_threads', + name: 'JMETER_num_threads', defaultValue: '2', - description: """number of jmeter threads. Resulting ASG stable sizes for t2.large instances are: - - 2 threads, 3 instances; - - 4 threads, 7 instances; - """ + description: "number of jmeter threads." ), string( - name: 'JMETER_ramp_duration', + name: 'JMETER_ramp_time', defaultValue: '900', description: 'period in seconds of ramp-up time.' ), @@ -182,7 +184,7 @@ stage('Build CodeDeploy Archive') { node { unstash 'src' wrap.call({ - sh ("./codedeploy/bin/build.sh") + sh ("./bin/build-codedeploy.sh") }) } } @@ -246,7 +248,7 @@ if (params.Run_JMeter) { wrap.call({ sh (""" HOST=\$(./bin/terraform.sh output route53-dns) - ./bin/jmeter.sh -Jthreads=${params.JMETER_threads} -Jramp_duration=${params.JMETER_ramp_duration} -Jduration=${params.JMETER_duration} -Jhost=\$HOST + ./bin/jmeter.sh -Jnum_threads=${params.JMETER_num_threads} -Jramp_time=${params.JMETER_ramp_time} -Jduration=${params.JMETER_duration} -Jhost=\$HOST ls -l build """) archiveArtifacts artifacts: 'build/*.jtl, build/*.xml, build/*.csv, build/*.html', fingerprint: true diff --git a/README.md b/README.md index ec6515f..30b6912 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Instructions * [Docker](https://docker.com/) (tested with 18.05.0-ce) * [Packer](https://www.packer.io/) (tested with 1.0.3) * [Terraform](https://www.terraform.io/) (tested with v0.11.7) +* [JQ](https://stedolan.github.io/jq/) (tested with 1.3 and 1.5) Optionally, you can use Vagrant to test ansible playbooks locally and Jenkins to orchestrate creation of AMIs in conjunction with GitHub branches and pull requests. @@ -116,6 +117,8 @@ A JMeter test harness that will allow testing of a the application A `Jenkinsfile` is provided that will allow Jenkins to execute Packer and Terraform, package a CodeDeploy application, and even run JMeter performance tests. In order for Jenkins to do this, it needs to have AWS credentials set up, preferably through an IAM role, granting full control of EC2 and VPC resources in that account, and write access to the S3 bucket used for storing CodeDeploy applications. Packer needs this in order to create AMIs, key pairs, etc, Terraform needs this to create a VPC and EC2 resources, and CodeDeploy needs this to store the artifact it creates. This could be pared down further through some careful logging and role work. +The Jenkins executor running this job needs to have both a recent Docker and the jq utility (version 1.3 or higher) installed. + The scripts here assume that Jenkins is running on EC2 and uses instance data from the Jenkins executor to infer what VPC and subnet to launch the new EC2 instance into. The AWS profile IAM user associated with your Jenkins instance or the Jenkins user's AWS credentials should have full control of EC2 in the account you are using. This script relies on Jenkins having a secret file containing the Google application credentials in JSON with the id "terraform-demo.json". You will need to add that to your Jenkins server's credentials. diff --git a/ansible/bakery.yml b/ansible/bakery.yml index 93f8b58..59827fe 100644 --- a/ansible/bakery.yml +++ b/ansible/bakery.yml @@ -3,6 +3,20 @@ # Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for # the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Install New Relic Infrastructure + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - newrelic.newrelic-infra + vars: + nrinfragent_os_name: CentOS + nrinfragent_os_version: 7 + nrinfragent_config: + license_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ + log_file: /var/log/newrelic-infra/nr-infra.log + log_to_stdout: false + - name: Install Web Application hosts: 127.0.0.1 connection: local @@ -12,6 +26,7 @@ - prepare-web-content - prepare-codedeploy + - name: Harden Server hosts: 127.0.0.1 connection: local diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml new file mode 100644 index 0000000..78e7568 --- /dev/null +++ b/ansible/newrelic-infrastructure.yml @@ -0,0 +1,21 @@ +--- +# Use ansible to install the newrelic infra agent. + +# Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for +# the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" + +- name: Set the newrelic license key + hosts: 127.0.0.1 + connection: local + become: yes + tasks: + - command: bash /app/bin/set-newrelic-license-key.sh + +- name: Restart newrelic-infra + hosts: 127.0.0.1 + connection: local + become: yes + service: + name: newerelic-infra + state: restarted + diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 05be059..5978c53 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -15,3 +15,6 @@ # And the ModusCreateOrg fork has been fixed to avoid running stuff from /tmp - src: https://github.com/ModusCreateOrg/ansible-aws-codedeploy-agent +# New Relic Infrastructure +- src: newrelic.newrelic-infra + diff --git a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 index e3e76e1..0afc0b8 100644 --- a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 +++ b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 @@ -3,12 +3,14 @@ venv = /app/venv wsgi-file = /app/src/wsgi.py chdir = /app/src master = 1 -workers = 2 -threads = 8 +workers = 64 +threads = 1 lazy-apps = 1 wsgi-env-behaviour = holy -enable-threads = 1 +enable-threads = 0 http-auto-chunked = 1 http-keepalive = 1 uwsgi-socket = 127.0.0.1:8008 - +harakiri = 120 +harakiri-verbose +max-requests = 2048 diff --git a/codedeploy/bin/build.sh b/bin/build-codedeploy.sh similarity index 78% rename from codedeploy/bin/build.sh rename to bin/build-codedeploy.sh index e61b0de..126fe6c 100755 --- a/codedeploy/bin/build.sh +++ b/bin/build-codedeploy.sh @@ -13,11 +13,9 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" BASE_DIR="$DIR/.." BUILD_DIR="$BASE_DIR/build" -ANSIBLE_DIR="$BASE_DIR/../ansible" -APPLICTION_DIR="$BASE_DIR/../application" -SRC_DIR="$BASE_DIR/../src" -VENV_DIR="$BASE_DIR/../venv" -DOCKER_DIR="$BASE_DIR/.." +ANSIBLE_DIR="$BASE_DIR/ansible" +APPLICTION_DIR="$BASE_DIR/application" +SRC_DIR="$BASE_DIR/src" GIT_REV="$(git rev-parse --short HEAD)" BUILD_NUMBER=${BUILD_NUMBER:-0} @@ -39,20 +37,22 @@ fi mkdir -p "$BUILD_DIR/socket" echo Build docker container $CONTAINERNAME -docker build -f=Dockerfile -t "$CONTAINERNAME" "$DOCKER_DIR" +docker build -f=Dockerfile -t "$CONTAINERNAME" "$BASE_DIR" echo Create python virtual environment -docker run --rm -v "$DOCKER_DIR:/src" "$CONTAINERNAME" /bin/bash -c \ - "mkdir -p /src/venv ; \ - cp -fa /app/venv/* /src/venv" +docker run \ + --rm \ + -v "$BASE_DIR:/src" \ + "$CONTAINERNAME" \ + /bin/bash -c \ + "mkdir -p /src/build/venv ; \ + cp -fa /app/venv/* /src/build/venv" SOURCES="$BASE_DIR/bin $ANSIBLE_DIR $APPLICTION_DIR $SRC_DIR -$BASE_DIR/appspec.yml -$BASE_DIR/bin -$VENV_DIR" +$BASE_DIR/codedeploy/appspec.yml" for src in $SOURCES; do cp -a "$src" "$BUILD_DIR" done @@ -70,7 +70,7 @@ done ) echo Remove docker generated files -docker run --rm -v "$DOCKER_DIR:/src" "$CONTAINERNAME" /bin/bash -c \ +docker run --rm -v "$BASE_DIR:/src" "$CONTAINERNAME" /bin/bash -c \ "rm -rf /src/venv" cd "$BUILD_DIR" diff --git a/codedeploy/bin/AfterInstall.sh b/bin/clean-workspace.sh similarity index 70% rename from codedeploy/bin/AfterInstall.sh rename to bin/clean-workspace.sh index 91f49d4..a227196 100755 --- a/codedeploy/bin/AfterInstall.sh +++ b/bin/clean-workspace.sh @@ -1,12 +1,8 @@ #!/usr/bin/env bash +# clean workspace # -# AfterInstall.sh -# -# AWS CodeDeploy After Install hook script - # Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ set -euo pipefail -IFS=$'\n\t' # Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" ${DEBUG:-false} && set -vx @@ -17,7 +13,13 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' # Credit to http://stackoverflow.com/a/246128/424301 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" BASE_DIR="$DIR/.." -ANSIBLE_DIR="$BASE_DIR/ansible" +BUILD_DIR="$BASE_DIR/build" +export BASE_DIR + +# shellcheck disable=SC1090 +. "$DIR/common.sh" -# Invoke Ansible for final set up -ansible-playbook -l localhost "$ANSIBLE_DIR/app-AfterInstall.yml" +cp "$BASE_DIR/env.sh.sample" "$BASE_DIR/env.sh" +clean_root_owned_docker_files +rm -rf "$BUILD_DIR" +mkdir "$BUILD_DIR" diff --git a/bin/codedeploy/AfterInstall.sh b/bin/codedeploy/AfterInstall.sh new file mode 100755 index 0000000..2fcbca4 --- /dev/null +++ b/bin/codedeploy/AfterInstall.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# AfterInstall.sh +# +# AWS CodeDeploy After Install hook script + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/../.." +ANSIBLE_DIR="$BASE_DIR/ansible" + +# Invoke Ansible for final set up +ansible-playbook -l localhost "$ANSIBLE_DIR/app-AfterInstall.yml" + +# Configure New Relic +# TODO: move into Ansible playbook app-AfterInstall.yml +NEWRELIC_CONFIG_DIR=/app +VENV_DIR=/app/venv +# Thanks Stack Overflow https://stackoverflow.com/a/9735663/424301 +EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) +EC2_REGION="$(sed 's/[a-z]$//' <<<"$EC2_AVAIL_ZONE")" + +NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \ + --region="$EC2_REGION" \ + --secret-id newrelic_license \ + --output text \ + --query '[SecretString]') +set +u +#shellcheck disable=SC1090 +source ${VENV_DIR}/bin/activate +set -u +newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" +sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini" diff --git a/codedeploy/bin/ApplicationStart.sh b/bin/codedeploy/ApplicationStart.sh similarity index 97% rename from codedeploy/bin/ApplicationStart.sh rename to bin/codedeploy/ApplicationStart.sh index 5d08139..c2698ca 100755 --- a/codedeploy/bin/ApplicationStart.sh +++ b/bin/codedeploy/ApplicationStart.sh @@ -16,7 +16,7 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' # Credit to http://stackoverflow.com/a/246128/424301 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -BASE_DIR="$DIR/.." +BASE_DIR="$DIR/../.." ANSIBLE_DIR="$BASE_DIR/ansible" # Invoke Ansible for final set up diff --git a/codedeploy/bin/ApplicationStop.sh b/bin/codedeploy/ApplicationStop.sh similarity index 100% rename from codedeploy/bin/ApplicationStop.sh rename to bin/codedeploy/ApplicationStop.sh diff --git a/codedeploy/bin/BeforeInstall.sh b/bin/codedeploy/BeforeInstall.sh similarity index 100% rename from codedeploy/bin/BeforeInstall.sh rename to bin/codedeploy/BeforeInstall.sh diff --git a/codedeploy/bin/ValidateService.sh b/bin/codedeploy/ValidateService.sh similarity index 100% rename from codedeploy/bin/ValidateService.sh rename to bin/codedeploy/ValidateService.sh diff --git a/bin/set-newrelic-license-key.sh b/bin/set-newrelic-license-key.sh new file mode 100755 index 0000000..6d60f5b --- /dev/null +++ b/bin/set-newrelic-license-key.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Set the newrelic license key fron AWS credentials +# +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." +export BASE_DIR + +NEWRELIC_CONFIG_FILE="/etc/newrelic-infra.yml" +# Thanks Stack Overflow https://stackoverflow.com/a/9735663/424301 +EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) +EC2_REGION="$(sed 's/[a-z]$//' <<<"$EC2_AVAIL_ZONE")" + +NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \ + --region="$EC2_REGION" \ + --secret-id newrelic_license \ + --output text \ + --query '[SecretString]') + +cp -a "${NEWRELIC_CONFIG_FILE}" "${NEWRELIC_CONFIG_FILE}.orig" +sed "s/ZZZZ*ZZZZ/${NEWRELIC_LICENSE_KEY}/" "${NEWRELIC_CONFIG_FILE}.orig" > "${NEWRELIC_CONFIG_FILE}" diff --git a/bin/terraform.sh b/bin/terraform.sh index 81d00bd..fcd9af9 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -39,6 +39,21 @@ if [[ -n "$GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE" ]]; then echo "Overriding Google Application Credentials" 1>&2 GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE" fi +NEWRELIC_LICENSE_KEY_OVERRIDE=${NEWRELIC_LICENSE_KEY_OVERRIDE:-} +if [[ -n "$NEWRELIC_LICENSE_KEY_OVERRIDE" ]]; then + echo "Overriding New Relic License Key" 1>&2 + NEWRELIC_LICENSE_KEY="$NEWRELIC_LICENSE_KEY_OVERRIDE" +fi +NEWRELIC_API_KEY_OVERRIDE=${NEWRELIC_API_KEY_OVERRIDE:-} +if [[ -n "$NEWRELIC_API_KEY_OVERRIDE" ]]; then + echo "Overriding New Relic API Key" 1>&2 + NEWRELIC_API_KEY="$NEWRELIC_API_KEY_OVERRIDE" +fi +NEWRELIC_ALERT_EMAIL_OVERRIDE=${NEWRELIC_ALERT_EMAIL_OVERRIDE:-} +if [[ -n "$NEWRELIC_ALERT_EMAIL_OVERRIDE" ]]; then + echo "Overriding New Relic Alert Email" 1>&2 + NEWRELIC_ALERT_EMAIL="$NEWRELIC_ALERT_EMAIL_OVERRIDE" +fi # Set up Google creds in build dir for docker terraform mkdir -p "$BUILD_DIR" @@ -55,6 +70,9 @@ GOOGLE_PROJECT_OVERRIDE=$(awk 'BEGIN { FS = "\"" } /project_id/{print $4}' <$GOO cat <>"$ENV_FILE" GOOGLE_APPLICATION_CREDENTIALS=/app/build/google.json GOOGLE_PROJECT=$GOOGLE_PROJECT_OVERRIDE +NEWRELIC_LICENSE_KEY=$NEWRELIC_LICENSE_KEY +NEWRELIC_API_KEY=$NEWRELIC_API_KEY +NEWRELIC_ALERT_EMAIL=$NEWRELIC_ALERT_EMAIL EOF # http://redsymbol.net/articles/bash-exit-traps/ @@ -75,14 +93,24 @@ function output () { function plan() { local extra + local nr_app_id local output local -i retcode local targets - extra=${1:-} + extra="$*" output="$(mktemp)" targets=$(get_targets) set +e + + nr_app_id=$(curl \ + -s \ + -X GET \ + 'https://api.newrelic.com/v2/applications.json' \ + -H "X-Api-Key:${NEWRELIC_API_KEY}" \ + -d "filter[name]=${NEWRELIC_APP_NAME}" \ + | jq '.applications[].id') + #shellcheck disable=SC2086 $DOCKER_TERRAFORM plan \ $extra \ @@ -90,6 +118,10 @@ function plan() { -lock=true \ -input="$INPUT_ENABLED" \ -var project_name="$PROJECT_NAME" \ + -var newrelic_license_key="$NEWRELIC_LICENSE_KEY" \ + -var newrelic_api_key="$NEWRELIC_API_KEY" \ + -var newrelic_alert_email="$NEWRELIC_ALERT_EMAIL" \ + -var newrelic_apm_entities="[$nr_app_id]" \ -var-file="/app/build/extra.tfvars" \ -out="$TF_PLAN" \ "$TF_DIR" \ diff --git a/bin/validate.sh b/bin/validate.sh index b26b761..3ce3976 100755 --- a/bin/validate.sh +++ b/bin/validate.sh @@ -30,7 +30,10 @@ $DOCKER_PACKER validate app/packer/machines/web-server.json echo "Linting terraform files for correctness" DOCKER_TERRAFORM=$(get_docker_terraform) init_terraform -$DOCKER_TERRAFORM validate +$DOCKER_TERRAFORM validate \ + -var 'newrelic_license_key=ZZZZ' \ + -var 'newrelic_api_key=ZZZZ' \ + -var 'newrelic_alert_email=ferd.berferd@example.com' \ echo "Linting terraform files for formatting" fmt=$($DOCKER_TERRAFORM fmt) if [[ -n "$fmt" ]]; then diff --git a/codedeploy/appspec.yml b/codedeploy/appspec.yml index e0f94aa..6c7275f 100644 --- a/codedeploy/appspec.yml +++ b/codedeploy/appspec.yml @@ -6,19 +6,23 @@ files: destination: /app/application - source: src destination: /app/src + - source: bin + destination: /app/bin + - source: ansible + destination: /app/ansible hooks: ApplicationStop: - - location: bin/ApplicationStop.sh + - location: bin/codedeploy/ApplicationStop.sh timeout: 600 BeforeInstall: - - location: bin/BeforeInstall.sh + - location: bin/codedeploy/BeforeInstall.sh timeout: 30 AfterInstall: - - location: bin/AfterInstall.sh + - location: bin/codedeploy/AfterInstall.sh timeout: 300 ApplicationStart: - - location: bin/ApplicationStart.sh + - location: bin/codedeploy/ApplicationStart.sh timeout: 120 ValidateService: - - location: bin/ValidateService.sh + - location: bin/codedeploy/ValidateService.sh timeout: 60 diff --git a/env.sh.sample b/env.sh.sample index 4057992..4894803 100644 --- a/env.sh.sample +++ b/env.sh.sample @@ -31,3 +31,9 @@ fi export GOOGLE_CLOUD_KEYFILE_JSON #export GOOGLE_PROJECT=example-terraform-demo-999999 export GOOGLE_REGION=us-east1 + +# Fill in your New Relic license key and API key +export NEWRELIC_LICENSE_KEY=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ +export NEWRELIC_API_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +export NEWRELIC_ALERT_EMAIL=nobody@example.com +export NEWRELIC_APP_NAME=Spin diff --git a/src/requirements.txt b/src/requirements.txt index d7d87aa..7bbc8b5 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,2 +1,4 @@ bottle==0.12.13 uwsgi +newrelic + diff --git a/src/run.py b/src/run.py index 81ba46c..5dac1b1 100755 --- a/src/run.py +++ b/src/run.py @@ -5,4 +5,4 @@ from spin import spin from bottle import run, default_app - run(host='localhost', port=8080, debug=True) + run(host="localhost", port=8080, debug=True) diff --git a/src/spin.py b/src/spin.py index 4070b85..5aadbf2 100755 --- a/src/spin.py +++ b/src/spin.py @@ -2,27 +2,68 @@ """This module spins the CPU.""" import os import time -from bottle import route, default_app, response +import random +import socket +import urllib2 +import newrelic.agent +from bottle import route, default_app, response, HTTPError -@route('/spin') -def spin(delay=5.0): +NEWRELIC_INI = "../newrelic.ini" +if os.path.isfile(NEWRELIC_INI): + newrelic.agent.initialize(NEWRELIC_INI) + + +@route("/spin") +def spin(delay=0.05, max_duration=10.0, simulate_congestion=True): """Spin the CPU, return the process id at the end""" + spin.invocations += 1 child_pid = os.getpid() upper_max = 100000000000000000000000000000000 + + # Use a pareto distribution to give additional + # variation to the delay + # See https://en.wikipedia.org/wiki/Pareto_distribution + alpha = 2 + pareto_factor = random.paretovariate(alpha) start_time = time.time() + current_time = start_time scratch = 42 + int(current_time) - end_time = start_time + delay + congestion_slowdown = 0.0 + if simulate_congestion: + congestion_slowdown = delay * 2 / (current_time - spin.last_time) + end_time = start_time + (delay + congestion_slowdown) * pareto_factor + time_limit = start_time + (max_duration) calcs = 0 while current_time < end_time: calcs += 1 scratch = (scratch * scratch) % upper_max current_time = time.time() - final_time = time.time() - interval = final_time - start_time + interval = current_time - start_time + if current_time > time_limit: + raise HTTPError( + 500, + "Allowed transaction time exceeded ({} ms elapsed)".format(interval), + ) + spin.last_time = current_time rate = calcs / interval - response.set_header('Content-Type', 'text/plain') - return ('pid {0} spun {1} times over {2}s (rate {3}/s)\n' - .format(child_pid, calcs, interval, rate)) + response.set_header("Content-Type", "text/plain") + return "node {0} pid {1} spun {2} times over {3}s (rate {4} invoked {5} times. Congestion slowdown {6}s)\n".format( + spin.node, child_pid, calcs, interval, rate, spin.invocations, congestion_slowdown + ) + + +spin.invocations = 0 +spin.last_time = time.time() - 10 +spin.slowdown = 0 +try: + # Thanks stack overflow https://stackoverflow.com/a/43816449/424301 + spin.node = urllib2.urlopen( + "http://169.254.169.254/latest/meta-data/instance-id", timeout=1 + ).read() +# Thanks stack overflow https://stackoverflow.com/questions/2712524/handling-urllib2s-timeout-python +except urllib2.URLError: + # Thanks stack overflow: https://stackoverflow.com/a/4271755/424301 + spin.node = socket.gethostname() application = default_app() diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index 3872c61..984164f 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,3 +1,3 @@ #cloud-config runcmd: - - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml + - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml diff --git a/terraform/elb.tf b/terraform/elb.tf index 56874e2..d2b67bd 100644 --- a/terraform/elb.tf +++ b/terraform/elb.tf @@ -19,7 +19,7 @@ resource "aws_elb" "infra-demo-elb" { healthy_threshold = 2 unhealthy_threshold = 2 timeout = 3 - target = "HTTP:80/" + target = "HTTP:80/api/spin" interval = 30 } diff --git a/terraform/infra-demo-role-policy.json b/terraform/infra-demo-role-policy.json index 05a5fe5..8cd8b0a 100644 --- a/terraform/infra-demo-role-policy.json +++ b/terraform/infra-demo-role-policy.json @@ -64,6 +64,15 @@ "arn:aws:s3:::aws-codedeploy-us-west-2/*", "arn:aws:s3:::aws-codedeploy-us-east-1/*" ] + }, + { + "Effect": "Allow", + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Resource": [ + "arn:aws:secretsmanager:*:*:secret:newrelic_license*" + ] } ] } diff --git a/terraform/instances.tf b/terraform/instances.tf index 15d339f..8403b8e 100644 --- a/terraform/instances.tf +++ b/terraform/instances.tf @@ -93,7 +93,7 @@ resource "aws_autoscaling_group" "infra-demo-web-asg" { max_size = "${var.max_size}" launch_configuration = "${aws_launch_configuration.infra-demo-web-lc.name}" - health_check_type = "EC2" + health_check_type = "ELB" vpc_zone_identifier = [ "${module.vpc.public_subnets}", diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf new file mode 100644 index 0000000..518e193 --- /dev/null +++ b/terraform/newrelic.tf @@ -0,0 +1,110 @@ +# Configure the New Relic provider + +# Adapted from https://www.terraform.io/docs/providers/newrelic/index.html + +provider "newrelic" { + api_key = "${var.newrelic_api_key}" + version = "~> 1.2" +} + +# Create an alert policy +resource "newrelic_alert_policy" "alert" { + name = "Alert" +} + +# Add a condition +resource "newrelic_alert_condition" "spin-appdex" { + policy_id = "${newrelic_alert_policy.alert.id}" + + name = "spin-appdex" + type = "apm_app_metric" + entities = "${var.newrelic_apm_entities}" + metric = "apdex" + runbook_url = "${var.newrelic_runbook_url}" + + term { + duration = 5 + operator = "below" + priority = "critical" + threshold = "0.75" + time_function = "all" + } + + condition_scope = "application" + + count = "${length(var.newrelic_apm_entities) > 0 ? 1 : 0}" +} + +# Add a notification channel +resource "newrelic_alert_channel" "email" { + name = "email" + type = "email" + + configuration = { + recipients = "richard+devops.infra.demo@moduscreate.com" + include_json_attachment = "1" + } + + count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" +} + +# Link the channel to the policy +resource "newrelic_alert_policy_channel" "alert_email" { + policy_id = "${newrelic_alert_policy.alert.id}" + channel_id = "${newrelic_alert_channel.email.id}" + + count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" +} + +# Add a dashboard +resource "newrelic_dashboard" "spindash" { + title = "Spin Dashboard" + + widget { + title = "Average Transaction Duration" + row = 1 + column = 1 + width = 2 + visualization = "faceted_line_chart" + nrql = "SELECT AVERAGE(duration) from Transaction FACET appName TIMESERIES auto" + } + + widget { + title = "Average Apdex" + row = 2 + column = 1 + width = 2 + visualization = "faceted_line_chart" + nrql = "SELECT apdex(duration, t: 0.4) from Transaction FACET appName TIMESERIES auto" + } + + widget { + title = "Average CPU Percent" + row = 3 + column = 1 + height = 1 + width = 2 + visualization = "line_chart" + nrql = "SELECT average(cpuPercent) FROM SystemSample TIMESERIES auto" + } + + widget { + title = "Throughput (by host)" + row = 4 + column = 1 + height = 1 + width = 2 + visualization = "faceted_line_chart" + nrql = "SELECT count(*) AS 'rpm' FROM Transaction WHERE appName = 'Spin' FACET host SINCE 30 minutes ago TIMESERIES 1 minute" + } + + widget { + title = "Throughput (total)" + row = 4 + column = 1 + height = 1 + width = 2 + visualization = "line_chart" + nrql = "SELECT count(*) AS 'rpm' FROM Transaction WHERE appName = 'Spin' SINCE 30 minutes ago TIMESERIES 1 minute" + } +} diff --git a/terraform/secrets.tf b/terraform/secrets.tf new file mode 100644 index 0000000..191edc0 --- /dev/null +++ b/terraform/secrets.tf @@ -0,0 +1,8 @@ +resource "aws_secretsmanager_secret" "newrelic_license" { + name = "newrelic_license" +} + +resource "aws_secretsmanager_secret_version" "newrelic_license" { + secret_id = "${aws_secretsmanager_secret.newrelic_license.id}" + secret_string = "${var.newrelic_license_key}" +} diff --git a/terraform/variables.tf b/terraform/variables.tf index 9a1d7fc..ba95f70 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -33,7 +33,7 @@ variable "host" { variable "instance_type" { description = "EC2 instance type for servers" - default = "t2.large" + default = "c5.large" } # Public key variables - you can specify another file for the public key @@ -81,3 +81,27 @@ variable "project_name" { description = "Project name" default = "devops-infra-demo" } + +variable "newrelic_license_key" { + description = "New Relic license key" +} + +variable "newrelic_api_key" { + description = "New Relic api key" +} + +variable "newrelic_apm_entities" { + description = "New Relic APM entity IDs" + type = "list" + default = [] +} + +variable "newrelic_runbook_url" { + description = "New Relic runbook URL" + default = "https://github.com/ModusCreateOrg/devops-infra-demo/wiki/runbook" +} + +variable "newrelic_alert_email" { + description = "New Relic alert email" + default = "" +}