From 6768f8f80e4d2f3784387337facf5c3eb530e0b6 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Tue, 29 Jan 2019 23:41:10 -0500 Subject: [PATCH 1/9] Add start of New Relic terraform work --- Jenkinsfile | 15 ++++++++---- bin/terraform.sh | 16 ++++++++++++- env.sh.sample | 4 +++- terraform/newrelic.tf | 54 ++++++++++++++++++++++++++++++++++++++++++ terraform/variables.tf | 15 ++++++++++++ 5 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 terraform/newrelic.tf diff --git a/Jenkinsfile b/Jenkinsfile index 32495ca..c69e26b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,10 +27,17 @@ def get_captcha(Long hash_const) { def wrap = { fn-> ansiColor('xterm') { - withCredentials([file(credentialsId: 'terraform-demo.json', - variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE')]) { - withCredentials([string(credentialsId: 'newrelic.license.key', - variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE')]) { + withCredentials( + [ + file(credentialsId: 'terraform-demo.json', + variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'), + string(credentialsId: 'newrelic.license.key', + variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.api.key', + variable: 'NEWRELIC_API_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.alert.email', + variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'), + ]) { sh (""" cp env.sh.sample env.sh rm -rf build diff --git a/bin/terraform.sh b/bin/terraform.sh index 6555680..118e928 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -44,6 +44,16 @@ if [[ -n "$NEWRELIC_LICENSE_KEY_OVERRIDE" ]]; then echo "Overriding New Relic License Key" 1>&2 NEWRELIC_LICENSE_KEY="$NEWRELIC_LICENSE_KEY_OVERRIDE" fi +NEWRELIC_API_KEY_OVERRIDE=${NEWRELIC_API_KEY_OVERRIDE:-} +if [[ -n "$NEWRELIC_API_KEY_OVERRIDE" ]]; then + echo "Overriding New Relic API Key" 1>&2 + NEWRELIC_API_KEY="$NEWRELIC_API_KEY_OVERRIDE" +fi +NEWRELIC_ALERT_EMAIL_OVERRIDE=${NEWRELIC_ALERT_EMAIL_OVERRIDE:-} +if [[ -n "$NEWRELIC_ALERT_EMAIL_OVERRIDE" ]]; then + echo "Overriding New Relic Alert Email" 1>&2 + NEWRELIC_ALERT_EMAIL="$NEWRELIC_ALERT_EMAIL_OVERRIDE" +fi # Set up Google creds in build dir for docker terraform mkdir -p "$BUILD_DIR" @@ -61,6 +71,8 @@ cat <>"$ENV_FILE" GOOGLE_APPLICATION_CREDENTIALS=/app/build/google.json GOOGLE_PROJECT=$GOOGLE_PROJECT_OVERRIDE NEWRELIC_LICENSE_KEY=$NEWRELIC_LICENSE_KEY +NEWRELIC_API_KEY=$NEWRELIC_API_KEY +NEWRELIC_ALERT_EMAIL=$NEWRELIC_ALERT_EMAIL EOF # http://redsymbol.net/articles/bash-exit-traps/ @@ -84,7 +96,7 @@ function plan() { local output local -i retcode local targets - extra=${1:-} + extra="$*" output="$(mktemp)" targets=$(get_targets) @@ -97,6 +109,8 @@ function plan() { -input="$INPUT_ENABLED" \ -var project_name="$PROJECT_NAME" \ -var newrelic_license_key="$NEWRELIC_LICENSE_KEY" \ + -var newrelic_api_key="$NEWRELIC_API_KEY" \ + -var newrelic_alert_email="$NEWRELIC_ALERT_EMAIL" \ -var-file="/app/build/extra.tfvars" \ -out="$TF_PLAN" \ "$TF_DIR" \ diff --git a/env.sh.sample b/env.sh.sample index 63ec330..2cafc39 100644 --- a/env.sh.sample +++ b/env.sh.sample @@ -32,5 +32,7 @@ export GOOGLE_CLOUD_KEYFILE_JSON #export GOOGLE_PROJECT=example-terraform-demo-999999 export GOOGLE_REGION=us-east1 -# Fill in your New Relic license key +# Fill in your New Relic license key and API key export NEWRELIC_LICENSE_KEY=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ +export NEWRELIC_API_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +export NEWRELIC_ALERT_EMAIL=nobody@example.com diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf new file mode 100644 index 0000000..5718962 --- /dev/null +++ b/terraform/newrelic.tf @@ -0,0 +1,54 @@ +# Configure the New Relic provider + +# Adapted from https://www.terraform.io/docs/providers/newrelic/index.html + +provider "newrelic" { + api_key = "${var.newrelic_api_key}" +} + +# Create an alert policy +resource "newrelic_alert_policy" "alert" { + name = "Alert" +} + +# Add a condition +resource "newrelic_alert_condition" "spin-appdex" { + policy_id = "${newrelic_alert_policy.alert.id}" + + name = "spin-appdex" + type = "apm_app_metric" + entities = ["179953338"] # You can look this up in New Relic + metric = "apdex" + runbook_url = "https://github.com/devops-infra-demo/wiki/runbook" + + term { + duration = 5 + operator = "below" + priority = "critical" + threshold = "0.75" + time_function = "all" + } + + count = "${length(var.newrelic_apm_entities) > 0 ? 1 : 0}" +} + +# Add a notification channel +resource "newrelic_alert_channel" "email" { + name = "email" + type = "email" + + configuration = { + recipients = "richard+devops.infra.demo@moduscreate.com" + include_json_attachment = "1" + } + + count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" +} + +# Link the channel to the policy +resource "newrelic_alert_policy_channel" "alert_email" { + policy_id = "${newrelic_alert_policy.alert.id}" + channel_id = "${newrelic_alert_channel.email.id}" + + count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" +} diff --git a/terraform/variables.tf b/terraform/variables.tf index a6333b2..f8eadc2 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -85,3 +85,18 @@ variable "project_name" { variable "newrelic_license_key" { description = "New Relic license key" } + +variable "newrelic_api_key" { + description = "New Relic api key" +} + +variable "newrelic_apm_entities" { + description = "New Relic APM entity IDs" + type = "list" + default = [] +} + +variable "newrelic_alert_email" { + description = "New Relic alert email" + default = "" +} From 840741183a1678bb8eac5d235606c082b80000d7 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 00:05:34 -0500 Subject: [PATCH 2/9] Try to fix up wrap function nesting --- Jenkinsfile | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c69e26b..e358a8a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,27 +26,26 @@ def get_captcha(Long hash_const) { } def wrap = { fn-> - ansiColor('xterm') { - withCredentials( - [ - file(credentialsId: 'terraform-demo.json', - variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'), - string(credentialsId: 'newrelic.license.key', - variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'), - string(credentialsId: 'newrelic.api.key', - variable: 'NEWRELIC_API_KEY_OVERRIDE'), - string(credentialsId: 'newrelic.alert.email', - variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'), - ]) { - sh (""" - cp env.sh.sample env.sh - rm -rf build - mkdir build - """) - fn() - } + ansiColor('xterm') { + withCredentials( + [ + file(credentialsId: 'terraform-demo.json', + variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'), + string(credentialsId: 'newrelic.license.key', + variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.api.key', + variable: 'NEWRELIC_API_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.alert.email', + variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'), + ]) { + sh (""" + cp env.sh.sample env.sh + rm -rf build + mkdir build + """) + fn() } - } + } } final Long XOR_CONST = 3735928559 // 0xdeadbeef From 3214df9109765cfd2a348102acfa6bd9f6fd1a8b Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 00:15:34 -0500 Subject: [PATCH 3/9] Add more mandatory variables to validate for lint --- bin/validate.sh | 4 +++- terraform/newrelic.tf | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/validate.sh b/bin/validate.sh index e882a5a..3ce3976 100755 --- a/bin/validate.sh +++ b/bin/validate.sh @@ -31,7 +31,9 @@ echo "Linting terraform files for correctness" DOCKER_TERRAFORM=$(get_docker_terraform) init_terraform $DOCKER_TERRAFORM validate \ - -var 'newrelic_license_key=ZZZZ' + -var 'newrelic_license_key=ZZZZ' \ + -var 'newrelic_api_key=ZZZZ' \ + -var 'newrelic_alert_email=ferd.berferd@example.com' \ echo "Linting terraform files for formatting" fmt=$($DOCKER_TERRAFORM fmt) if [[ -n "$fmt" ]]; then diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 5718962..3d4730b 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -4,6 +4,7 @@ provider "newrelic" { api_key = "${var.newrelic_api_key}" + version = "~> 1.2" } # Create an alert policy From 0afe2436a9516646e4944ce3fbfd18a63b1dff0d Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 00:29:21 -0500 Subject: [PATCH 4/9] Get New Relic app ID dynamically --- bin/terraform.sh | 11 +++++++++++ env.sh.sample | 1 + 2 files changed, 12 insertions(+) diff --git a/bin/terraform.sh b/bin/terraform.sh index 118e928..fcd9af9 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -93,6 +93,7 @@ function output () { function plan() { local extra + local nr_app_id local output local -i retcode local targets @@ -101,6 +102,15 @@ function plan() { targets=$(get_targets) set +e + + nr_app_id=$(curl \ + -s \ + -X GET \ + 'https://api.newrelic.com/v2/applications.json' \ + -H "X-Api-Key:${NEWRELIC_API_KEY}" \ + -d "filter[name]=${NEWRELIC_APP_NAME}" \ + | jq '.applications[].id') + #shellcheck disable=SC2086 $DOCKER_TERRAFORM plan \ $extra \ @@ -111,6 +121,7 @@ function plan() { -var newrelic_license_key="$NEWRELIC_LICENSE_KEY" \ -var newrelic_api_key="$NEWRELIC_API_KEY" \ -var newrelic_alert_email="$NEWRELIC_ALERT_EMAIL" \ + -var newrelic_apm_entities="[$nr_app_id]" \ -var-file="/app/build/extra.tfvars" \ -out="$TF_PLAN" \ "$TF_DIR" \ diff --git a/env.sh.sample b/env.sh.sample index 2cafc39..4894803 100644 --- a/env.sh.sample +++ b/env.sh.sample @@ -36,3 +36,4 @@ export GOOGLE_REGION=us-east1 export NEWRELIC_LICENSE_KEY=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ export NEWRELIC_API_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA export NEWRELIC_ALERT_EMAIL=nobody@example.com +export NEWRELIC_APP_NAME=Spin From 2e41388cc11510fc37e9ef245594f99b2d25b9c2 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 01:05:46 -0500 Subject: [PATCH 5/9] Add note about Jenkins server requiring jq --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ec6515f..30b6912 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Instructions * [Docker](https://docker.com/) (tested with 18.05.0-ce) * [Packer](https://www.packer.io/) (tested with 1.0.3) * [Terraform](https://www.terraform.io/) (tested with v0.11.7) +* [JQ](https://stedolan.github.io/jq/) (tested with 1.3 and 1.5) Optionally, you can use Vagrant to test ansible playbooks locally and Jenkins to orchestrate creation of AMIs in conjunction with GitHub branches and pull requests. @@ -116,6 +117,8 @@ A JMeter test harness that will allow testing of a the application A `Jenkinsfile` is provided that will allow Jenkins to execute Packer and Terraform, package a CodeDeploy application, and even run JMeter performance tests. In order for Jenkins to do this, it needs to have AWS credentials set up, preferably through an IAM role, granting full control of EC2 and VPC resources in that account, and write access to the S3 bucket used for storing CodeDeploy applications. Packer needs this in order to create AMIs, key pairs, etc, Terraform needs this to create a VPC and EC2 resources, and CodeDeploy needs this to store the artifact it creates. This could be pared down further through some careful logging and role work. +The Jenkins executor running this job needs to have both a recent Docker and the jq utility (version 1.3 or higher) installed. + The scripts here assume that Jenkins is running on EC2 and uses instance data from the Jenkins executor to infer what VPC and subnet to launch the new EC2 instance into. The AWS profile IAM user associated with your Jenkins instance or the Jenkins user's AWS credentials should have full control of EC2 in the account you are using. This script relies on Jenkins having a secret file containing the Google application credentials in JSON with the id "terraform-demo.json". You will need to add that to your Jenkins server's credentials. From cc7ba04643d51894649651d2bd7d10408fb0f2d3 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 01:25:17 -0500 Subject: [PATCH 6/9] Run ansible-playbook as centos Otherwise it can't find the ansible-galaxy stuff in /home/centos/.ansible --- ansible/newrelic-infrastructure.yml | 9 +++++---- terraform/cloud-config.yml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml index b0ec75d..78e7568 100644 --- a/ansible/newrelic-infrastructure.yml +++ b/ansible/newrelic-infrastructure.yml @@ -11,10 +11,11 @@ tasks: - command: bash /app/bin/set-newrelic-license-key.sh -- name: Restart newrelic +- name: Restart newrelic-infra hosts: 127.0.0.1 connection: local become: yes - tasks: - - command: service newrelic-infra restart - \ No newline at end of file + service: + name: newerelic-infra + state: restarted + diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index 9e449bb..984164f 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,3 +1,3 @@ #cloud-config runcmd: - - ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml + - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml From 776bf6bfcf123963b13bc508474facf5eb235024 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 01:59:28 -0500 Subject: [PATCH 7/9] condtion_scope is not really optional --- terraform/newrelic.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 3d4730b..7c57b69 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -30,6 +30,8 @@ resource "newrelic_alert_condition" "spin-appdex" { time_function = "all" } + condition_scope = "application" + count = "${length(var.newrelic_apm_entities) > 0 ? 1 : 0}" } From 06951826d857387a10dd8f8c91419ef23d178ca3 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 03:20:38 -0500 Subject: [PATCH 8/9] Get spin to slow down and fail when it is pummeled --- .../templates/infra-demo.ini.j2 | 2 +- src/spin.py | 22 ++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 index f1f388b..699d286 100644 --- a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 +++ b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 @@ -3,7 +3,7 @@ venv = /app/venv wsgi-file = /app/src/wsgi.py chdir = /app/src master = 1 -workers = 150 +workers = 16 threads = 1 lazy-apps = 1 wsgi-env-behaviour = holy diff --git a/src/spin.py b/src/spin.py index 3a3eabc..6d393ef 100755 --- a/src/spin.py +++ b/src/spin.py @@ -4,15 +4,17 @@ import time import newrelic.agent import random -from bottle import route, default_app, response +from bottle import route, default_app, response, HTTPError newrelic_ini = '../newrelic.ini' if os.path.isfile(newrelic_ini): newrelic.agent.initialize(newrelic_ini) + @route('/spin') def spin(delay=0.1): """Spin the CPU, return the process id at the end""" + spin.invocations += 1 child_pid = os.getpid() upper_max = 100000000000000000000000000000000 @@ -23,19 +25,27 @@ def spin(delay=0.1): pareto_factor = random.paretovariate(alpha) start_time = time.time() + current_time = start_time scratch = 42 + int(current_time) - end_time = start_time + (delay * pareto_factor) + end_time = start_time + pareto_factor * (delay + (delay * 10 / (current_time - spin.last_time))) + time_limit = start_time + (delay * 50) calcs = 0 while current_time < end_time: calcs += 1 scratch = (scratch * scratch) % upper_max current_time = time.time() - final_time = time.time() - interval = final_time - start_time + interval = current_time - start_time + if current_time > time_limit: + raise HTTPError(500, "Allowed transaction time exceeded ({} ms elapsed)".format(interval)) + spin.last_time = current_time rate = calcs / interval response.set_header('Content-Type', 'text/plain') - return ('pid {0} spun {1} times over {2}s (rate {3}/s)\n' - .format(child_pid, calcs, interval, rate)) + return ('pid {0} spun {1} times over {2}s (rate {3} invoked {4} times/s)\n' + .format(child_pid, calcs, interval, rate, spin.invocations)) + +spin.invocations = 0 +spin.last_time = time.time() - 10 +spin.slowdown = 0 application = default_app() From d2915bc58eac2b3adeb1b505f5d08b23248af1ad Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 03:36:41 -0500 Subject: [PATCH 9/9] Bump up max allowed transaction time --- src/spin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/spin.py b/src/spin.py index 6d393ef..8ea72cc 100755 --- a/src/spin.py +++ b/src/spin.py @@ -28,8 +28,9 @@ def spin(delay=0.1): current_time = start_time scratch = 42 + int(current_time) - end_time = start_time + pareto_factor * (delay + (delay * 10 / (current_time - spin.last_time))) - time_limit = start_time + (delay * 50) + congestion_slowdown = delay * 10 / (current_time - spin.last_time) + end_time = start_time + (delay + congestion_slowdown) * pareto_factor + time_limit = start_time + (delay * 100) calcs = 0 while current_time < end_time: calcs += 1