diff --git a/Jenkinsfile b/Jenkinsfile index a846962..859efb1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,16 +26,22 @@ def get_captcha(Long hash_const) { } def wrap = { fn-> - ansiColor('xterm') { - withCredentials([file(credentialsId: 'terraform-demo.json', - variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE')]) { - withCredentials([string(credentialsId: 'newrelic.license.key', - variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE')]) { - sh ("bin/clean-workspace.sh") - fn() - } + ansiColor('xterm') { + withCredentials( + [ + file(credentialsId: 'terraform-demo.json', + variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'), + string(credentialsId: 'newrelic.license.key', + variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.api.key', + variable: 'NEWRELIC_API_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.alert.email', + variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'), + ]) { + sh ("bin/clean-workspace.sh") + fn() } - } + } } final Long XOR_CONST = 3735928559 // 0xdeadbeef diff --git a/README.md b/README.md index ec6515f..30b6912 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Instructions * [Docker](https://docker.com/) (tested with 18.05.0-ce) * [Packer](https://www.packer.io/) (tested with 1.0.3) * [Terraform](https://www.terraform.io/) (tested with v0.11.7) +* [JQ](https://stedolan.github.io/jq/) (tested with 1.3 and 1.5) Optionally, you can use Vagrant to test ansible playbooks locally and Jenkins to orchestrate creation of AMIs in conjunction with GitHub branches and pull requests. @@ -116,6 +117,8 @@ A JMeter test harness that will allow testing of a the application A `Jenkinsfile` is provided that will allow Jenkins to execute Packer and Terraform, package a CodeDeploy application, and even run JMeter performance tests. In order for Jenkins to do this, it needs to have AWS credentials set up, preferably through an IAM role, granting full control of EC2 and VPC resources in that account, and write access to the S3 bucket used for storing CodeDeploy applications. Packer needs this in order to create AMIs, key pairs, etc, Terraform needs this to create a VPC and EC2 resources, and CodeDeploy needs this to store the artifact it creates. This could be pared down further through some careful logging and role work. +The Jenkins executor running this job needs to have both a recent Docker and the jq utility (version 1.3 or higher) installed. + The scripts here assume that Jenkins is running on EC2 and uses instance data from the Jenkins executor to infer what VPC and subnet to launch the new EC2 instance into. The AWS profile IAM user associated with your Jenkins instance or the Jenkins user's AWS credentials should have full control of EC2 in the account you are using. This script relies on Jenkins having a secret file containing the Google application credentials in JSON with the id "terraform-demo.json". You will need to add that to your Jenkins server's credentials. diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml index b0ec75d..78e7568 100644 --- a/ansible/newrelic-infrastructure.yml +++ b/ansible/newrelic-infrastructure.yml @@ -11,10 +11,11 @@ tasks: - command: bash /app/bin/set-newrelic-license-key.sh -- name: Restart newrelic +- name: Restart newrelic-infra hosts: 127.0.0.1 connection: local become: yes - tasks: - - command: service newrelic-infra restart - \ No newline at end of file + service: + name: newerelic-infra + state: restarted + diff --git a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 index f1f388b..699d286 100644 --- a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 +++ b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 @@ -3,7 +3,7 @@ venv = /app/venv wsgi-file = /app/src/wsgi.py chdir = /app/src master = 1 -workers = 150 +workers = 16 threads = 1 lazy-apps = 1 wsgi-env-behaviour = holy diff --git a/bin/terraform.sh b/bin/terraform.sh index 6555680..fcd9af9 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -44,6 +44,16 @@ if [[ -n "$NEWRELIC_LICENSE_KEY_OVERRIDE" ]]; then echo "Overriding New Relic License Key" 1>&2 NEWRELIC_LICENSE_KEY="$NEWRELIC_LICENSE_KEY_OVERRIDE" fi +NEWRELIC_API_KEY_OVERRIDE=${NEWRELIC_API_KEY_OVERRIDE:-} +if [[ -n "$NEWRELIC_API_KEY_OVERRIDE" ]]; then + echo "Overriding New Relic API Key" 1>&2 + NEWRELIC_API_KEY="$NEWRELIC_API_KEY_OVERRIDE" +fi +NEWRELIC_ALERT_EMAIL_OVERRIDE=${NEWRELIC_ALERT_EMAIL_OVERRIDE:-} +if [[ -n "$NEWRELIC_ALERT_EMAIL_OVERRIDE" ]]; then + echo "Overriding New Relic Alert Email" 1>&2 + NEWRELIC_ALERT_EMAIL="$NEWRELIC_ALERT_EMAIL_OVERRIDE" +fi # Set up Google creds in build dir for docker terraform mkdir -p "$BUILD_DIR" @@ -61,6 +71,8 @@ cat <>"$ENV_FILE" GOOGLE_APPLICATION_CREDENTIALS=/app/build/google.json GOOGLE_PROJECT=$GOOGLE_PROJECT_OVERRIDE NEWRELIC_LICENSE_KEY=$NEWRELIC_LICENSE_KEY +NEWRELIC_API_KEY=$NEWRELIC_API_KEY +NEWRELIC_ALERT_EMAIL=$NEWRELIC_ALERT_EMAIL EOF # http://redsymbol.net/articles/bash-exit-traps/ @@ -81,14 +93,24 @@ function output () { function plan() { local extra + local nr_app_id local output local -i retcode local targets - extra=${1:-} + extra="$*" output="$(mktemp)" targets=$(get_targets) set +e + + nr_app_id=$(curl \ + -s \ + -X GET \ + 'https://api.newrelic.com/v2/applications.json' \ + -H "X-Api-Key:${NEWRELIC_API_KEY}" \ + -d "filter[name]=${NEWRELIC_APP_NAME}" \ + | jq '.applications[].id') + #shellcheck disable=SC2086 $DOCKER_TERRAFORM plan \ $extra \ @@ -97,6 +119,9 @@ function plan() { -input="$INPUT_ENABLED" \ -var project_name="$PROJECT_NAME" \ -var newrelic_license_key="$NEWRELIC_LICENSE_KEY" \ + -var newrelic_api_key="$NEWRELIC_API_KEY" \ + -var newrelic_alert_email="$NEWRELIC_ALERT_EMAIL" \ + -var newrelic_apm_entities="[$nr_app_id]" \ -var-file="/app/build/extra.tfvars" \ -out="$TF_PLAN" \ "$TF_DIR" \ diff --git a/bin/validate.sh b/bin/validate.sh index e882a5a..3ce3976 100755 --- a/bin/validate.sh +++ b/bin/validate.sh @@ -31,7 +31,9 @@ echo "Linting terraform files for correctness" DOCKER_TERRAFORM=$(get_docker_terraform) init_terraform $DOCKER_TERRAFORM validate \ - -var 'newrelic_license_key=ZZZZ' + -var 'newrelic_license_key=ZZZZ' \ + -var 'newrelic_api_key=ZZZZ' \ + -var 'newrelic_alert_email=ferd.berferd@example.com' \ echo "Linting terraform files for formatting" fmt=$($DOCKER_TERRAFORM fmt) if [[ -n "$fmt" ]]; then diff --git a/env.sh.sample b/env.sh.sample index 63ec330..4894803 100644 --- a/env.sh.sample +++ b/env.sh.sample @@ -32,5 +32,8 @@ export GOOGLE_CLOUD_KEYFILE_JSON #export GOOGLE_PROJECT=example-terraform-demo-999999 export GOOGLE_REGION=us-east1 -# Fill in your New Relic license key +# Fill in your New Relic license key and API key export NEWRELIC_LICENSE_KEY=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ +export NEWRELIC_API_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +export NEWRELIC_ALERT_EMAIL=nobody@example.com +export NEWRELIC_APP_NAME=Spin diff --git a/src/spin.py b/src/spin.py index 3a3eabc..8ea72cc 100755 --- a/src/spin.py +++ b/src/spin.py @@ -4,15 +4,17 @@ import time import newrelic.agent import random -from bottle import route, default_app, response +from bottle import route, default_app, response, HTTPError newrelic_ini = '../newrelic.ini' if os.path.isfile(newrelic_ini): newrelic.agent.initialize(newrelic_ini) + @route('/spin') def spin(delay=0.1): """Spin the CPU, return the process id at the end""" + spin.invocations += 1 child_pid = os.getpid() upper_max = 100000000000000000000000000000000 @@ -23,19 +25,28 @@ def spin(delay=0.1): pareto_factor = random.paretovariate(alpha) start_time = time.time() + current_time = start_time scratch = 42 + int(current_time) - end_time = start_time + (delay * pareto_factor) + congestion_slowdown = delay * 10 / (current_time - spin.last_time) + end_time = start_time + (delay + congestion_slowdown) * pareto_factor + time_limit = start_time + (delay * 100) calcs = 0 while current_time < end_time: calcs += 1 scratch = (scratch * scratch) % upper_max current_time = time.time() - final_time = time.time() - interval = final_time - start_time + interval = current_time - start_time + if current_time > time_limit: + raise HTTPError(500, "Allowed transaction time exceeded ({} ms elapsed)".format(interval)) + spin.last_time = current_time rate = calcs / interval response.set_header('Content-Type', 'text/plain') - return ('pid {0} spun {1} times over {2}s (rate {3}/s)\n' - .format(child_pid, calcs, interval, rate)) + return ('pid {0} spun {1} times over {2}s (rate {3} invoked {4} times/s)\n' + .format(child_pid, calcs, interval, rate, spin.invocations)) + +spin.invocations = 0 +spin.last_time = time.time() - 10 +spin.slowdown = 0 application = default_app() diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index 9e449bb..984164f 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,3 +1,3 @@ #cloud-config runcmd: - - ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml + - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf new file mode 100644 index 0000000..7c57b69 --- /dev/null +++ b/terraform/newrelic.tf @@ -0,0 +1,57 @@ +# Configure the New Relic provider + +# Adapted from https://www.terraform.io/docs/providers/newrelic/index.html + +provider "newrelic" { + api_key = "${var.newrelic_api_key}" + version = "~> 1.2" +} + +# Create an alert policy +resource "newrelic_alert_policy" "alert" { + name = "Alert" +} + +# Add a condition +resource "newrelic_alert_condition" "spin-appdex" { + policy_id = "${newrelic_alert_policy.alert.id}" + + name = "spin-appdex" + type = "apm_app_metric" + entities = ["179953338"] # You can look this up in New Relic + metric = "apdex" + runbook_url = "https://github.com/devops-infra-demo/wiki/runbook" + + term { + duration = 5 + operator = "below" + priority = "critical" + threshold = "0.75" + time_function = "all" + } + + condition_scope = "application" + + count = "${length(var.newrelic_apm_entities) > 0 ? 1 : 0}" +} + +# Add a notification channel +resource "newrelic_alert_channel" "email" { + name = "email" + type = "email" + + configuration = { + recipients = "richard+devops.infra.demo@moduscreate.com" + include_json_attachment = "1" + } + + count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" +} + +# Link the channel to the policy +resource "newrelic_alert_policy_channel" "alert_email" { + policy_id = "${newrelic_alert_policy.alert.id}" + channel_id = "${newrelic_alert_channel.email.id}" + + count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" +} diff --git a/terraform/variables.tf b/terraform/variables.tf index a6333b2..f8eadc2 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -85,3 +85,18 @@ variable "project_name" { variable "newrelic_license_key" { description = "New Relic license key" } + +variable "newrelic_api_key" { + description = "New Relic api key" +} + +variable "newrelic_apm_entities" { + description = "New Relic APM entity IDs" + type = "list" + default = [] +} + +variable "newrelic_alert_email" { + description = "New Relic alert email" + default = "" +}