Add New Relic terraform support

Jenkinsfile

@@ -26,16 +26,22 @@ def get_captcha(Long hash_const) {
 }
 
 def wrap = { fn->
-        ansiColor('xterm') {
-            withCredentials([file(credentialsId: 'terraform-demo.json',
-                variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE')]) {
-                withCredentials([string(credentialsId: 'newrelic.license.key',
-                    variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE')]) {
-                    sh ("bin/clean-workspace.sh")
-                    fn()
-                }
+    ansiColor('xterm') {
+        withCredentials(
+            [
+                file(credentialsId: 'terraform-demo.json',
+                     variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'),
+                string(credentialsId: 'newrelic.license.key',
+                       variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'),
+                string(credentialsId: 'newrelic.api.key',
+                       variable: 'NEWRELIC_API_KEY_OVERRIDE'),
+                string(credentialsId: 'newrelic.alert.email',
+                       variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'),
+            ]) {
+                sh ("bin/clean-workspace.sh")
+                fn()
             }
-        }
+    }
 }
 
 final Long XOR_CONST = 3735928559 // 0xdeadbeef

README.md

@@ -23,6 +23,7 @@ Instructions
 * [Docker](https://docker.com/) (tested with 18.05.0-ce)
 * [Packer](https://www.packer.io/) (tested with 1.0.3)
 * [Terraform](https://www.terraform.io/) (tested with  v0.11.7)
+* [JQ](https://stedolan.github.io/jq/) (tested with 1.3 and 1.5)
 
 Optionally, you can use Vagrant to test ansible playbooks locally and Jenkins to orchestrate creation of AMIs in conjunction with GitHub branches and pull requests.
 
@@ -116,6 +117,8 @@ A JMeter test harness that will allow testing of a the application
 
 A `Jenkinsfile` is provided that will allow Jenkins to execute Packer and Terraform, package a CodeDeploy application, and even run JMeter performance tests. In order for Jenkins to do this, it needs to have AWS credentials set up, preferably through an IAM role, granting full control of EC2 and VPC resources in that account, and write access to the S3 bucket used for storing CodeDeploy applications. Packer needs this in order to create AMIs, key pairs, etc, Terraform needs this to create a VPC and EC2 resources, and CodeDeploy needs this to store the artifact it creates. This could be pared down further through some careful logging and role work.
 
+The Jenkins executor running this job needs to have both a recent Docker and the jq utility (version 1.3 or higher) installed.
+
 The scripts here assume that Jenkins is running on EC2 and uses instance data from the Jenkins executor to infer what VPC and subnet to launch the new EC2 instance into.  The AWS profile IAM user associated with your Jenkins instance or the Jenkins user's AWS credentials should have full control of EC2 in the account you are using.
 
 This script relies on Jenkins having a secret file containing the Google application credentials in JSON with the id "terraform-demo.json". You will need to add that to your Jenkins server's credentials.

ansible/newrelic-infrastructure.yml

@@ -11,10 +11,11 @@
   tasks:
     - command: bash /app/bin/set-newrelic-license-key.sh
 
-- name: Restart newrelic
+- name: Restart newrelic-infra
   hosts: 127.0.0.1
   connection: local
   become: yes
-  tasks:
-    - command: service newrelic-infra restart
-
+  service:
+    name: newerelic-infra
+    state: restarted
+

ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2

@@ -3,7 +3,7 @@ venv = /app/venv
 wsgi-file = /app/src/wsgi.py
 chdir = /app/src
 master = 1
-workers = 150
+workers = 16
 threads = 1
 lazy-apps = 1
 wsgi-env-behaviour = holy

bin/terraform.sh

@@ -44,6 +44,16 @@ if [[ -n "$NEWRELIC_LICENSE_KEY_OVERRIDE" ]]; then
     echo "Overriding New Relic License Key" 1>&2
     NEWRELIC_LICENSE_KEY="$NEWRELIC_LICENSE_KEY_OVERRIDE"
 fi
+NEWRELIC_API_KEY_OVERRIDE=${NEWRELIC_API_KEY_OVERRIDE:-}
+if [[ -n "$NEWRELIC_API_KEY_OVERRIDE" ]]; then
+    echo "Overriding New Relic API Key" 1>&2
+    NEWRELIC_API_KEY="$NEWRELIC_API_KEY_OVERRIDE"
+fi
+NEWRELIC_ALERT_EMAIL_OVERRIDE=${NEWRELIC_ALERT_EMAIL_OVERRIDE:-}
+if [[ -n "$NEWRELIC_ALERT_EMAIL_OVERRIDE" ]]; then
+    echo "Overriding New Relic Alert Email" 1>&2
+    NEWRELIC_ALERT_EMAIL="$NEWRELIC_ALERT_EMAIL_OVERRIDE"
+fi
 
 # Set up Google creds in build dir for docker terraform
 mkdir -p "$BUILD_DIR"
@@ -61,6 +71,8 @@ cat <<EOF >>"$ENV_FILE"
 GOOGLE_APPLICATION_CREDENTIALS=/app/build/google.json
 GOOGLE_PROJECT=$GOOGLE_PROJECT_OVERRIDE
 NEWRELIC_LICENSE_KEY=$NEWRELIC_LICENSE_KEY
+NEWRELIC_API_KEY=$NEWRELIC_API_KEY
+NEWRELIC_ALERT_EMAIL=$NEWRELIC_ALERT_EMAIL
 EOF
 
 # http://redsymbol.net/articles/bash-exit-traps/
@@ -81,14 +93,24 @@ function output () {
 
 function plan() {
     local extra
+    local nr_app_id
     local output
     local -i retcode
     local targets
-    extra=${1:-}
+    extra="$*"
     output="$(mktemp)"
     targets=$(get_targets)
 
     set +e
+
+    nr_app_id=$(curl \
+        -s \
+        -X GET \
+        'https://api.newrelic.com/v2/applications.json' \
+        -H "X-Api-Key:${NEWRELIC_API_KEY}" \
+        -d "filter[name]=${NEWRELIC_APP_NAME}" \
+        | jq '.applications[].id')
+
     #shellcheck disable=SC2086
     $DOCKER_TERRAFORM plan \
         $extra \
@@ -97,6 +119,9 @@ function plan() {
         -input="$INPUT_ENABLED" \
         -var project_name="$PROJECT_NAME" \
         -var newrelic_license_key="$NEWRELIC_LICENSE_KEY" \
+        -var newrelic_api_key="$NEWRELIC_API_KEY" \
+        -var newrelic_alert_email="$NEWRELIC_ALERT_EMAIL" \
+        -var newrelic_apm_entities="[$nr_app_id]" \
         -var-file="/app/build/extra.tfvars" \
         -out="$TF_PLAN" \
         "$TF_DIR" \

bin/validate.sh

@@ -31,7 +31,9 @@ echo "Linting terraform files for correctness"
 DOCKER_TERRAFORM=$(get_docker_terraform)
 init_terraform
 $DOCKER_TERRAFORM validate \
-    -var 'newrelic_license_key=ZZZZ'
+    -var 'newrelic_license_key=ZZZZ' \
+    -var 'newrelic_api_key=ZZZZ' \
+    -var 'newrelic_alert_email=ferd.berferd@example.com' \
 echo "Linting terraform files for formatting"
 fmt=$($DOCKER_TERRAFORM fmt)
 if [[ -n "$fmt" ]]; then

env.sh.sample

@@ -32,5 +32,8 @@ export GOOGLE_CLOUD_KEYFILE_JSON
 #export GOOGLE_PROJECT=example-terraform-demo-999999
 export GOOGLE_REGION=us-east1
 
-# Fill in your New Relic license key
+# Fill in your New Relic license key and API key
 export NEWRELIC_LICENSE_KEY=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ
+export NEWRELIC_API_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
+export NEWRELIC_ALERT_EMAIL=nobody@example.com
+export NEWRELIC_APP_NAME=Spin

src/spin.py

@@ -4,15 +4,17 @@
 import time
 import newrelic.agent
 import random
-from bottle import route, default_app, response
+from bottle import route, default_app, response, HTTPError
 
 newrelic_ini = '../newrelic.ini'
 if os.path.isfile(newrelic_ini):
     newrelic.agent.initialize(newrelic_ini)
 
+
 @route('/spin')
 def spin(delay=0.1):
     """Spin the CPU, return the process id at the end"""
+    spin.invocations += 1
     child_pid = os.getpid()
     upper_max = 100000000000000000000000000000000
 
@@ -23,19 +25,28 @@ def spin(delay=0.1):
     pareto_factor = random.paretovariate(alpha)
     start_time = time.time()
 
+
     current_time = start_time
     scratch = 42 + int(current_time)
-    end_time = start_time + (delay * pareto_factor)
+    congestion_slowdown = delay * 10 / (current_time - spin.last_time)
+    end_time = start_time + (delay + congestion_slowdown) * pareto_factor
+    time_limit = start_time + (delay * 100)
     calcs = 0
     while current_time < end_time:
         calcs += 1
         scratch = (scratch * scratch) % upper_max
         current_time = time.time()
-    final_time = time.time()
-    interval = final_time - start_time
+        interval = current_time - start_time
+        if current_time > time_limit:
+            raise HTTPError(500, "Allowed transaction time exceeded ({} ms elapsed)".format(interval))
+    spin.last_time = current_time
     rate = calcs / interval
     response.set_header('Content-Type', 'text/plain')
-    return ('pid {0} spun {1} times over {2}s (rate {3}/s)\n'
-            .format(child_pid, calcs, interval, rate))
+    return ('pid {0} spun {1} times over {2}s (rate {3} invoked {4} times/s)\n'
+            .format(child_pid, calcs, interval, rate, spin.invocations))
+
+spin.invocations = 0
+spin.last_time = time.time() - 10
+spin.slowdown = 0
 
 application = default_app()

terraform/cloud-config.yml

@@ -1,3 +1,3 @@
 #cloud-config
 runcmd:
- - ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml
+ - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml

terraform/newrelic.tf

@@ -0,0 +1,57 @@
+# Configure the New Relic provider
+
+# Adapted from https://www.terraform.io/docs/providers/newrelic/index.html
+
+provider "newrelic" {
+  api_key = "${var.newrelic_api_key}"
+  version = "~> 1.2"
+}
+
+# Create an alert policy
+resource "newrelic_alert_policy" "alert" {
+  name = "Alert"
+}
+
+# Add a condition
+resource "newrelic_alert_condition" "spin-appdex" {
+  policy_id = "${newrelic_alert_policy.alert.id}"
+
+  name        = "spin-appdex"
+  type        = "apm_app_metric"
+  entities    = ["179953338"]                                       # You can look this up in New Relic
+  metric      = "apdex"
+  runbook_url = "https://github.com/devops-infra-demo/wiki/runbook"
+
+  term {
+    duration      = 5
+    operator      = "below"
+    priority      = "critical"
+    threshold     = "0.75"
+    time_function = "all"
+  }
+
+  condition_scope = "application"
+
+  count = "${length(var.newrelic_apm_entities) > 0 ? 1 : 0}"
+}
+
+# Add a notification channel
+resource "newrelic_alert_channel" "email" {
+  name = "email"
+  type = "email"
+
+  configuration = {
+    recipients              = "richard+devops.infra.demo@moduscreate.com"
+    include_json_attachment = "1"
+  }
+
+  count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}"
+}
+
+# Link the channel to the policy
+resource "newrelic_alert_policy_channel" "alert_email" {
+  policy_id  = "${newrelic_alert_policy.alert.id}"
+  channel_id = "${newrelic_alert_channel.email.id}"
+
+  count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}"
+}

terraform/variables.tf

@@ -85,3 +85,18 @@ variable "project_name" {
 variable "newrelic_license_key" {
   description = "New Relic license key"
 }
+
+variable "newrelic_api_key" {
+  description = "New Relic api key"
+}
+
+variable "newrelic_apm_entities" {
+  description = "New Relic APM entity IDs"
+  type        = "list"
+  default     = []
+}
+
+variable "newrelic_alert_email" {
+  description = "New Relic alert email"
+  default     = ""
+}