From 33bfbe23720d27167a8305ab61dc0f71f2b6ce11 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Mon, 28 Jan 2019 17:32:37 -0500 Subject: [PATCH 01/52] Poke New Relic license key in as a secret * Use AWS Secrets Manager to store New Relic license key * Add New Relic license key oerride to Jenkinsfile * Make Terraform put New Relic license in env.sh --- Jenkinsfile | 15 +++++++++------ ansible/bakery.yml | 15 +++++++++++++++ ansible/requirements.yml | 3 +++ bin/terraform.sh | 7 +++++++ bin/validate.sh | 3 ++- env.sh.sample | 3 +++ terraform/infra-demo-role-policy.json | 9 +++++++++ terraform/secrets.tf | 8 ++++++++ terraform/variables.tf | 4 ++++ 9 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 terraform/secrets.tf diff --git a/Jenkinsfile b/Jenkinsfile index f1533ad..c190ec8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,12 +29,15 @@ def wrap = { fn-> ansiColor('xterm') { withCredentials([file(credentialsId: 'terraform-demo.json', variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE')]) { - sh (""" - cp env.sh.sample env.sh - rm -rf build - mkdir build - """) - fn() + withCredentials([string(credentialsId: 'newrelic.license.key', + variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE')]) { + sh (""" + cp env.sh.sample env.sh + rm -rf build + mkdir build + """) + fn() + } } } } diff --git a/ansible/bakery.yml b/ansible/bakery.yml index 93f8b58..606dcc1 100644 --- a/ansible/bakery.yml +++ b/ansible/bakery.yml @@ -3,6 +3,20 @@ # Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for # the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Install New Relic Infrastructure + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - newrelic.newrelic-infra + vars: + nrinfragent_os_name: CentOS + nrinfragent_os_version: 7 + nrinfragent_config: + license_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ + log_file: /var/log/newrelic/nr-infra.log + log_to_stdout: false + - name: Install Web Application hosts: 127.0.0.1 connection: local @@ -12,6 +26,7 @@ - prepare-web-content - prepare-codedeploy + - name: Harden Server hosts: 127.0.0.1 connection: local diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 05be059..5978c53 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -15,3 +15,6 @@ # And the ModusCreateOrg fork has been fixed to avoid running stuff from /tmp - src: https://github.com/ModusCreateOrg/ansible-aws-codedeploy-agent +# New Relic Infrastructure +- src: newrelic.newrelic-infra + diff --git a/bin/terraform.sh b/bin/terraform.sh index 81d00bd..6555680 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -39,6 +39,11 @@ if [[ -n "$GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE" ]]; then echo "Overriding Google Application Credentials" 1>&2 GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE" fi +NEWRELIC_LICENSE_KEY_OVERRIDE=${NEWRELIC_LICENSE_KEY_OVERRIDE:-} +if [[ -n "$NEWRELIC_LICENSE_KEY_OVERRIDE" ]]; then + echo "Overriding New Relic License Key" 1>&2 + NEWRELIC_LICENSE_KEY="$NEWRELIC_LICENSE_KEY_OVERRIDE" +fi # Set up Google creds in build dir for docker terraform mkdir -p "$BUILD_DIR" @@ -55,6 +60,7 @@ GOOGLE_PROJECT_OVERRIDE=$(awk 'BEGIN { FS = "\"" } /project_id/{print $4}' <$GOO cat <>"$ENV_FILE" GOOGLE_APPLICATION_CREDENTIALS=/app/build/google.json GOOGLE_PROJECT=$GOOGLE_PROJECT_OVERRIDE +NEWRELIC_LICENSE_KEY=$NEWRELIC_LICENSE_KEY EOF # http://redsymbol.net/articles/bash-exit-traps/ @@ -90,6 +96,7 @@ function plan() { -lock=true \ -input="$INPUT_ENABLED" \ -var project_name="$PROJECT_NAME" \ + -var newrelic_license_key="$NEWRELIC_LICENSE_KEY" \ -var-file="/app/build/extra.tfvars" \ -out="$TF_PLAN" \ "$TF_DIR" \ diff --git a/bin/validate.sh b/bin/validate.sh index b26b761..e882a5a 100755 --- a/bin/validate.sh +++ b/bin/validate.sh @@ -30,7 +30,8 @@ $DOCKER_PACKER validate app/packer/machines/web-server.json echo "Linting terraform files for correctness" DOCKER_TERRAFORM=$(get_docker_terraform) init_terraform -$DOCKER_TERRAFORM validate +$DOCKER_TERRAFORM validate \ + -var 'newrelic_license_key=ZZZZ' echo "Linting terraform files for formatting" fmt=$($DOCKER_TERRAFORM fmt) if [[ -n "$fmt" ]]; then diff --git a/env.sh.sample b/env.sh.sample index 4057992..63ec330 100644 --- a/env.sh.sample +++ b/env.sh.sample @@ -31,3 +31,6 @@ fi export GOOGLE_CLOUD_KEYFILE_JSON #export GOOGLE_PROJECT=example-terraform-demo-999999 export GOOGLE_REGION=us-east1 + +# Fill in your New Relic license key +export NEWRELIC_LICENSE_KEY=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ diff --git a/terraform/infra-demo-role-policy.json b/terraform/infra-demo-role-policy.json index 05a5fe5..8cd8b0a 100644 --- a/terraform/infra-demo-role-policy.json +++ b/terraform/infra-demo-role-policy.json @@ -64,6 +64,15 @@ "arn:aws:s3:::aws-codedeploy-us-west-2/*", "arn:aws:s3:::aws-codedeploy-us-east-1/*" ] + }, + { + "Effect": "Allow", + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Resource": [ + "arn:aws:secretsmanager:*:*:secret:newrelic_license*" + ] } ] } diff --git a/terraform/secrets.tf b/terraform/secrets.tf new file mode 100644 index 0000000..191edc0 --- /dev/null +++ b/terraform/secrets.tf @@ -0,0 +1,8 @@ +resource "aws_secretsmanager_secret" "newrelic_license" { + name = "newrelic_license" +} + +resource "aws_secretsmanager_secret_version" "newrelic_license" { + secret_id = "${aws_secretsmanager_secret.newrelic_license.id}" + secret_string = "${var.newrelic_license_key}" +} diff --git a/terraform/variables.tf b/terraform/variables.tf index 9a1d7fc..a6333b2 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -81,3 +81,7 @@ variable "project_name" { description = "Project name" default = "devops-infra-demo" } + +variable "newrelic_license_key" { + description = "New Relic license key" +} From 64aa0cbc375bc3543d7ce63fd4d32cc19c5727f4 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Mon, 28 Jan 2019 17:28:46 -0500 Subject: [PATCH 02/52] Initial Initial changes --- codedeploy/bin/AfterInstall.sh | 9 +++++++++ src/requirements.txt | 2 ++ src/spin.py | 3 +++ 3 files changed, 14 insertions(+) diff --git a/codedeploy/bin/AfterInstall.sh b/codedeploy/bin/AfterInstall.sh index 91f49d4..914a65c 100755 --- a/codedeploy/bin/AfterInstall.sh +++ b/codedeploy/bin/AfterInstall.sh @@ -21,3 +21,12 @@ ANSIBLE_DIR="$BASE_DIR/ansible" # Invoke Ansible for final set up ansible-playbook -l localhost "$ANSIBLE_DIR/app-AfterInstall.yml" + +# Configure New Relic +NEWRELIC_CONFIG_DIR=/app +VENV_DIR=/app/venv +NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value --secret-id newrelic_license --output text --query '[SecretString]') +${VENV_DIR}/bin/activate +newrelic-admin generate-config ${NEWRELIC_LICENSE_KEY} ${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig +sed 's/^app_name =.*$/app_name = Spin/' ${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig > ${NEWRELIC_CONFIG_DIR}/newrelic.ini +#rm -f ${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index d7d87aa..7bbc8b5 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,2 +1,4 @@ bottle==0.12.13 uwsgi +newrelic + diff --git a/src/spin.py b/src/spin.py index 4070b85..7696508 100755 --- a/src/spin.py +++ b/src/spin.py @@ -2,8 +2,11 @@ """This module spins the CPU.""" import os import time +import newrelic.agent from bottle import route, default_app, response +newrelic.agent.initialize('../newrelic.ini') + @route('/spin') def spin(delay=5.0): """Spin the CPU, return the process id at the end""" From fe960c1f4170dc4ce2f8502cf4ead11f517d235e Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Mon, 28 Jan 2019 17:54:56 -0500 Subject: [PATCH 03/52] quote vars quote vars --- codedeploy/bin/AfterInstall.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codedeploy/bin/AfterInstall.sh b/codedeploy/bin/AfterInstall.sh index 914a65c..623867d 100755 --- a/codedeploy/bin/AfterInstall.sh +++ b/codedeploy/bin/AfterInstall.sh @@ -27,6 +27,6 @@ NEWRELIC_CONFIG_DIR=/app VENV_DIR=/app/venv NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value --secret-id newrelic_license --output text --query '[SecretString]') ${VENV_DIR}/bin/activate -newrelic-admin generate-config ${NEWRELIC_LICENSE_KEY} ${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig -sed 's/^app_name =.*$/app_name = Spin/' ${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig > ${NEWRELIC_CONFIG_DIR}/newrelic.ini -#rm -f ${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig \ No newline at end of file +newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" +sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini" +#rm -f "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" \ No newline at end of file From d702a9d85069fe0e2c6f7e78da428c271a446a17 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Mon, 28 Jan 2019 19:30:00 -0500 Subject: [PATCH 04/52] Fix up New Relic license key retrieval --- .gitignore | 2 ++ codedeploy/bin/AfterInstall.sh | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ad4f921..8409a05 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ __pycache__ build/ jmeter.log +scan-xccdf-results.html +scan-xccdf-results.xml venv/ diff --git a/codedeploy/bin/AfterInstall.sh b/codedeploy/bin/AfterInstall.sh index 623867d..b56f90b 100755 --- a/codedeploy/bin/AfterInstall.sh +++ b/codedeploy/bin/AfterInstall.sh @@ -25,8 +25,17 @@ ansible-playbook -l localhost "$ANSIBLE_DIR/app-AfterInstall.yml" # Configure New Relic NEWRELIC_CONFIG_DIR=/app VENV_DIR=/app/venv -NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value --secret-id newrelic_license --output text --query '[SecretString]') +# Thanks Stack Overflow https://stackoverflow.com/a/9735663/424301 +EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) +#shellcheck disable=SC2001 +EC2_REGION="$(echo "$EC2_AVAIL_ZONE" | sed 's/[a-z]$//')" + +NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \ + --region="$EC2_REGION" \ + --secret-id newrelic_license \ + --output text \ + --query '[SecretString]') ${VENV_DIR}/bin/activate newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini" -#rm -f "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" \ No newline at end of file +#rm -f "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" From a8dee19b0705a42b1df75be2d43e9186672bb5ed Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Mon, 28 Jan 2019 19:38:06 -0500 Subject: [PATCH 05/52] Ensure we source virtualenv activate --- codedeploy/bin/AfterInstall.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codedeploy/bin/AfterInstall.sh b/codedeploy/bin/AfterInstall.sh index b56f90b..9c64d93 100755 --- a/codedeploy/bin/AfterInstall.sh +++ b/codedeploy/bin/AfterInstall.sh @@ -35,7 +35,8 @@ NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \ --secret-id newrelic_license \ --output text \ --query '[SecretString]') -${VENV_DIR}/bin/activate +#shellcheck disable=SC1090 +source ${VENV_DIR}/bin/activate newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini" #rm -f "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" From 3dcb7b422b1fc3232e769a89f8a304c29fabd29c Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Mon, 28 Jan 2019 19:42:37 -0500 Subject: [PATCH 06/52] Allow unbound shell variables in activate See http://redsymbol.net/articles/unofficial-bash-strict-mode/#sourcing-nonconforming-document --- codedeploy/bin/AfterInstall.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codedeploy/bin/AfterInstall.sh b/codedeploy/bin/AfterInstall.sh index 9c64d93..29944af 100755 --- a/codedeploy/bin/AfterInstall.sh +++ b/codedeploy/bin/AfterInstall.sh @@ -35,8 +35,10 @@ NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \ --secret-id newrelic_license \ --output text \ --query '[SecretString]') +set +u #shellcheck disable=SC1090 source ${VENV_DIR}/bin/activate +set -u newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini" #rm -f "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" From f0f33d9d413306bafa9f1523b18d5d1ffa8e835b Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Mon, 28 Jan 2019 22:37:21 -0500 Subject: [PATCH 07/52] Simplify Jenkinsfile, spin for less time --- Jenkinsfile | 5 +---- src/spin.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c190ec8..32495ca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,10 +92,7 @@ properties([ string( name: 'JMETER_threads', defaultValue: '2', - description: """number of jmeter threads. Resulting ASG stable sizes for t2.large instances are: - - 2 threads, 3 instances; - - 4 threads, 7 instances; - """ + description: "number of jmeter threads." ), string( name: 'JMETER_ramp_duration', diff --git a/src/spin.py b/src/spin.py index 7696508..3a3eabc 100755 --- a/src/spin.py +++ b/src/spin.py @@ -3,19 +3,29 @@ import os import time import newrelic.agent +import random from bottle import route, default_app, response -newrelic.agent.initialize('../newrelic.ini') +newrelic_ini = '../newrelic.ini' +if os.path.isfile(newrelic_ini): + newrelic.agent.initialize(newrelic_ini) @route('/spin') -def spin(delay=5.0): +def spin(delay=0.1): """Spin the CPU, return the process id at the end""" child_pid = os.getpid() upper_max = 100000000000000000000000000000000 + + # Use a pareto distribution to give additional + # variation to the delay + # See https://en.wikipedia.org/wiki/Pareto_distribution + alpha = 2 + pareto_factor = random.paretovariate(alpha) start_time = time.time() + current_time = start_time scratch = 42 + int(current_time) - end_time = start_time + delay + end_time = start_time + (delay * pareto_factor) calcs = 0 while current_time < end_time: calcs += 1 From c444de771eb38fc787a640217c6e4c38a897aeb6 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Tue, 29 Jan 2019 00:17:58 -0500 Subject: [PATCH 08/52] Enhance uwsgi config; ASG: use ELB health check --- ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 | 6 +++--- codedeploy/bin/AfterInstall.sh | 5 ++--- terraform/elb.tf | 2 +- terraform/instances.tf | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 index e3e76e1..f1f388b 100644 --- a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 +++ b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 @@ -3,11 +3,11 @@ venv = /app/venv wsgi-file = /app/src/wsgi.py chdir = /app/src master = 1 -workers = 2 -threads = 8 +workers = 150 +threads = 1 lazy-apps = 1 wsgi-env-behaviour = holy -enable-threads = 1 +enable-threads = 0 http-auto-chunked = 1 http-keepalive = 1 uwsgi-socket = 127.0.0.1:8008 diff --git a/codedeploy/bin/AfterInstall.sh b/codedeploy/bin/AfterInstall.sh index 29944af..93d1db5 100755 --- a/codedeploy/bin/AfterInstall.sh +++ b/codedeploy/bin/AfterInstall.sh @@ -23,12 +23,12 @@ ANSIBLE_DIR="$BASE_DIR/ansible" ansible-playbook -l localhost "$ANSIBLE_DIR/app-AfterInstall.yml" # Configure New Relic +# TODO: move into Ansible playbook app-AfterInstall.yml NEWRELIC_CONFIG_DIR=/app VENV_DIR=/app/venv # Thanks Stack Overflow https://stackoverflow.com/a/9735663/424301 EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) -#shellcheck disable=SC2001 -EC2_REGION="$(echo "$EC2_AVAIL_ZONE" | sed 's/[a-z]$//')" +EC2_REGION="$(sed 's/[a-z]$//' <<<"$EC2_AVAIL_ZONE")" NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \ --region="$EC2_REGION" \ @@ -41,4 +41,3 @@ source ${VENV_DIR}/bin/activate set -u newrelic-admin generate-config "${NEWRELIC_LICENSE_KEY}" "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" sed 's/^app_name =.*$/app_name = Spin/' "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" > "${NEWRELIC_CONFIG_DIR}/newrelic.ini" -#rm -f "${NEWRELIC_CONFIG_DIR}/newrelic.ini.orig" diff --git a/terraform/elb.tf b/terraform/elb.tf index 56874e2..d2b67bd 100644 --- a/terraform/elb.tf +++ b/terraform/elb.tf @@ -19,7 +19,7 @@ resource "aws_elb" "infra-demo-elb" { healthy_threshold = 2 unhealthy_threshold = 2 timeout = 3 - target = "HTTP:80/" + target = "HTTP:80/api/spin" interval = 30 } diff --git a/terraform/instances.tf b/terraform/instances.tf index 15d339f..8403b8e 100644 --- a/terraform/instances.tf +++ b/terraform/instances.tf @@ -93,7 +93,7 @@ resource "aws_autoscaling_group" "infra-demo-web-asg" { max_size = "${var.max_size}" launch_configuration = "${aws_launch_configuration.infra-demo-web-lc.name}" - health_check_type = "EC2" + health_check_type = "ELB" vpc_zone_identifier = [ "${module.vpc.public_subnets}", From 7d1a5cf54e843aa4a66a47a1c4d6cd9044a0f2a1 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Tue, 29 Jan 2019 12:48:56 -0500 Subject: [PATCH 09/52] Initial Initial --- ansible/bakery.yml | 14 -------------- ansible/newrelic-infrastructure.yml | 21 ++++++++++++++++++++ bin/set-newrelic-license-key.sh | 30 +++++++++++++++++++++++++++++ codedeploy/appspec.yml | 4 ++++ terraform/cloud-config.yml | 1 + 5 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 ansible/newrelic-infrastructure.yml create mode 100755 bin/set-newrelic-license-key.sh diff --git a/ansible/bakery.yml b/ansible/bakery.yml index 606dcc1..3051c39 100644 --- a/ansible/bakery.yml +++ b/ansible/bakery.yml @@ -3,19 +3,6 @@ # Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for # the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" -- name: Install New Relic Infrastructure - hosts: 127.0.0.1 - connection: local - become: yes - roles: - - newrelic.newrelic-infra - vars: - nrinfragent_os_name: CentOS - nrinfragent_os_version: 7 - nrinfragent_config: - license_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ - log_file: /var/log/newrelic/nr-infra.log - log_to_stdout: false - name: Install Web Application hosts: 127.0.0.1 @@ -26,7 +13,6 @@ - prepare-web-content - prepare-codedeploy - - name: Harden Server hosts: 127.0.0.1 connection: local diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml new file mode 100644 index 0000000..f489fa6 --- /dev/null +++ b/ansible/newrelic-infrastructure.yml @@ -0,0 +1,21 @@ +--- +# Use ansible to install the newrelic infra agent. + +# Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for +# the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Install New Relic Infrastructure + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - newrelic.newrelic-infra + vars: + nrinfragent_os_name: CentOS + nrinfragent_os_version: 7 + nrinfragent_config: + license_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ + log_file: /var/log/newrelic/nr-infra.log + log_to_stdout: false + +-name: Set the newrelic license key + local_action: bash /app/bin/set-newrelic-license-key.sh \ No newline at end of file diff --git a/bin/set-newrelic-license-key.sh b/bin/set-newrelic-license-key.sh new file mode 100755 index 0000000..6d60f5b --- /dev/null +++ b/bin/set-newrelic-license-key.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Set the newrelic license key fron AWS credentials +# +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." +export BASE_DIR + +NEWRELIC_CONFIG_FILE="/etc/newrelic-infra.yml" +# Thanks Stack Overflow https://stackoverflow.com/a/9735663/424301 +EC2_AVAIL_ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) +EC2_REGION="$(sed 's/[a-z]$//' <<<"$EC2_AVAIL_ZONE")" + +NEWRELIC_LICENSE_KEY=$(aws secretsmanager get-secret-value \ + --region="$EC2_REGION" \ + --secret-id newrelic_license \ + --output text \ + --query '[SecretString]') + +cp -a "${NEWRELIC_CONFIG_FILE}" "${NEWRELIC_CONFIG_FILE}.orig" +sed "s/ZZZZ*ZZZZ/${NEWRELIC_LICENSE_KEY}/" "${NEWRELIC_CONFIG_FILE}.orig" > "${NEWRELIC_CONFIG_FILE}" diff --git a/codedeploy/appspec.yml b/codedeploy/appspec.yml index e0f94aa..2538144 100644 --- a/codedeploy/appspec.yml +++ b/codedeploy/appspec.yml @@ -6,6 +6,10 @@ files: destination: /app/application - source: src destination: /app/src + - source: bin + destination: /app/bin + - source: ansible + destination: /app/ansible hooks: ApplicationStop: - location: bin/ApplicationStop.sh diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index 3872c61..ecf0cea 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,3 +1,4 @@ #cloud-config runcmd: - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml + - sudo -u centos ansible-playbook -l localhost /app/ansible/newrelic-infrastructure.yml From 95364b0a8ba2debd601ee5e84be469c2b63c7f97 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Tue, 29 Jan 2019 16:04:09 -0500 Subject: [PATCH 10/52] Fix newrelic install Fix newrelic install --- ansible/newrelic-infrastructure.yml | 9 +++++++-- terraform/cloud-config.yml | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml index f489fa6..2c80d36 100644 --- a/ansible/newrelic-infrastructure.yml +++ b/ansible/newrelic-infrastructure.yml @@ -17,5 +17,10 @@ log_file: /var/log/newrelic/nr-infra.log log_to_stdout: false --name: Set the newrelic license key - local_action: bash /app/bin/set-newrelic-license-key.sh \ No newline at end of file +- name: Set the newrelic license key + hosts: 127.0.0.1 + connection: local + become: yes + tasks: + - command: ansible-galaxy install newrelic.newrelic-infra + - command: bash /app/bin/set-newrelic-license-key.sh \ No newline at end of file diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index ecf0cea..ec641ea 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,4 +1,4 @@ #cloud-config runcmd: - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml - - sudo -u centos ansible-playbook -l localhost /app/ansible/newrelic-infrastructure.yml + - ansible-playbook -l localhost /app/ansible/newrelic-infrastructure.yml From 7436b3bc05a4b1bca3fe5aa8e6be92071c40da02 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Tue, 29 Jan 2019 16:14:56 -0500 Subject: [PATCH 11/52] Install newrelic.newrelic-infra newrelic.newrelic-infra first --- ansible/newrelic-infrastructure.yml | 11 +++++++++-- terraform/cloud-config.yml | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml index 2c80d36..544605b 100644 --- a/ansible/newrelic-infrastructure.yml +++ b/ansible/newrelic-infrastructure.yml @@ -3,6 +3,13 @@ # Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for # the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Install newrelic.newrelic-infra + hosts: 127.0.0.1 + connection: local + become: yes + tasks: + - command: ansible-galaxy install newrelic.newrelic-infra + - name: Install New Relic Infrastructure hosts: 127.0.0.1 connection: local @@ -22,5 +29,5 @@ connection: local become: yes tasks: - - command: ansible-galaxy install newrelic.newrelic-infra - - command: bash /app/bin/set-newrelic-license-key.sh \ No newline at end of file + - command: bash /app/bin/set-newrelic-license-key.sh + \ No newline at end of file diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index ec641ea..ecf0cea 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,4 +1,4 @@ #cloud-config runcmd: - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml - - ansible-playbook -l localhost /app/ansible/newrelic-infrastructure.yml + - sudo -u centos ansible-playbook -l localhost /app/ansible/newrelic-infrastructure.yml From 67bfa325dbdd62fdc3182445d63bef63c57c8ac5 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Tue, 29 Jan 2019 20:50:38 -0500 Subject: [PATCH 12/52] move newrelic install move newrelic install back to bakery --- ansible/bakery.yml | 14 ++++++++++++++ ansible/newrelic-infrastructure.yml | 20 -------------------- terraform/cloud-config.yml | 2 +- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/ansible/bakery.yml b/ansible/bakery.yml index 3051c39..606dcc1 100644 --- a/ansible/bakery.yml +++ b/ansible/bakery.yml @@ -3,6 +3,19 @@ # Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for # the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Install New Relic Infrastructure + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - newrelic.newrelic-infra + vars: + nrinfragent_os_name: CentOS + nrinfragent_os_version: 7 + nrinfragent_config: + license_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ + log_file: /var/log/newrelic/nr-infra.log + log_to_stdout: false - name: Install Web Application hosts: 127.0.0.1 @@ -13,6 +26,7 @@ - prepare-web-content - prepare-codedeploy + - name: Harden Server hosts: 127.0.0.1 connection: local diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml index 544605b..5ea7139 100644 --- a/ansible/newrelic-infrastructure.yml +++ b/ansible/newrelic-infrastructure.yml @@ -3,26 +3,6 @@ # Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for # the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" -- name: Install newrelic.newrelic-infra - hosts: 127.0.0.1 - connection: local - become: yes - tasks: - - command: ansible-galaxy install newrelic.newrelic-infra - -- name: Install New Relic Infrastructure - hosts: 127.0.0.1 - connection: local - become: yes - roles: - - newrelic.newrelic-infra - vars: - nrinfragent_os_name: CentOS - nrinfragent_os_version: 7 - nrinfragent_config: - license_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ - log_file: /var/log/newrelic/nr-infra.log - log_to_stdout: false - name: Set the newrelic license key hosts: 127.0.0.1 diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index ecf0cea..ec641ea 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,4 +1,4 @@ #cloud-config runcmd: - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml - - sudo -u centos ansible-playbook -l localhost /app/ansible/newrelic-infrastructure.yml + - ansible-playbook -l localhost /app/ansible/newrelic-infrastructure.yml From dce63c7d47f9a7cd18aaeb3bc2e8859d18bb03ea Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Tue, 29 Jan 2019 20:59:42 -0500 Subject: [PATCH 13/52] restart newrelic after key restart newrelic after key --- ansible/newrelic-infrastructure.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml index 5ea7139..b0ec75d 100644 --- a/ansible/newrelic-infrastructure.yml +++ b/ansible/newrelic-infrastructure.yml @@ -10,4 +10,11 @@ become: yes tasks: - command: bash /app/bin/set-newrelic-license-key.sh + +- name: Restart newrelic + hosts: 127.0.0.1 + connection: local + become: yes + tasks: + - command: service newrelic-infra restart \ No newline at end of file From 6740dfd93b6d9c15769a7ef180a626ae037b95e1 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Tue, 29 Jan 2019 21:00:47 -0500 Subject: [PATCH 14/52] Move CodeDeploy build around to fix before / after When CodeDeploy was running, it was leaving a different set of files in /app than Ansible & Packer originally left there. There were also duplicate virtualenv files in /app/venv and /app/build/venv. This fixes that too. --- Jenkinsfile | 2 +- .../bin/build.sh => bin/build-codedeploy.sh | 26 +++++++++---------- .../bin => bin/codedeploy}/AfterInstall.sh | 0 .../codedeploy}/ApplicationStart.sh | 0 .../bin => bin/codedeploy}/ApplicationStop.sh | 0 .../bin => bin/codedeploy}/BeforeInstall.sh | 0 .../bin => bin/codedeploy}/ValidateService.sh | 0 codedeploy/appspec.yml | 10 +++---- 8 files changed, 19 insertions(+), 19 deletions(-) rename codedeploy/bin/build.sh => bin/build-codedeploy.sh (78%) rename {codedeploy/bin => bin/codedeploy}/AfterInstall.sh (100%) rename {codedeploy/bin => bin/codedeploy}/ApplicationStart.sh (100%) rename {codedeploy/bin => bin/codedeploy}/ApplicationStop.sh (100%) rename {codedeploy/bin => bin/codedeploy}/BeforeInstall.sh (100%) rename {codedeploy/bin => bin/codedeploy}/ValidateService.sh (100%) diff --git a/Jenkinsfile b/Jenkinsfile index 32495ca..e8bf12d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -182,7 +182,7 @@ stage('Build CodeDeploy Archive') { node { unstash 'src' wrap.call({ - sh ("./codedeploy/bin/build.sh") + sh ("./bin/build-codedeploy.sh") }) } } diff --git a/codedeploy/bin/build.sh b/bin/build-codedeploy.sh similarity index 78% rename from codedeploy/bin/build.sh rename to bin/build-codedeploy.sh index e61b0de..126fe6c 100755 --- a/codedeploy/bin/build.sh +++ b/bin/build-codedeploy.sh @@ -13,11 +13,9 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" BASE_DIR="$DIR/.." BUILD_DIR="$BASE_DIR/build" -ANSIBLE_DIR="$BASE_DIR/../ansible" -APPLICTION_DIR="$BASE_DIR/../application" -SRC_DIR="$BASE_DIR/../src" -VENV_DIR="$BASE_DIR/../venv" -DOCKER_DIR="$BASE_DIR/.." +ANSIBLE_DIR="$BASE_DIR/ansible" +APPLICTION_DIR="$BASE_DIR/application" +SRC_DIR="$BASE_DIR/src" GIT_REV="$(git rev-parse --short HEAD)" BUILD_NUMBER=${BUILD_NUMBER:-0} @@ -39,20 +37,22 @@ fi mkdir -p "$BUILD_DIR/socket" echo Build docker container $CONTAINERNAME -docker build -f=Dockerfile -t "$CONTAINERNAME" "$DOCKER_DIR" +docker build -f=Dockerfile -t "$CONTAINERNAME" "$BASE_DIR" echo Create python virtual environment -docker run --rm -v "$DOCKER_DIR:/src" "$CONTAINERNAME" /bin/bash -c \ - "mkdir -p /src/venv ; \ - cp -fa /app/venv/* /src/venv" +docker run \ + --rm \ + -v "$BASE_DIR:/src" \ + "$CONTAINERNAME" \ + /bin/bash -c \ + "mkdir -p /src/build/venv ; \ + cp -fa /app/venv/* /src/build/venv" SOURCES="$BASE_DIR/bin $ANSIBLE_DIR $APPLICTION_DIR $SRC_DIR -$BASE_DIR/appspec.yml -$BASE_DIR/bin -$VENV_DIR" +$BASE_DIR/codedeploy/appspec.yml" for src in $SOURCES; do cp -a "$src" "$BUILD_DIR" done @@ -70,7 +70,7 @@ done ) echo Remove docker generated files -docker run --rm -v "$DOCKER_DIR:/src" "$CONTAINERNAME" /bin/bash -c \ +docker run --rm -v "$BASE_DIR:/src" "$CONTAINERNAME" /bin/bash -c \ "rm -rf /src/venv" cd "$BUILD_DIR" diff --git a/codedeploy/bin/AfterInstall.sh b/bin/codedeploy/AfterInstall.sh similarity index 100% rename from codedeploy/bin/AfterInstall.sh rename to bin/codedeploy/AfterInstall.sh diff --git a/codedeploy/bin/ApplicationStart.sh b/bin/codedeploy/ApplicationStart.sh similarity index 100% rename from codedeploy/bin/ApplicationStart.sh rename to bin/codedeploy/ApplicationStart.sh diff --git a/codedeploy/bin/ApplicationStop.sh b/bin/codedeploy/ApplicationStop.sh similarity index 100% rename from codedeploy/bin/ApplicationStop.sh rename to bin/codedeploy/ApplicationStop.sh diff --git a/codedeploy/bin/BeforeInstall.sh b/bin/codedeploy/BeforeInstall.sh similarity index 100% rename from codedeploy/bin/BeforeInstall.sh rename to bin/codedeploy/BeforeInstall.sh diff --git a/codedeploy/bin/ValidateService.sh b/bin/codedeploy/ValidateService.sh similarity index 100% rename from codedeploy/bin/ValidateService.sh rename to bin/codedeploy/ValidateService.sh diff --git a/codedeploy/appspec.yml b/codedeploy/appspec.yml index 2538144..6c7275f 100644 --- a/codedeploy/appspec.yml +++ b/codedeploy/appspec.yml @@ -12,17 +12,17 @@ files: destination: /app/ansible hooks: ApplicationStop: - - location: bin/ApplicationStop.sh + - location: bin/codedeploy/ApplicationStop.sh timeout: 600 BeforeInstall: - - location: bin/BeforeInstall.sh + - location: bin/codedeploy/BeforeInstall.sh timeout: 30 AfterInstall: - - location: bin/AfterInstall.sh + - location: bin/codedeploy/AfterInstall.sh timeout: 300 ApplicationStart: - - location: bin/ApplicationStart.sh + - location: bin/codedeploy/ApplicationStart.sh timeout: 120 ValidateService: - - location: bin/ValidateService.sh + - location: bin/codedeploy/ValidateService.sh timeout: 60 From a118bb87cde258024493555d66277c219f7cffbf Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Tue, 29 Jan 2019 21:11:49 -0500 Subject: [PATCH 15/52] Clean root owned docker files before every step --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index e8bf12d..08c45f0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,6 +33,8 @@ def wrap = { fn-> variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE')]) { sh (""" cp env.sh.sample env.sh + . bin/common.sh + clean_root_owned_docker_files rm -rf build mkdir build """) From 1d187c8c75efc6b3e8677199876bbd5ba0c2c192 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Tue, 29 Jan 2019 21:18:25 -0500 Subject: [PATCH 16/52] Move clean workspace scriptlet to full script --- Jenkinsfile | 8 +------- bin/clean-workspace.sh | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) create mode 100755 bin/clean-workspace.sh diff --git a/Jenkinsfile b/Jenkinsfile index 08c45f0..a846962 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -31,13 +31,7 @@ def wrap = { fn-> variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE')]) { withCredentials([string(credentialsId: 'newrelic.license.key', variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE')]) { - sh (""" - cp env.sh.sample env.sh - . bin/common.sh - clean_root_owned_docker_files - rm -rf build - mkdir build - """) + sh ("bin/clean-workspace.sh") fn() } } diff --git a/bin/clean-workspace.sh b/bin/clean-workspace.sh new file mode 100755 index 0000000..a227196 --- /dev/null +++ b/bin/clean-workspace.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# clean workspace +# +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." +BUILD_DIR="$BASE_DIR/build" +export BASE_DIR + +# shellcheck disable=SC1090 +. "$DIR/common.sh" + +cp "$BASE_DIR/env.sh.sample" "$BASE_DIR/env.sh" +clean_root_owned_docker_files +rm -rf "$BUILD_DIR" +mkdir "$BUILD_DIR" From fc63fa7175d45ea866c7aa42f4b037c16479c033 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Tue, 29 Jan 2019 21:22:05 -0500 Subject: [PATCH 17/52] Fix up CodeDeploy BASE_DIR --- bin/codedeploy/AfterInstall.sh | 2 +- bin/codedeploy/ApplicationStart.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/codedeploy/AfterInstall.sh b/bin/codedeploy/AfterInstall.sh index 93d1db5..2fcbca4 100755 --- a/bin/codedeploy/AfterInstall.sh +++ b/bin/codedeploy/AfterInstall.sh @@ -16,7 +16,7 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' # Credit to http://stackoverflow.com/a/246128/424301 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -BASE_DIR="$DIR/.." +BASE_DIR="$DIR/../.." ANSIBLE_DIR="$BASE_DIR/ansible" # Invoke Ansible for final set up diff --git a/bin/codedeploy/ApplicationStart.sh b/bin/codedeploy/ApplicationStart.sh index 5d08139..c2698ca 100755 --- a/bin/codedeploy/ApplicationStart.sh +++ b/bin/codedeploy/ApplicationStart.sh @@ -16,7 +16,7 @@ export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' # Credit to http://stackoverflow.com/a/246128/424301 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -BASE_DIR="$DIR/.." +BASE_DIR="$DIR/../.." ANSIBLE_DIR="$BASE_DIR/ansible" # Invoke Ansible for final set up From 63c330e5145fa472a5c8bf9a435630c48d793cc0 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Tue, 29 Jan 2019 22:15:48 -0500 Subject: [PATCH 18/52] fix newrelic log location fix newrelic log location --- ansible/bakery.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/bakery.yml b/ansible/bakery.yml index 606dcc1..59827fe 100644 --- a/ansible/bakery.yml +++ b/ansible/bakery.yml @@ -14,7 +14,7 @@ nrinfragent_os_version: 7 nrinfragent_config: license_key: ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ - log_file: /var/log/newrelic/nr-infra.log + log_file: /var/log/newrelic-infra/nr-infra.log log_to_stdout: false - name: Install Web Application From 3230f3459a18e0a1bb64c9e92ed620684f5341a6 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Tue, 29 Jan 2019 23:28:12 -0500 Subject: [PATCH 19/52] Simplify by running more than one playbook --- terraform/cloud-config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index ec641ea..9e449bb 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,4 +1,3 @@ #cloud-config runcmd: - - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml - - ansible-playbook -l localhost /app/ansible/newrelic-infrastructure.yml + - ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml From 6768f8f80e4d2f3784387337facf5c3eb530e0b6 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Tue, 29 Jan 2019 23:41:10 -0500 Subject: [PATCH 20/52] Add start of New Relic terraform work --- Jenkinsfile | 15 ++++++++---- bin/terraform.sh | 16 ++++++++++++- env.sh.sample | 4 +++- terraform/newrelic.tf | 54 ++++++++++++++++++++++++++++++++++++++++++ terraform/variables.tf | 15 ++++++++++++ 5 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 terraform/newrelic.tf diff --git a/Jenkinsfile b/Jenkinsfile index 32495ca..c69e26b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,10 +27,17 @@ def get_captcha(Long hash_const) { def wrap = { fn-> ansiColor('xterm') { - withCredentials([file(credentialsId: 'terraform-demo.json', - variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE')]) { - withCredentials([string(credentialsId: 'newrelic.license.key', - variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE')]) { + withCredentials( + [ + file(credentialsId: 'terraform-demo.json', + variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'), + string(credentialsId: 'newrelic.license.key', + variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.api.key', + variable: 'NEWRELIC_API_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.alert.email', + variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'), + ]) { sh (""" cp env.sh.sample env.sh rm -rf build diff --git a/bin/terraform.sh b/bin/terraform.sh index 6555680..118e928 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -44,6 +44,16 @@ if [[ -n "$NEWRELIC_LICENSE_KEY_OVERRIDE" ]]; then echo "Overriding New Relic License Key" 1>&2 NEWRELIC_LICENSE_KEY="$NEWRELIC_LICENSE_KEY_OVERRIDE" fi +NEWRELIC_API_KEY_OVERRIDE=${NEWRELIC_API_KEY_OVERRIDE:-} +if [[ -n "$NEWRELIC_API_KEY_OVERRIDE" ]]; then + echo "Overriding New Relic API Key" 1>&2 + NEWRELIC_API_KEY="$NEWRELIC_API_KEY_OVERRIDE" +fi +NEWRELIC_ALERT_EMAIL_OVERRIDE=${NEWRELIC_ALERT_EMAIL_OVERRIDE:-} +if [[ -n "$NEWRELIC_ALERT_EMAIL_OVERRIDE" ]]; then + echo "Overriding New Relic Alert Email" 1>&2 + NEWRELIC_ALERT_EMAIL="$NEWRELIC_ALERT_EMAIL_OVERRIDE" +fi # Set up Google creds in build dir for docker terraform mkdir -p "$BUILD_DIR" @@ -61,6 +71,8 @@ cat <>"$ENV_FILE" GOOGLE_APPLICATION_CREDENTIALS=/app/build/google.json GOOGLE_PROJECT=$GOOGLE_PROJECT_OVERRIDE NEWRELIC_LICENSE_KEY=$NEWRELIC_LICENSE_KEY +NEWRELIC_API_KEY=$NEWRELIC_API_KEY +NEWRELIC_ALERT_EMAIL=$NEWRELIC_ALERT_EMAIL EOF # http://redsymbol.net/articles/bash-exit-traps/ @@ -84,7 +96,7 @@ function plan() { local output local -i retcode local targets - extra=${1:-} + extra="$*" output="$(mktemp)" targets=$(get_targets) @@ -97,6 +109,8 @@ function plan() { -input="$INPUT_ENABLED" \ -var project_name="$PROJECT_NAME" \ -var newrelic_license_key="$NEWRELIC_LICENSE_KEY" \ + -var newrelic_api_key="$NEWRELIC_API_KEY" \ + -var newrelic_alert_email="$NEWRELIC_ALERT_EMAIL" \ -var-file="/app/build/extra.tfvars" \ -out="$TF_PLAN" \ "$TF_DIR" \ diff --git a/env.sh.sample b/env.sh.sample index 63ec330..2cafc39 100644 --- a/env.sh.sample +++ b/env.sh.sample @@ -32,5 +32,7 @@ export GOOGLE_CLOUD_KEYFILE_JSON #export GOOGLE_PROJECT=example-terraform-demo-999999 export GOOGLE_REGION=us-east1 -# Fill in your New Relic license key +# Fill in your New Relic license key and API key export NEWRELIC_LICENSE_KEY=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ +export NEWRELIC_API_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +export NEWRELIC_ALERT_EMAIL=nobody@example.com diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf new file mode 100644 index 0000000..5718962 --- /dev/null +++ b/terraform/newrelic.tf @@ -0,0 +1,54 @@ +# Configure the New Relic provider + +# Adapted from https://www.terraform.io/docs/providers/newrelic/index.html + +provider "newrelic" { + api_key = "${var.newrelic_api_key}" +} + +# Create an alert policy +resource "newrelic_alert_policy" "alert" { + name = "Alert" +} + +# Add a condition +resource "newrelic_alert_condition" "spin-appdex" { + policy_id = "${newrelic_alert_policy.alert.id}" + + name = "spin-appdex" + type = "apm_app_metric" + entities = ["179953338"] # You can look this up in New Relic + metric = "apdex" + runbook_url = "https://github.com/devops-infra-demo/wiki/runbook" + + term { + duration = 5 + operator = "below" + priority = "critical" + threshold = "0.75" + time_function = "all" + } + + count = "${length(var.newrelic_apm_entities) > 0 ? 1 : 0}" +} + +# Add a notification channel +resource "newrelic_alert_channel" "email" { + name = "email" + type = "email" + + configuration = { + recipients = "richard+devops.infra.demo@moduscreate.com" + include_json_attachment = "1" + } + + count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" +} + +# Link the channel to the policy +resource "newrelic_alert_policy_channel" "alert_email" { + policy_id = "${newrelic_alert_policy.alert.id}" + channel_id = "${newrelic_alert_channel.email.id}" + + count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" +} diff --git a/terraform/variables.tf b/terraform/variables.tf index a6333b2..f8eadc2 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -85,3 +85,18 @@ variable "project_name" { variable "newrelic_license_key" { description = "New Relic license key" } + +variable "newrelic_api_key" { + description = "New Relic api key" +} + +variable "newrelic_apm_entities" { + description = "New Relic APM entity IDs" + type = "list" + default = [] +} + +variable "newrelic_alert_email" { + description = "New Relic alert email" + default = "" +} From 840741183a1678bb8eac5d235606c082b80000d7 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 00:05:34 -0500 Subject: [PATCH 21/52] Try to fix up wrap function nesting --- Jenkinsfile | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c69e26b..e358a8a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,27 +26,26 @@ def get_captcha(Long hash_const) { } def wrap = { fn-> - ansiColor('xterm') { - withCredentials( - [ - file(credentialsId: 'terraform-demo.json', - variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'), - string(credentialsId: 'newrelic.license.key', - variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'), - string(credentialsId: 'newrelic.api.key', - variable: 'NEWRELIC_API_KEY_OVERRIDE'), - string(credentialsId: 'newrelic.alert.email', - variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'), - ]) { - sh (""" - cp env.sh.sample env.sh - rm -rf build - mkdir build - """) - fn() - } + ansiColor('xterm') { + withCredentials( + [ + file(credentialsId: 'terraform-demo.json', + variable: 'GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE'), + string(credentialsId: 'newrelic.license.key', + variable: 'NEWRELIC_LICENSE_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.api.key', + variable: 'NEWRELIC_API_KEY_OVERRIDE'), + string(credentialsId: 'newrelic.alert.email', + variable: 'NEWRELIC_ALERT_EMAIL_OVERRIDE'), + ]) { + sh (""" + cp env.sh.sample env.sh + rm -rf build + mkdir build + """) + fn() } - } + } } final Long XOR_CONST = 3735928559 // 0xdeadbeef From 3214df9109765cfd2a348102acfa6bd9f6fd1a8b Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 00:15:34 -0500 Subject: [PATCH 22/52] Add more mandatory variables to validate for lint --- bin/validate.sh | 4 +++- terraform/newrelic.tf | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/validate.sh b/bin/validate.sh index e882a5a..3ce3976 100755 --- a/bin/validate.sh +++ b/bin/validate.sh @@ -31,7 +31,9 @@ echo "Linting terraform files for correctness" DOCKER_TERRAFORM=$(get_docker_terraform) init_terraform $DOCKER_TERRAFORM validate \ - -var 'newrelic_license_key=ZZZZ' + -var 'newrelic_license_key=ZZZZ' \ + -var 'newrelic_api_key=ZZZZ' \ + -var 'newrelic_alert_email=ferd.berferd@example.com' \ echo "Linting terraform files for formatting" fmt=$($DOCKER_TERRAFORM fmt) if [[ -n "$fmt" ]]; then diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 5718962..3d4730b 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -4,6 +4,7 @@ provider "newrelic" { api_key = "${var.newrelic_api_key}" + version = "~> 1.2" } # Create an alert policy From 0afe2436a9516646e4944ce3fbfd18a63b1dff0d Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 00:29:21 -0500 Subject: [PATCH 23/52] Get New Relic app ID dynamically --- bin/terraform.sh | 11 +++++++++++ env.sh.sample | 1 + 2 files changed, 12 insertions(+) diff --git a/bin/terraform.sh b/bin/terraform.sh index 118e928..fcd9af9 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -93,6 +93,7 @@ function output () { function plan() { local extra + local nr_app_id local output local -i retcode local targets @@ -101,6 +102,15 @@ function plan() { targets=$(get_targets) set +e + + nr_app_id=$(curl \ + -s \ + -X GET \ + 'https://api.newrelic.com/v2/applications.json' \ + -H "X-Api-Key:${NEWRELIC_API_KEY}" \ + -d "filter[name]=${NEWRELIC_APP_NAME}" \ + | jq '.applications[].id') + #shellcheck disable=SC2086 $DOCKER_TERRAFORM plan \ $extra \ @@ -111,6 +121,7 @@ function plan() { -var newrelic_license_key="$NEWRELIC_LICENSE_KEY" \ -var newrelic_api_key="$NEWRELIC_API_KEY" \ -var newrelic_alert_email="$NEWRELIC_ALERT_EMAIL" \ + -var newrelic_apm_entities="[$nr_app_id]" \ -var-file="/app/build/extra.tfvars" \ -out="$TF_PLAN" \ "$TF_DIR" \ diff --git a/env.sh.sample b/env.sh.sample index 2cafc39..4894803 100644 --- a/env.sh.sample +++ b/env.sh.sample @@ -36,3 +36,4 @@ export GOOGLE_REGION=us-east1 export NEWRELIC_LICENSE_KEY=ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ export NEWRELIC_API_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA export NEWRELIC_ALERT_EMAIL=nobody@example.com +export NEWRELIC_APP_NAME=Spin From 2e41388cc11510fc37e9ef245594f99b2d25b9c2 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 01:05:46 -0500 Subject: [PATCH 24/52] Add note about Jenkins server requiring jq --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ec6515f..30b6912 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Instructions * [Docker](https://docker.com/) (tested with 18.05.0-ce) * [Packer](https://www.packer.io/) (tested with 1.0.3) * [Terraform](https://www.terraform.io/) (tested with v0.11.7) +* [JQ](https://stedolan.github.io/jq/) (tested with 1.3 and 1.5) Optionally, you can use Vagrant to test ansible playbooks locally and Jenkins to orchestrate creation of AMIs in conjunction with GitHub branches and pull requests. @@ -116,6 +117,8 @@ A JMeter test harness that will allow testing of a the application A `Jenkinsfile` is provided that will allow Jenkins to execute Packer and Terraform, package a CodeDeploy application, and even run JMeter performance tests. In order for Jenkins to do this, it needs to have AWS credentials set up, preferably through an IAM role, granting full control of EC2 and VPC resources in that account, and write access to the S3 bucket used for storing CodeDeploy applications. Packer needs this in order to create AMIs, key pairs, etc, Terraform needs this to create a VPC and EC2 resources, and CodeDeploy needs this to store the artifact it creates. This could be pared down further through some careful logging and role work. +The Jenkins executor running this job needs to have both a recent Docker and the jq utility (version 1.3 or higher) installed. + The scripts here assume that Jenkins is running on EC2 and uses instance data from the Jenkins executor to infer what VPC and subnet to launch the new EC2 instance into. The AWS profile IAM user associated with your Jenkins instance or the Jenkins user's AWS credentials should have full control of EC2 in the account you are using. This script relies on Jenkins having a secret file containing the Google application credentials in JSON with the id "terraform-demo.json". You will need to add that to your Jenkins server's credentials. From cc7ba04643d51894649651d2bd7d10408fb0f2d3 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 01:25:17 -0500 Subject: [PATCH 25/52] Run ansible-playbook as centos Otherwise it can't find the ansible-galaxy stuff in /home/centos/.ansible --- ansible/newrelic-infrastructure.yml | 9 +++++---- terraform/cloud-config.yml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ansible/newrelic-infrastructure.yml b/ansible/newrelic-infrastructure.yml index b0ec75d..78e7568 100644 --- a/ansible/newrelic-infrastructure.yml +++ b/ansible/newrelic-infrastructure.yml @@ -11,10 +11,11 @@ tasks: - command: bash /app/bin/set-newrelic-license-key.sh -- name: Restart newrelic +- name: Restart newrelic-infra hosts: 127.0.0.1 connection: local become: yes - tasks: - - command: service newrelic-infra restart - \ No newline at end of file + service: + name: newerelic-infra + state: restarted + diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml index 9e449bb..984164f 100644 --- a/terraform/cloud-config.yml +++ b/terraform/cloud-config.yml @@ -1,3 +1,3 @@ #cloud-config runcmd: - - ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml + - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml /app/ansible/newrelic-infrastructure.yml From 776bf6bfcf123963b13bc508474facf5eb235024 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 01:59:28 -0500 Subject: [PATCH 26/52] condtion_scope is not really optional --- terraform/newrelic.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 3d4730b..7c57b69 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -30,6 +30,8 @@ resource "newrelic_alert_condition" "spin-appdex" { time_function = "all" } + condition_scope = "application" + count = "${length(var.newrelic_apm_entities) > 0 ? 1 : 0}" } From 06951826d857387a10dd8f8c91419ef23d178ca3 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 03:20:38 -0500 Subject: [PATCH 27/52] Get spin to slow down and fail when it is pummeled --- .../templates/infra-demo.ini.j2 | 2 +- src/spin.py | 22 ++++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 index f1f388b..699d286 100644 --- a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 +++ b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 @@ -3,7 +3,7 @@ venv = /app/venv wsgi-file = /app/src/wsgi.py chdir = /app/src master = 1 -workers = 150 +workers = 16 threads = 1 lazy-apps = 1 wsgi-env-behaviour = holy diff --git a/src/spin.py b/src/spin.py index 3a3eabc..6d393ef 100755 --- a/src/spin.py +++ b/src/spin.py @@ -4,15 +4,17 @@ import time import newrelic.agent import random -from bottle import route, default_app, response +from bottle import route, default_app, response, HTTPError newrelic_ini = '../newrelic.ini' if os.path.isfile(newrelic_ini): newrelic.agent.initialize(newrelic_ini) + @route('/spin') def spin(delay=0.1): """Spin the CPU, return the process id at the end""" + spin.invocations += 1 child_pid = os.getpid() upper_max = 100000000000000000000000000000000 @@ -23,19 +25,27 @@ def spin(delay=0.1): pareto_factor = random.paretovariate(alpha) start_time = time.time() + current_time = start_time scratch = 42 + int(current_time) - end_time = start_time + (delay * pareto_factor) + end_time = start_time + pareto_factor * (delay + (delay * 10 / (current_time - spin.last_time))) + time_limit = start_time + (delay * 50) calcs = 0 while current_time < end_time: calcs += 1 scratch = (scratch * scratch) % upper_max current_time = time.time() - final_time = time.time() - interval = final_time - start_time + interval = current_time - start_time + if current_time > time_limit: + raise HTTPError(500, "Allowed transaction time exceeded ({} ms elapsed)".format(interval)) + spin.last_time = current_time rate = calcs / interval response.set_header('Content-Type', 'text/plain') - return ('pid {0} spun {1} times over {2}s (rate {3}/s)\n' - .format(child_pid, calcs, interval, rate)) + return ('pid {0} spun {1} times over {2}s (rate {3} invoked {4} times/s)\n' + .format(child_pid, calcs, interval, rate, spin.invocations)) + +spin.invocations = 0 +spin.last_time = time.time() - 10 +spin.slowdown = 0 application = default_app() From d2915bc58eac2b3adeb1b505f5d08b23248af1ad Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 03:36:41 -0500 Subject: [PATCH 28/52] Bump up max allowed transaction time --- src/spin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/spin.py b/src/spin.py index 6d393ef..8ea72cc 100755 --- a/src/spin.py +++ b/src/spin.py @@ -28,8 +28,9 @@ def spin(delay=0.1): current_time = start_time scratch = 42 + int(current_time) - end_time = start_time + pareto_factor * (delay + (delay * 10 / (current_time - spin.last_time))) - time_limit = start_time + (delay * 50) + congestion_slowdown = delay * 10 / (current_time - spin.last_time) + end_time = start_time + (delay + congestion_slowdown) * pareto_factor + time_limit = start_time + (delay * 100) calcs = 0 while current_time < end_time: calcs += 1 From f9a636539c21eca126cc8e394eab598a66a708fd Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 06:23:14 -0500 Subject: [PATCH 29/52] Increase number of workers for next load test --- ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 index 699d286..f579f0b 100644 --- a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 +++ b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 @@ -3,7 +3,7 @@ venv = /app/venv wsgi-file = /app/src/wsgi.py chdir = /app/src master = 1 -workers = 16 +workers = 64 threads = 1 lazy-apps = 1 wsgi-env-behaviour = holy From 3040ecce670d8624ec4f66fde942e4ebe131271b Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 06:29:26 -0500 Subject: [PATCH 30/52] Bump instance size up to c5.large It is actually cheaper than a t2.large... --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index f8eadc2..d5936c0 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -33,7 +33,7 @@ variable "host" { variable "instance_type" { description = "EC2 instance type for servers" - default = "t2.large" + default = "c5.large" } # Public key variables - you can specify another file for the public key From cfa1bd918b41ecf0c08ea32d3f69d9316e09f94b Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 06:34:30 -0500 Subject: [PATCH 31/52] Fix mismatch in Jenkins num_threads --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 859efb1..a6c1013 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -248,7 +248,7 @@ if (params.Run_JMeter) { wrap.call({ sh (""" HOST=\$(./bin/terraform.sh output route53-dns) - ./bin/jmeter.sh -Jthreads=${params.JMETER_threads} -Jramp_duration=${params.JMETER_ramp_duration} -Jduration=${params.JMETER_duration} -Jhost=\$HOST + ./bin/jmeter.sh -Jnum_threads=${params.JMETER_threads} -Jramp_duration=${params.JMETER_ramp_duration} -Jduration=${params.JMETER_duration} -Jhost=\$HOST ls -l build """) archiveArtifacts artifacts: 'build/*.jtl, build/*.xml, build/*.csv, build/*.html', fingerprint: true From c97ddf1395b6eb614f3978e11bb47769b8100f9d Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 06:57:12 -0500 Subject: [PATCH 32/52] All the JMeter things have to match up Otherwise the tests are invalid --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a6c1013..c0f6a61 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,12 +92,12 @@ properties([ description: "Execute a JMeter load test against the stack" ), string( - name: 'JMETER_threads', + name: 'JMETER_num_threads', defaultValue: '2', description: "number of jmeter threads." ), string( - name: 'JMETER_ramp_duration', + name: 'JMETER_ramp_time', defaultValue: '900', description: 'period in seconds of ramp-up time.' ), @@ -248,7 +248,7 @@ if (params.Run_JMeter) { wrap.call({ sh (""" HOST=\$(./bin/terraform.sh output route53-dns) - ./bin/jmeter.sh -Jnum_threads=${params.JMETER_threads} -Jramp_duration=${params.JMETER_ramp_duration} -Jduration=${params.JMETER_duration} -Jhost=\$HOST + ./bin/jmeter.sh -Jnum_threads=${params.JMETER_num_threads} -Jramp_time=${params.JMETER_ramp_time} -Jduration=${params.JMETER_duration} -Jhost=\$HOST ls -l build """) archiveArtifacts artifacts: 'build/*.jtl, build/*.xml, build/*.csv, build/*.html', fingerprint: true From f1f7f57ab452205f17ae4d614af3621c06faa194 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 07:21:59 -0500 Subject: [PATCH 33/52] Fix up runbook URL, use var for New Relic entities --- terraform/newrelic.tf | 4 ++-- terraform/variables.tf | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 7c57b69..f4a76ad 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -18,9 +18,9 @@ resource "newrelic_alert_condition" "spin-appdex" { name = "spin-appdex" type = "apm_app_metric" - entities = ["179953338"] # You can look this up in New Relic + entities = "${var.newrelic_apm_entities}" metric = "apdex" - runbook_url = "https://github.com/devops-infra-demo/wiki/runbook" + runbook_url = "${var.newrelic_runbook_url}" term { duration = 5 diff --git a/terraform/variables.tf b/terraform/variables.tf index d5936c0..0d8f3f0 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -96,6 +96,11 @@ variable "newrelic_apm_entities" { default = [] } +variable "newrelic_runbook_url" { + description = "New Relic runbook URL" + default = "https://github.com/ModusCreateOrg/devops-infra-demo/wiki/runbook" +} + variable "newrelic_alert_email" { description = "New Relic alert email" default = "" From f404c2ea2bea37ac8cbdd8d45a5991abc8bd9713 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 07:23:43 -0500 Subject: [PATCH 34/52] Fix terraform fmt --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 0d8f3f0..ba95f70 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -98,7 +98,7 @@ variable "newrelic_apm_entities" { variable "newrelic_runbook_url" { description = "New Relic runbook URL" - default = "https://github.com/ModusCreateOrg/devops-infra-demo/wiki/runbook" + default = "https://github.com/ModusCreateOrg/devops-infra-demo/wiki/runbook" } variable "newrelic_alert_email" { From 9c1737143acbeb97a3187fb0b281c0df6bd5cd2a Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 10:39:32 -0500 Subject: [PATCH 35/52] Add node to spin output, reformat with black See https://github.com/ambv/black --- src/run.py | 2 +- src/spin.py | 43 ++++++++++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/run.py b/src/run.py index 81ba46c..5dac1b1 100755 --- a/src/run.py +++ b/src/run.py @@ -5,4 +5,4 @@ from spin import spin from bottle import run, default_app - run(host='localhost', port=8080, debug=True) + run(host="localhost", port=8080, debug=True) diff --git a/src/spin.py b/src/spin.py index 8ea72cc..c80efc9 100755 --- a/src/spin.py +++ b/src/spin.py @@ -2,17 +2,19 @@ """This module spins the CPU.""" import os import time -import newrelic.agent import random +import socket +import urllib2 +import newrelic.agent from bottle import route, default_app, response, HTTPError -newrelic_ini = '../newrelic.ini' -if os.path.isfile(newrelic_ini): - newrelic.agent.initialize(newrelic_ini) +NEWRELIC_INI = "../newrelic.ini" +if os.path.isfile(NEWRELIC_INI): + newrelic.agent.initialize(NEWRELIC_INI) -@route('/spin') -def spin(delay=0.1): +@route("/spin") +def spin(delay=0.1, max_duration=10.0, simulate_congestion=True): """Spin the CPU, return the process id at the end""" spin.invocations += 1 child_pid = os.getpid() @@ -25,12 +27,13 @@ def spin(delay=0.1): pareto_factor = random.paretovariate(alpha) start_time = time.time() - current_time = start_time scratch = 42 + int(current_time) - congestion_slowdown = delay * 10 / (current_time - spin.last_time) + congestion_slowdown = 0.0 + if simulate_congestion: + congestion_slowdown = delay * 10 / (current_time - spin.last_time) end_time = start_time + (delay + congestion_slowdown) * pareto_factor - time_limit = start_time + (delay * 100) + time_limit = start_time + (max_duration) calcs = 0 while current_time < end_time: calcs += 1 @@ -38,15 +41,29 @@ def spin(delay=0.1): current_time = time.time() interval = current_time - start_time if current_time > time_limit: - raise HTTPError(500, "Allowed transaction time exceeded ({} ms elapsed)".format(interval)) + raise HTTPError( + 500, + "Allowed transaction time exceeded ({} ms elapsed)".format(interval), + ) spin.last_time = current_time rate = calcs / interval - response.set_header('Content-Type', 'text/plain') - return ('pid {0} spun {1} times over {2}s (rate {3} invoked {4} times/s)\n' - .format(child_pid, calcs, interval, rate, spin.invocations)) + response.set_header("Content-Type", "text/plain") + return "node {0} pid {1} spun {2} times over {3}s (rate {4} invoked {5} times/s)\n".format( + spin.node, child_pid, calcs, interval, rate, spin.invocations + ) + spin.invocations = 0 spin.last_time = time.time() - 10 spin.slowdown = 0 +try: + # Thanks stack overflow https://stackoverflow.com/a/43816449/424301 + spin.node = urllib2.urlopen( + "http://169.254.169.254/latest/meta-data/instance-id", timeout=1 + ).read() +# Thanks stack overflow https://stackoverflow.com/questions/2712524/handling-urllib2s-timeout-python +except urllib2.URLError: + # Thanks stack overflow: https://stackoverflow.com/a/4271755/424301 + spin.node = socket.gethostname() application = default_app() From 80ed96a7b7aa42d6956bbe7938fd77d94d08d98d Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 10:45:32 -0500 Subject: [PATCH 36/52] Add dashboard Add dashboard --- terraform/newrelic.tf | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index f4a76ad..7e0df54 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -55,3 +55,25 @@ resource "newrelic_alert_policy_channel" "alert_email" { count = "${length(var.newrelic_alert_email) > 0 ? 1 : 0}" } + +# Add a dashboard +resource "newrelic_dashboard" "spindash" { + title = "Spin Dashboard" + + widget { + title = "Average Transaction Duration" + row = 1 + column = 1 + width = 2 + visualization = "faceted_line_chart" + nrql = "SELECT AVERAGE(duration) from Transaction FACET appName TIMESERIES auto" + } + + widget { + title = "Page Views" + row = 1 + column = 3 + visualization = "billboard" + nrql = "SELECT count(*) FROM PageView SINCE 1 week ago" + } +} From 8d4233834b390ccdb5524f0e38d7fd1402a9b4fc Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 11:24:35 -0500 Subject: [PATCH 37/52] more widgets more widgets --- terraform/newrelic.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 7e0df54..85839d6 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -69,6 +69,24 @@ resource "newrelic_dashboard" "spindash" { nrql = "SELECT AVERAGE(duration) from Transaction FACET appName TIMESERIES auto" } + widget { + title = "Average Apdex" + row = 1 + column = 1 + width = 2 + visualization = "faceted_line_chart" + nrql = "SELECT apdex(duration, t: 0.4) from Transaction FACET appName TIMESERIES auto" + } + + widget { + title = "Average CPU Percent" + row = 1 + column = 1 + width = 2 + visualization = "faceted_line_chart" + nrql = "SELECT average(cpuPercent) FROM SystemSample SINCE 1 HOUR AGO COMPARE WITH 3 HOUR AGO TIMESERIES" + } + widget { title = "Page Views" row = 1 From 5c953182fc78aa6d9f443538b6f8ec8254b1676c Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 11:29:33 -0500 Subject: [PATCH 38/52] more widgets more widgets --- terraform/newrelic.tf | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 85839d6..271fbbc 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -70,8 +70,17 @@ resource "newrelic_dashboard" "spindash" { } widget { - title = "Average Apdex" - row = 1 + title = "Average Apdex 1" + row = 2 + column = 1 + width = 2 + visualization = "faceted_line_chart" + nrql = "SELECT apdex(duration, 0.5) FROM Transaction SINCE 3 HOURS AGO COMPARE WITH 6 HOURS AGO TIMESERIES" + } + + widget { + title = "Average Apdex 2" + row = 3 column = 1 width = 2 visualization = "faceted_line_chart" @@ -80,18 +89,18 @@ resource "newrelic_dashboard" "spindash" { widget { title = "Average CPU Percent" - row = 1 + row = 4 column = 1 width = 2 visualization = "faceted_line_chart" nrql = "SELECT average(cpuPercent) FROM SystemSample SINCE 1 HOUR AGO COMPARE WITH 3 HOUR AGO TIMESERIES" } - widget { - title = "Page Views" - row = 1 - column = 3 - visualization = "billboard" - nrql = "SELECT count(*) FROM PageView SINCE 1 week ago" - } +# widget { +# title = "Page Views" +# row = 1 +# column = 3 +# visualization = "billboard" +# nrql = "SELECT count(*) FROM PageView SINCE 1 week ago" +# } } From e0af2c0fa0cf1a05294a8d7e5b8d4dcd0f66e0b3 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 11:30:50 -0500 Subject: [PATCH 39/52] syntax syntax --- terraform/newrelic.tf | 7 ------- 1 file changed, 7 deletions(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 271fbbc..aa31a8f 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -96,11 +96,4 @@ resource "newrelic_dashboard" "spindash" { nrql = "SELECT average(cpuPercent) FROM SystemSample SINCE 1 HOUR AGO COMPARE WITH 3 HOUR AGO TIMESERIES" } -# widget { -# title = "Page Views" -# row = 1 -# column = 3 -# visualization = "billboard" -# nrql = "SELECT count(*) FROM PageView SINCE 1 week ago" -# } } From c9614f0724d20b2921614071e52d0dc591eeb5e4 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 11:38:34 -0500 Subject: [PATCH 40/52] syntax syntax --- terraform/newrelic.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index aa31a8f..11225d3 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -95,5 +95,5 @@ resource "newrelic_dashboard" "spindash" { visualization = "faceted_line_chart" nrql = "SELECT average(cpuPercent) FROM SystemSample SINCE 1 HOUR AGO COMPARE WITH 3 HOUR AGO TIMESERIES" } - + } From c1c30fff981d1e4de21d679d298c7544656b9612 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 11:41:39 -0500 Subject: [PATCH 41/52] Fix up terraform fmt issue --- terraform/newrelic.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 11225d3..09c9330 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -95,5 +95,4 @@ resource "newrelic_dashboard" "spindash" { visualization = "faceted_line_chart" nrql = "SELECT average(cpuPercent) FROM SystemSample SINCE 1 HOUR AGO COMPARE WITH 3 HOUR AGO TIMESERIES" } - } From 5caab1fc1a6a78a5c2c32673acfc53fede87e7a6 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 11:55:51 -0500 Subject: [PATCH 42/52] removed widget removed widget --- terraform/newrelic.tf | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 09c9330..97fff20 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -70,16 +70,7 @@ resource "newrelic_dashboard" "spindash" { } widget { - title = "Average Apdex 1" - row = 2 - column = 1 - width = 2 - visualization = "faceted_line_chart" - nrql = "SELECT apdex(duration, 0.5) FROM Transaction SINCE 3 HOURS AGO COMPARE WITH 6 HOURS AGO TIMESERIES" - } - - widget { - title = "Average Apdex 2" + title = "Average Apdex" row = 3 column = 1 width = 2 From 7bbbb9c57d5415c015432a95a4233438ecab1964 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 13:10:46 -0500 Subject: [PATCH 43/52] save save --- terraform/newrelic.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 97fff20..1c88b58 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -71,7 +71,7 @@ resource "newrelic_dashboard" "spindash" { widget { title = "Average Apdex" - row = 3 + row = 2 column = 1 width = 2 visualization = "faceted_line_chart" @@ -80,7 +80,7 @@ resource "newrelic_dashboard" "spindash" { widget { title = "Average CPU Percent" - row = 4 + row = 3 column = 1 width = 2 visualization = "faceted_line_chart" From 520a0c8cdd03fd426999848cfda9c906eda7ec8d Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 13:13:07 -0500 Subject: [PATCH 44/52] simplify CPU simplify CPU --- terraform/newrelic.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 1c88b58..ae66f0e 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -84,6 +84,6 @@ resource "newrelic_dashboard" "spindash" { column = 1 width = 2 visualization = "faceted_line_chart" - nrql = "SELECT average(cpuPercent) FROM SystemSample SINCE 1 HOUR AGO COMPARE WITH 3 HOUR AGO TIMESERIES" + nrql = "SELECT average(cpuPercent) FROM SystemSample SINCE 1 HOUR AGO TIMESERIES" } } From 035c1acc488387dc52456501f4043f9c048c342e Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 13:14:13 -0500 Subject: [PATCH 45/52] Simplify query --- terraform/newrelic.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 1c88b58..c45dc2a 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -84,6 +84,6 @@ resource "newrelic_dashboard" "spindash" { column = 1 width = 2 visualization = "faceted_line_chart" - nrql = "SELECT average(cpuPercent) FROM SystemSample SINCE 1 HOUR AGO COMPARE WITH 3 HOUR AGO TIMESERIES" + nrql = "SELECT average(cpuPercent) FROM SystemSample TIMESERIES auto" } } From 952b25d1d4525590960e80359b61cb36cf20e512 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 13:22:47 -0500 Subject: [PATCH 46/52] Tweak chart type --- terraform/newrelic.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index c45dc2a..b187435 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -82,8 +82,9 @@ resource "newrelic_dashboard" "spindash" { title = "Average CPU Percent" row = 3 column = 1 + height = 1 width = 2 - visualization = "faceted_line_chart" + visualization = "line_chart" nrql = "SELECT average(cpuPercent) FROM SystemSample TIMESERIES auto" } } From d6ca53d9c998e132e2cf8ba18de386760f9f5c12 Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 13:49:37 -0500 Subject: [PATCH 47/52] Make uwsgi processes more mortal Force them to die if they take more than 120 seconds Force them to process a max number of requests --- ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 index f579f0b..0afc0b8 100644 --- a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 +++ b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 @@ -11,4 +11,6 @@ enable-threads = 0 http-auto-chunked = 1 http-keepalive = 1 uwsgi-socket = 127.0.0.1:8008 - +harakiri = 120 +harakiri-verbose +max-requests = 2048 From 632bcfec27de448b8eefb95950aa62b840f53670 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 14:36:53 -0500 Subject: [PATCH 48/52] throughput throughput --- terraform/newrelic.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index b187435..8498ab0 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -87,4 +87,14 @@ resource "newrelic_dashboard" "spindash" { visualization = "line_chart" nrql = "SELECT average(cpuPercent) FROM SystemSample TIMESERIES auto" } + + widget { + title = "Throughput" + row = 4 + column = 1 + height = 1 + width = 2 + visualization = "line_chart" + nrql = "SELECT count(*) AS 'rpm' FROM Transaction WHERE appName = 'Spin' FACET host SINCE 30 minutes ago TIMESERIES 1 minute" + } } From 4db7b7283335d5ce1aff429af18e7b987b265f2f Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 14:52:18 -0500 Subject: [PATCH 49/52] new widget new widget --- terraform/newrelic.tf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 8498ab0..f7862dc 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -88,6 +88,16 @@ resource "newrelic_dashboard" "spindash" { nrql = "SELECT average(cpuPercent) FROM SystemSample TIMESERIES auto" } + widget { + title = "Throughput faceted" + row = 4 + column = 1 + height = 1 + width = 2 + visualization = "faceted_line_chart" + nrql = "SELECT count(*) AS 'rpm' FROM Transaction WHERE appName = 'Spin' FACET host SINCE 30 minutes ago TIMESERIES 1 minute" + } + widget { title = "Throughput" row = 4 @@ -95,6 +105,6 @@ resource "newrelic_dashboard" "spindash" { height = 1 width = 2 visualization = "line_chart" - nrql = "SELECT count(*) AS 'rpm' FROM Transaction WHERE appName = 'Spin' FACET host SINCE 30 minutes ago TIMESERIES 1 minute" + nrql = "SELECT count(*) AS 'rpm' FROM Transaction WHERE appName = 'Spin' SINCE 30 minutes ago TIMESERIES 1 minute" } } From ed8fd330369d368a33078f83c1769ca820175bb8 Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 14:58:36 -0500 Subject: [PATCH 50/52] faceted --- terraform/newrelic.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index f7862dc..247dbab 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -89,7 +89,7 @@ resource "newrelic_dashboard" "spindash" { } widget { - title = "Throughput faceted" + title = "Throughput (faceted)" row = 4 column = 1 height = 1 From ff7df3a4c66b2b2eeb853cb78f41d16917e1d21c Mon Sep 17 00:00:00 2001 From: dranderson <4012994+dranderson@users.noreply.github.com> Date: Wed, 30 Jan 2019 15:01:23 -0500 Subject: [PATCH 51/52] renamed --- terraform/newrelic.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/newrelic.tf b/terraform/newrelic.tf index 247dbab..518e193 100644 --- a/terraform/newrelic.tf +++ b/terraform/newrelic.tf @@ -89,7 +89,7 @@ resource "newrelic_dashboard" "spindash" { } widget { - title = "Throughput (faceted)" + title = "Throughput (by host)" row = 4 column = 1 height = 1 @@ -99,7 +99,7 @@ resource "newrelic_dashboard" "spindash" { } widget { - title = "Throughput" + title = "Throughput (total)" row = 4 column = 1 height = 1 From 4d16fa361146a1a7d0b7b3abccd02ff14a0073dc Mon Sep 17 00:00:00 2001 From: Richard Bullington-McGuire Date: Wed, 30 Jan 2019 17:08:24 -0500 Subject: [PATCH 52/52] Tweak congestion slowdown to be less acute --- src/spin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/spin.py b/src/spin.py index c80efc9..5aadbf2 100755 --- a/src/spin.py +++ b/src/spin.py @@ -14,7 +14,7 @@ @route("/spin") -def spin(delay=0.1, max_duration=10.0, simulate_congestion=True): +def spin(delay=0.05, max_duration=10.0, simulate_congestion=True): """Spin the CPU, return the process id at the end""" spin.invocations += 1 child_pid = os.getpid() @@ -31,7 +31,7 @@ def spin(delay=0.1, max_duration=10.0, simulate_congestion=True): scratch = 42 + int(current_time) congestion_slowdown = 0.0 if simulate_congestion: - congestion_slowdown = delay * 10 / (current_time - spin.last_time) + congestion_slowdown = delay * 2 / (current_time - spin.last_time) end_time = start_time + (delay + congestion_slowdown) * pareto_factor time_limit = start_time + (max_duration) calcs = 0 @@ -48,8 +48,8 @@ def spin(delay=0.1, max_duration=10.0, simulate_congestion=True): spin.last_time = current_time rate = calcs / interval response.set_header("Content-Type", "text/plain") - return "node {0} pid {1} spun {2} times over {3}s (rate {4} invoked {5} times/s)\n".format( - spin.node, child_pid, calcs, interval, rate, spin.invocations + return "node {0} pid {1} spun {2} times over {3}s (rate {4} invoked {5} times. Congestion slowdown {6}s)\n".format( + spin.node, child_pid, calcs, interval, rate, spin.invocations, congestion_slowdown )