Terminate the instance directly rather than mark it unhealthy

This will allow us to have more control over the timing and capacity
of the ASG.

Signed-off-by: Tom Duffield <tom@chef.io>

packer/conf/bin/bk-install-elastic-stack.sh

@@ -108,7 +108,7 @@ EOF
 
   cat << EOF > /etc/systemd/system/buildkite-agent@.service.d/10-power-off-stop.conf
 [Service]
-ExecStopPost=/usr/local/bin/mark-asg-unhealthy
+ExecStopPost=/usr/local/bin/terminate-instance ${BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB_DECREASE_DESIRED_CAPACITY}
 ExecStopPost=/bin/sudo poweroff
 EOF
 fi

packer/conf/buildkite-agent/scripts/terminate-instance

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ "${1:-false}" == "true" ]] ; then
+  should_decrement="should"
+else
+  should_decrement="no-should"
+fi
+
+instance_id=$(curl -fsSL http://169.254.169.254/latest/meta-data/instance-id)
+region=$(curl -fsSL http://169.254.169.254/latest/meta-data/placement/availability-zone | head -c -1)
+
+aws autoscaling terminate-instance-in-auto-scaling-group --region "$region" --instance-id "$instance_id" "--${should_decrement}-decrement-desired-capacity"

packer/scripts/install-buildkite-agent.sh

@@ -51,7 +51,7 @@ sudo cp /tmp/conf/buildkite-agent/systemd/buildkite-agent@.service /etc/systemd/
 
 echo "Adding termination scripts..."
 sudo cp /tmp/conf/buildkite-agent/scripts/stop-agent-gracefully /usr/local/bin/stop-agent-gracefully
-sudo cp /tmp/conf/buildkite-agent/scripts/mark-asg-unhealthy /usr/local/bin/mark-asg-unhealthy
+sudo cp /tmp/conf/buildkite-agent/scripts/terminate-instance /usr/local/bin/terminate-instance
 
 echo "Copying built-in plugins..."
 sudo mkdir -p /usr/local/buildkite-aws-stack/plugins

templates/aws-stack.yml

@@ -21,6 +21,7 @@ Metadata:
         - BuildkiteAgentExperiments
         - BuildkiteTerminateInstanceAfterJob
         - BuildkiteTerminateInstanceAfterJobTimeout
+        - BuildkiteTerminateInstanceAfterJobShouldDecreaseCapacity
 
       - Label:
           default: Network Configuration
@@ -127,7 +128,7 @@ Parameters:
     Default: ""
 
   BuildkiteTerminateInstanceAfterJob:
-    Description: Whether or not to terminate the instance after the job has completed.
+    Description: Set to "true" to terminate the instance after a job has completed.
     Type: String
     AllowedValues:
       - "true"
@@ -140,6 +141,14 @@ Parameters:
     Default: 1800
     MinValue: 1
 
+  BuildkiteTerminateInstanceAfterJobDecreaseDesiredCapacity:
+    Description: Set to "true" to decrease the desired capacity of the autoscaling group by 1 when terminating an instance after the job completes.
+    Type: String
+    AllowedValues:
+      - "true"
+      - "false"
+    Default: "false"
+
   BuildkiteQueue:
     Description: Queue name that agents will use, targeted in pipeline steps using "queue={value}"
     Type: String
@@ -715,6 +724,7 @@ Resources:
             BUILDKITE_LIFECYCLE_TOPIC=${AgentLifecycleTopic} \
             BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB=${BuildkiteTerminateInstanceAfterJob} \
             BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB_TIMEOUT=${BuildkiteTerminateInstanceAfterJobTimeout} \
+            BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB_DECREASE_DESIRED_CAPACITY=${BuildkiteTerminateInstanceAfterJobDecreaseDesiredCapacity} \
             AWS_DEFAULT_REGION=${AWS::Region} \
             SECRETS_PLUGIN_ENABLED=${EnableSecretsPlugin} \
             ECR_PLUGIN_ENABLED=${EnableECRPlugin} \