diff --git a/packer/conf/bin/bk-install-elastic-stack.sh b/packer/conf/bin/bk-install-elastic-stack.sh index 5ca977b22..e752f08af 100755 --- a/packer/conf/bin/bk-install-elastic-stack.sh +++ b/packer/conf/bin/bk-install-elastic-stack.sh @@ -108,7 +108,7 @@ EOF cat << EOF > /etc/systemd/system/buildkite-agent@.service.d/10-power-off-stop.conf [Service] -ExecStopPost=/usr/local/bin/mark-asg-unhealthy +ExecStopPost=/usr/local/bin/terminate-instance ${BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB_DECREASE_DESIRED_CAPACITY} ExecStopPost=/bin/sudo poweroff EOF fi diff --git a/packer/conf/buildkite-agent/scripts/mark-asg-unhealthy b/packer/conf/buildkite-agent/scripts/mark-asg-unhealthy deleted file mode 100755 index f9dafab91..000000000 --- a/packer/conf/buildkite-agent/scripts/mark-asg-unhealthy +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -instance_id=$(curl -fsSL http://169.254.169.254/latest/meta-data/instance-id) -region=$(curl -fsSL http://169.254.169.254/latest/meta-data/placement/availability-zone | head -c -1) - -aws autoscaling set-instance-health --instance-id "$instance_id" --region "$region" --health-status "Unhealthy" diff --git a/packer/conf/buildkite-agent/scripts/terminate-instance b/packer/conf/buildkite-agent/scripts/terminate-instance new file mode 100755 index 000000000..ed847f8ea --- /dev/null +++ b/packer/conf/buildkite-agent/scripts/terminate-instance @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +if [[ "${1:-false}" == "true" ]] ; then + should_decrement="should" +else + should_decrement="no-should" +fi + +instance_id=$(curl -fsSL http://169.254.169.254/latest/meta-data/instance-id) +region=$(curl -fsSL http://169.254.169.254/latest/meta-data/placement/availability-zone | head -c -1) + +aws autoscaling terminate-instance-in-auto-scaling-group --region "$region" --instance-id "$instance_id" "--${should_decrement}-decrement-desired-capacity" diff --git a/packer/scripts/install-buildkite-agent.sh b/packer/scripts/install-buildkite-agent.sh index 92dd14f52..42736643f 100755 --- a/packer/scripts/install-buildkite-agent.sh +++ b/packer/scripts/install-buildkite-agent.sh @@ -51,7 +51,7 @@ sudo cp /tmp/conf/buildkite-agent/systemd/buildkite-agent@.service /etc/systemd/ echo "Adding termination scripts..." sudo cp /tmp/conf/buildkite-agent/scripts/stop-agent-gracefully /usr/local/bin/stop-agent-gracefully -sudo cp /tmp/conf/buildkite-agent/scripts/mark-asg-unhealthy /usr/local/bin/mark-asg-unhealthy +sudo cp /tmp/conf/buildkite-agent/scripts/terminate-instance /usr/local/bin/terminate-instance echo "Copying built-in plugins..." sudo mkdir -p /usr/local/buildkite-aws-stack/plugins diff --git a/templates/aws-stack.yml b/templates/aws-stack.yml index aa231efb0..88fbc3c8e 100644 --- a/templates/aws-stack.yml +++ b/templates/aws-stack.yml @@ -21,6 +21,7 @@ Metadata: - BuildkiteAgentExperiments - BuildkiteTerminateInstanceAfterJob - BuildkiteTerminateInstanceAfterJobTimeout + - BuildkiteTerminateInstanceAfterJobDecreaseDesiredCapacity - Label: default: Network Configuration @@ -127,7 +128,7 @@ Parameters: Default: "" BuildkiteTerminateInstanceAfterJob: - Description: Whether or not to terminate the instance after the job has completed. + Description: Set to "true" to terminate the instance after a job has completed. Type: String AllowedValues: - "true" @@ -140,6 +141,14 @@ Parameters: Default: 1800 MinValue: 1 + BuildkiteTerminateInstanceAfterJobDecreaseDesiredCapacity: + Description: Set to "true" to decrease the desired capacity of the autoscaling group by 1 when terminating an instance after the job completes. + Type: String + AllowedValues: + - "true" + - "false" + Default: "false" + BuildkiteQueue: Description: Queue name that agents will use, targeted in pipeline steps using "queue={value}" Type: String @@ -715,6 +724,7 @@ Resources: BUILDKITE_LIFECYCLE_TOPIC=${AgentLifecycleTopic} \ BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB=${BuildkiteTerminateInstanceAfterJob} \ BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB_TIMEOUT=${BuildkiteTerminateInstanceAfterJobTimeout} \ + BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB_DECREASE_DESIRED_CAPACITY=${BuildkiteTerminateInstanceAfterJobDecreaseDesiredCapacity} \ AWS_DEFAULT_REGION=${AWS::Region} \ SECRETS_PLUGIN_ENABLED=${EnableSecretsPlugin} \ ECR_PLUGIN_ENABLED=${EnableECRPlugin} \