Initial script to start removing zombie build instances. (#11088)

* Set up script to remove old instances.

* Switch to push for testing.

* Debugging.

* Authenticate.

* Try again.

* Set region.

* Add JQ.

* Add iso check.

* Add curr time. Add JQ query.

* Correctly set env.

* Format command.

* Format command.

* Should be inverted.

* Only get running.

* Filter for instance name.

* Space instead of comma.

* Debug.

* Put back running instance filter.

* Undo changes.

* Test.

* Move comment.

.github/workflows/gradle.yml

@@ -368,7 +368,6 @@ jobs:
           label: ${{ needs.start-platform-new-scheduler-acceptance-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-platform-new-scheduler-acceptance-runner.outputs.ec2-instance-id }}
 
-
   # In case of self-hosted EC2 errors, remove this block.
   stop-platform-build-runner:
     name: "Platform: Stop Build EC2 Runner"

.github/workflows/terminate-zombie-build-instances.yml

@@ -0,0 +1,27 @@
+name: Terminate Zombie Build Instances
+
+on:
+  push:
+  schedule:
+    - cron: "0 */1 * * *"
+
+jobs:
+  terminate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: List Instances Older Than an Hour
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          # See https://github.com/aws/aws-cli/issues/5623
+          AWS_EC2_METADATA_DISABLED: true
+        run: |
+          set -euxo pipefail
+
+          aws configure set default.region us-east-2
+
+          # Since the AWS cli returns an ISO HH:MM timestamp, and Jq only accepts Z timestamps, we define a function toZ to convert this.
+          aws ec2 describe-instances --no-paginate --filters Name=instance-type,Values=c5.2xlarge Name=instance-state-name,Values=running \
+            --query 'Reservations[*].Instances[*].{Instance:InstanceId,LaunchTime:LaunchTime}' --output json \
+          | jq 'def toZ(str): str | (split("+")[0] + "Z") | fromdate ;
+              flatten | map( { InstanceId: .Instance, LaunchTime: toZ(.LaunchTime) } ) | map( select ( .LaunchTime < (now - 3600) ) )'