Self-hosted EC2 Runners (#3019)

Use self-hosted ec2 runners for the main Airbyte build as discussed today. Will bring this up for evaluation after a week. Anecdotal testing shows builds generally finish in 9 - 10 mins vs 15 - 20 mins on the regular runner, even after accounting for the time taken to spin up the instance (about 1 min). Removed this from the frontend tests since Cypress is running into an issue that seems to be related to how the tests are structured and I don't think we need to debug now. The slash commands will continue running on the github runner. Our AWS limit is currently 160 vCPU, which gives us 20 parallel builds. AWS says we can increase a limit increase if we run into any issues.
airbytehq · Apr 25, 2021 · 2bbb9c8 · 2bbb9c8
1 parent 49296ef
commit 2bbb9c8
Showing 1 changed file with 110 additions and 3 deletions.
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -16,8 +16,38 @@ jobs:
         run: ./tools/bin/ci_integration_workflow_launcher.sh
         env:
           GITHUB_TOKEN: ${{ secrets.SLASH_COMMAND_PAT }}
-  build:
+
+  ## Gradle Build
+  # In case of self-hosted EC2 errors, remove the `start-build-runner` block.
+  start-build-runner:
+    name: Start Build EC2 Runner
     runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Start EC2 Runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2.1.0
+        with:
+          mode: start
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          ec2-image-id: ami-0b39c2b1b65f75ca8
+          ec2-instance-type: c5.2xlarge
+          subnet-id: subnet-01d35f948346c05bd
+          security-group-id: sg-04399aaf4d51a35f9
+  build:
+    # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
+    needs: start-build-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner
+    # runs-on: ubuntu-latest
+    name: Build Airbyte
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
@@ -128,8 +158,60 @@ jobs:
           SLACK_USERNAME: Buildbot
           SLACK_TITLE: "Build Success"
           SLACK_FOOTER: ""
-  test_docker:
+  # In case of self-hosted EC2 errors, remove the `stop-build-runner` block.
+  stop-build-runner:
+    name: Stop Build EC2 Runner
+    needs:
+      - start-build-runner # required to get output from the start-runner job
+      - build # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2.1.0
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-build-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }}
+
+  ## Acceptance Test
+  # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
+  start-acceptance-test-runner:
+    name: Start Acceptance Test EC2 Runner
     runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2.1.0
+        with:
+          mode: start
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          ec2-image-id: ami-0b39c2b1b65f75ca8
+          ec2-instance-type: c5.2xlarge
+          subnet-id: subnet-01d35f948346c05bd
+          security-group-id: sg-04399aaf4d51a35f9
+  acceptance-test:
+    # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
+    needs: start-acceptance-test-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
+    # runs-on: ubuntu-latest
+    name: Run Acceptance Tests
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
@@ -155,9 +237,33 @@ jobs:
       - name: Run Docker End-to-End Acceptance Tests
         run: |
           ./tools/bin/acceptance_test.sh
+  # In case of self-hosted EC2 errors, remove the `stop-build-runner` block.
+  stop-acceptance-test-runner:
+    name: Stop Acceptance Test EC2 Runner
+    needs:
+      - start-acceptance-test-runner # required to get output from the start-runner job
+      - acceptance-test # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2.1.0
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-acceptance-test-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-acceptance-test-runner.outputs.ec2-instance-id }}
 
-  launch_e2e_frontend_tests:
+  ## Frontend Test
+  frontend-test:
     runs-on: ubuntu-latest
+    name: Run Frontend Test
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
@@ -179,6 +285,7 @@ jobs:
 
       - name: Run End-to-End Frontend Tests
         run: ./tools/bin/e2e_test.sh
+
 # DISABLED UNTIL WE HAVE TEMPORAL ON KUBE
 #  test_kube:
 #    runs-on: ubuntu-latest