From 2bbb9c816dea1ca91ddd61fc56ffd0772cbca851 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Sun, 25 Apr 2021 17:22:04 +0800 Subject: [PATCH] Self-hosted EC2 Runners (#3019) Use self-hosted ec2 runners for the main Airbyte build as discussed today. Will bring this up for evaluation after a week. Anecdotal testing shows builds generally finish in 9 - 10 mins vs 15 - 20 mins on the regular runner, even after accounting for the time taken to spin up the instance (about 1 min). Removed this from the frontend tests since Cypress is running into an issue that seems to be related to how the tests are structured and I don't think we need to debug now. The slash commands will continue running on the github runner. Our AWS limit is currently 160 vCPU, which gives us 20 parallel builds. AWS says we can increase a limit increase if we run into any issues. --- .github/workflows/gradle.yml | 113 ++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 08377e1e1f193b..a82a481ea2e390 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -16,8 +16,38 @@ jobs: run: ./tools/bin/ci_integration_workflow_launcher.sh env: GITHUB_TOKEN: ${{ secrets.SLASH_COMMAND_PAT }} - build: + + ## Gradle Build + # In case of self-hosted EC2 errors, remove the `start-build-runner` block. + start-build-runner: + name: Start Build EC2 Runner runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Start EC2 Runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2.1.0 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-0b39c2b1b65f75ca8 + ec2-instance-type: c5.2xlarge + subnet-id: subnet-01d35f948346c05bd + security-group-id: sg-04399aaf4d51a35f9 + build: + # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. + needs: start-build-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner + # runs-on: ubuntu-latest + name: Build Airbyte steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -128,8 +158,60 @@ jobs: SLACK_USERNAME: Buildbot SLACK_TITLE: "Build Success" SLACK_FOOTER: "" - test_docker: + # In case of self-hosted EC2 errors, remove the `stop-build-runner` block. + stop-build-runner: + name: Stop Build EC2 Runner + needs: + - start-build-runner # required to get output from the start-runner job + - build # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2.1.0 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + + ## Acceptance Test + # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. + start-acceptance-test-runner: + name: Start Acceptance Test EC2 Runner runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Start EC2 runner + id: start-ec2-runner + uses: machulav/ec2-github-runner@v2.1.0 + with: + mode: start + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + ec2-image-id: ami-0b39c2b1b65f75ca8 + ec2-instance-type: c5.2xlarge + subnet-id: subnet-01d35f948346c05bd + security-group-id: sg-04399aaf4d51a35f9 + acceptance-test: + # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. + needs: start-acceptance-test-runner # required to start the main job when the runner is ready + runs-on: ${{ needs.start-acceptance-test-runner.outputs.label }} # run the job on the newly created runner + # runs-on: ubuntu-latest + name: Run Acceptance Tests steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -155,9 +237,33 @@ jobs: - name: Run Docker End-to-End Acceptance Tests run: | ./tools/bin/acceptance_test.sh + # In case of self-hosted EC2 errors, remove the `stop-build-runner` block. + stop-acceptance-test-runner: + name: Stop Acceptance Test EC2 Runner + needs: + - start-acceptance-test-runner # required to get output from the start-runner job + - acceptance-test # required to wait when the main job is done + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@v2.1.0 + with: + mode: stop + github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + label: ${{ needs.start-acceptance-test-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-acceptance-test-runner.outputs.ec2-instance-id }} - launch_e2e_frontend_tests: + ## Frontend Test + frontend-test: runs-on: ubuntu-latest + name: Run Frontend Test steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -179,6 +285,7 @@ jobs: - name: Run End-to-End Frontend Tests run: ./tools/bin/e2e_test.sh + # DISABLED UNTIL WE HAVE TEMPORAL ON KUBE # test_kube: # runs-on: ubuntu-latest