From a562c2963bebe99c8f8b64297fa9a5a4e709376e Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Tue, 29 Mar 2022 17:30:21 -0500 Subject: [PATCH] More ec2 pipelines using dynamic PATs (#11525) parker noticed we missed a few reviewer: please help me make sure the function exists for `find_valid_pat` in each place it's called --- .github/workflows/gradle.yml | 33 ++++++++++++------- .../workflows/publish-external-command.yml | 3 +- .github/workflows/release-airbyte-os.yml | 3 +- .github/workflows/test-command.yml | 3 +- .../workflows/test-performance-command.yml | 3 +- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8f5ee6807c9e..e584cbeed282 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -413,6 +413,7 @@ jobs: - start-frontend-runner # required to get output from the start-runner job - frontend-test # required to wait when the e2e-test job is done - frontend-build # required to wait when then build job is done + - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -428,7 +429,7 @@ jobs: uses: supertopher/ec2-github-runner@base64v1.0.5 with: mode: stop - github-token: ${{ secrets.AIRBYTEIO_PAT }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} label: ${{ needs.start-frontend-runner.outputs.label }} ec2-instance-id: ${{ needs.start-frontend-runner.outputs.ec2-instance-id }} @@ -557,6 +558,7 @@ jobs: needs: - start-platform-build-runner # required to get output from the start-runner job - platform-build # required to wait when the main job is done + - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -572,7 +574,7 @@ jobs: uses: supertopher/ec2-github-runner@base64v1.0.5 with: mode: stop - github-token: ${{ secrets.AIRBYTEIO_PAT }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} label: ${{ needs.start-platform-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} @@ -580,7 +582,9 @@ jobs: # In case of self-hosted EC2 errors, remove this block. start-platform-new-scheduler-acceptance-runner: name: "Platform: Start Docker w/ Scheduler v2 Test Runner" - needs: changes + needs: + - changes + - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -595,9 +599,9 @@ jobs: id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: + github-token: ${{ needs.find_valid_pat.outputs.pat }} aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} platform-new-scheduler-acceptance: # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. needs: start-platform-new-scheduler-acceptance-runner # required to start the main job when the runner is ready @@ -681,6 +685,7 @@ jobs: needs: - start-platform-new-scheduler-acceptance-runner # required to get output from the start-runner job - platform-new-scheduler-acceptance # required to wait when the main job is done + - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -696,8 +701,8 @@ jobs: uses: supertopher/ec2-github-runner@base64v1.0.5 with: mode: stop - github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} label: ${{ needs.start-platform-new-scheduler-acceptance-runner.outputs.label }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} ec2-instance-id: ${{ needs.start-platform-new-scheduler-acceptance-runner.outputs.ec2-instance-id }} ## Kube Acceptance Tests @@ -705,7 +710,9 @@ jobs: # In case of self-hosted EC2 errors, remove this block. start-kube-acceptance-test-runner: name: "Platform: Start Kube Acceptance Test EC2 Runner" - needs: changes + needs: + - changes + - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -724,7 +731,7 @@ jobs: ec2-image-id: ami-0c1a9bc22624339d8 aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} kube-acceptance-test: name: "Platform: Acceptance Tests (Kube)" # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. @@ -847,6 +854,7 @@ jobs: needs: - start-kube-acceptance-test-runner # required to get output from the start-runner job - kube-acceptance-test # required to wait when the main job is done + - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -862,7 +870,7 @@ jobs: uses: supertopher/ec2-github-runner@base64v1.0.5 with: mode: stop - github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }} @@ -871,7 +879,9 @@ jobs: # In case of self-hosted EC2 errors, remove this block. start-kube-acceptance-test-runner-v2: name: "Platform: Start Scheduler V2 Kube Acceptance Test Runner" - needs: changes + needs: + - changes + - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -890,7 +900,7 @@ jobs: ec2-image-id: ami-0c1a9bc22624339d8 aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} kube-acceptance-test-v2: name: "Platform: Acceptance Tests (Kube v2)" # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. @@ -980,6 +990,7 @@ jobs: needs: - start-kube-acceptance-test-runner-v2 # required to get output from the start-runner job - kube-acceptance-test-v2 # required to wait when the main job is done + - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -995,6 +1006,6 @@ jobs: uses: supertopher/ec2-github-runner@base64v1.0.5 with: mode: stop - github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} label: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }} ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish-external-command.yml b/.github/workflows/publish-external-command.yml index 86fb2df9aa9d..0abd91369287 100644 --- a/.github/workflows/publish-external-command.yml +++ b/.github/workflows/publish-external-command.yml @@ -42,6 +42,7 @@ jobs: # In case of self-hosted EC2 errors, remove this block. start-publish-image-runner: name: Start Build EC2 Runner + needs: find_valid_pat runs-on: ubuntu-latest outputs: label: ${{ steps.start-ec2-runner.outputs.label }} @@ -58,7 +59,7 @@ jobs: with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} # 80 gb disk ec2-image-id: ami-0d648081937c75a73 publish-image: diff --git a/.github/workflows/release-airbyte-os.yml b/.github/workflows/release-airbyte-os.yml index 5156359c4485..e2786bd6e0f6 100644 --- a/.github/workflows/release-airbyte-os.yml +++ b/.github/workflows/release-airbyte-os.yml @@ -29,6 +29,7 @@ jobs: # In case of self-hosted EC2 errors, remove this block. start-release-airbyte-runner: name: "Release Airbyte: Start EC2 Runner" + needs: find_valid_pat timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -43,7 +44,7 @@ jobs: with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} releaseAirbyte: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 8845651fca33..c63df20ab48b 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -37,6 +37,7 @@ jobs: ${{ secrets.DAVINCHIA_PAT }} start-test-runner: name: Start Build EC2 Runner + need: find_valid_pat timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -54,7 +55,7 @@ jobs: with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} # 80 gb disk ec2-image-id: ami-0d648081937c75a73 integration-test: diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index 6dfd0cba963c..818b1d495eca 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -43,6 +43,7 @@ jobs: ${{ secrets.DAVINCHIA_PAT }} start-test-runner: name: Start Build EC2 Runner + needs: find_valid_pat timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -60,7 +61,7 @@ jobs: with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} # 80 gb disk ec2-image-id: ami-0d648081937c75a73 performance-test: