More ec2 pipelines using dynamic PATs (#11525)

parker noticed we missed a few reviewer: please help me make sure the function exists for `find_valid_pat` in each place it's called
airbytehq · Mar 29, 2022 · a562c29 · a562c29
1 parent 707b2a2
commit a562c29
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 15 deletions.
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -413,6 +413,7 @@ jobs:
       - start-frontend-runner # required to get output from the start-runner job
       - frontend-test # required to wait when the e2e-test job is done
       - frontend-build # required to wait when then build job is done
+      - find_valid_pat
     runs-on: ubuntu-latest
     # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
     # Thus, we check for skipped here.
@@ -428,7 +429,7 @@ jobs:
         uses: supertopher/ec2-github-runner@base64v1.0.5
         with:
           mode: stop
-          github-token: ${{ secrets.AIRBYTEIO_PAT }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           label: ${{ needs.start-frontend-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-frontend-runner.outputs.ec2-instance-id }}
 
@@ -557,6 +558,7 @@ jobs:
     needs:
       - start-platform-build-runner # required to get output from the start-runner job
       - platform-build # required to wait when the main job is done
+      - find_valid_pat
     runs-on: ubuntu-latest
     # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
     # Thus, we check for skipped here.
@@ -572,15 +574,17 @@ jobs:
         uses: supertopher/ec2-github-runner@base64v1.0.5
         with:
           mode: stop
-          github-token: ${{ secrets.AIRBYTEIO_PAT }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           label: ${{ needs.start-platform-build-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }}
 
   # Scheduler V2
   # In case of self-hosted EC2 errors, remove this block.
   start-platform-new-scheduler-acceptance-runner:
     name: "Platform: Start Docker w/ Scheduler v2 Test Runner"
-    needs: changes
+    needs:
+      - changes
+      - find_valid_pat
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
     timeout-minutes: 10
@@ -595,9 +599,9 @@ jobs:
         id: start-ec2-runner
         uses: ./.github/actions/start-aws-runner
         with:
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }}
   platform-new-scheduler-acceptance:
     # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
     needs: start-platform-new-scheduler-acceptance-runner # required to start the main job when the runner is ready
@@ -681,6 +685,7 @@ jobs:
     needs:
       - start-platform-new-scheduler-acceptance-runner # required to get output from the start-runner job
       - platform-new-scheduler-acceptance # required to wait when the main job is done
+      - find_valid_pat
     runs-on: ubuntu-latest
     # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
     # Thus, we check for skipped here.
@@ -696,16 +701,18 @@ jobs:
         uses: supertopher/ec2-github-runner@base64v1.0.5
         with:
           mode: stop
-          github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }}
           label: ${{ needs.start-platform-new-scheduler-acceptance-runner.outputs.label }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           ec2-instance-id: ${{ needs.start-platform-new-scheduler-acceptance-runner.outputs.ec2-instance-id }}
 
   ## Kube Acceptance Tests
   # Docker acceptance tests run as part of the build job.
   # In case of self-hosted EC2 errors, remove this block.
   start-kube-acceptance-test-runner:
     name: "Platform: Start Kube Acceptance Test EC2 Runner"
-    needs: changes
+    needs:
+      - changes
+      - find_valid_pat
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
     timeout-minutes: 10
@@ -724,7 +731,7 @@ jobs:
           ec2-image-id: ami-0c1a9bc22624339d8
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
   kube-acceptance-test:
     name: "Platform: Acceptance Tests (Kube)"
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
@@ -847,6 +854,7 @@ jobs:
     needs:
       - start-kube-acceptance-test-runner # required to get output from the start-runner job
       - kube-acceptance-test # required to wait when the main job is done
+      - find_valid_pat
     runs-on: ubuntu-latest
     # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
     # Thus, we check for skipped here.
@@ -862,7 +870,7 @@ jobs:
         uses: supertopher/ec2-github-runner@base64v1.0.5
         with:
           mode: stop
-          github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }}
 
@@ -871,7 +879,9 @@ jobs:
   # In case of self-hosted EC2 errors, remove this block.
   start-kube-acceptance-test-runner-v2:
     name: "Platform: Start Scheduler V2 Kube Acceptance Test Runner"
-    needs: changes
+    needs:
+      - changes
+      - find_valid_pat
     # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
     if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
     timeout-minutes: 10
@@ -890,7 +900,7 @@ jobs:
           ec2-image-id: ami-0c1a9bc22624339d8
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
   kube-acceptance-test-v2:
     name: "Platform: Acceptance Tests (Kube v2)"
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
@@ -980,6 +990,7 @@ jobs:
     needs:
       - start-kube-acceptance-test-runner-v2 # required to get output from the start-runner job
       - kube-acceptance-test-v2 # required to wait when the main job is done
+      - find_valid_pat
     runs-on: ubuntu-latest
     # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
     # Thus, we check for skipped here.
@@ -995,6 +1006,6 @@ jobs:
         uses: supertopher/ec2-github-runner@base64v1.0.5
         with:
           mode: stop
-          github-token: ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           label: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }}
           ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.ec2-instance-id }}
diff --git a/.github/workflows/publish-external-command.yml b/.github/workflows/publish-external-command.yml
@@ -42,6 +42,7 @@ jobs:
   # In case of self-hosted EC2 errors, remove this block.
   start-publish-image-runner:
     name: Start Build EC2 Runner
+    needs: find_valid_pat
     runs-on: ubuntu-latest
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
@@ -58,7 +59,7 @@ jobs:
         with:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           # 80 gb disk
           ec2-image-id: ami-0d648081937c75a73
   publish-image:

diff --git a/.github/workflows/release-airbyte-os.yml b/.github/workflows/release-airbyte-os.yml
@@ -29,6 +29,7 @@ jobs:
   # In case of self-hosted EC2 errors, remove this block.
   start-release-airbyte-runner:
     name: "Release Airbyte: Start EC2 Runner"
+    needs: find_valid_pat
     timeout-minutes: 10
     runs-on: ubuntu-latest
     outputs:
@@ -43,7 +44,7 @@ jobs:
         with:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
 
   releaseAirbyte:
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.

diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -37,6 +37,7 @@ jobs:
             ${{ secrets.DAVINCHIA_PAT }}
   start-test-runner:
     name: Start Build EC2 Runner
+    need: find_valid_pat
     timeout-minutes: 10
     runs-on: ubuntu-latest
     outputs:
@@ -54,7 +55,7 @@ jobs:
         with:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           # 80 gb disk
           ec2-image-id: ami-0d648081937c75a73
   integration-test:

diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml
@@ -43,6 +43,7 @@ jobs:
             ${{ secrets.DAVINCHIA_PAT }}
   start-test-runner:
     name: Start Build EC2 Runner
+    needs: find_valid_pat
     timeout-minutes: 10
     runs-on: ubuntu-latest
     outputs:
@@ -60,7 +61,7 @@ jobs:
         with:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
           # 80 gb disk
           ec2-image-id: ami-0d648081937c75a73
   performance-test: