From 2764fb36f47953a8acf145c83b964a3263829064 Mon Sep 17 00:00:00 2001 From: git-phu Date: Tue, 22 Nov 2022 16:04:16 -0800 Subject: [PATCH 1/2] find_non_rate_limited_PAT --- .github/workflows/gke-kube-test-command.yml | 42 +++--- .github/workflows/gradle.yml | 128 ++++++++++++------ .github/workflows/publish-command.yml | 91 ++++++++----- .../workflows/publish-connector-command.yml | 39 +++--- .github/workflows/publish-oss-for-cloud.yml | 21 +-- .github/workflows/release-airbyte-os.yml | 35 +++-- .github/workflows/test-command.yml | 38 +++--- .../workflows/test-performance-command.yml | 39 +++--- 8 files changed, 249 insertions(+), 184 deletions(-) diff --git a/.github/workflows/gke-kube-test-command.yml b/.github/workflows/gke-kube-test-command.yml index 997241ed11562..a99b66fd57044 100644 --- a/.github/workflows/gke-kube-test-command.yml +++ b/.github/workflows/gke-kube-test-command.yml @@ -17,28 +17,10 @@ on: required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} start-gke-kube-acceptance-test-runner: timeout-minutes: 10 name: Start GKE Kube Acceptance Test EC2 Runner runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -48,13 +30,20 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} gke-kube-acceptance-test: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. needs: start-gke-kube-acceptance-test-runner # required to start the main job when the runner is ready @@ -146,7 +135,6 @@ jobs: needs: - start-gke-kube-acceptance-test-runner # required to get output from the start-runner job - gke-kube-acceptance-test # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest if: ${{ always() }} steps: @@ -156,10 +144,22 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-gke-kube-acceptance-test-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index c90a714d48e55..24e10263c198c 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -81,23 +81,6 @@ jobs: - 'airbyte-api/src/main/openapi/config.yaml' - 'airbyte-webapp/**' - 'airbyte-webapp-e2e-tests/**' - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} # Uncomment to debug. # changes-output: @@ -172,7 +155,6 @@ jobs: name: "Connectors Base: Start Build EC2 Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: | needs.changes.outputs.build == 'true' || needs.changes.outputs.connectors == 'true' || needs.changes.outputs.db == 'true' || (always() && github.ref == 'refs/heads/master') @@ -184,13 +166,20 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} build-connectors-base: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. @@ -299,7 +288,6 @@ jobs: needs: - start-connectors-base-build-runner # required to get output from the start-runner job - build-connectors-base # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -311,11 +299,18 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-connectors-base-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }} @@ -325,7 +320,6 @@ jobs: name: "Frontend: Start EC2 Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: | needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/master' @@ -338,18 +332,24 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} frontend-build: name: "Frontend: Build" needs: - start-frontend-runner - - find_valid_pat runs-on: ${{ needs.start-frontend-runner.outputs.label }} steps: - name: Checkout Airbyte @@ -459,7 +459,6 @@ jobs: - start-frontend-runner # required to get output from the start-runner job - frontend-test # required to wait when the e2e-test job is done - frontend-build # required to wait when then build job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -471,11 +470,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-frontend-runner.outputs.label }} ec2-instance-id: ${{ needs.start-frontend-runner.outputs.ec2-instance-id }} @@ -487,7 +495,6 @@ jobs: name: "Platform: Start Build EC2 Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -498,13 +505,20 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} platform-build: name: "Platform: Build" # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. @@ -646,7 +660,6 @@ jobs: needs: - start-platform-build-runner # required to get output from the start-runner job - platform-build # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -658,11 +671,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-platform-build-runner.outputs.label }} ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} @@ -673,7 +695,6 @@ jobs: name: "Platform: Start Kube Acceptance Test Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -684,6 +705,13 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner @@ -692,7 +720,7 @@ jobs: ec2-image-id: ami-0c1a9bc22624339d8 aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} kube-acceptance-test: name: "Platform: Acceptance Tests (Kube)" # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. @@ -842,7 +870,6 @@ jobs: needs: - start-kube-acceptance-test-runner # required to get output from the start-runner job - kube-acceptance-test # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -854,11 +881,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }} @@ -870,7 +906,6 @@ jobs: name: "Platform: Start Helm Acceptance Test Runner" needs: - changes - - find_valid_pat # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') timeout-minutes: 10 @@ -880,7 +915,14 @@ jobs: ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner @@ -889,7 +931,7 @@ jobs: ec2-image-id: ami-0c1a9bc22624339d8 aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} helm-acceptance-test: name: "Platform: Acceptance Tests (Helm)" @@ -1039,7 +1081,6 @@ jobs: needs: - start-helm-acceptance-test-runner # required to get output from the start-runner job - helm-acceptance-test # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. # Thus, we check for skipped here. @@ -1051,11 +1092,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 14ca7de9d5ed1..72f4f7e801250 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -30,29 +30,11 @@ on: default: "true" jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} ## Gradle Build # In case of self-hosted EC2 errors, remove this block. start-publish-image-runner-0: name: Start Build EC2 Runner 0 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -62,19 +44,25 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher start-publish-image-runner-1: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 1 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -84,19 +72,25 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher start-publish-image-runner-2: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 2 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -106,19 +100,25 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher start-publish-image-runner-3: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 3 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -128,19 +128,25 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher start-publish-image-runner-4: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 4 runs-on: ubuntu-latest - needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -150,13 +156,20 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher preprocess-matrix: needs: start-publish-image-runner-0 @@ -392,7 +405,6 @@ jobs: - start-publish-image-runner-0 # required to get output from the start-runner job - preprocess-matrix - publish-image # required to wait when the main job is done - - find_valid_pat - add-helpful-info-to-git-comment runs-on: ubuntu-latest steps: @@ -402,11 +414,20 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-publish-image-runner-0.outputs.label }} ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} stop-publish-image-runner-multi: @@ -420,7 +441,6 @@ jobs: - start-publish-image-runner-4 - preprocess-matrix - publish-image # required to wait when the main job is done - - find_valid_pat strategy: fail-fast: false matrix: @@ -451,10 +471,19 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ matrix.ec2-instance.label }} ec2-instance-id: ${{ matrix.ec2-instance.id }} diff --git a/.github/workflows/publish-connector-command.yml b/.github/workflows/publish-connector-command.yml index cc695436f7ad6..3fcc2daef8f05 100644 --- a/.github/workflows/publish-connector-command.yml +++ b/.github/workflows/publish-connector-command.yml @@ -26,30 +26,12 @@ on: required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} ## Gradle Build # In case of self-hosted EC2 errors, remove this block. # start-bump-build-test-connector-runner: # name: Start Build EC2 Runner # runs-on: ubuntu-latest -# needs: find_valid_pat # outputs: # label: ${{ steps.start-ec2-runner.outputs.label }} # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} @@ -59,13 +41,20 @@ jobs: # with: # repository: ${{ github.event.inputs.repo }} # ref: ${{ github.event.inputs.gitref }} +# - name: Check PAT rate limits +# run: | +# ./tools/bin/find_non_rate_limited_PAT \ +# ${{ secrets.AIRBYTEIO_PAT }} \ +# ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ +# ${{ secrets.SUPERTOPHER_PAT }} \ +# ${{ secrets.DAVINCHIA_PAT }} # - name: Start AWS Runner # id: start-ec2-runner # uses: ./.github/actions/start-aws-runner # with: # aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} -# github-token: ${{ needs.find_valid_pat.outputs.pat }} +# github-token: ${{ env.PAT }} # # 80 gb disk # ec2-image-id: ami-06cf12549e3d9c522 # bump-build-test-connector: @@ -192,7 +181,6 @@ jobs: # needs: # - start-bump-build-test-connector-runner # required to get output from the start-runner job # - bump-build-test-connector # required to wait when the main job is done -# - find_valid_pat # runs-on: ubuntu-latest # if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs # steps: @@ -202,10 +190,19 @@ jobs: # aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} # aws-region: us-east-2 +# - name: Checkout Airbyte +# uses: actions/checkout@v3 +# - name: Check PAT rate limits +# run: | +# ./tools/bin/find_non_rate_limited_PAT \ +# ${{ secrets.AIRBYTEIO_PAT }} \ +# ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ +# ${{ secrets.SUPERTOPHER_PAT }} \ +# ${{ secrets.DAVINCHIA_PAT }} # - name: Stop EC2 runner # uses: supertopher/ec2-github-runner@base64v1.0.10 # with: # mode: stop -# github-token: ${{ needs.find_valid_pat.outputs.pat }} +# github-token: ${{ env.PAT }} # label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }} # ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish-oss-for-cloud.yml b/.github/workflows/publish-oss-for-cloud.yml index 74fb5486c6b3a..ad31e647fc100 100644 --- a/.github/workflows/publish-oss-for-cloud.yml +++ b/.github/workflows/publish-oss-for-cloud.yml @@ -9,41 +9,30 @@ on: description: "Publish artifacts for the following git ref (if unspecified, uses the latest commit for the current branch):" required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" + start-runner: + name: "Start Runner on AWS" timeout-minutes: 10 runs-on: ubuntu-latest outputs: - pat: ${{ steps.variables.outputs.pat }} + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 - name: Check PAT rate limits - id: variables run: | ./tools/bin/find_non_rate_limited_PAT \ ${{ secrets.AIRBYTEIO_PAT }} \ ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ ${{ secrets.SUPERTOPHER_PAT }} \ ${{ secrets.DAVINCHIA_PAT }} - start-runner: - name: "Start Runner on AWS" - needs: find_valid_pat - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} generate-tags: name: "Generate Dev and Master Tags" diff --git a/.github/workflows/release-airbyte-os.yml b/.github/workflows/release-airbyte-os.yml index 2165dc688f51a..48e6651681854 100644 --- a/.github/workflows/release-airbyte-os.yml +++ b/.github/workflows/release-airbyte-os.yml @@ -9,42 +9,31 @@ on: required: true default: "patch" jobs: - find_valid_pat: - name: "Find a PAT with room for actions" + # In case of self-hosted EC2 errors, remove this block. + start-release-airbyte-runner: + name: "Release Airbyte: Start EC2 Runner" timeout-minutes: 10 runs-on: ubuntu-latest outputs: - pat: ${{ steps.variables.outputs.pat }} + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 - name: Check PAT rate limits - id: variables run: | ./tools/bin/find_non_rate_limited_PAT \ ${{ secrets.AIRBYTEIO_PAT }} \ ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ ${{ secrets.SUPERTOPHER_PAT }} \ ${{ secrets.DAVINCHIA_PAT }} - # In case of self-hosted EC2 errors, remove this block. - start-release-airbyte-runner: - name: "Release Airbyte: Start EC2 Runner" - needs: find_valid_pat - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} releaseAirbyte: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. @@ -163,7 +152,6 @@ jobs: needs: - start-release-airbyte-runner # required to get output from the start-runner job - releaseAirbyte # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -173,10 +161,19 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-release-airbyte-runner.outputs.label }} ec2-instance-id: ${{ needs.start-release-airbyte-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 23b0ff27469a4..05f81c845957e 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -21,8 +21,8 @@ on: required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" + uuid: + name: "Custom UUID of workflow run" timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -30,19 +30,9 @@ jobs: steps: - name: UUID ${{ github.event.inputs.uuid }} run: true - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} start-test-runner: name: Start Build EC2 Runner - needs: find_valid_pat + needs: uuid timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -51,6 +41,13 @@ jobs: steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} @@ -60,7 +57,7 @@ jobs: with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} # 80 gb disk ec2-image-id: ami-06cf12549e3d9c522 integration-test: @@ -188,7 +185,7 @@ jobs: needs: - start-test-runner # required to get output from the start-runner job - integration-test # required to wait when the main job is done - - find_valid_pat + - uuid runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -198,10 +195,19 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index c420ebc928118..11e51f2b2d105 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -26,26 +26,8 @@ on: required: false jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} start-test-runner: name: Start Build EC2 Runner - needs: find_valid_pat timeout-minutes: 10 runs-on: ubuntu-latest outputs: @@ -57,13 +39,20 @@ jobs: with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Start AWS Runner id: start-ec2-runner uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} # 80 gb disk ec2-image-id: ami-06cf12549e3d9c522 performance-test: @@ -188,7 +177,6 @@ jobs: needs: - start-test-runner # required to get output from the start-runner job - performance-test # required to wait when the main job is done - - find_valid_pat runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -198,10 +186,19 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} - name: Stop EC2 runner uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} + github-token: ${{ env.PAT }} label: ${{ needs.start-test-runner.outputs.label }} ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }} From 7273874c73fffb6ffc637f7ba012d7243b3070f2 Mon Sep 17 00:00:00 2001 From: git-phu Date: Tue, 22 Nov 2022 16:29:56 -0800 Subject: [PATCH 2/2] checkout repo to find pat --- .github/workflows/gradle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 24e10263c198c..869ff11d411ba 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -299,6 +299,8 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 - name: Check PAT rate limits run: | ./tools/bin/find_non_rate_limited_PAT \