airbyte-ci: compute GHA runs-on from --ci-requirements

.github/actions/airbyte-ci-requirements/action.yml

@@ -0,0 +1,104 @@
+name: "Get airbyte-ci runner name"
+description: "Runs a given airbyte-ci command with the --ci-requirements flag to get the CI requirements for a given command"
+inputs:
+  runner_type:
+    description: "Type of runner to get requirements for. One of: format, test, nightly, publish"
+    required: true
+  runner_size:
+    description: "One of: format, test, nightly, publish"
+    required: true
+  airbyte_ci_command:
+    description: "airbyte-ci command to get CI requirements for."
+    required: true
+  runner_name_prefix:
+    description: "Prefix of runner name"
+    required: false
+    default: ci-runner-connector
+  github_token:
+    description: "GitHub token"
+    required: true
+  sentry_dsn:
+    description: "Sentry DSN"
+    required: false
+  airbyte_ci_binary_url:
+    description: "URL to airbyte-ci binary"
+    required: false
+    default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check if PR is from a fork
+      if: github.event_name == 'pull_request'
+      shell: bash
+      run: |
+        if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
+          echo "PR is from a fork. Exiting workflow..."
+          exit 78
+        fi
+
+    - name: Get changed files
+      uses: tj-actions/changed-files@v39
+      id: changes
+      with:
+        files_yaml: |
+          pipelines:
+            - 'airbyte-ci/connectors/pipelines/**'
+
+    - name: Determine how Airbyte CI should be installed
+      shell: bash
+      id: determine-install-mode
+      run: |
+        if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
+          echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
+          echo "install-mode=dev" >> $GITHUB_OUTPUT
+        else
+          echo "install-mode=production" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Install airbyte-ci binary
+      id: install-airbyte-ci
+      if: steps.determine-install-mode.outputs.install-mode == 'production'
+      shell: bash
+      run: |
+        curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
+        sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+        sudo chmod +x /usr/local/bin/airbyte-ci
+
+    - name: Install Python 3.10
+      uses: actions/setup-python@v4
+      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      with:
+        python-version: "3.10"
+        token: ${{ inputs.github_token }}
+
+    - name: Install ci-connector-ops package
+      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      shell: bash
+      run: |
+        pip install pipx
+        pipx ensurepath
+        pipx install airbyte-ci/connectors/pipelines/
+
+    - name: Get dagger version from airbyte-ci
+      id: get-dagger-version
+      shell: bash
+      run: |
+        dagger_version=$(airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
+        echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT"
+
+    - name: Get runner name
+      id: get-runner-name
+      shell: bash
+      run: |
+        runner_name_prefix=${{ inputs.runner_name_prefix }}
+        runner_type=${{ inputs.runner_type }}
+        runner_size=${{ inputs.runner_size }}
+        dashed_dagger_version=$(echo "${{ steps.get-dagger-version.outputs.dagger_version }}" | tr '.' '-')
+        runner_name="${runner_name_prefix}-${runner_type}-${runner_size}-dagger-${dashed_dagger_version}"
+        echo ${runner_name}
+        echo "runner_name=${runner_name}" >> "$GITHUB_OUTPUT"
+outputs:
+  runner_name:
+    description: "Name of self hosted CI runner to use"
+    value: ${{ steps.get-runner-name.outputs.runner_name }}

.github/workflows/airbyte-ci-tests.yml

@@ -17,9 +17,31 @@ on:
       - reopened
       - synchronize
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "publish"
+          runner_size: "large"
+          airbyte_ci_command: "publish"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   run-airbyte-ci-tests:
     name: Run Airbyte CI tests
-    runs-on: "ci-runner-connector-test-large-dagger-0-9-5"
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/cat-tests.yml

@@ -14,9 +14,31 @@ on:
     paths:
       - airbyte-integrations/bases/connector-acceptance-test/**
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "test"
+          runner_size: "large"
+          airbyte_ci_command: "test"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   run-cat-unit-tests:
     name: Run CAT unit tests
-    runs-on: "ci-runner-connector-test-large-dagger-0-9-5"
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/connectors_nightly_build.yml

@@ -6,21 +6,38 @@ on:
     - cron: "0 0 * * *"
   workflow_dispatch:
     inputs:
-      runs-on:
-        type: string
-        default: ci-runner-connector-nightly-xlarge-dagger-0-9-5
-        required: true
       test-connectors-options:
         default: --concurrency=5 --support-level=certified
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}"
 
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "nightly"
+          runner_size: "xlarge"
+          airbyte_ci_command: "connectors test"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}"
     timeout-minutes: 720 # 12 hours
-    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/connectors_tests.yml

@@ -17,9 +17,6 @@ on:
       test-connectors-options:
         description: "Options to pass to the 'airbyte-ci connectors test' command"
         default: "--modified"
-      runner:
-        description: "The runner to use for this job"
-        default: "ci-runner-connector-test-large-dagger-0-9-5"
       airbyte_ci_binary_url:
         description: "The URL to download the airbyte-ci binary from"
         required: false
@@ -30,10 +27,32 @@ on:
       - synchronize
       - ready_for_review
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "test"
+          runner_size: "large"
+          airbyte_ci_command: "connectors test"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   connectors_ci:
     name: Connectors CI
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     timeout-minutes: 1440 # 24 hours
-    runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-9-5'}}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/connectors_weekly_build.yml

@@ -6,21 +6,39 @@ on:
     - cron: "0 12 * * 0"
   workflow_dispatch:
     inputs:
-      runs-on:
-        type: string
-        default: ci-runner-connector-nightly-xlarge-dagger-0-9-5
-        required: true
       test-connectors-options:
         default: --concurrency=3 --support-level=community
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}"
 
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "publish"
+          runner_size: "large"
+          airbyte_ci_command: "publish"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}"
     timeout-minutes: 8640 # 6 days
-    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/format_check.yml

@@ -16,17 +16,42 @@ on:
   pull_request:
 
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "format"
+          runner_size: "medium"
+          airbyte_ci_command: "format"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          # Pin to a specific version of airbyte-ci to avoid transient failures
+          # Mentioned in issue https://github.com/airbytehq/airbyte/issues/34041
+          airbyte_ci_binary_url: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/2.14.1/airbyte-ci
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   format-check:
-    runs-on: "ci-runner-connector-format-medium-dagger-0-6-4"
     # IMPORTANT: This name must match the require check name on the branch protection settings
     name: "Check for formatting errors"
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
           token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
-
+          fetch-depth: 1
       - name: Run airbyte-ci format check [MASTER]
         id: airbyte_ci_format_check_all_master
         if: github.ref == 'refs/heads/master'

.github/workflows/format_fix.yml

@@ -9,9 +9,31 @@ concurrency:
 on:
   workflow_dispatch:
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "publish"
+          runner_size: "large"
+          airbyte_ci_command: "publish"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   format-fix:
-    runs-on: "ci-runner-connector-format-medium-dagger-0-9-5"
     name: "Run airbyte-ci format fix all"
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3