airbyte-ci: compute GHA runs-on from --ci-requirements

.github/actions/airbyte-ci-requirements/action.yml

@@ -0,0 +1,104 @@
+name: "Get airbyte-ci runner name"
+description: "Runs a given airbyte-ci command with the --ci-requirements flag to get the CI requirements for a given command"
+inputs:
+  runner_type:
+    description: "Type of runner to get requirements for. One of: format, test, nightly, publish"
+    required: true
+  runner_size:
+    description: "One of: format, test, nightly, publish"
+    required: true
+  airbyte_ci_command:
+    description: "airbyte-ci command to get CI requirements for."
+    required: true
+  runner_name_prefix:
+    description: "Prefix of runner name"
+    required: false
+    default: ci-runner-connector
+  github_token:
+    description: "GitHub token"
+    required: true
+  sentry_dsn:
+    description: "Sentry DSN"
+    required: false
+  airbyte_ci_binary_url:
+    description: "URL to airbyte-ci binary"
+    required: false
+    default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check if PR is from a fork
+      if: github.event_name == 'pull_request'
+      shell: bash
+      run: |
+        if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
+          echo "PR is from a fork. Exiting workflow..."
+          exit 78
+        fi
+
+    - name: Get changed files
+      uses: tj-actions/changed-files@v39
+      id: changes
+      with:
+        files_yaml: |
+          pipelines:
+            - 'airbyte-ci/connectors/pipelines/**'
+
+    - name: Determine how Airbyte CI should be installed
+      shell: bash
+      id: determine-install-mode
+      run: |
+        if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
+          echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
+          echo "install-mode=dev" >> $GITHUB_OUTPUT
+        else
+          echo "install-mode=production" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Install airbyte-ci binary
+      id: install-airbyte-ci
+      if: steps.determine-install-mode.outputs.install-mode == 'production'
+      shell: bash
+      run: |
+        curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
+        sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+        sudo chmod +x /usr/local/bin/airbyte-ci
+
+    - name: Install Python 3.10
+      uses: actions/setup-python@v4
+      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      with:
+        python-version: "3.10"
+        token: ${{ inputs.github_token }}
+
+    - name: Install ci-connector-ops package
+      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      shell: bash
+      run: |
+        pip install pipx
+        pipx ensurepath
+        pipx install airbyte-ci/connectors/pipelines/
+
+    - name: Get dagger version from airbyte-ci
+      id: get-dagger-version
+      shell: bash
+      run: |
+        dagger_version=$(airbyte-ci ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
+        echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT"
+
+    - name: Get runner name
+      id: get-runner-name
+      shell: bash
+      run: |
+        runner_name_prefix=${{ inputs.runner_name_prefix }}
+        runner_type=${{ inputs.runner_type }}
+        runner_size=${{ inputs.runner_size }}
+        dashed_dagger_version=$(echo "${{ steps.get-dagger-version.outputs.dagger_version }}" | tr '.' '-')
+        runner_name="${runner_name_prefix}-${runner_type}-${runner_size}-dagger-${dashed_dagger_version}"
+        echo ${runner_name}
+        echo "runner_name=${runner_name}" >> "$GITHUB_OUTPUT"
+outputs:
+  runner_name:
+    description: "Name of self hosted CI runner to use"
+    value: ${{ steps.get-runner-name.outputs.runner_name }}

.github/workflows/airbyte-ci-tests.yml

@@ -6,20 +6,37 @@ concurrency:
 
 on:
   workflow_dispatch:
-    inputs:
-      airbyte_ci_binary_url:
-        description: "URL to airbyte-ci binary"
-        required: false
-        default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
   pull_request:
     types:
       - opened
       - reopened
       - synchronize
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "test"
+          runner_size: "large"
+          airbyte_ci_command: "test"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   run-airbyte-ci-tests:
     name: Run Airbyte CI tests
-    runs-on: "ci-runner-connector-test-large-dagger-0-9-5"
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -78,7 +95,6 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'"
-          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
 
       - name: Run airbyte-ci/connectors/pipelines tests
@@ -94,7 +110,6 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines --disallow-untyped-defs' --poetry-run-command='ruff check pipelines'"
-          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
 
       - name: Run airbyte-ci/connectors/base_images tests
@@ -110,7 +125,6 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'"
-          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
 
       - name: Run test pipeline for the metadata lib
@@ -124,7 +138,6 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
 
       - name: Run test for the metadata orchestrator
@@ -138,7 +151,6 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
 
       - name: Run airbyte-lib tests
@@ -153,5 +165,4 @@ jobs:
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
           subcommand: "test airbyte-lib"
-          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}

.github/workflows/cat-tests.yml

@@ -14,9 +14,31 @@ on:
     paths:
       - airbyte-integrations/bases/connector-acceptance-test/**
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "test"
+          runner_size: "large"
+          airbyte_ci_command: "test"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   run-cat-unit-tests:
     name: Run CAT unit tests
-    runs-on: "ci-runner-connector-test-large-dagger-0-9-5"
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/connectors_nightly_build.yml

@@ -6,21 +6,38 @@ on:
     - cron: "0 0 * * *"
   workflow_dispatch:
     inputs:
-      runs-on:
-        type: string
-        default: ci-runner-connector-nightly-xlarge-dagger-0-9-5
-        required: true
       test-connectors-options:
         default: --concurrency=5 --support-level=certified
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}"
 
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "nightly"
+          runner_size: "xlarge"
+          airbyte_ci_command: "connectors test"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}"
     timeout-minutes: 720 # 12 hours
-    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3

.github/workflows/connectors_tests.yml

@@ -17,23 +17,38 @@ on:
       test-connectors-options:
         description: "Options to pass to the 'airbyte-ci connectors test' command"
         default: "--modified"
-      runner:
-        description: "The runner to use for this job"
-        default: "ci-runner-connector-test-large-dagger-0-9-5"
-      airbyte_ci_binary_url:
-        description: "The URL to download the airbyte-ci binary from"
-        required: false
-        default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
   pull_request:
     types:
       - opened
       - synchronize
       - ready_for_review
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "test"
+          runner_size: "large"
+          airbyte_ci_command: "connectors test"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   connectors_ci:
     name: Connectors CI
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     timeout-minutes: 1440 # 24 hours
-    runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-9-5'}}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
@@ -71,7 +86,6 @@ jobs:
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test"
-          airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
       - name: Test connectors [PULL REQUESTS]
         if: github.event_name == 'pull_request'

.github/workflows/connectors_weekly_build.yml

@@ -6,21 +6,39 @@ on:
     - cron: "0 12 * * 0"
   workflow_dispatch:
     inputs:
-      runs-on:
-        type: string
-        default: ci-runner-connector-nightly-xlarge-dagger-0-9-5
-        required: true
       test-connectors-options:
         default: --concurrency=3 --support-level=community
         required: true
 
-run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
+run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}"
 
 jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "test"
+          runner_size: "large"
+          airbyte_ci_command: "connectors test"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
   test_connectors:
-    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}"
+    name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}"
     timeout-minutes: 8640 # 6 days
-    runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3