airbyte-ci: CLI exposes CI requirements

.github/actions/airbyte-ci-requirements/action.yml

@@ -0,0 +1,102 @@
+name: "Get airbyte-ci runner name"
+description: "Runs a given airbyte-ci command with the --ci-requirements flag to get the CI requirements for a given command"
+inputs:
+  runner_type:
+    description: "Type of runner to get requirements for. One of: format, test, nightly, publish"
+    required: true
+  runner_size:
+    description: "One of: format, test, nightly, publish"
+    required: true
+  airbyte_ci_command:
+    description: "airbyte-ci command to get CI requirements for."
+    required: true
+  runner_name_prefix:
+    description: "Prefix of runner name"
+    required: false
+    default: ci-runner-connector
+  github_token:
+    description: "GitHub token"
+    required: true
+  sentry_dsn:
+    description: "Sentry DSN"
+    required: false
+  airbyte_ci_binary_url:
+    description: "URL to airbyte-ci binary"
+    required: false
+    default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check if PR is from a fork
+      if: github.event_name == 'pull_request'
+      shell: bash
+      run: |
+        if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
+          echo "PR is from a fork. Exiting workflow..."
+          exit 78
+        fi
+
+    - name: Get changed files
+      uses: tj-actions/changed-files@v39
+      id: changes
+      with:
+        files_yaml: |
+          pipelines:
+            - 'airbyte-ci/connectors/pipelines/**'
+
+    - name: Determine how Airbyte CI should be installed
+      shell: bash
+      id: determine-install-mode
+      run: |
+        if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
+          echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
+          echo "install-mode=dev" >> $GITHUB_OUTPUT
+        else
+          echo "install-mode=production" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Install airbyte-ci binary
+      id: install-airbyte-ci
+      if: steps.determine-install-mode.outputs.install-mode == 'production'
+      shell: bash
+      run: |
+        curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin
+        sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+        sudo chmod +x /usr/local/bin/airbyte-ci
+
+    - name: Install Python 3.10
+      uses: actions/setup-python@v4
+      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      with:
+        python-version: "3.10"
+        token: ${{ inputs.github_token }}
+
+    - name: Install ci-connector-ops package
+      if: steps.determine-install-mode.outputs.install-mode == 'dev'
+      shell: bash
+      run: |
+        pip install pipx
+        pipx ensurepath
+        pipx install airbyte-ci/connectors/pipelines/
+
+    - name: Get dagger version from airbyte-ci
+      id: get-dagger-version
+      shell: bash
+      run: |
+        dagger_version=$(airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} --ci-requirements | tail -n 1 | jq -r '.dagger_version')
+        echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT"
+
+    - name: Get runner name
+      id: get-runner-name
+      shell: bash
+      run: |
+        runner_name_prefix=${{ inputs.runner_name_prefix }}
+        runner_type=${{ inputs.runner_type }}
+        runner_size=${{ inputs.runner_size }}
+        dashed_dagger_version=$(echo "${{ steps.get-dagger-version.outputs.dagger_version }}" | tr '.' '-')
+        runner_name="${runner_name_prefix}-${runner_type}-dagger-${runner_size}-${dashed_dagger_version}"
+        echo ${runner_name}
+        echo "runner_name=${runner_name}" >> "$GITHUB_OUTPUT"
+  outputs:
+    runner_name: ${{ steps.get-runner-name.outputs.runner_name }}

.github/workflows/format_check.yml

@@ -16,17 +16,35 @@ on:
   pull_request:
 
 jobs:
-  format-check:
-    runs-on: "ci-runner-connector-format-medium-dagger-0-6-4"
-    # IMPORTANT: This name must match the require check name on the branch protection settings
-    name: "Check for formatting errors"
+  get-ci-runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
           token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
-
+      - name: Get CI runner
+        id: get-ci-runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "format"
+          runner_size: "medium"
+          airbyte_ci_command: "format"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          # Pin to a specific version of airbyte-ci to avoid transient failures
+          # Mentioned in issue https://github.com/airbytehq/airbyte/issues/34041
+          airbyte_ci_binary_url: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/2.14.1/airbyte-ci
+    outputs:
+      runner-name: ${{ steps.get-ci-runner.outputs.runner-name }}
+  format-check:
+    needs: get-ci-runner
+    runs-on: ${{ needs.get-ci-runner.outputs.runner-name }}
+    # IMPORTANT: This name must match the require check name on the branch protection settings
+    name: "Check for formatting errors"
+    steps:
       - name: Run airbyte-ci format check [MASTER]
         id: airbyte_ci_format_check_all_master
         if: github.ref == 'refs/heads/master'

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py

@@ -8,13 +8,15 @@
 from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
 from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext
 from pipelines.airbyte_ci.connectors.publish.pipeline import reorder_contexts, run_connector_publish_pipeline
+from pipelines.cli.click_decorators import click_ci_requirements_option
 from pipelines.cli.confirm_prompt import confirm
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 from pipelines.consts import ContextState
 from pipelines.helpers.utils import fail_if_missing_docker_hub_creds
 
 
 @click.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.")
+@click_ci_requirements_option()
 @click.option("--pre-release/--main-release", help="Use this flag if you want to publish pre-release images.", default=True, type=bool)
 @click.option(
     "--spec-cache-gcs-credentials",

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py

@@ -11,6 +11,7 @@
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
 from pipelines.airbyte_ci.connectors.test.pipeline import run_connector_test_pipeline
+from pipelines.cli.click_decorators import click_ci_requirements_option
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 from pipelines.consts import LOCAL_BUILD_PLATFORM, ContextState
 from pipelines.helpers.github import update_global_commit_status_check_for_tests
@@ -19,6 +20,7 @@
 
 
 @click.command(cls=DaggerPipelineCommand, help="Test all the selected connectors.")
+@click_ci_requirements_option()
 @click.option(
     "--code-tests-only",
     is_flag=True,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py

@@ -14,7 +14,7 @@
 import asyncclick as click
 from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter
 from pipelines.airbyte_ci.format.format_command import FormatCommand
-from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj
 from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results
 from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
 from pipelines.models.steps import StepStatus
@@ -25,6 +25,7 @@
     help="Commands related to formatting.",
 )
 @click.option("--quiet", "-q", help="Hide details of the formatter execution.", default=False, is_flag=True)
+@click_ci_requirements_option()
 @click_merge_args_into_context_obj
 @pass_pipeline_context
 @click_ignore_unused_kwargs

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py

@@ -9,7 +9,7 @@
 
 import asyncclick as click
 import asyncer
-from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj
 from pipelines.consts import DOCKER_VERSION
 from pipelines.helpers.utils import sh_dash_c
 from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
@@ -36,6 +36,7 @@ async def run_poetry_command(container: dagger.Container, command: str) -> Tuple
 
 @click.command()
 @click.argument("poetry_package_path")
+@click_ci_requirements_option()
 @click.option(
     "-c",
     "--poetry-run-command",

airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py

@@ -23,7 +23,12 @@
 from github import PullRequest
 from pipelines import main_logger
 from pipelines.cli.auto_update import __installed_version__, check_for_upgrade, pre_confirm_auto_update_flag
-from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.cli.click_decorators import (
+    CI_REQUIREMENTS_OPTION_NAME,
+    click_append_to_context_object,
+    click_ignore_unused_kwargs,
+    click_merge_args_into_context_obj,
+)
 from pipelines.cli.confirm_prompt import pre_confirm_all_flag
 from pipelines.cli.lazy_group import LazyGroup
 from pipelines.cli.telemetry import click_track_command
@@ -82,6 +87,9 @@ def check_local_docker_configuration() -> None:
 
 
 def is_dagger_run_enabled_by_default() -> bool:
+    if CI_REQUIREMENTS_OPTION_NAME in sys.argv:
+        return False
+
     dagger_run_by_default = [
         ["connectors", "test"],
         ["connectors", "build"],

airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py

@@ -5,9 +5,15 @@
 import functools
 import inspect
 from functools import wraps
-from typing import Any, Callable, Type
+from importlib import metadata
+from typing import Any, Callable, Type, TypeVar
 
 import asyncclick as click
+from pipelines.models.ci_requirements import CIRequirements
+
+_AnyCallable = Callable[..., Any]
+FC = TypeVar("FC", bound="_AnyCallable | click.core.Command")
+CI_REQUIREMENTS_OPTION_NAME = "--ci-requirements"
 
 
 def _contains_var_kwarg(f: Callable) -> bool:
@@ -121,3 +127,27 @@ def decorated_function(*args: Any, **kwargs: Any) -> Any:  # noqa: ANN401
             return f(*args, **kwargs)
 
         return decorated_function
+
+
+def click_ci_requirements_option() -> Callable[[FC], FC]:
+    """Add a --ci-requirements option to the command.
+
+    Returns:
+        Callable[[FC], FC]: The decorated command.
+    """
+
+    def callback(ctx: click.Context, param: click.Parameter, value: bool) -> None:
+        if value:
+            ci_requirements = CIRequirements()
+            click.echo(ci_requirements.to_json())
+            ctx.exit()
+
+    return click.decorators.option(
+        CI_REQUIREMENTS_OPTION_NAME,
+        is_flag=True,
+        expose_value=False,
+        is_eager=True,
+        flag_value=True,
+        help="Show the CI requirements and exit. It used to make airbyte-ci client define the CI runners it will run on.",
+        callback=callback,
+    )

airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import json
+from dataclasses import dataclass
+from importlib import metadata
+
+INFRA_SUPPORTED_DAGGER_VERSIONS = {
+    "0.6.4",
+    "0.9.5",
+}
+
+
+@dataclass
+class CIRequirements:
+    """
+    A dataclass to store the CI requirements.
+    It used to make airbyte-ci client define the CI runners it will run on.
+    """
+
+    dagger_version = metadata.version("dagger-io")
+
+    def __post_init__(self):
+        if self.dagger_version not in INFRA_SUPPORTED_DAGGER_VERSIONS:
+            raise ValueError(
+                f"Unsupported dagger version: {self.dagger_version}. " f"Supported versions are: {INFRA_SUPPORTED_DAGGER_VERSIONS}."
+            )
+
+    def to_json(self):
+        return json.dumps(
+            {
+                "dagger_version": self.dagger_version,
+            }
+        )

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.1.3"
+version = "3.2.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]