diff --git a/.github/actions/airbyte-ci-requirements/action.yml b/.github/actions/airbyte-ci-requirements/action.yml new file mode 100644 index 00000000000000..8172bf5a8ee1d3 --- /dev/null +++ b/.github/actions/airbyte-ci-requirements/action.yml @@ -0,0 +1,104 @@ +name: "Get airbyte-ci runner name" +description: "Runs a given airbyte-ci command with the --ci-requirements flag to get the CI requirements for a given command" +inputs: + runner_type: + description: "Type of runner to get requirements for. One of: format, test, nightly, publish" + required: true + runner_size: + description: "One of: format, test, nightly, publish" + required: true + airbyte_ci_command: + description: "airbyte-ci command to get CI requirements for." + required: true + runner_name_prefix: + description: "Prefix of runner name" + required: false + default: ci-runner-connector + github_token: + description: "GitHub token" + required: true + sentry_dsn: + description: "Sentry DSN" + required: false + airbyte_ci_binary_url: + description: "URL to airbyte-ci binary" + required: false + default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + +runs: + using: "composite" + steps: + - name: Check if PR is from a fork + if: github.event_name == 'pull_request' + shell: bash + run: | + if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then + echo "PR is from a fork. Exiting workflow..." + exit 78 + fi + + - name: Get changed files + uses: tj-actions/changed-files@v39 + id: changes + with: + files_yaml: | + pipelines: + - 'airbyte-ci/connectors/pipelines/**' + + - name: Determine how Airbyte CI should be installed + shell: bash + id: determine-install-mode + run: | + if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then + echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source." + echo "install-mode=dev" >> $GITHUB_OUTPUT + else + echo "install-mode=production" >> $GITHUB_OUTPUT + fi + + - name: Install airbyte-ci binary + id: install-airbyte-ci + if: steps.determine-install-mode.outputs.install-mode == 'production' + shell: bash + run: | + curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin + sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci + sudo chmod +x /usr/local/bin/airbyte-ci + + - name: Install Python 3.10 + uses: actions/setup-python@v4 + if: steps.determine-install-mode.outputs.install-mode == 'dev' + with: + python-version: "3.10" + token: ${{ inputs.github_token }} + + - name: Install ci-connector-ops package + if: steps.determine-install-mode.outputs.install-mode == 'dev' + shell: bash + run: | + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/pipelines/ + + - name: Get dagger version from airbyte-ci + id: get-dagger-version + shell: bash + run: | + dagger_version=$(airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version') + echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT" + + - name: Get runner name + id: get-runner-name + shell: bash + run: | + runner_name_prefix=${{ inputs.runner_name_prefix }} + runner_type=${{ inputs.runner_type }} + runner_size=${{ inputs.runner_size }} + dashed_dagger_version=$(echo "${{ steps.get-dagger-version.outputs.dagger_version }}" | tr '.' '-') + runner_name="${runner_name_prefix}-${runner_type}-${runner_size}-dagger-${dashed_dagger_version}" + echo ${runner_name} + echo "runner_name=${runner_name}" >> "$GITHUB_OUTPUT" +outputs: + runner_name: + description: "Name of self hosted CI runner to use" + value: ${{ steps.get-runner-name.outputs.runner_name }} diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index 1b5f32810b6f3d..2f9da503509c9a 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -17,9 +17,31 @@ on: - reopened - synchronize jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "publish" + runner_size: "large" + airbyte_ci_command: "publish" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} run-airbyte-ci-tests: name: Run Airbyte CI tests - runs-on: "ci-runner-connector-test-large-dagger-0-9-5" + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/cat-tests.yml b/.github/workflows/cat-tests.yml index ebc803b212d64a..2f8cda400060b3 100644 --- a/.github/workflows/cat-tests.yml +++ b/.github/workflows/cat-tests.yml @@ -14,9 +14,31 @@ on: paths: - airbyte-integrations/bases/connector-acceptance-test/** jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "test" + runner_size: "large" + airbyte_ci_command: "test" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} run-cat-unit-tests: name: Run CAT unit tests - runs-on: "ci-runner-connector-test-large-dagger-0-9-5" + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/connectors_nightly_build.yml b/.github/workflows/connectors_nightly_build.yml index d89ad8ad4a6f4c..a2724c063d6b7e 100644 --- a/.github/workflows/connectors_nightly_build.yml +++ b/.github/workflows/connectors_nightly_build.yml @@ -6,21 +6,38 @@ on: - cron: "0 0 * * *" workflow_dispatch: inputs: - runs-on: - type: string - default: ci-runner-connector-nightly-xlarge-dagger-0-9-5 - required: true test-connectors-options: default: --concurrency=5 --support-level=certified required: true -run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}" +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}" jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "nightly" + runner_size: "xlarge" + airbyte_ci_command: "connectors test" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} test_connectors: - name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}" + name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}" timeout-minutes: 720 # 12 hours - runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }} + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml index 190851778ab8be..a19b45473b4dd4 100644 --- a/.github/workflows/connectors_tests.yml +++ b/.github/workflows/connectors_tests.yml @@ -17,9 +17,6 @@ on: test-connectors-options: description: "Options to pass to the 'airbyte-ci connectors test' command" default: "--modified" - runner: - description: "The runner to use for this job" - default: "ci-runner-connector-test-large-dagger-0-9-5" airbyte_ci_binary_url: description: "The URL to download the airbyte-ci binary from" required: false @@ -30,10 +27,32 @@ on: - synchronize - ready_for_review jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "test" + runner_size: "large" + airbyte_ci_command: "connectors test" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} connectors_ci: name: Connectors CI + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} timeout-minutes: 1440 # 24 hours - runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-9-5'}} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/connectors_weekly_build.yml b/.github/workflows/connectors_weekly_build.yml index 55ecd461bdd22b..e69bb6beae36c8 100644 --- a/.github/workflows/connectors_weekly_build.yml +++ b/.github/workflows/connectors_weekly_build.yml @@ -6,21 +6,39 @@ on: - cron: "0 12 * * 0" workflow_dispatch: inputs: - runs-on: - type: string - default: ci-runner-connector-nightly-xlarge-dagger-0-9-5 - required: true test-connectors-options: default: --concurrency=3 --support-level=community required: true -run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}" +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}" jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "publish" + runner_size: "large" + airbyte_ci_command: "publish" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} test_connectors: - name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }}" + name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}" timeout-minutes: 8640 # 6 days - runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-9-5' }} + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 44ceac22e0b87a..ba0de58a5e408e 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -16,17 +16,42 @@ on: pull_request: jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "format" + runner_size: "medium" + airbyte_ci_command: "format" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + # Pin to a specific version of airbyte-ci to avoid transient failures + # Mentioned in issue https://github.com/airbytehq/airbyte/issues/34041 + airbyte_ci_binary_url: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/2.14.1/airbyte-ci + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} format-check: - runs-on: "ci-runner-connector-format-medium-dagger-0-6-4" # IMPORTANT: This name must match the require check name on the branch protection settings name: "Check for formatting errors" + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - + fetch-depth: 1 - name: Run airbyte-ci format check [MASTER] id: airbyte_ci_format_check_all_master if: github.ref == 'refs/heads/master' diff --git a/.github/workflows/format_fix.yml b/.github/workflows/format_fix.yml index fdc33d8fd405b4..3ea52c3ccbc933 100644 --- a/.github/workflows/format_fix.yml +++ b/.github/workflows/format_fix.yml @@ -9,9 +9,31 @@ concurrency: on: workflow_dispatch: jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "publish" + runner_size: "large" + airbyte_ci_command: "publish" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} format-fix: - runs-on: "ci-runner-connector-format-medium-dagger-0-9-5" name: "Run airbyte-ci format fix all" + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml index 34d8a42490dd24..eaaab555b4e3d9 100644 --- a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml +++ b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml @@ -8,9 +8,31 @@ on: paths: - "airbyte-ci/connectors/metadata_service/orchestrator/**" jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "test" # We don't have a specific runner for metadata, let's use the test one + runner_size: "large" + airbyte_ci_command: "metadata" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} connector_metadata_service_deploy_orchestrator: name: Connector metadata service deploy orchestrator - runs-on: ci-runner-connector-test-large-dagger-0-9-5 + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v2 diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 228faae2abd94d..73a674eedddd14 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -14,18 +14,36 @@ on: publish-options: description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. " default: "--pre-release" - runs-on: - type: string - default: ci-runner-connector-publish-large-dagger-0-9-5 - required: true airbyte-ci-binary-url: description: "URL to airbyte-ci binary" required: false default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "publish" + runner_size: "large" + airbyte_ci_command: "connectors publish" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} publish_connectors: name: Publish connectors - runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-9-5' }} + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3