-
Notifications
You must be signed in to change notification settings - Fork 3.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
airbyte-ci: run poe tasks declared in pyproject.toml file of internal…
… poetry packages (#34736)
- Loading branch information
1 parent
cee3a19
commit 5af9696
Showing
28 changed files
with
646 additions
and
442 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,142 +1,80 @@ | ||
| name: Connector Ops CI - Pipeline Unit Test | ||
| name: Internal Poetry packages CI | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| airbyte_ci_subcommand: | ||
| description: "Subcommand to pass to the 'airbyte-ci test' command" | ||
| default: "--poetry-package-path=airbyte-ci/connectors/pipelines" | ||
| pull_request: | ||
| types: | ||
| - opened | ||
| - reopened | ||
| - synchronize | ||
| jobs: | ||
| run-airbyte-ci-tests: | ||
| run-airbyte-ci-poetry-ci: | ||
| #name: Internal Poetry packages CI | ||
| # To rename in a follow up PR | ||
| name: Run Airbyte CI tests | ||
| runs-on: tooling-test-large | ||
| permissions: | ||
| pull-requests: read | ||
| statuses: write | ||
| steps: | ||
| - name: Checkout Airbyte | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| fetch-depth: 0 | ||
| ref: ${{ github.event.pull_request.head.ref }} | ||
|
|
||
| # IMPORTANT! This is nessesary to make sure that a status is reported on the PR | ||
| # even if the workflow is skipped. If we used github actions filters, the workflow | ||
| # would not be reported as skipped, but instead would be forever pending. | ||
| # | ||
| # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. | ||
| # | ||
| # Also it gets worse | ||
| # | ||
| # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. | ||
| # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. | ||
| - name: Get changed files | ||
| uses: tj-actions/changed-files@v39 | ||
| id: changes | ||
| with: | ||
| files_yaml: | | ||
| ops: | ||
| - 'airbyte-ci/connectors/connector_ops/**' | ||
| - '!**/*.md' | ||
| base_images: | ||
| - 'airbyte-ci/connectors/connector_ops/**' | ||
| - 'airbyte-ci/connectors/base_images/**' | ||
| - '!**/*.md' | ||
| pipelines: | ||
| - 'airbyte-ci/connectors/connector_ops/**' | ||
| - 'airbyte-ci/connectors/base_images/**' | ||
| - 'airbyte-ci/connectors/pipelines/**' | ||
| - '!**/*.md' | ||
| metadata_lib: | ||
| - 'airbyte-ci/connectors/metadata_service/lib/**' | ||
| - '!**/*.md' | ||
| metadata_orchestrator: | ||
| - 'airbyte-ci/connectors/metadata_service/lib/**' | ||
| - 'airbyte-ci/connectors/metadata_service/orchestrator/**' | ||
| - '!**/*.md' | ||
| airbyte_lib: | ||
| - 'airbyte-lib/**' | ||
| - '!**/*.md' | ||
| - name: Run airbyte-ci/connectors/connector_ops tests | ||
| if: steps.changes.outputs.ops_any_changed == 'true' | ||
| id: run-airbyte-ci-connectors-connector-ops-tests | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| context: "pull_request" | ||
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | ||
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | ||
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | ||
| subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'" | ||
|
|
||
| - name: Run airbyte-ci/connectors/pipelines tests | ||
| id: run-airbyte-ci-connectors-pipelines-tests | ||
| if: steps.changes.outputs.pipelines_any_changed == 'true' | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| context: "pull_request" | ||
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | ||
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | ||
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | ||
| subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines --disallow-untyped-defs' --poetry-run-command='ruff check pipelines'" | ||
| - name: Extract branch name [WORKFLOW DISPATCH] | ||
| shell: bash | ||
| if: github.event_name == 'workflow_dispatch' | ||
| run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT | ||
| id: extract_branch | ||
| - name: Fetch last commit id from remote branch [PULL REQUESTS] | ||
| if: github.event_name == 'pull_request' | ||
| id: fetch_last_commit_id_pr | ||
| run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT | ||
| - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH] | ||
| if: github.event_name == 'workflow_dispatch' | ||
| id: fetch_last_commit_id_wd | ||
| run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT | ||
|
|
||
| - name: Run airbyte-ci/connectors/base_images tests | ||
| id: run-airbyte-ci-connectors-base-images-tests | ||
| if: steps.changes.outputs.base_images_any_changed == 'true' | ||
| - name: Run poe tasks for modified internal packages [PULL REQUEST] | ||
| if: github.event_name == 'pull_request' | ||
| id: run-airbyte-ci-test-pr | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| context: "pull_request" | ||
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | ||
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | ||
| git_branch: ${{ github.head_ref }} | ||
| git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} | ||
| github_token: ${{ github.token }} | ||
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | ||
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | ||
| subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'" | ||
|
|
||
| - name: Run test pipeline for the metadata lib | ||
| id: metadata_lib-test-pipeline | ||
| if: steps.changes.outputs.metadata_lib_any_changed == 'true' | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'" | ||
| context: "pull_request" | ||
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
|
|
||
| - name: Run test for the metadata orchestrator | ||
| id: metadata_orchestrator-test-pipeline | ||
| if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true' | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'" | ||
| context: "pull_request" | ||
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| subcommand: "test --modified" | ||
|
|
||
| - name: Run airbyte-lib tests | ||
| if: steps.changes.outputs.airbyte_lib_any_changed == 'true' | ||
| id: run-airbyte-lib-tests | ||
| - name: Run poe tasks for requested internal packages [WORKFLOW DISPATCH] | ||
| id: run-airbyte-ci-test-workflow-dispatch | ||
| if: github.event_name == 'workflow_dispatch' | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| context: "pull_request" | ||
| context: "manual" | ||
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | ||
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | ||
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | ||
| git_branch: ${{ steps.extract_branch.outputs.branch }} | ||
| git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} | ||
| github_token: ${{ github.token }} | ||
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | ||
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | ||
| subcommand: "test airbyte-lib --side-car-docker-engine --pass-env-var=GCP_GSM_CREDENTIALS --poetry-run-command='pytest'" | ||
| subcommand: "test ${{ inputs.airbyte_ci_subcommand}}" |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.