airbyte-ci: run poe tasks declared in pyproject.toml file of internal poetry packages

.github/workflows/airbyte-ci-tests.yml

@@ -1,142 +1,80 @@
-name: Connector Ops CI - Pipeline Unit Test
+name: Internal Poetry packages CI
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 on:
   workflow_dispatch:
+    inputs:
+      airbyte_ci_subcommand:
+        description: "Subcommand to pass to the 'airbyte-ci test' command"
+        default: "--poetry-package-path=airbyte-ci/connectors/pipelines"
   pull_request:
     types:
       - opened
       - reopened
       - synchronize
 jobs:
-  run-airbyte-ci-tests:
+  run-airbyte-ci-poetry-ci:
+    #name: Internal Poetry packages CI
+    # To rename in a follow up PR
     name: Run Airbyte CI tests
     runs-on: tooling-test-large
+    permissions:
+      pull-requests: read
+      statuses: write
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.ref }}
 
-      # IMPORTANT! This is nessesary to make sure that a status is reported on the PR
-      # even if the workflow is skipped. If we used github actions filters, the workflow
-      # would not be reported as skipped, but instead would be forever pending.
-      #
-      # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE.
-      #
-      # Also it gets worse
-      #
-      # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED.
-      # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING.
-      - name: Get changed files
-        uses: tj-actions/changed-files@v39
-        id: changes
-        with:
-          files_yaml: |
-            ops:
-              - 'airbyte-ci/connectors/connector_ops/**'
-              - '!**/*.md'
-            base_images:
-              - 'airbyte-ci/connectors/connector_ops/**'
-              - 'airbyte-ci/connectors/base_images/**'
-              - '!**/*.md'
-            pipelines:
-              - 'airbyte-ci/connectors/connector_ops/**'
-              - 'airbyte-ci/connectors/base_images/**'
-              - 'airbyte-ci/connectors/pipelines/**'
-              - '!**/*.md'
-            metadata_lib:
-              - 'airbyte-ci/connectors/metadata_service/lib/**'
-              - '!**/*.md'
-            metadata_orchestrator:
-              - 'airbyte-ci/connectors/metadata_service/lib/**'
-              - 'airbyte-ci/connectors/metadata_service/orchestrator/**'
-              - '!**/*.md'
-            airbyte_lib:
-              - 'airbyte-lib/**'
-              - '!**/*.md'
-
-      - name: Run airbyte-ci/connectors/connector_ops tests
-        if: steps.changes.outputs.ops_any_changed == 'true'
-        id: run-airbyte-ci-connectors-connector-ops-tests
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'"
-
-      - name: Run airbyte-ci/connectors/pipelines tests
-        id: run-airbyte-ci-connectors-pipelines-tests
-        if: steps.changes.outputs.pipelines_any_changed == 'true'
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines --disallow-untyped-defs' --poetry-run-command='ruff check pipelines'"
+      - name: Extract branch name [WORKFLOW DISPATCH]
+        shell: bash
+        if: github.event_name == 'workflow_dispatch'
+        run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+        id: extract_branch
+      - name: Fetch last commit id from remote branch [PULL REQUESTS]
+        if: github.event_name == 'pull_request'
+        id: fetch_last_commit_id_pr
+        run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT
+      - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
+        if: github.event_name == 'workflow_dispatch'
+        id: fetch_last_commit_id_wd
+        run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
 
-      - name: Run airbyte-ci/connectors/base_images tests
-        id: run-airbyte-ci-connectors-base-images-tests
-        if: steps.changes.outputs.base_images_any_changed == 'true'
+      - name: Run poe tasks for modified internal packages [PULL REQUEST]
+        if: github.event_name == 'pull_request'
+        id: run-airbyte-ci-test-pr
         uses: ./.github/actions/run-airbyte-ci
         with:
           context: "pull_request"
           dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          git_branch: ${{ github.head_ref }}
+          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
+          github_token: ${{ github.token }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'"
-
-      - name: Run test pipeline for the metadata lib
-        id: metadata_lib-test-pipeline
-        if: steps.changes.outputs.metadata_lib_any_changed == 'true'
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'"
-          context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-      - name: Run test for the metadata orchestrator
-        id: metadata_orchestrator-test-pipeline
-        if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true'
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'"
-          context: "pull_request"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          subcommand: "test --modified"
 
-      - name: Run airbyte-lib tests
-        if: steps.changes.outputs.airbyte_lib_any_changed == 'true'
-        id: run-airbyte-lib-tests
+      - name: Run poe tasks for requested internal packages [WORKFLOW DISPATCH]
+        id: run-airbyte-ci-test-workflow-dispatch
+        if: github.event_name == 'workflow_dispatch'
         uses: ./.github/actions/run-airbyte-ci
         with:
-          context: "pull_request"
+          context: "manual"
           dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          git_branch: ${{ steps.extract_branch.outputs.branch }}
+          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
+          github_token: ${{ github.token }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-lib --side-car-docker-engine --pass-env-var=GCP_GSM_CREDENTIALS --poetry-run-command='pytest'"
+          subcommand: "test ${{ inputs.airbyte_ci_subcommand}}"

airbyte-ci/connectors/pipelines/README.md

@@ -613,39 +613,34 @@ flowchart TD
 
 ### <a id="tests-command"></a>`tests` command
 
-This command runs the Python tests for a airbyte-ci poetry package.
+This command runs the poe tasks declared in the `[tool.airbyte-ci]` section of our internal poetry packages.
+Feel free to checkout this [Pydantic model](https://github.com/airbytehq/airbyte/blob/main/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py#L9) to see the list of available options in `[tool.airbyte-ci]` section.
 
-#### Arguments
-
-| Option                | Required | Default | Mapped environment variable | Description                         |
-| --------------------- | -------- | ------- | --------------------------- | ----------------------------------- |
-| `poetry_package_path` | True     |         |                             | The path to poetry package to test. |
+You can find the list of internal packages [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py#L1)
 
 #### Options
 
-| Option                    | Required | Default | Mapped environment variable | Description                                                                                 |
-| ------------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- |
-| `-c/--poetry-run-command` | True     | None    |                             | The command to run with `poetry run`                                                        |
-| `-e/--pass-env-var`       | False    | None    |                             | Host environment variable that is passed to the container running the poetry command        |
-| `--ci-requirements`       | False    |         |                             | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. |
+| Option                    | Required | Multiple| Description                                                                                 |
+| ------------------------- | -------- | ------- | ------------------------------------------------------------------------------------------- |
+| `--poetry-package-path/-p`| False    | True    | Poetry packages path to run the poe tasks for.                                              |
+| `--modified`              | False    | False   | Run poe tasks of modified internal poetry packages.                                         |
+| `--ci-requirements`       | False    | False   | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. |
 
 #### Examples
+You can pass multiple `--poetry-package-path` options to run poe tasks.
 
-You can pass multiple `-c/--poetry-run-command` options to run multiple commands.
-
-E.G.: running `pytest` and `mypy`:
-`airbyte-ci test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines'`
+E.G.: running Poe tasks on `airbyte-lib` and `airbyte-ci/connectors/pipelines`:
+`airbyte-ci test --poetry-package-path=airbyte-ci/connectors/pipelines --poetry-package-path=airbyte-lib`
 
-E.G.: passing the environment variable `GCP_GSM_CREDENTIALS` environment variable to the container
-running the poetry command: `airbyte-ci test airbyte-lib --pass-env-var='GCP_GSM_CREDENTIALS'`
+E.G.: running Poe tasks on the modified internal packages of the current branch:
+`airbyte-ci test --modified`
 
-E.G.: running `pytest` on a specific test folder:
-`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --poetry-run-command='pytest tests/unit_tests'`
 
 ## Changelog
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 4.0.0  | [#34736](https://github.com/airbytehq/airbyte/pull/34736)  | Run poe tasks declared in internal poetry packages.                                         |
 | 3.10.4  | [#34867](https://github.com/airbytehq/airbyte/pull/34867)  | Remove connector ops team                                                                                                  |
 | 3.10.3  | [#34836](https://github.com/airbytehq/airbyte/pull/34836)  | Add check for python registry publishing enabled for certified python sources.                                             |
 | 3.10.2  | [#34044](https://github.com/airbytehq/airbyte/pull/34044)  | Add pypi validation testing.                                                                                               |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py

@@ -39,16 +39,16 @@ async def _run(self, *args: Any) -> StepResult:
                     await connector.with_exec(["spec"])
                 except ExecError:
                     return StepResult(
-                        self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {platform}."
+                        step=self, status=StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {platform}."
                     )
                 build_results_per_platform[platform] = connector
             except QueryError as e:
-                return StepResult(self, StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}")
+                return StepResult(step=self, status=StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}")
         success_message = (
             f"The {self.context.connector.technical_name} docker image "
             f"was successfully built for platform(s) {', '.join(self.build_platforms)}"
         )
-        return StepResult(self, StepStatus.SUCCESS, stdout=success_message, output_artifact=build_results_per_platform)
+        return StepResult(step=self, status=StepStatus.SUCCESS, stdout=success_message, output_artifact=build_results_per_platform)
 
     async def _build_connector(self, platform: Platform, *args: Any, **kwargs: Any) -> Container:
         """Implement the generation of the image for the platform and return the corresponding container.
@@ -89,8 +89,8 @@ async def _run(self) -> StepResult:
             _, exported_tar_path = await export_container_to_tarball(self.context, container, platform)
             if not exported_tar_path:
                 return StepResult(
-                    self,
-                    StepStatus.FAILURE,
+                    step=self,
+                    status=StepStatus.FAILURE,
                     stderr=f"Failed to export the connector image {self.image_name}:{self.image_tag} to a tarball.",
                 )
             try:
@@ -104,7 +104,7 @@ async def _run(self) -> StepResult:
                     loaded_images.append(full_image_name)
             except docker.errors.DockerException as e:
                 return StepResult(
-                    self, StepStatus.FAILURE, stderr=f"Something went wrong while interacting with the local docker client: {e}"
+                    step=self, status=StepStatus.FAILURE, stderr=f"Something went wrong while interacting with the local docker client: {e}"
                 )
 
-        return StepResult(self, StepStatus.SUCCESS, stdout=f"Loaded image {','.join(loaded_images)} to your Docker host ({image_sha}).")
+        return StepResult(step=self, status=StepStatus.SUCCESS, stdout=f"Loaded image {','.join(loaded_images)} to your Docker host ({image_sha}).")

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py

@@ -36,10 +36,10 @@ async def _run(self, dist_dir: Directory) -> StepResult:
                     if num_files == 0
                     else "More than one distribution tar file was built for the current java connector."
                 )
-                return StepResult(self, StepStatus.FAILURE, stderr=error_message)
+                return StepResult(step=self, status=StepStatus.FAILURE, stderr=error_message)
             dist_tar = dist_dir.file(tar_files[0])
         except QueryError as e:
-            return StepResult(self, StepStatus.FAILURE, stderr=str(e))
+            return StepResult(step=self, status=StepStatus.FAILURE, stderr=str(e))
         return await super()._run(dist_tar)
 
     async def _build_connector(self, platform: Platform, dist_tar: File) -> Container:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py

@@ -36,4 +36,4 @@ async def _run(self) -> StepResult:
             build_normalization_container = normalization.with_normalization(self.context, self.build_platform)
         else:
             build_normalization_container = self.context.dagger_client.container().from_(self.normalization_image)
-        return StepResult(self, StepStatus.SUCCESS, output_artifact=build_normalization_container)
+        return StepResult(step=self, status=StepStatus.SUCCESS, output_artifact=build_normalization_container)