Airbyte-ci: Add ClickPipelineContext and update test to use

airbyte-ci/connectors/pipelines/README.md

@@ -52,8 +52,8 @@ If you face any installation problems feel free to reach out the Airbyte Connect
 ### Setting up connector secrets access
 
 If you plan to use Airbyte CI to run CAT (Connector Acceptance Tests), we recommend setting up GSM
-access so that Airbyte CI can pull remote secrets from GSM. For setup instructions, see the 
-CI Credentials package (which Airbyte CI uses under the hood) README's 
+access so that Airbyte CI can pull remote secrets from GSM. For setup instructions, see the
+CI Credentials package (which Airbyte CI uses under the hood) README's
 [Get GSM Access](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md#get-gsm-access)
 instructions.
 
@@ -155,7 +155,7 @@ Available commands:
 | `--enable-dependency-scanning / --disable-dependency-scanning` | False    | ` --disable-dependency-scanning` |                             | When enabled the dependency scanning will be performed to detect the connectors to select according to a dependency change.                                                                                                                                                                                                                                                                                                                                                               |
 | `--docker-hub-username`                                        |          |                                  | DOCKER_HUB_USERNAME         | Your username to connect to DockerHub. Required for the publish subcommand.                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `--docker-hub-password`                                        |          |                                  | DOCKER_HUB_PASSWORD         | Your password to connect to DockerHub. Required for the publish subcommand.                                                                                                                                                                                                                                                                                                                                                                                                               |
-                                                                                                                                           
+
 
 ### <a id="connectors-list-command"></a>`connectors list` command
 Retrieve the list of connectors satisfying the provided filters.
@@ -408,6 +408,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                         | Description                                                                                               |
 | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
+| 2.5.7   | [#31628](https://github.com/airbytehq/airbyte/pull/31628)  | Add ClickPipelineContext class                                                                            |
 | 2.5.6   | [#32139](https://github.com/airbytehq/airbyte/pull/32139)  | Test coverage report on Python connector UnitTest.                                                        |
 | 2.5.5   | [#32114](https://github.com/airbytehq/airbyte/pull/32114)  | Create cache mount for `/var/lib/docker` to store images in `dind` context.                               |
 | 2.5.4   | [#32090](https://github.com/airbytehq/airbyte/pull/32090)  | Do not cache `docker login`.                                                                              |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py

@@ -37,7 +37,7 @@ async def build(ctx: click.Context, use_host_gradle_dist_tar: bool) -> bool:
             ci_context=ctx.obj.get("ci_context"),
             ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
             use_local_cdk=ctx.obj.get("use_local_cdk"),
-            open_report_in_browser=ctx.obj.get("open_report_in_browser"),
+            enable_report_auto_open=ctx.obj.get("enable_report_auto_open"),
             use_host_gradle_dist_tar=use_host_gradle_dist_tar,
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
             s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py

@@ -39,7 +39,7 @@ async def bump_version(
             ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
             ci_git_user=ctx.obj["ci_git_user"],
             ci_github_access_token=ctx.obj["ci_github_access_token"],
-            open_report_in_browser=False,
+            enable_report_auto_open=False,
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
             s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),
         )

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py

@@ -9,6 +9,7 @@
 import asyncclick as click
 from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo
 from pipelines import main_logger
+from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj
 from pipelines.cli.lazy_group import LazyGroup
 from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors
 
@@ -225,43 +226,24 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool:
     required=False,
     envvar="DOCKER_HUB_PASSWORD",
 )
+@click_merge_args_into_context_obj
+@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"]))
 @click.pass_context
+@click_ignore_unused_kwargs
 async def connectors(
     ctx: click.Context,
-    use_remote_secrets: Optional[bool],
-    names: Tuple[str],
-    languages: Tuple[ConnectorLanguage],
-    support_levels: Tuple[str],
-    modified: bool,
-    metadata_changes_only: bool,
-    metadata_query: str,
-    concurrency: int,
-    execute_timeout: int,
-    enable_dependency_scanning: bool,
-    use_local_cdk: bool,
-    enable_report_auto_open: bool,
-    docker_hub_username: str,
-    docker_hub_password: str,
 ):
     """Group all the connectors-ci command."""
     validate_environment(ctx.obj["is_local"])
 
-    ctx.ensure_object(dict)
-    ctx.obj["use_remote_secrets"] = should_use_remote_secrets(use_remote_secrets)
-    ctx.obj["concurrency"] = concurrency
-    ctx.obj["execute_timeout"] = execute_timeout
-    ctx.obj["use_local_cdk"] = use_local_cdk
-    ctx.obj["open_report_in_browser"] = enable_report_auto_open
-    ctx.obj["docker_hub_username"] = docker_hub_username
-    ctx.obj["docker_hub_password"] = docker_hub_password
     ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files(
-        names,
-        support_levels,
-        languages,
-        modified,
-        metadata_changes_only,
-        metadata_query,
+        ctx.obj["names"],
+        ctx.obj["support_levels"],
+        ctx.obj["languages"],
+        ctx.obj["modified"],
+        ctx.obj["metadata_changes_only"],
+        ctx.obj["metadata_query"],
         ctx.obj["modified_files"],
-        enable_dependency_scanning,
+        ctx.obj["enable_dependency_scanning"],
     )
     log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"])

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py

@@ -19,7 +19,7 @@
 from pipelines.helpers.github import update_commit_status_check
 from pipelines.helpers.slack import send_message_to_webhook
 from pipelines.helpers.utils import METADATA_FILE_NAME
-from pipelines.models.contexts import PipelineContext
+from pipelines.models.contexts.pipeline_context import PipelineContext
 
 
 class ConnectorContext(PipelineContext):
@@ -54,7 +54,7 @@ def __init__(
         code_tests_only: bool = False,
         use_local_cdk: bool = False,
         use_host_gradle_dist_tar: bool = False,
-        open_report_in_browser: bool = True,
+        enable_report_auto_open: bool = True,
         docker_hub_username: Optional[str] = None,
         docker_hub_password: Optional[str] = None,
         s3_build_cache_access_key_id: Optional[str] = None,
@@ -82,7 +82,7 @@ def __init__(
             fast_tests_only (bool, optional): Whether to run only fast tests. Defaults to False.
             code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False.
             use_host_gradle_dist_tar (bool, optional): Used when developing java connectors with gradle. Defaults to False.
-            open_report_in_browser (bool, optional): Open HTML report in browser window. Defaults to True.
+            enable_report_auto_open (bool, optional): Open HTML report in browser window. Defaults to True.
             docker_hub_username (Optional[str], optional): Docker Hub username to use to read registries. Defaults to None.
             docker_hub_password (Optional[str], optional): Docker Hub password to use to read registries. Defaults to None.
             s3_build_cache_access_key_id (Optional[str], optional): Gradle S3 Build Cache credentials. Defaults to None.
@@ -104,7 +104,7 @@ def __init__(
         self.code_tests_only = code_tests_only
         self.use_local_cdk = use_local_cdk
         self.use_host_gradle_dist_tar = use_host_gradle_dist_tar
-        self.open_report_in_browser = open_report_in_browser
+        self.enable_report_auto_open = enable_report_auto_open
         self.docker_hub_username = docker_hub_username
         self.docker_hub_password = docker_hub_password
         self.s3_build_cache_access_key_id = s3_build_cache_access_key_id
@@ -127,7 +127,7 @@ def __init__(
             ci_gcs_credentials=ci_gcs_credentials,
             ci_git_user=ci_git_user,
             ci_github_access_token=ci_github_access_token,
-            open_report_in_browser=open_report_in_browser,
+            enable_report_auto_open=enable_report_auto_open,
         )
 
     @property

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py

@@ -41,7 +41,7 @@ async def migrate_to_base_image(
             ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
             ci_git_user=ctx.obj["ci_git_user"],
             ci_github_access_token=ctx.obj["ci_github_access_token"],
-            open_report_in_browser=False,
+            enable_report_auto_open=False,
             docker_hub_username=ctx.obj.get("docker_hub_username"),
             docker_hub_password=ctx.obj.get("docker_hub_password"),
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py

@@ -107,9 +107,9 @@ async def to_html(self) -> str:
     async def save(self) -> None:
         local_html_path = await self.save_local(self.html_report_file_name, await self.to_html())
         absolute_path = await local_html_path.resolve()
-        if self.pipeline_context.open_report_in_browser:
+        if self.pipeline_context.enable_report_auto_open:
             self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}")
-            if self.pipeline_context.open_report_in_browser:
+            if self.pipeline_context.enable_report_auto_open:
                 self.pipeline_context.logger.info("Opening HTML report in browser.")
                 webbrowser.open(absolute_path.as_uri())
         if self.remote_storage_enabled:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py

@@ -20,7 +20,7 @@
 from pipelines.dagger.actions import secrets
 from pipelines.dagger.containers import internal_tools
 from pipelines.helpers.utils import METADATA_FILE_NAME
-from pipelines.models.contexts import PipelineContext
+from pipelines.models.contexts.pipeline_context import PipelineContext
 from pipelines.models.steps import Step, StepResult, StepStatus
 
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py

@@ -35,7 +35,7 @@ async def upgrade_base_image(ctx: click.Context, set_if_not_exists: bool, docker
             ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
             ci_git_user=ctx.obj["ci_git_user"],
             ci_github_access_token=ctx.obj["ci_github_access_token"],
-            open_report_in_browser=False,
+            enable_report_auto_open=False,
             docker_hub_username=ctx.obj.get("docker_hub_username"),
             docker_hub_password=ctx.obj.get("docker_hub_password"),
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py

@@ -7,7 +7,7 @@
 import dagger
 from pipelines.dagger.actions.python.pipx import with_installed_pipx_package
 from pipelines.dagger.containers.python import with_python_base
-from pipelines.models.contexts import PipelineContext
+from pipelines.models.contexts.pipeline_context import PipelineContext
 from pipelines.models.steps import MountPath, Step, StepResult
 
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py

@@ -11,7 +11,7 @@
 from pipelines.consts import AMAZONCORRETTO_IMAGE
 from pipelines.dagger.actions import secrets
 from pipelines.helpers.utils import sh_dash_c
-from pipelines.models.contexts import PipelineContext
+from pipelines.models.contexts.pipeline_context import PipelineContext
 from pipelines.models.steps import Step, StepResult
 
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from pipelines.models.contexts import PipelineContext
+from pipelines.models.contexts.pipeline_context import PipelineContext
 from pipelines.models.steps import Step, StepResult, StepStatus
 
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py

@@ -3,7 +3,7 @@
 #
 
 from pipelines.dagger.actions.python.poetry import with_poetry_module
-from pipelines.models.contexts import PipelineContext
+from pipelines.models.contexts.pipeline_context import PipelineContext
 from pipelines.models.steps import Step, StepResult
 
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py

@@ -2,23 +2,68 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+import logging
+
 import asyncclick as click
-from pipelines.airbyte_ci.test.pipeline import run_test
+from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj
+from pipelines.consts import DOCKER_VERSION
+from pipelines.helpers.utils import sh_dash_c
+from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
 
 
 @click.command()
 @click.argument("poetry_package_path")
 @click.option("--test-directory", default="tests", help="The directory containing the tests to run.")
-async def test(
-    poetry_package_path: str,
-    test_directory: str,
-):
+@click_merge_args_into_context_obj
+@pass_pipeline_context
+@click_ignore_unused_kwargs
+async def test(pipeline_context: ClickPipelineContext):
     """Runs the tests for the given airbyte-ci package.
 
     Args:
-        poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory.
-        test_directory (str): The directory containing the tests to run.
+        pipeline_context (ClickPipelineContext): The context object.
     """
-    success = await run_test(poetry_package_path, test_directory)
-    if not success:
-        click.Abort()
+    poetry_package_path = pipeline_context.params["poetry_package_path"]
+    test_directory = pipeline_context.params["test_directory"]
+
+    logger = logging.getLogger(f"{poetry_package_path}.tests")
+    logger.info(f"Running tests for {poetry_package_path}")
+
+    # The following directories are always mounted because a lot of tests rely on them
+    directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"]
+    directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount]))
+
+    pipeline_name = f"Unit tests for {poetry_package_path}"
+    dagger_client = await pipeline_context.get_dagger_client(pipeline_name=pipeline_name)
+    pytest_container = await (
+        dagger_client.container()
+        .from_("python:3.10.12")
+        .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin")
+        .with_exec(
+            sh_dash_c(
+                [
+                    "apt-get update",
+                    "apt-get install -y bash git curl",
+                    "pip install pipx",
+                    "pipx ensurepath",
+                    "pipx install poetry",
+                ]
+            )
+        )
+        .with_env_variable("VERSION", DOCKER_VERSION)
+        .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"]))
+        .with_mounted_directory(
+            "/airbyte",
+            dagger_client.host().directory(
+                ".",
+                exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"],
+                include=directories_to_mount,
+            ),
+        )
+        .with_workdir(f"/airbyte/{poetry_package_path}")
+        .with_exec(["poetry", "install"])
+        .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock"))
+        .with_exec(["poetry", "run", "pytest", test_directory])
+    )
+
+    await pytest_container