From a4994f485aef073176893862b01ec53e904d7004 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 17:42:42 -0700 Subject: [PATCH 01/30] Add lazy decorator --- .../pipelines/pipelines/cli/airbyte_ci.py | 12 +++++++++ .../pipelines/pipelines/cli/lazy_decorator.py | 25 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 7da72b02ee6651..683f8b25a0e480 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -32,6 +32,17 @@ __installed_version__ = importlib.metadata.version("pipelines") +def display_welcome_message() -> None: + print(''' + █████╗ ██╗██████╗ ██████╗ ██╗ ██╗████████╗███████╗ + ██╔══██╗██║██╔══██╗██╔══██╗╚██╗ ██╔╝╚══██╔══╝██╔════╝ + ███████║██║██████╔╝██████╔╝ ╚████╔╝ ██║ █████╗ + ██╔══██║██║██╔══██╗██╔══██╗ ╚██╔╝ ██║ ██╔══╝ + ██║ ██║██║██║ ██║██████╔╝ ██║ ██║ ███████╗ + ╚═╝ ╚═╝╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝ ╚══════╝ + ''') + + def check_up_to_date() -> bool: """Check if the installed version of pipelines is up to date.""" latest_version = get_latest_version() @@ -193,6 +204,7 @@ def airbyte_ci( ci_job_key: str, show_dagger_logs: bool, ): # noqa D103 + display_welcome_message() ctx.ensure_object(dict) check_up_to_date() ctx.obj["is_local"] = is_local diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py new file mode 100644 index 00000000000000..b65e610c52128d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py @@ -0,0 +1,25 @@ +from functools import wraps +from typing import Any, Callable, Type + + +class LazyPassDecorator: + def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: + self.cls = cls + self.args = args + self.kwargs = kwargs + + def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: + @wraps(f) + def decorated_function(*args: Any, **kwargs: Any) -> Any: + # Check if the kwargs already contain the arguments being passed by the decorator + decorator_kwargs = {k: v for k, v in self.kwargs.items() if k not in kwargs} + # Create an instance of the class + instance = self.cls(*self.args, **decorator_kwargs) + # If function has **kwargs, we can put the instance there + if 'kwargs' in kwargs: + kwargs['kwargs'] = instance + # Otherwise, add it to positional arguments + else: + args = (*args, instance) + return f(*args, **kwargs) + return decorated_function From ccddd26128eec4542c7d5d8cb0375c5286b0ad98 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 18:13:28 -0700 Subject: [PATCH 02/30] Add playground subcommands --- .../pipelines/pipelines/airbyte_ci/playground/__init__.py | 3 +++ .../pipelines/pipelines/airbyte_ci/playground/commands.py | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# From 96643f192b4659d9c0fc9d33e5cd9cbe48cdad18 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 18:41:46 -0700 Subject: [PATCH 03/30] steal --- .../airbyte_ci/playground/commands.py | 32 +++ .../pipelines/pipelines/cli/airbyte_ci.py | 1 + .../pipelines/pipelines/stolen/__init__.py | 3 + .../pipelines/pipelines/stolen/base.py | 69 ++++++ .../{cli => stolen}/lazy_decorator.py | 0 .../pipelines/pipelines/stolen/settings.py | 228 ++++++++++++++++++ .../pipelines/pipelines/stolen/singleton.py | 22 ++ 7 files changed, 355 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/base.py rename airbyte-ci/connectors/pipelines/pipelines/{cli => stolen}/lazy_decorator.py (100%) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index c941b30457953b..ccff9988add53a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -1,3 +1,35 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +import click +from pipelines.stolen.base import PipelineContext +from pipelines.stolen.lazy_decorator import LazyPassDecorator + +from pipelines.stolen.settings import GlobalSettings + +# Stolen +settings = GlobalSettings() +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(PipelineContext, global_settings=settings) +pass_global_settings: LazyPassDecorator = LazyPassDecorator(GlobalSettings) + +# NEW + +@click.command() +@click.argument("arg1") +@click.option("--opt", default="default_value") +@pass_pipeline_context +@pass_global_settings +def playground( + ctx, + arg1: str, + opt: str, +): + """Runs the tests for the given airbyte-ci package. + + Args: + poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. + test_directory (str): The directory containing the tests to run. + """ + print(f"playground: {arg1} {opt}") + print(f"ctx: {dir(ctx)}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 683f8b25a0e480..e1b55fefa8abd8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -157,6 +157,7 @@ def get_modified_files( "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", + "playground": "pipelines.airbyte_ci.playground.commands.playground", }, ) @click.version_option(__installed_version__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py new file mode 100644 index 00000000000000..40758a09658796 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py @@ -0,0 +1,69 @@ +import sys +from typing import Any, Callable, Optional, Union + +import dagger + +# TODO ben set up for async +# from asyncclick import Context, get_current_context + +from click import Context, get_current_context +from dagger.api.gen import Client, Container +from pydantic import BaseModel, Field, PrivateAttr + +from .settings import GlobalSettings +from .singleton import Singleton + + +# this is a bit of a hack to get around how prefect resolves parameters +# basically without this, prefect will attempt to access the context +# before we create it in main.py in order to resolve it as a parameter +# wrapping it in a function like this prevents that from happening +def get_context() -> Context: + return get_current_context() + +class PipelineContext(BaseModel, Singleton): + global_settings: GlobalSettings + dockerd_service: Optional[Container] = Field(default=None) + _dagger_client: Optional[Client] = PrivateAttr(default=None) + _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_context) + + class Config: + arbitrary_types_allowed=True + + def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): + """ + Initialize the PipelineContext instance. + + This method checks the _initialized flag for the PipelineContext class in the Singleton base class. + If the flag is False, the initialization logic is executed and the flag is set to True. + If the flag is True, the initialization logic is skipped. + + This ensures that the initialization logic is only executed once, even if the PipelineContext instance is retrieved multiple times. + This can be useful if the initialization logic is expensive (e.g., it involves network requests or database queries). + """ + if not Singleton._initialized[PipelineContext]: + super().__init__(global_settings=global_settings, **data) + self.set_global_prefect_tag_context() + Singleton._initialized[PipelineContext] = True + + import asyncio + + _dagger_client_lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock) + + async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name: Optional[str] = None) -> Client: + if not self._dagger_client: + async with self._dagger_client_lock: + if not self._dagger_client: + connection = dagger.Connection(dagger.Config(log_output=sys.stdout)) + self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore + client = self._dagger_client + assert client, "Error initializing Dagger client" + return client.pipeline(pipeline_name) if pipeline_name else client + + +class GlobalContext(BaseModel, Singleton): + pipeline_context: Optional[PipelineContext] = Field(default=None) + click_context: Optional[Context] = Field(default=None) + + class Config: + arbitrary_types_allowed = True diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/cli/lazy_decorator.py rename to airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py new file mode 100644 index 00000000000000..16a20542c17117 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py @@ -0,0 +1,228 @@ +import os +import platform +from typing import Any, Callable, List, Optional + +import platformdirs +from dagger import Client, Container +from pydantic import BaseSettings, Field, SecretBytes, SecretStr +from pygit2 import Commit, Repository #type: ignore + +from .singleton import Singleton + + +def get_git_revision() -> str: + repo = Repository(".") + commit_hash:str = os.environ.get("LAST_COMMIT_SHA", repo.revparse_single("HEAD").hex ) + return commit_hash + +def get_current_branch() -> str: + repo = Repository(".") + return str(repo.head.shorthand) + +def get_latest_commit_message() -> str: + repo = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.message) + +def get_latest_commit_author() -> str: + repo: Repository = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.author.name) + +def get_latest_commit_time() -> str: + repo = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.commit_time) + +def get_repo_root_path() -> str: + repo = Repository(".") + return str(os.path.dirname(os.path.dirname(repo.path))) + +def get_repo_fullname() -> str: + repo = Repository(".") + repo_url:str = repo.remotes["origin"].url + + # Handle HTTPS URLs + if "https://" in repo_url: + parts = repo_url.split("/") + owner = parts[-2] + repo_name = parts[-1].replace(".git", "") + + # Handle SSH URLs + else: + repo_url = repo_url.replace("git@github.com:", "") + owner, repo_name = repo_url.split("/")[:2] + repo_name = repo_name.replace(".git", "") + + return f"{owner}/{repo_name}" + +# Immutable. Use this for application configuration. Created at bootstrap. +class GlobalSettings(BaseSettings, Singleton): + DAGGER: bool = Field(True, env="DAGGER") + GITHUB_TOKEN: Optional[SecretStr] = Field(None, env="GITHUB_CUSTOM_TOKEN") + GIT_CURRENT_REVISION: str = Field(default_factory=get_git_revision) + GIT_CURRENT_BRANCH: str = Field(default_factory=get_current_branch) + GIT_LATEST_COMMIT_MESSAGE: str = Field(default_factory=get_latest_commit_message) + GIT_LATEST_COMMIT_AUTHOR: str = Field(default_factory=get_latest_commit_author) + GIT_LATEST_COMMIT_TIME: str = Field(default_factory=get_latest_commit_time) + GIT_REPOSITORY: str = Field(default_factory=get_repo_fullname) + GIT_REPO_ROOT_PATH: str = Field(default_factory=get_repo_root_path) + CI: bool = Field(False, env="CI") + LOG_LEVEL: str = Field("WARNING", env="LOG_LEVEL") + PLATFORM: str = platform.system() + DEBUG: bool = Field(False, env="AIRCMD_DEBUG") + + # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 + GITHUB_EVENT_NAME: str = Field("push", env="GITHUB_EVENT_NAME") + GITHUB_ACTION: str = Field("local_action", env="GITHUB_ACTION") + GITHUB_ACTOR: str = Field("local_actor", env="GITHUB_ACTOR") + GITHUB_JOB: str = Field("local_job", env="GITHUB_JOB") + GITHUB_RUN_NUMBER: int = Field(0, env="GITHUB_RUN_NUMBER") + GITHUB_RUN_ID: int = Field(0, env="GITHUB_RUN_ID") + GITHUB_API_URL: str = Field("https://api.github.com", env="GITHUB_API_URL") + GITHUB_SERVER_URL: str = Field("https://github.com", env="GITHUB_SERVER_URL") + GITHUB_GRAPHQL_URL: str = Field("https://api.github.com/graphql", env="GITHUB_GRAPHQL_URL") + GITHUB_EVENT_PATH: Optional[str] = Field("/tmp/mockevents", env="GITHUB_EVENT_PATH") + + POETRY_CACHE_DIR: str = Field( + default_factory=lambda: platformdirs.user_cache_dir("pypoetry"), + env="POETRY_CACHE_DIR" + ) + MYPY_CACHE_DIR: str = Field("~/.cache/.mypy_cache", env="MYPY_CACHE_DIR") + DEFAULT_PYTHON_EXCLUDE: List[str] = Field(["**/.venv", "**/__pycache__"], env="DEFAULT_PYTHON_EXCLUDE") + DEFAULT_EXCLUDED_FILES: List[str] = Field( + [ + ".git", + "**/build", + "**/.venv", + "**/secrets", + "**/__pycache__", + "**/*.egg-info", + "**/.vscode", + "**/.pytest_cache", + "**/.eggs", + "**/.mypy_cache", + "**/.DS_Store", + ], + env="DEFAULT_EXCLUDED_FILES" + ) + DOCKER_VERSION:str = Field("20.10.23", env="DOCKER_VERSION") + DOCKER_DIND_IMAGE: str = Field("docker:dind", env="DOCKER_DIND_IMAGE") + DOCKER_CLI_IMAGE: str = Field("docker:cli", env="DOCKER_CLI_IMAGE") + GRADLE_HOMEDIR_PATH: str = Field("/root/.gradle", env="GRADLE_HOMEDIR_PATH") + GRADLE_CACHE_VOLUME_PATH: str = Field("/root/gradle-cache", env="GRADLE_CACHE_VOLUME_PATH") + + PREFECT_API_URL: str = Field("http://127.0.0.1:4200/api", env="PREFECT_API_URL") + PREFECT_COMMA_DELIMITED_USER_TAGS: str = Field("", env="PREFECT_COMMA_DELIMITED_USER_TAGS") + PREFECT_COMMA_DELIMITED_SYSTEM_TAGS: str = Field("CI:False", env="PREFECT_COMMA_DELIMITED_SYSTEM_TAGS") + + SECRET_DOCKER_HUB_USERNAME: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_USERNAME") + SECRET_DOCKER_HUB_PASSWORD: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_PASSWORD") + SECRET_TAILSCALE_AUTHKEY: Optional[SecretStr] = Field(None, env="SECRET_TAILSCALE_AUTHKEY") + + PIP_CACHE_DIR: str = Field( + default_factory=lambda: platformdirs.user_cache_dir("pip"), + env="PIP_CACHE_DIR" + ) + + class Config: + arbitrary_types_allowed = True + env_file = '.env' + allow_mutation = False + + +''' +If both include and exclude are supplied, the load_settings function will first filter the environment variables based on the include list, and then it will +further filter the resulting environment variables based on the exclude list. + +Here's the order of operations: + + 1 If include is provided, only the environment variables with keys in the include list will be considered. + 2 If exclude is provided, any environment variables with keys in the exclude list will be removed from the filtered list obtained in step 1. + 3 The remaining environment variables will be loaded into the container. +''' + +def load_settings(client: Client, settings: BaseSettings, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None) -> Callable[[Container], Container]: + def load_envs(ctr: Container) -> Container: + settings_dict = {key: value for key, value in settings.dict().items() if value is not None} + + if include is not None: + settings_dict = {key: settings_dict[key] for key in include if key in settings_dict} + + if exclude is not None: + settings_dict = {key: value for key, value in settings_dict.items() if key not in exclude} + + for key, value in settings_dict.items(): + env_key = key.upper() + if isinstance(value, SecretStr) or isinstance(value, SecretBytes): # env var can be stored in buildkit layer cache, so we must use client.secret instead + secret = client.set_secret(env_key, str(value.get_secret_value())) + ctr = ctr.with_secret_variable(env_key, secret) + else: + ctr = ctr.with_env_variable(env_key, str(value)) + + return ctr + + return load_envs + + +class GithubActionsInputSettings(BaseSettings): + """ + A Pydantic BaseSettings subclass that transforms input names to the format expected by GitHub Actions. + + GitHub Actions converts input names to environment variables in a specific way: + - The input name is converted to uppercase. + - Any '-' characters are converted to '_'. + - The prefix 'INPUT_' is added to the start. + + This class automatically applies these transformations when you create an instance of it. + + Example: + If you create an instance with the input {'project-token': 'abc'}, it will be transformed to {'INPUT_PROJECT_TOKEN': 'abc'}. + """ + + # Github action specific fields + GITHUB_ACTION: str + GITHUB_ACTOR: str + GITHUB_API_URL: str + GITHUB_EVENT_NAME: str + GITHUB_GRAPHQL_URL: str + GITHUB_JOB: str + GITHUB_REF: str + GITHUB_REPOSITORY: str + GITHUB_RUN_ID: str + GITHUB_RUN_NUMBER: str + GITHUB_SERVER_URL: str + GITHUB_SHA: str + GITHUB_EVENT_PATH: str + + class Config: + env_prefix = "INPUT_" + extra = "allow" + + def __init__(self, global_settings: GlobalSettings, **data: Any): + + # transform input names to the format expected by GitHub Actions and prepare them to be injected as environment variables. + + transformed_data = {self.Config.env_prefix + k.replace("-", "_").upper(): v for k, v in data.items()} + + # inject the context that github actions wants via environment variables. + # in typescript, it is injected here: + # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 + + transformed_data.update({ + "GITHUB_SHA": global_settings.GIT_CURRENT_REVISION, + "GITHUB_REF": global_settings.GIT_CURRENT_BRANCH, + "GITHUB_EVENT_NAME": global_settings.GITHUB_EVENT_NAME, + "GITHUB_ACTION": global_settings.GITHUB_ACTION, + "GITHUB_ACTOR": global_settings.GITHUB_ACTOR, + "GITHUB_JOB": global_settings.GITHUB_JOB, + "GITHUB_RUN_NUMBER": global_settings.GITHUB_RUN_NUMBER, + "GITHUB_RUN_ID": global_settings.GITHUB_RUN_ID, + "GITHUB_API_URL": global_settings.GITHUB_API_URL, + "GITHUB_SERVER_URL": global_settings.GITHUB_SERVER_URL, + "GITHUB_GRAPHQL_URL": global_settings.GITHUB_GRAPHQL_URL, + "GITHUB_REPOSITORY": global_settings.GIT_REPOSITORY, + "GITHUB_EVENT_PATH": global_settings.GITHUB_EVENT_PATH + }) + super().__init__(**transformed_data) + diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py new file mode 100644 index 00000000000000..3fb7f62aedbdaf --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py @@ -0,0 +1,22 @@ +from typing import Any, Type + + +class Singleton: + """ + A base class for implementing the Singleton pattern. + + This class stores instances and initialization flags for each subclass in dictionaries. + This allows each subclass to have its own unique instance and control over its initialization process. + + The __new__ method ensures that only one instance of each subclass is created. + The _initialized dictionary is used to control when the initialization logic of each subclass is executed. + """ + _instances: dict[Type['Singleton'], Any] = {} + _initialized: dict[Type['Singleton'], bool] = {} + + def __new__(cls: Type['Singleton'], *args: Any, **kwargs: Any) -> Any: + + if cls not in cls._instances: + cls._instances[cls] = super().__new__(cls) + cls._initialized[cls] = False + return cls._instances[cls] From efa0786d1214008a6e2cf636a3b6bff236c84ce8 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 19:01:57 -0700 Subject: [PATCH 04/30] Show demo of context coming through --- .../airbyte_ci/playground/commands.py | 24 +- .../pipelines/pipelines/stolen/base.py | 1 - .../pipelines/pipelines/stolen/settings.py | 238 +++++++++--------- airbyte-ci/connectors/pipelines/poetry.lock | 35 ++- .../connectors/pipelines/pyproject.toml | 1 + 5 files changed, 172 insertions(+), 127 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index ccff9988add53a..9de4337cbec7ed 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -16,14 +16,14 @@ # NEW @click.command() -@click.argument("arg1") +@click.argument("hold") @click.option("--opt", default="default_value") @pass_pipeline_context @pass_global_settings def playground( - ctx, - arg1: str, - opt: str, + ctx: PipelineContext, + args, + **kwargs, ): """Runs the tests for the given airbyte-ci package. @@ -31,5 +31,17 @@ def playground( poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. test_directory (str): The directory containing the tests to run. """ - print(f"playground: {arg1} {opt}") - print(f"ctx: {dir(ctx)}") + + # ctx = PipelineContext(global_settings=GlobalSettings(PLATFORM='Darwin'), dockerd_service=None, asyncio=) + # args = GlobalSettings(PLATFORM='Darwin') + # kwargs = {'opt': 'tight', 'hold': 'holdme'} + + import pdb; pdb.set_trace() + print(f"playground: {args} {kwargs}") + print(f"ctx: {ctx._click_context().obj}") + + # (Pdb) ctx._click_context().args + # [] + # (Pdb) ctx._click_context().params + # {'opt': 'tight', 'hold': 'holdme'} + # (Pdb) diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py index 40758a09658796..ab3ef205feb34b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py @@ -43,7 +43,6 @@ def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): """ if not Singleton._initialized[PipelineContext]: super().__init__(global_settings=global_settings, **data) - self.set_global_prefect_tag_context() Singleton._initialized[PipelineContext] = True import asyncio diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py index 16a20542c17117..4596b6ba5d5c4f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py @@ -11,147 +11,147 @@ def get_git_revision() -> str: - repo = Repository(".") + repo = Repository(".") commit_hash:str = os.environ.get("LAST_COMMIT_SHA", repo.revparse_single("HEAD").hex ) return commit_hash -def get_current_branch() -> str: - repo = Repository(".") - return str(repo.head.shorthand) - -def get_latest_commit_message() -> str: - repo = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.message) - -def get_latest_commit_author() -> str: - repo: Repository = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.author.name) - -def get_latest_commit_time() -> str: - repo = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.commit_time) +def get_current_branch() -> str: + repo = Repository(".") + return str(repo.head.shorthand) + +def get_latest_commit_message() -> str: + repo = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.message) + +def get_latest_commit_author() -> str: + repo: Repository = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.author.name) + +def get_latest_commit_time() -> str: + repo = Repository(".") + commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] + return str(commit.commit_time) def get_repo_root_path() -> str: repo = Repository(".") return str(os.path.dirname(os.path.dirname(repo.path))) -def get_repo_fullname() -> str: +def get_repo_fullname() -> str: repo = Repository(".") repo_url:str = repo.remotes["origin"].url - + # Handle HTTPS URLs if "https://" in repo_url: parts = repo_url.split("/") owner = parts[-2] repo_name = parts[-1].replace(".git", "") - + # Handle SSH URLs else: repo_url = repo_url.replace("git@github.com:", "") owner, repo_name = repo_url.split("/")[:2] repo_name = repo_name.replace(".git", "") - + return f"{owner}/{repo_name}" # Immutable. Use this for application configuration. Created at bootstrap. class GlobalSettings(BaseSettings, Singleton): - DAGGER: bool = Field(True, env="DAGGER") - GITHUB_TOKEN: Optional[SecretStr] = Field(None, env="GITHUB_CUSTOM_TOKEN") - GIT_CURRENT_REVISION: str = Field(default_factory=get_git_revision) - GIT_CURRENT_BRANCH: str = Field(default_factory=get_current_branch) - GIT_LATEST_COMMIT_MESSAGE: str = Field(default_factory=get_latest_commit_message) - GIT_LATEST_COMMIT_AUTHOR: str = Field(default_factory=get_latest_commit_author) - GIT_LATEST_COMMIT_TIME: str = Field(default_factory=get_latest_commit_time) - GIT_REPOSITORY: str = Field(default_factory=get_repo_fullname) - GIT_REPO_ROOT_PATH: str = Field(default_factory=get_repo_root_path) - CI: bool = Field(False, env="CI") - LOG_LEVEL: str = Field("WARNING", env="LOG_LEVEL") + # DAGGER: bool = Field(True, env="DAGGER") + # GITHUB_TOKEN: Optional[SecretStr] = Field(None, env="GITHUB_CUSTOM_TOKEN") + # GIT_CURRENT_REVISION: str = Field(default_factory=get_git_revision) + # GIT_CURRENT_BRANCH: str = Field(default_factory=get_current_branch) + # GIT_LATEST_COMMIT_MESSAGE: str = Field(default_factory=get_latest_commit_message) + # GIT_LATEST_COMMIT_AUTHOR: str = Field(default_factory=get_latest_commit_author) + # GIT_LATEST_COMMIT_TIME: str = Field(default_factory=get_latest_commit_time) + # GIT_REPOSITORY: str = Field(default_factory=get_repo_fullname) + # GIT_REPO_ROOT_PATH: str = Field(default_factory=get_repo_root_path) + # CI: bool = Field(False, env="CI") + # LOG_LEVEL: str = Field("WARNING", env="LOG_LEVEL") PLATFORM: str = platform.system() - DEBUG: bool = Field(False, env="AIRCMD_DEBUG") - - # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 - GITHUB_EVENT_NAME: str = Field("push", env="GITHUB_EVENT_NAME") - GITHUB_ACTION: str = Field("local_action", env="GITHUB_ACTION") - GITHUB_ACTOR: str = Field("local_actor", env="GITHUB_ACTOR") - GITHUB_JOB: str = Field("local_job", env="GITHUB_JOB") - GITHUB_RUN_NUMBER: int = Field(0, env="GITHUB_RUN_NUMBER") - GITHUB_RUN_ID: int = Field(0, env="GITHUB_RUN_ID") - GITHUB_API_URL: str = Field("https://api.github.com", env="GITHUB_API_URL") - GITHUB_SERVER_URL: str = Field("https://github.com", env="GITHUB_SERVER_URL") - GITHUB_GRAPHQL_URL: str = Field("https://api.github.com/graphql", env="GITHUB_GRAPHQL_URL") - GITHUB_EVENT_PATH: Optional[str] = Field("/tmp/mockevents", env="GITHUB_EVENT_PATH") - - POETRY_CACHE_DIR: str = Field( - default_factory=lambda: platformdirs.user_cache_dir("pypoetry"), - env="POETRY_CACHE_DIR" - ) - MYPY_CACHE_DIR: str = Field("~/.cache/.mypy_cache", env="MYPY_CACHE_DIR") - DEFAULT_PYTHON_EXCLUDE: List[str] = Field(["**/.venv", "**/__pycache__"], env="DEFAULT_PYTHON_EXCLUDE") - DEFAULT_EXCLUDED_FILES: List[str] = Field( - [ - ".git", - "**/build", - "**/.venv", - "**/secrets", - "**/__pycache__", - "**/*.egg-info", - "**/.vscode", - "**/.pytest_cache", - "**/.eggs", - "**/.mypy_cache", - "**/.DS_Store", - ], - env="DEFAULT_EXCLUDED_FILES" - ) - DOCKER_VERSION:str = Field("20.10.23", env="DOCKER_VERSION") - DOCKER_DIND_IMAGE: str = Field("docker:dind", env="DOCKER_DIND_IMAGE") - DOCKER_CLI_IMAGE: str = Field("docker:cli", env="DOCKER_CLI_IMAGE") - GRADLE_HOMEDIR_PATH: str = Field("/root/.gradle", env="GRADLE_HOMEDIR_PATH") - GRADLE_CACHE_VOLUME_PATH: str = Field("/root/gradle-cache", env="GRADLE_CACHE_VOLUME_PATH") - - PREFECT_API_URL: str = Field("http://127.0.0.1:4200/api", env="PREFECT_API_URL") - PREFECT_COMMA_DELIMITED_USER_TAGS: str = Field("", env="PREFECT_COMMA_DELIMITED_USER_TAGS") - PREFECT_COMMA_DELIMITED_SYSTEM_TAGS: str = Field("CI:False", env="PREFECT_COMMA_DELIMITED_SYSTEM_TAGS") - - SECRET_DOCKER_HUB_USERNAME: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_USERNAME") - SECRET_DOCKER_HUB_PASSWORD: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_PASSWORD") - SECRET_TAILSCALE_AUTHKEY: Optional[SecretStr] = Field(None, env="SECRET_TAILSCALE_AUTHKEY") - - PIP_CACHE_DIR: str = Field( - default_factory=lambda: platformdirs.user_cache_dir("pip"), - env="PIP_CACHE_DIR" - ) - - class Config: - arbitrary_types_allowed = True - env_file = '.env' + # DEBUG: bool = Field(False, env="AIRCMD_DEBUG") + + # # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 + # GITHUB_EVENT_NAME: str = Field("push", env="GITHUB_EVENT_NAME") + # GITHUB_ACTION: str = Field("local_action", env="GITHUB_ACTION") + # GITHUB_ACTOR: str = Field("local_actor", env="GITHUB_ACTOR") + # GITHUB_JOB: str = Field("local_job", env="GITHUB_JOB") + # GITHUB_RUN_NUMBER: int = Field(0, env="GITHUB_RUN_NUMBER") + # GITHUB_RUN_ID: int = Field(0, env="GITHUB_RUN_ID") + # GITHUB_API_URL: str = Field("https://api.github.com", env="GITHUB_API_URL") + # GITHUB_SERVER_URL: str = Field("https://github.com", env="GITHUB_SERVER_URL") + # GITHUB_GRAPHQL_URL: str = Field("https://api.github.com/graphql", env="GITHUB_GRAPHQL_URL") + # GITHUB_EVENT_PATH: Optional[str] = Field("/tmp/mockevents", env="GITHUB_EVENT_PATH") + + # POETRY_CACHE_DIR: str = Field( + # default_factory=lambda: platformdirs.user_cache_dir("pypoetry"), + # env="POETRY_CACHE_DIR" + # ) + # MYPY_CACHE_DIR: str = Field("~/.cache/.mypy_cache", env="MYPY_CACHE_DIR") + # DEFAULT_PYTHON_EXCLUDE: List[str] = Field(["**/.venv", "**/__pycache__"], env="DEFAULT_PYTHON_EXCLUDE") + # DEFAULT_EXCLUDED_FILES: List[str] = Field( + # [ + # ".git", + # "**/build", + # "**/.venv", + # "**/secrets", + # "**/__pycache__", + # "**/*.egg-info", + # "**/.vscode", + # "**/.pytest_cache", + # "**/.eggs", + # "**/.mypy_cache", + # "**/.DS_Store", + # ], + # env="DEFAULT_EXCLUDED_FILES" + # ) + # DOCKER_VERSION:str = Field("20.10.23", env="DOCKER_VERSION") + # DOCKER_DIND_IMAGE: str = Field("docker:dind", env="DOCKER_DIND_IMAGE") + # DOCKER_CLI_IMAGE: str = Field("docker:cli", env="DOCKER_CLI_IMAGE") + # GRADLE_HOMEDIR_PATH: str = Field("/root/.gradle", env="GRADLE_HOMEDIR_PATH") + # GRADLE_CACHE_VOLUME_PATH: str = Field("/root/gradle-cache", env="GRADLE_CACHE_VOLUME_PATH") + + # PREFECT_API_URL: str = Field("http://127.0.0.1:4200/api", env="PREFECT_API_URL") + # PREFECT_COMMA_DELIMITED_USER_TAGS: str = Field("", env="PREFECT_COMMA_DELIMITED_USER_TAGS") + # PREFECT_COMMA_DELIMITED_SYSTEM_TAGS: str = Field("CI:False", env="PREFECT_COMMA_DELIMITED_SYSTEM_TAGS") + + # SECRET_DOCKER_HUB_USERNAME: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_USERNAME") + # SECRET_DOCKER_HUB_PASSWORD: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_PASSWORD") + # SECRET_TAILSCALE_AUTHKEY: Optional[SecretStr] = Field(None, env="SECRET_TAILSCALE_AUTHKEY") + + # PIP_CACHE_DIR: str = Field( + # default_factory=lambda: platformdirs.user_cache_dir("pip"), + # env="PIP_CACHE_DIR" + # ) + + class Config: + arbitrary_types_allowed = True + # env_file = '.env' allow_mutation = False ''' -If both include and exclude are supplied, the load_settings function will first filter the environment variables based on the include list, and then it will -further filter the resulting environment variables based on the exclude list. - -Here's the order of operations: - - 1 If include is provided, only the environment variables with keys in the include list will be considered. - 2 If exclude is provided, any environment variables with keys in the exclude list will be removed from the filtered list obtained in step 1. - 3 The remaining environment variables will be loaded into the container. -''' - -def load_settings(client: Client, settings: BaseSettings, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None) -> Callable[[Container], Container]: - def load_envs(ctr: Container) -> Container: - settings_dict = {key: value for key, value in settings.dict().items() if value is not None} - - if include is not None: - settings_dict = {key: settings_dict[key] for key in include if key in settings_dict} - - if exclude is not None: - settings_dict = {key: value for key, value in settings_dict.items() if key not in exclude} - +If both include and exclude are supplied, the load_settings function will first filter the environment variables based on the include list, and then it will +further filter the resulting environment variables based on the exclude list. + +Here's the order of operations: + + 1 If include is provided, only the environment variables with keys in the include list will be considered. + 2 If exclude is provided, any environment variables with keys in the exclude list will be removed from the filtered list obtained in step 1. + 3 The remaining environment variables will be loaded into the container. +''' + +def load_settings(client: Client, settings: BaseSettings, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None) -> Callable[[Container], Container]: + def load_envs(ctr: Container) -> Container: + settings_dict = {key: value for key, value in settings.dict().items() if value is not None} + + if include is not None: + settings_dict = {key: settings_dict[key] for key in include if key in settings_dict} + + if exclude is not None: + settings_dict = {key: value for key, value in settings_dict.items() if key not in exclude} + for key, value in settings_dict.items(): env_key = key.upper() if isinstance(value, SecretStr) or isinstance(value, SecretBytes): # env var can be stored in buildkit layer cache, so we must use client.secret instead @@ -159,10 +159,10 @@ def load_envs(ctr: Container) -> Container: ctr = ctr.with_secret_variable(env_key, secret) else: ctr = ctr.with_env_variable(env_key, str(value)) - - return ctr - - return load_envs + + return ctr + + return load_envs class GithubActionsInputSettings(BaseSettings): @@ -197,14 +197,14 @@ class GithubActionsInputSettings(BaseSettings): class Config: env_prefix = "INPUT_" - extra = "allow" + extra = "allow" def __init__(self, global_settings: GlobalSettings, **data: Any): # transform input names to the format expected by GitHub Actions and prepare them to be injected as environment variables. transformed_data = {self.Config.env_prefix + k.replace("-", "_").upper(): v for k, v in data.items()} - + # inject the context that github actions wants via environment variables. # in typescript, it is injected here: # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 @@ -221,7 +221,7 @@ def __init__(self, global_settings: GlobalSettings, **data: Any): "GITHUB_API_URL": global_settings.GITHUB_API_URL, "GITHUB_SERVER_URL": global_settings.GITHUB_SERVER_URL, "GITHUB_GRAPHQL_URL": global_settings.GITHUB_GRAPHQL_URL, - "GITHUB_REPOSITORY": global_settings.GIT_REPOSITORY, + "GITHUB_REPOSITORY": global_settings.GIT_REPOSITORY, "GITHUB_EVENT_PATH": global_settings.GITHUB_EVENT_PATH }) super().__init__(**transformed_data) diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index 68b610e709d2d2..4dfc6816e6d52c 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1525,6 +1525,39 @@ typing-extensions = ">=3.10,<4.6.0 || >4.6.0" [package.extras] dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] +[[package]] +name = "pygit2" +version = "1.13.1" +description = "Python bindings for libgit2." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygit2-1.13.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:30db67f73ef28b07864f2509978b7396d1ed3b72f6f252d301db12a4c9f90f5b"}, + {file = "pygit2-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825f22a1bbf73c7a11c69e53a29485d10b4df6a635ccd120cf2966e6535a5b52"}, + {file = "pygit2-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7356d4e41f122a066fa1cce3f5dbedf73c03781692f5eab3687bc355a083575"}, + {file = "pygit2-1.13.1-cp310-cp310-win32.whl", hash = "sha256:cf47de2e21cdeb5d8c35f0d1a381b56fdb365dac3dcd8ea7fa057b390ce83d40"}, + {file = "pygit2-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:6ede9700fdbf78a5a1513549f37884233f29d3343412272c0800cda40c4c2c56"}, + {file = "pygit2-1.13.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9c84eff2223e5fd442b746785b9cd21f98c1f53a0f3fe8d4ed06aee60a09ea35"}, + {file = "pygit2-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bd649b9ef17564b642f59e1a2751e30fdd07d3707b0642d8012062615651039"}, + {file = "pygit2-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:787ea717bb9fadb3ee2836ed32a9ed2110ef861862bfe6b693becda75a2eaa5c"}, + {file = "pygit2-1.13.1-cp311-cp311-win32.whl", hash = "sha256:d2dbf3d6976b0626fafd7d1c7363ae92dcacaa63789e8c432bc8caea86132235"}, + {file = "pygit2-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce8618e5876b4c54942587d72a0d84f6e6a5b0e69db5f8d06dc5f567abd07ed1"}, + {file = "pygit2-1.13.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8d2d97cfe2bf2abbb0ef5984771578d1b05053942bfe1b46d4ac48d19c5eda56"}, + {file = "pygit2-1.13.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a162db1afdc5bae608d739395a248a373165176f83c7fe57a1073e9168b459"}, + {file = "pygit2-1.13.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357c11b30d6c63ff58401a897df39b27903beffebb24842c8ce9ce77b90fe0b1"}, + {file = "pygit2-1.13.1-cp38-cp38-win32.whl", hash = "sha256:ed7fc70bc8f6db227c9919958d064cb49eaa68cc97f51c1f9de920a4500c6766"}, + {file = "pygit2-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:949ad31e0fab408449721cc5b582350f6c5c56ab068bfa10cd6d10c2830deaa9"}, + {file = "pygit2-1.13.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ea2b870675ef1a2bef3300dda725aae9f8c68265e633ed683fce85588cfb4d37"}, + {file = "pygit2-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74af678b98f6a08ef4315f5b64889011e05ad702e340cc6cde59926906650039"}, + {file = "pygit2-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33bbe4d7501a600be320147c63a8d1a966fb7424595e6eb53fdc30259b8921dc"}, + {file = "pygit2-1.13.1-cp39-cp39-win32.whl", hash = "sha256:697044df77c8b3849fec8d7dd454acd347b180212c6bc5526eeb9309eff63a65"}, + {file = "pygit2-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:8004244da8183fcefcf7c3d4d119806e9c705543bcf24045b97e3eddaa869aef"}, + {file = "pygit2-1.13.1.tar.gz", hash = "sha256:d8e6d540aad9ded1cf2c6bda31ba48b1e20c18525807dbd837317bef4dccb994"}, +] + +[package.dependencies] +cffi = ">=1.9.1" + [[package]] name = "pygithub" version = "1.59.1" @@ -2248,4 +2281,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "76b53958ee6dd19c2816b023cce1585bb198e14c79124d56de2445c494d51cf2" +content-hash = "7b6c32239940ee2c73e44c3f75267a56f46d7a513a14efe676b7a6da4c74f8c3" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 84d8dabca86d07..bb70ce815cc205 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -25,6 +25,7 @@ connector-ops = {path = "../connector_ops", develop = true} toml = "^0.10.2" sentry-sdk = "^1.28.1" segment-analytics-python = "^2.2.3" +pygit2 = "^1.13.1" [tool.poetry.group.test.dependencies] pytest = "^6.2.5" From 76458dc154f954cb54d14e961e9ab986a6dffe2a Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 19 Oct 2023 13:11:18 -0700 Subject: [PATCH 05/30] Add pass_click_context_and_args_to_children --- .../airbyte_ci/playground/commands.py | 14 +++--- .../pipelines/pipelines/cli/airbyte_ci.py | 49 ++++++++++++++----- .../pipelines/pipelines/stolen/base.py | 30 +++++++++--- 3 files changed, 65 insertions(+), 28 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 9de4337cbec7ed..5c3580617a547c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -3,15 +3,15 @@ # import click -from pipelines.stolen.base import PipelineContext +from pipelines.stolen.base import ClickPipelineContext from pipelines.stolen.lazy_decorator import LazyPassDecorator from pipelines.stolen.settings import GlobalSettings # Stolen settings = GlobalSettings() -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(PipelineContext, global_settings=settings) -pass_global_settings: LazyPassDecorator = LazyPassDecorator(GlobalSettings) +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext, global_settings=settings) +# pass_global_settings: LazyPassDecorator = LazyPassDecorator(GlobalSettings) # NEW @@ -19,10 +19,9 @@ @click.argument("hold") @click.option("--opt", default="default_value") @pass_pipeline_context -@pass_global_settings +# @pass_global_settings def playground( - ctx: PipelineContext, - args, + ctx: ClickPipelineContext, **kwargs, ): """Runs the tests for the given airbyte-ci package. @@ -32,12 +31,11 @@ def playground( test_directory (str): The directory containing the tests to run. """ - # ctx = PipelineContext(global_settings=GlobalSettings(PLATFORM='Darwin'), dockerd_service=None, asyncio=) + # ctx = ClickPipelineContext(global_settings=GlobalSettings(PLATFORM='Darwin'), dockerd_service=None, asyncio=) # args = GlobalSettings(PLATFORM='Darwin') # kwargs = {'opt': 'tight', 'hold': 'holdme'} import pdb; pdb.set_trace() - print(f"playground: {args} {kwargs}") print(f"ctx: {ctx._click_context().obj}") # (Pdb) ctx._click_context().args diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index e1b55fefa8abd8..8d96e9ecc5cab8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -146,6 +146,28 @@ def get_modified_files( return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) +# TO MOVE + +def pass_click_context_and_args_to_children(f): + """ + TODO + """ + + def wrapper(*args, **kwargs): + ctx = args[0] + ctx.ensure_object(dict) + click_obj = ctx.obj + click_params = ctx.params + command_name = ctx.command.name + + # Error if click_obj and click_params have the same key + intersection = set(click_obj.keys()) & set(click_params.keys()) + if intersection: + raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") + + return f(*args, **kwargs) + + return wrapper # COMMANDS @@ -187,6 +209,7 @@ def get_modified_files( @click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) @click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) @click.pass_context +@pass_click_context_and_args_to_children @track_command def airbyte_ci( ctx: click.Context, @@ -206,24 +229,24 @@ def airbyte_ci( show_dagger_logs: bool, ): # noqa D103 display_welcome_message() - ctx.ensure_object(dict) + check_up_to_date() - ctx.obj["is_local"] = is_local + # ctx.obj["is_local"] = is_local ctx.obj["is_ci"] = not is_local - ctx.obj["git_branch"] = git_branch - ctx.obj["git_revision"] = git_revision - ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id + # ctx.obj["git_branch"] = git_branch + # ctx.obj["git_revision"] = git_revision + # ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id ctx.obj["gha_workflow_run_url"] = ( f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None ) - ctx.obj["ci_context"] = ci_context - ctx.obj["ci_report_bucket_name"] = ci_report_bucket_name - ctx.obj["ci_gcs_credentials"] = ci_gcs_credentials - ctx.obj["ci_git_user"] = ci_git_user - ctx.obj["ci_github_access_token"] = ci_github_access_token - ctx.obj["ci_job_key"] = ci_job_key - ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp - ctx.obj["show_dagger_logs"] = show_dagger_logs + # ctx.obj["ci_context"] = ci_context + # ctx.obj["ci_report_bucket_name"] = ci_report_bucket_name + # ctx.obj["ci_gcs_credentials"] = ci_gcs_credentials + # ctx.obj["ci_git_user"] = ci_git_user + # ctx.obj["ci_github_access_token"] = ci_github_access_token + # ctx.obj["ci_job_key"] = ci_job_key + # ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp + # ctx.obj["show_dagger_logs"] = show_dagger_logs if pull_request_number and ci_github_access_token: ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py index ab3ef205feb34b..188e9639d72d8e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py @@ -21,29 +21,45 @@ def get_context() -> Context: return get_current_context() -class PipelineContext(BaseModel, Singleton): +class ClickPipelineContext(BaseModel, Singleton): global_settings: GlobalSettings dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_context) + @property + def params(self): + ctx = self._click_context() + click_obj = ctx.obj + click_params = ctx.params + command_name = ctx.command.name + + + # Error if click_obj and click_params have the same key + all_click_params_keys = [p.name for p in ctx.command.params.keys] + intersection = set(click_obj.keys()) & set(all_click_params_keys) + if intersection: + raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") + + return {**click_obj, **click_params} + class Config: arbitrary_types_allowed=True def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): """ - Initialize the PipelineContext instance. + Initialize the ClickPipelineContext instance. - This method checks the _initialized flag for the PipelineContext class in the Singleton base class. + This method checks the _initialized flag for the ClickPipelineContext class in the Singleton base class. If the flag is False, the initialization logic is executed and the flag is set to True. If the flag is True, the initialization logic is skipped. - This ensures that the initialization logic is only executed once, even if the PipelineContext instance is retrieved multiple times. + This ensures that the initialization logic is only executed once, even if the ClickPipelineContext instance is retrieved multiple times. This can be useful if the initialization logic is expensive (e.g., it involves network requests or database queries). """ - if not Singleton._initialized[PipelineContext]: + if not Singleton._initialized[ClickPipelineContext]: super().__init__(global_settings=global_settings, **data) - Singleton._initialized[PipelineContext] = True + Singleton._initialized[ClickPipelineContext] = True import asyncio @@ -61,7 +77,7 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name class GlobalContext(BaseModel, Singleton): - pipeline_context: Optional[PipelineContext] = Field(default=None) + pipeline_context: Optional[ClickPipelineContext] = Field(default=None) click_context: Optional[Context] = Field(default=None) class Config: From 58845d36175a90bea9e0c8d3d44c076d83e7f88a Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 19 Oct 2023 13:56:37 -0700 Subject: [PATCH 06/30] Add click decorator to reduce boilerplate --- .../connectors/build_image/commands.py | 2 +- .../connectors/bump_version/commands.py | 2 +- .../airbyte_ci/connectors/commands.py | 13 +- .../airbyte_ci/connectors/context.py | 8 +- .../migrate_to_base_image/commands.py | 2 +- .../airbyte_ci/connectors/reports.py | 4 +- .../connectors/upgrade_base_image/commands.py | 2 +- .../airbyte_ci/playground/commands.py | 12 +- .../pipelines/pipelines/cli/airbyte_ci.py | 82 ++----- .../pipelines/cli/click_decorators.py | 68 ++++++ .../pipelines/pipelines/cli/telemetry.py | 6 +- .../pipelines/pipelines/models/contexts.py | 4 +- .../pipelines/pipelines/stolen/base.py | 16 +- .../pipelines/pipelines/stolen/settings.py | 228 ------------------ 14 files changed, 113 insertions(+), 336 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py index 0cdf469c9d9ccf..d391b2f62a2533 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py @@ -38,7 +38,7 @@ def build(ctx: click.Context, use_host_gradle_dist_tar: bool) -> bool: ci_context=ctx.obj.get("ci_context"), ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], use_local_cdk=ctx.obj.get("use_local_cdk"), - open_report_in_browser=ctx.obj.get("open_report_in_browser"), + enable_report_auto_open=ctx.obj.get("enable_report_auto_open"), use_host_gradle_dist_tar=use_host_gradle_dist_tar, ) for connector in ctx.obj["selected_connectors_with_modified_files"] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py index 1da52905c82dbc..d93b9e4385a79b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py @@ -40,7 +40,7 @@ def bump_version( ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], ci_git_user=ctx.obj["ci_git_user"], ci_github_access_token=ctx.obj["ci_github_access_token"], - open_report_in_browser=False, + enable_report_auto_open=False, ) for connector in ctx.obj["selected_connectors_with_modified_files"] ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index f985348db32a53..58796ee414e7c5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -9,6 +9,7 @@ import click from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo from pipelines import main_logger +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors @@ -177,7 +178,8 @@ def validate_environment(is_local: bool, use_remote_secrets: bool): default=True, type=bool, ) -@click.pass_context +@click_pass_context_and_args_to_children +@click_ignore_unused_kwargs def connectors( ctx: click.Context, use_remote_secrets: bool, @@ -187,21 +189,12 @@ def connectors( modified: bool, metadata_changes_only: bool, metadata_query: str, - concurrency: int, - execute_timeout: int, enable_dependency_scanning: bool, - use_local_cdk: bool, - enable_report_auto_open: bool, ): """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"], use_remote_secrets) ctx.ensure_object(dict) - ctx.obj["use_remote_secrets"] = use_remote_secrets - ctx.obj["concurrency"] = concurrency - ctx.obj["execute_timeout"] = execute_timeout - ctx.obj["use_local_cdk"] = use_local_cdk - ctx.obj["open_report_in_browser"] = enable_report_auto_open ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( names, support_levels, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index 1c7dc061550960..db39c29b96555e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -54,7 +54,7 @@ def __init__( code_tests_only: bool = False, use_local_cdk: bool = False, use_host_gradle_dist_tar: bool = False, - open_report_in_browser: bool = True, + enable_report_auto_open: bool = True, docker_hub_username: Optional[str] = None, docker_hub_password: Optional[str] = None, ): @@ -79,7 +79,7 @@ def __init__( fast_tests_only (bool, optional): Whether to run only fast tests. Defaults to False. code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False. use_host_gradle_dist_tar (bool, optional): Used when developing java connectors with gradle. Defaults to False. - open_report_in_browser (bool, optional): Open HTML report in browser window. Defaults to True. + enable_report_auto_open (bool, optional): Open HTML report in browser window. Defaults to True. docker_hub_username (Optional[str], optional): Docker Hub username to use to read registries. Defaults to None. docker_hub_password (Optional[str], optional): Docker Hub password to use to read registries. Defaults to None. """ @@ -98,7 +98,7 @@ def __init__( self.code_tests_only = code_tests_only self.use_local_cdk = use_local_cdk self.use_host_gradle_dist_tar = use_host_gradle_dist_tar - self.open_report_in_browser = open_report_in_browser + self.enable_report_auto_open = enable_report_auto_open self.docker_hub_username = docker_hub_username self.docker_hub_password = docker_hub_password @@ -118,7 +118,7 @@ def __init__( ci_gcs_credentials=ci_gcs_credentials, ci_git_user=ci_git_user, ci_github_access_token=ci_github_access_token, - open_report_in_browser=open_report_in_browser, + enable_report_auto_open=enable_report_auto_open, ) @property diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py index b57afc0e000593..1ce65625a97184 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py @@ -55,7 +55,7 @@ def migrate_to_base_image( ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], ci_git_user=ctx.obj["ci_git_user"], ci_github_access_token=ctx.obj["ci_github_access_token"], - open_report_in_browser=False, + enable_report_auto_open=False, docker_hub_username=docker_hub_username, docker_hub_password=docker_hub_password, ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py index 4762f508d30d09..2b64f13eaf9153 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py @@ -139,9 +139,9 @@ async def to_html(self) -> str: async def save(self) -> None: local_html_path = await self.save_local(self.html_report_file_name, await self.to_html()) absolute_path = await local_html_path.resolve() - if self.pipeline_context.open_report_in_browser: + if self.pipeline_context.enable_report_auto_open: self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}") - if self.pipeline_context.open_report_in_browser: + if self.pipeline_context.enable_report_auto_open: self.pipeline_context.logger.info("Opening HTML report in browser.") webbrowser.open(absolute_path.as_uri()) if self.remote_storage_enabled: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py index 7c857bf617a49c..e7f4e36bd88378 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py @@ -47,7 +47,7 @@ def upgrade_base_image(ctx: click.Context, set_if_not_exists: bool, docker_hub_u ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], ci_git_user=ctx.obj["ci_git_user"], ci_github_access_token=ctx.obj["ci_github_access_token"], - open_report_in_browser=False, + enable_report_auto_open=False, docker_hub_username=docker_hub_username, docker_hub_password=docker_hub_password, ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 5c3580617a547c..77640f7c426d40 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -3,26 +3,20 @@ # import click +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children from pipelines.stolen.base import ClickPipelineContext from pipelines.stolen.lazy_decorator import LazyPassDecorator -from pipelines.stolen.settings import GlobalSettings -# Stolen -settings = GlobalSettings() -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext, global_settings=settings) -# pass_global_settings: LazyPassDecorator = LazyPassDecorator(GlobalSettings) - -# NEW +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() @click.argument("hold") @click.option("--opt", default="default_value") @pass_pipeline_context -# @pass_global_settings +@click_ignore_unused_kwargs def playground( ctx: ClickPipelineContext, - **kwargs, ): """Runs the tests for the given airbyte-ci package. diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 8d96e9ecc5cab8..9a2e5bb00eecfa 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -14,8 +14,9 @@ import git from github import PullRequest from pipelines import main_logger +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children from pipelines.cli.lazy_group import LazyGroup -from pipelines.cli.telemetry import track_command +from pipelines.cli.telemetry import click_track_command from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH, CIContext from pipelines.helpers import github from pipelines.helpers.git import ( @@ -146,32 +147,20 @@ def get_modified_files( return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) -# TO MOVE - -def pass_click_context_and_args_to_children(f): - """ - TODO - """ - - def wrapper(*args, **kwargs): - ctx = args[0] - ctx.ensure_object(dict) - click_obj = ctx.obj - click_params = ctx.params - command_name = ctx.command.name - - # Error if click_obj and click_params have the same key - intersection = set(click_obj.keys()) & set(click_params.keys()) - if intersection: - raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") - - return f(*args, **kwargs) - - return wrapper +def log_git_info(ctx: click.Context): + main_logger.info("Running airbyte-ci in CI mode.") + main_logger.info(f"CI Context: {ctx.obj['ci_context']}") + main_logger.info(f"CI Report Bucket Name: {ctx.obj['ci_report_bucket_name']}") + main_logger.info(f"Git Branch: {ctx.obj['git_branch']}") + main_logger.info(f"Git Revision: {ctx.obj['git_revision']}") + main_logger.info(f"GitHub Workflow Run ID: {ctx.obj['gha_workflow_run_id']}") + main_logger.info(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}") + main_logger.info(f"Pull Request Number: {ctx.obj['pull_request_number']}") + main_logger.info(f"Pipeline Start Timestamp: {ctx.obj['pipeline_start_timestamp']}") + main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") # COMMANDS - @click.group( cls=LazyGroup, help="Airbyte CI top-level command group.", @@ -208,9 +197,9 @@ def wrapper(*args, **kwargs): ) @click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) @click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) -@click.pass_context -@pass_click_context_and_args_to_children -@track_command +@click_track_command +@click_pass_context_and_args_to_children +@click_ignore_unused_kwargs def airbyte_ci( ctx: click.Context, is_local: bool, @@ -219,55 +208,26 @@ def airbyte_ci( diffed_branch: str, gha_workflow_run_id: str, ci_context: str, - pipeline_start_timestamp: int, pull_request_number: int, - ci_git_user: str, ci_github_access_token: str, - ci_report_bucket_name: str, - ci_gcs_credentials: str, - ci_job_key: str, - show_dagger_logs: bool, ): # noqa D103 display_welcome_message() - check_up_to_date() - # ctx.obj["is_local"] = is_local + ctx.obj["is_ci"] = not is_local - # ctx.obj["git_branch"] = git_branch - # ctx.obj["git_revision"] = git_revision - # ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id ctx.obj["gha_workflow_run_url"] = ( f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None ) - # ctx.obj["ci_context"] = ci_context - # ctx.obj["ci_report_bucket_name"] = ci_report_bucket_name - # ctx.obj["ci_gcs_credentials"] = ci_gcs_credentials - # ctx.obj["ci_git_user"] = ci_git_user - # ctx.obj["ci_github_access_token"] = ci_github_access_token - # ctx.obj["ci_job_key"] = ci_job_key - # ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp - # ctx.obj["show_dagger_logs"] = show_dagger_logs - - if pull_request_number and ci_github_access_token: - ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) - else: - ctx.obj["pull_request"] = None + + can_get_pull_request = pull_request_number and ci_github_access_token + ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) if can_get_pull_request else None ctx.obj["modified_files"] = transform_strs_to_paths( get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"]) ) if not is_local: - main_logger.info("Running airbyte-ci in CI mode.") - main_logger.info(f"CI Context: {ci_context}") - main_logger.info(f"CI Report Bucket Name: {ci_report_bucket_name}") - main_logger.info(f"Git Branch: {git_branch}") - main_logger.info(f"Git Revision: {git_revision}") - main_logger.info(f"GitHub Workflow Run ID: {gha_workflow_run_id}") - main_logger.info(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}") - main_logger.info(f"Pull Request Number: {pull_request_number}") - main_logger.info(f"Pipeline Start Timestamp: {pipeline_start_timestamp}") - main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") + log_git_info(ctx) set_working_directory_to_root() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py new file mode 100644 index 00000000000000..97b08e6351075d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -0,0 +1,68 @@ + + +import functools +import inspect +import click + + + +def _contains_var_kwarg(f): + return any( + param.kind == inspect.Parameter.VAR_KEYWORD + for param in inspect.signature(f).parameters.values() + ) + + +def _is_kwarg_of(key, f): + param = inspect.signature(f).parameters.get(key, False) + return param and ( + param.kind is inspect.Parameter.KEYWORD_ONLY or + param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + ) + + +def click_ignore_unused_kwargs(f): + """Make function ignore unmatched kwargs. + + If the function already has the catch all **kwargs, do nothing. + + Useful in the case that the argument is meant to be passed to a child command + and is not used by the parent command + """ + if _contains_var_kwarg(f): + return f + + @functools.wraps(f) + def inner(*args, **kwargs): + filtered_kwargs = { + key: value + for key, value in kwargs.items() + if _is_kwarg_of(key, f) + } + return f(*args, **filtered_kwargs) + return inner + + +def click_pass_context_and_args_to_children(f): + """ + Decorator to pass click context and args to children commands. + """ + + @click.pass_context + def wrapper(*args, **kwargs): + ctx = args[0] + ctx.ensure_object(dict) + click_obj = ctx.obj + click_params = ctx.params + command_name = ctx.command.name + + # Error if click_obj and click_params have the same key + intersection = set(click_obj.keys()) & set(click_params.keys()) + if intersection: + raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") + + ctx.obj = {**click_obj, **click_params} + + return f(*args, **kwargs) + + return wrapper diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py index 337a9182987d16..942d69cb5751a7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py @@ -9,6 +9,7 @@ import sys import segment.analytics as analytics +from click import get_current_context analytics.write_key = "G6G7whgro81g9xM00kN2buclGKvcOjFd" analytics.send = True @@ -34,13 +35,12 @@ def _get_anonymous_system_id(): return unique_id -def track_command(f): +def click_track_command(f): """ Decorator to track CLI commands with segment.io """ - def wrapper(*args, **kwargs): - ctx = args[0] + ctx = get_current_context() top_level_command = ctx.command_path full_cmd = " ".join(sys.argv) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py index 69a47f540f1c23..72a8581454765e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py @@ -62,7 +62,7 @@ def __init__( ci_gcs_credentials: Optional[str] = None, ci_git_user: Optional[str] = None, ci_github_access_token: Optional[str] = None, - open_report_in_browser: bool = True, + enable_report_auto_open: bool = True, ): """Initialize a pipeline context. @@ -105,7 +105,7 @@ def __init__( self.started_at = None self.stopped_at = None self.secrets_to_mask = [] - self.open_report_in_browser = open_report_in_browser + self.enable_report_auto_open = enable_report_auto_open update_commit_status_check(**self.github_commit_status) @property diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py index 188e9639d72d8e..efcf51ed7bcaf2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py +++ b/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py @@ -10,7 +10,6 @@ from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr -from .settings import GlobalSettings from .singleton import Singleton @@ -22,7 +21,6 @@ def get_context() -> Context: return get_current_context() class ClickPipelineContext(BaseModel, Singleton): - global_settings: GlobalSettings dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_context) @@ -34,9 +32,8 @@ def params(self): click_params = ctx.params command_name = ctx.command.name - # Error if click_obj and click_params have the same key - all_click_params_keys = [p.name for p in ctx.command.params.keys] + all_click_params_keys = [p.name for p in ctx.command.params] intersection = set(click_obj.keys()) & set(all_click_params_keys) if intersection: raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") @@ -46,7 +43,7 @@ def params(self): class Config: arbitrary_types_allowed=True - def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): + def __init__(self, **data: dict[str, Any]): """ Initialize the ClickPipelineContext instance. @@ -58,7 +55,7 @@ def __init__(self, global_settings: GlobalSettings, **data: dict[str, Any]): This can be useful if the initialization logic is expensive (e.g., it involves network requests or database queries). """ if not Singleton._initialized[ClickPipelineContext]: - super().__init__(global_settings=global_settings, **data) + super().__init__(**data) Singleton._initialized[ClickPipelineContext] = True import asyncio @@ -75,10 +72,3 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name assert client, "Error initializing Dagger client" return client.pipeline(pipeline_name) if pipeline_name else client - -class GlobalContext(BaseModel, Singleton): - pipeline_context: Optional[ClickPipelineContext] = Field(default=None) - click_context: Optional[Context] = Field(default=None) - - class Config: - arbitrary_types_allowed = True diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py deleted file mode 100644 index 4596b6ba5d5c4f..00000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/settings.py +++ /dev/null @@ -1,228 +0,0 @@ -import os -import platform -from typing import Any, Callable, List, Optional - -import platformdirs -from dagger import Client, Container -from pydantic import BaseSettings, Field, SecretBytes, SecretStr -from pygit2 import Commit, Repository #type: ignore - -from .singleton import Singleton - - -def get_git_revision() -> str: - repo = Repository(".") - commit_hash:str = os.environ.get("LAST_COMMIT_SHA", repo.revparse_single("HEAD").hex ) - return commit_hash - -def get_current_branch() -> str: - repo = Repository(".") - return str(repo.head.shorthand) - -def get_latest_commit_message() -> str: - repo = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.message) - -def get_latest_commit_author() -> str: - repo: Repository = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.author.name) - -def get_latest_commit_time() -> str: - repo = Repository(".") - commit: Commit = repo[os.environ.get("LAST_COMMIT_SHA", repo.head.target)] - return str(commit.commit_time) - -def get_repo_root_path() -> str: - repo = Repository(".") - return str(os.path.dirname(os.path.dirname(repo.path))) - -def get_repo_fullname() -> str: - repo = Repository(".") - repo_url:str = repo.remotes["origin"].url - - # Handle HTTPS URLs - if "https://" in repo_url: - parts = repo_url.split("/") - owner = parts[-2] - repo_name = parts[-1].replace(".git", "") - - # Handle SSH URLs - else: - repo_url = repo_url.replace("git@github.com:", "") - owner, repo_name = repo_url.split("/")[:2] - repo_name = repo_name.replace(".git", "") - - return f"{owner}/{repo_name}" - -# Immutable. Use this for application configuration. Created at bootstrap. -class GlobalSettings(BaseSettings, Singleton): - # DAGGER: bool = Field(True, env="DAGGER") - # GITHUB_TOKEN: Optional[SecretStr] = Field(None, env="GITHUB_CUSTOM_TOKEN") - # GIT_CURRENT_REVISION: str = Field(default_factory=get_git_revision) - # GIT_CURRENT_BRANCH: str = Field(default_factory=get_current_branch) - # GIT_LATEST_COMMIT_MESSAGE: str = Field(default_factory=get_latest_commit_message) - # GIT_LATEST_COMMIT_AUTHOR: str = Field(default_factory=get_latest_commit_author) - # GIT_LATEST_COMMIT_TIME: str = Field(default_factory=get_latest_commit_time) - # GIT_REPOSITORY: str = Field(default_factory=get_repo_fullname) - # GIT_REPO_ROOT_PATH: str = Field(default_factory=get_repo_root_path) - # CI: bool = Field(False, env="CI") - # LOG_LEVEL: str = Field("WARNING", env="LOG_LEVEL") - PLATFORM: str = platform.system() - # DEBUG: bool = Field(False, env="AIRCMD_DEBUG") - - # # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 - # GITHUB_EVENT_NAME: str = Field("push", env="GITHUB_EVENT_NAME") - # GITHUB_ACTION: str = Field("local_action", env="GITHUB_ACTION") - # GITHUB_ACTOR: str = Field("local_actor", env="GITHUB_ACTOR") - # GITHUB_JOB: str = Field("local_job", env="GITHUB_JOB") - # GITHUB_RUN_NUMBER: int = Field(0, env="GITHUB_RUN_NUMBER") - # GITHUB_RUN_ID: int = Field(0, env="GITHUB_RUN_ID") - # GITHUB_API_URL: str = Field("https://api.github.com", env="GITHUB_API_URL") - # GITHUB_SERVER_URL: str = Field("https://github.com", env="GITHUB_SERVER_URL") - # GITHUB_GRAPHQL_URL: str = Field("https://api.github.com/graphql", env="GITHUB_GRAPHQL_URL") - # GITHUB_EVENT_PATH: Optional[str] = Field("/tmp/mockevents", env="GITHUB_EVENT_PATH") - - # POETRY_CACHE_DIR: str = Field( - # default_factory=lambda: platformdirs.user_cache_dir("pypoetry"), - # env="POETRY_CACHE_DIR" - # ) - # MYPY_CACHE_DIR: str = Field("~/.cache/.mypy_cache", env="MYPY_CACHE_DIR") - # DEFAULT_PYTHON_EXCLUDE: List[str] = Field(["**/.venv", "**/__pycache__"], env="DEFAULT_PYTHON_EXCLUDE") - # DEFAULT_EXCLUDED_FILES: List[str] = Field( - # [ - # ".git", - # "**/build", - # "**/.venv", - # "**/secrets", - # "**/__pycache__", - # "**/*.egg-info", - # "**/.vscode", - # "**/.pytest_cache", - # "**/.eggs", - # "**/.mypy_cache", - # "**/.DS_Store", - # ], - # env="DEFAULT_EXCLUDED_FILES" - # ) - # DOCKER_VERSION:str = Field("20.10.23", env="DOCKER_VERSION") - # DOCKER_DIND_IMAGE: str = Field("docker:dind", env="DOCKER_DIND_IMAGE") - # DOCKER_CLI_IMAGE: str = Field("docker:cli", env="DOCKER_CLI_IMAGE") - # GRADLE_HOMEDIR_PATH: str = Field("/root/.gradle", env="GRADLE_HOMEDIR_PATH") - # GRADLE_CACHE_VOLUME_PATH: str = Field("/root/gradle-cache", env="GRADLE_CACHE_VOLUME_PATH") - - # PREFECT_API_URL: str = Field("http://127.0.0.1:4200/api", env="PREFECT_API_URL") - # PREFECT_COMMA_DELIMITED_USER_TAGS: str = Field("", env="PREFECT_COMMA_DELIMITED_USER_TAGS") - # PREFECT_COMMA_DELIMITED_SYSTEM_TAGS: str = Field("CI:False", env="PREFECT_COMMA_DELIMITED_SYSTEM_TAGS") - - # SECRET_DOCKER_HUB_USERNAME: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_USERNAME") - # SECRET_DOCKER_HUB_PASSWORD: Optional[SecretStr] = Field(None, env="SECRET_DOCKER_HUB_PASSWORD") - # SECRET_TAILSCALE_AUTHKEY: Optional[SecretStr] = Field(None, env="SECRET_TAILSCALE_AUTHKEY") - - # PIP_CACHE_DIR: str = Field( - # default_factory=lambda: platformdirs.user_cache_dir("pip"), - # env="PIP_CACHE_DIR" - # ) - - class Config: - arbitrary_types_allowed = True - # env_file = '.env' - allow_mutation = False - - -''' -If both include and exclude are supplied, the load_settings function will first filter the environment variables based on the include list, and then it will -further filter the resulting environment variables based on the exclude list. - -Here's the order of operations: - - 1 If include is provided, only the environment variables with keys in the include list will be considered. - 2 If exclude is provided, any environment variables with keys in the exclude list will be removed from the filtered list obtained in step 1. - 3 The remaining environment variables will be loaded into the container. -''' - -def load_settings(client: Client, settings: BaseSettings, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None) -> Callable[[Container], Container]: - def load_envs(ctr: Container) -> Container: - settings_dict = {key: value for key, value in settings.dict().items() if value is not None} - - if include is not None: - settings_dict = {key: settings_dict[key] for key in include if key in settings_dict} - - if exclude is not None: - settings_dict = {key: value for key, value in settings_dict.items() if key not in exclude} - - for key, value in settings_dict.items(): - env_key = key.upper() - if isinstance(value, SecretStr) or isinstance(value, SecretBytes): # env var can be stored in buildkit layer cache, so we must use client.secret instead - secret = client.set_secret(env_key, str(value.get_secret_value())) - ctr = ctr.with_secret_variable(env_key, secret) - else: - ctr = ctr.with_env_variable(env_key, str(value)) - - return ctr - - return load_envs - - -class GithubActionsInputSettings(BaseSettings): - """ - A Pydantic BaseSettings subclass that transforms input names to the format expected by GitHub Actions. - - GitHub Actions converts input names to environment variables in a specific way: - - The input name is converted to uppercase. - - Any '-' characters are converted to '_'. - - The prefix 'INPUT_' is added to the start. - - This class automatically applies these transformations when you create an instance of it. - - Example: - If you create an instance with the input {'project-token': 'abc'}, it will be transformed to {'INPUT_PROJECT_TOKEN': 'abc'}. - """ - - # Github action specific fields - GITHUB_ACTION: str - GITHUB_ACTOR: str - GITHUB_API_URL: str - GITHUB_EVENT_NAME: str - GITHUB_GRAPHQL_URL: str - GITHUB_JOB: str - GITHUB_REF: str - GITHUB_REPOSITORY: str - GITHUB_RUN_ID: str - GITHUB_RUN_NUMBER: str - GITHUB_SERVER_URL: str - GITHUB_SHA: str - GITHUB_EVENT_PATH: str - - class Config: - env_prefix = "INPUT_" - extra = "allow" - - def __init__(self, global_settings: GlobalSettings, **data: Any): - - # transform input names to the format expected by GitHub Actions and prepare them to be injected as environment variables. - - transformed_data = {self.Config.env_prefix + k.replace("-", "_").upper(): v for k, v in data.items()} - - # inject the context that github actions wants via environment variables. - # in typescript, it is injected here: - # https://github.com/actions/toolkit/blob/7b617c260dff86f8d044d5ab0425444b29fa0d18/packages/github/src/context.ts#L6 - - transformed_data.update({ - "GITHUB_SHA": global_settings.GIT_CURRENT_REVISION, - "GITHUB_REF": global_settings.GIT_CURRENT_BRANCH, - "GITHUB_EVENT_NAME": global_settings.GITHUB_EVENT_NAME, - "GITHUB_ACTION": global_settings.GITHUB_ACTION, - "GITHUB_ACTOR": global_settings.GITHUB_ACTOR, - "GITHUB_JOB": global_settings.GITHUB_JOB, - "GITHUB_RUN_NUMBER": global_settings.GITHUB_RUN_NUMBER, - "GITHUB_RUN_ID": global_settings.GITHUB_RUN_ID, - "GITHUB_API_URL": global_settings.GITHUB_API_URL, - "GITHUB_SERVER_URL": global_settings.GITHUB_SERVER_URL, - "GITHUB_GRAPHQL_URL": global_settings.GITHUB_GRAPHQL_URL, - "GITHUB_REPOSITORY": global_settings.GIT_REPOSITORY, - "GITHUB_EVENT_PATH": global_settings.GITHUB_EVENT_PATH - }) - super().__init__(**transformed_data) - From 435f44dd29c5e5ffa815ac9f8c906bb821d47a15 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 19 Oct 2023 14:22:43 -0700 Subject: [PATCH 07/30] Add click_append_to_context_object decorator --- .../airbyte_ci/connectors/commands.py | 5 +- .../airbyte_ci/playground/commands.py | 5 +- .../pipelines/pipelines/cli/airbyte_ci.py | 56 +++++++++++-------- .../pipelines/cli/click_decorators.py | 21 ++++++- 4 files changed, 58 insertions(+), 29 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 58796ee414e7c5..dcb14b318e1d70 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -9,7 +9,7 @@ import click from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo from pipelines import main_logger -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors @@ -178,7 +178,7 @@ def validate_environment(is_local: bool, use_remote_secrets: bool): default=True, type=bool, ) -@click_pass_context_and_args_to_children +@click_merge_args_into_context_obj @click_ignore_unused_kwargs def connectors( ctx: click.Context, @@ -194,7 +194,6 @@ def connectors( """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"], use_remote_secrets) - ctx.ensure_object(dict) ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( names, support_levels, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 77640f7c426d40..4cd43340e07ad5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -3,7 +3,7 @@ # import click -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.stolen.base import ClickPipelineContext from pipelines.stolen.lazy_decorator import LazyPassDecorator @@ -29,8 +29,7 @@ def playground( # args = GlobalSettings(PLATFORM='Darwin') # kwargs = {'opt': 'tight', 'hold': 'holdme'} - import pdb; pdb.set_trace() - print(f"ctx: {ctx._click_context().obj}") + print(f"params: {ctx.params}") # (Pdb) ctx._click_context().args # [] diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 9a2e5bb00eecfa..2be63e30698b48 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -8,13 +8,13 @@ import logging import os from pathlib import Path -from typing import List +from typing import List, Optional import click import git from github import PullRequest from pipelines import main_logger -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_pass_context_and_args_to_children +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.cli.telemetry import click_track_command from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH, CIContext @@ -159,6 +159,33 @@ def log_git_info(ctx: click.Context): main_logger.info(f"Pipeline Start Timestamp: {ctx.obj['pipeline_start_timestamp']}") main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") +def _get_gha_workflow_run_id(ctx: click.Context) -> Optional[str]: + if ctx.obj["is_local"]: + return "TESTEST" + return ctx.obj["gha_workflow_run_id"] + +def _get_pull_request(ctx: click.Context): + pull_request_number = ctx.obj["pull_request_number"] + ci_github_access_token = ctx.obj["ci_github_access_token"] + + can_get_pull_request = pull_request_number and ci_github_access_token + if not can_get_pull_request: + return None + + return github.get_pull_request(pull_request_number, ci_github_access_token) + +def _get_modified_files(ctx: click.Context) -> List[Path]: + return transform_strs_to_paths( + get_modified_files( + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["diffed_branch"], + ctx.obj["is_local"], + ctx.obj["ci_context"], + ctx.obj["pull_request"] + ) + ) + # COMMANDS @click.group( @@ -198,34 +225,19 @@ def log_git_info(ctx: click.Context): @click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) @click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) @click_track_command -@click_pass_context_and_args_to_children +@click_merge_args_into_context_obj +@click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) +@click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_id) +@click_append_to_context_object("pull_request", _get_pull_request) +@click_append_to_context_object("modified_files", _get_modified_files) @click_ignore_unused_kwargs def airbyte_ci( ctx: click.Context, is_local: bool, - git_branch: str, - git_revision: str, - diffed_branch: str, - gha_workflow_run_id: str, - ci_context: str, - pull_request_number: int, - ci_github_access_token: str, ): # noqa D103 display_welcome_message() check_up_to_date() - ctx.obj["is_ci"] = not is_local - ctx.obj["gha_workflow_run_url"] = ( - f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None - ) - - can_get_pull_request = pull_request_number and ci_github_access_token - ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) if can_get_pull_request else None - - ctx.obj["modified_files"] = transform_strs_to_paths( - get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"]) - ) - if not is_local: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 97b08e6351075d..dcd6516c14da21 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -2,6 +2,7 @@ import functools import inspect +from typing import Any, Callable import click @@ -43,7 +44,7 @@ def inner(*args, **kwargs): return inner -def click_pass_context_and_args_to_children(f): +def click_merge_args_into_context_obj(f): """ Decorator to pass click context and args to children commands. """ @@ -66,3 +67,21 @@ def wrapper(*args, **kwargs): return f(*args, **kwargs) return wrapper + +def click_append_to_context_object(key: str, value: Callable | Any): + """ + Decorator to append a value to the click context object. + """ + + def decorator(f): + def wrapper(*args, **kwargs): + ctx = click.get_current_context() + ctx.ensure_object(dict) + + if callable(value): + ctx.obj[key] = value(ctx) + else: + ctx.obj[key] = value + return f(*args, **kwargs) + return wrapper + return decorator From 55baf8f6c1660f5f512e9f7024827f82877701f3 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 19 Oct 2023 14:35:32 -0700 Subject: [PATCH 08/30] Move files --- .../airbyte_ci/connectors/context.py | 2 +- .../airbyte_ci/playground/commands.py | 59 ++++++++++++++----- .../pipelines/airbyte_ci/steps/docker.py | 2 +- .../pipelines/airbyte_ci/steps/gradle.py | 2 +- .../pipelines/airbyte_ci/steps/no_op.py | 2 +- .../pipelines/airbyte_ci/steps/poetry.py | 2 +- .../pipelines/cli/click_decorators.py | 26 +++++++- .../pipelines/models/contexts/__init__.py | 3 + .../contexts/click_pipeline_context.py} | 2 +- .../pipeline_context.py} | 0 .../pipelines/{stolen => models}/singleton.py | 0 .../pipelines/pipelines/models/steps.py | 2 +- .../pipelines/stolen/lazy_decorator.py | 25 -------- .../test_steps/test_simple_docker_step.py | 2 +- 14 files changed, 81 insertions(+), 48 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py rename airbyte-ci/connectors/pipelines/pipelines/{stolen/base.py => models/contexts/click_pipeline_context.py} (98%) rename airbyte-ci/connectors/pipelines/pipelines/models/{contexts.py => contexts/pipeline_context.py} (100%) rename airbyte-ci/connectors/pipelines/pipelines/{stolen => models}/singleton.py (100%) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index db39c29b96555e..18122e4fefb26f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -19,7 +19,7 @@ from pipelines.helpers.github import update_commit_status_check from pipelines.helpers.slack import send_message_to_webhook from pipelines.helpers.utils import METADATA_FILE_NAME -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext class ConnectorContext(PipelineContext): diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 4cd43340e07ad5..42fbcedd475108 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -4,8 +4,8 @@ import click from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.stolen.base import ClickPipelineContext -from pipelines.stolen.lazy_decorator import LazyPassDecorator +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext +from pipelines.cli.click_decorators import LazyPassDecorator pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @@ -18,21 +18,52 @@ def playground( ctx: ClickPipelineContext, ): - """Runs the tests for the given airbyte-ci package. + """ + TODO + 1. Make async + 1. Call a dagger pipeline - Args: - poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. - test_directory (str): The directory containing the tests to run. + Blockers: + 1. Need asyncio to run dagger pipeline """ - # ctx = ClickPipelineContext(global_settings=GlobalSettings(PLATFORM='Darwin'), dockerd_service=None, asyncio=) - # args = GlobalSettings(PLATFORM='Darwin') - # kwargs = {'opt': 'tight', 'hold': 'holdme'} + # dagger_client = await ctx.get_dagger_client(pipeline_name="format_ci") + # pytest_container = await ( + # dagger_client.container() + # .from_("python:3.10.12") + # .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + # .with_exec( + # sh_dash_c( + # [ + # "apt-get update", + # "apt-get install -y bash git curl", + # "pip install pipx", + # "pipx ensurepath", + # "pipx install poetry", + # ] + # ) + # ) + # .with_env_variable("VERSION", DOCKER_VERSION) + # .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) + # .with_mounted_directory( + # "/airbyte", + # dagger_client.host().directory( + # ".", + # exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], + # include=directories_to_mount, + # ), + # ) + # .with_workdir(f"/airbyte/{poetry_package_path}") + # .with_exec(["poetry", "install"]) + # .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + # .with_exec(["poetry", "run", "pytest", test_directory]) + # ) + + # await pytest_container + # return True + + print(f"params: {ctx.params}") - # (Pdb) ctx._click_context().args - # [] - # (Pdb) ctx._click_context().params - # {'opt': 'tight', 'hold': 'holdme'} - # (Pdb) + diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py index 1dd5c76c2af268..964ee2c93b3f1a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py @@ -7,7 +7,7 @@ import dagger from pipelines.dagger.actions.python.pipx import with_installed_pipx_package from pipelines.dagger.containers.python import with_python_base -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import MountPath, Step, StepResult diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 924fc8807fcfb8..d9def2b31873c3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -11,7 +11,7 @@ from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions import secrets from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py index 2a054c4df3dca3..23e70650f4059d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py index 3acaa28faff614..f7140f1e2ccd89 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py @@ -3,7 +3,7 @@ # from pipelines.dagger.actions.python.poetry import with_poetry_module -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index dcd6516c14da21..532d37ebda1517 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -1,8 +1,9 @@ import functools +from functools import wraps import inspect -from typing import Any, Callable +from typing import Any, Callable, Type import click @@ -85,3 +86,26 @@ def wrapper(*args, **kwargs): return f(*args, **kwargs) return wrapper return decorator + + +class LazyPassDecorator: + def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: + self.cls = cls + self.args = args + self.kwargs = kwargs + + def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: + @wraps(f) + def decorated_function(*args: Any, **kwargs: Any) -> Any: + # Check if the kwargs already contain the arguments being passed by the decorator + decorator_kwargs = {k: v for k, v in self.kwargs.items() if k not in kwargs} + # Create an instance of the class + instance = self.cls(*self.args, **decorator_kwargs) + # If function has **kwargs, we can put the instance there + if 'kwargs' in kwargs: + kwargs['kwargs'] = instance + # Otherwise, add it to positional arguments + else: + args = (*args, instance) + return f(*args, **kwargs) + return decorated_function diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py similarity index 98% rename from airbyte-ci/connectors/pipelines/pipelines/stolen/base.py rename to airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index efcf51ed7bcaf2..4cf262b4f3ae91 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/base.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -10,7 +10,7 @@ from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr -from .singleton import Singleton +from ..singleton import Singleton # this is a bit of a hack to get around how prefect resolves parameters diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/models/contexts.py rename to airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/stolen/singleton.py rename to airbyte-ci/connectors/pipelines/pipelines/models/singleton.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py index 4c5f9a523da5d2..5cc119ed6440c3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py @@ -21,7 +21,7 @@ from pipelines.helpers.utils import format_duration, get_exec_result if typing.TYPE_CHECKING: - from pipelines.models.contexts import PipelineContext + from pipelines.models.contexts.pipeline_context import PipelineContext from abc import ABC from typing import ClassVar diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py deleted file mode 100644 index b65e610c52128d..00000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/lazy_decorator.py +++ /dev/null @@ -1,25 +0,0 @@ -from functools import wraps -from typing import Any, Callable, Type - - -class LazyPassDecorator: - def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: - self.cls = cls - self.args = args - self.kwargs = kwargs - - def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: - @wraps(f) - def decorated_function(*args: Any, **kwargs: Any) -> Any: - # Check if the kwargs already contain the arguments being passed by the decorator - decorator_kwargs = {k: v for k, v in self.kwargs.items() if k not in kwargs} - # Create an instance of the class - instance = self.cls(*self.args, **decorator_kwargs) - # If function has **kwargs, we can put the instance there - if 'kwargs' in kwargs: - kwargs['kwargs'] = instance - # Otherwise, add it to positional arguments - else: - args = (*args, instance) - return f(*args, **kwargs) - return decorated_function diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py index a4b23cbaea51a9..27d5765037feca 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py @@ -7,7 +7,7 @@ import pytest from pipelines.airbyte_ci.steps.docker import SimpleDockerStep from pipelines.helpers.utils import get_exec_result -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import MountPath pytestmark = [ From 8bfe2aa610a2121059b879876eed4ab3da2915b6 Mon Sep 17 00:00:00 2001 From: bnchrch Date: Thu, 19 Oct 2023 21:48:50 +0000 Subject: [PATCH 09/30] Automated Commit - Formatting Changes --- .../airbyte_ci/playground/commands.py | 9 ++---- .../pipelines/pipelines/cli/airbyte_ci.py | 14 +++++++-- .../pipelines/cli/click_decorators.py | 31 +++++++------------ .../pipelines/pipelines/cli/telemetry.py | 1 + .../models/contexts/click_pipeline_context.py | 13 ++++---- .../pipelines/pipelines/models/singleton.py | 7 +++-- 6 files changed, 36 insertions(+), 39 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 42fbcedd475108..647a50cf6a20db 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -3,13 +3,12 @@ # import click -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext -from pipelines.cli.click_decorators import LazyPassDecorator - pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + @click.command() @click.argument("hold") @click.option("--opt", default="default_value") @@ -62,8 +61,4 @@ def playground( # await pytest_container # return True - - print(f"params: {ctx.params}") - - diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 2be63e30698b48..f7026278fce687 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -34,14 +34,16 @@ def display_welcome_message() -> None: - print(''' + print( + """ █████╗ ██╗██████╗ ██████╗ ██╗ ██╗████████╗███████╗ ██╔══██╗██║██╔══██╗██╔══██╗╚██╗ ██╔╝╚══██╔══╝██╔════╝ ███████║██║██████╔╝██████╔╝ ╚████╔╝ ██║ █████╗ ██╔══██║██║██╔══██╗██╔══██╗ ╚██╔╝ ██║ ██╔══╝ ██║ ██║██║██║ ██║██████╔╝ ██║ ██║ ███████╗ ╚═╝ ╚═╝╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝ ╚══════╝ - ''') + """ + ) def check_up_to_date() -> bool: @@ -147,6 +149,7 @@ def get_modified_files( return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) + def log_git_info(ctx: click.Context): main_logger.info("Running airbyte-ci in CI mode.") main_logger.info(f"CI Context: {ctx.obj['ci_context']}") @@ -159,11 +162,13 @@ def log_git_info(ctx: click.Context): main_logger.info(f"Pipeline Start Timestamp: {ctx.obj['pipeline_start_timestamp']}") main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") + def _get_gha_workflow_run_id(ctx: click.Context) -> Optional[str]: if ctx.obj["is_local"]: return "TESTEST" return ctx.obj["gha_workflow_run_id"] + def _get_pull_request(ctx: click.Context): pull_request_number = ctx.obj["pull_request_number"] ci_github_access_token = ctx.obj["ci_github_access_token"] @@ -174,6 +179,7 @@ def _get_pull_request(ctx: click.Context): return github.get_pull_request(pull_request_number, ci_github_access_token) + def _get_modified_files(ctx: click.Context) -> List[Path]: return transform_strs_to_paths( get_modified_files( @@ -182,12 +188,14 @@ def _get_modified_files(ctx: click.Context) -> List[Path]: ctx.obj["diffed_branch"], ctx.obj["is_local"], ctx.obj["ci_context"], - ctx.obj["pull_request"] + ctx.obj["pull_request"], ) ) + # COMMANDS + @click.group( cls=LazyGroup, help="Airbyte CI top-level command group.", diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 532d37ebda1517..47538c7a99de01 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -1,26 +1,18 @@ - - import functools -from functools import wraps import inspect +from functools import wraps from typing import Any, Callable, Type -import click +import click def _contains_var_kwarg(f): - return any( - param.kind == inspect.Parameter.VAR_KEYWORD - for param in inspect.signature(f).parameters.values() - ) + return any(param.kind == inspect.Parameter.VAR_KEYWORD for param in inspect.signature(f).parameters.values()) def _is_kwarg_of(key, f): param = inspect.signature(f).parameters.get(key, False) - return param and ( - param.kind is inspect.Parameter.KEYWORD_ONLY or - param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD - ) + return param and (param.kind is inspect.Parameter.KEYWORD_ONLY or param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD) def click_ignore_unused_kwargs(f): @@ -36,12 +28,9 @@ def click_ignore_unused_kwargs(f): @functools.wraps(f) def inner(*args, **kwargs): - filtered_kwargs = { - key: value - for key, value in kwargs.items() - if _is_kwarg_of(key, f) - } + filtered_kwargs = {key: value for key, value in kwargs.items() if _is_kwarg_of(key, f)} return f(*args, **filtered_kwargs) + return inner @@ -69,6 +58,7 @@ def wrapper(*args, **kwargs): return wrapper + def click_append_to_context_object(key: str, value: Callable | Any): """ Decorator to append a value to the click context object. @@ -84,7 +74,9 @@ def wrapper(*args, **kwargs): else: ctx.obj[key] = value return f(*args, **kwargs) + return wrapper + return decorator @@ -102,10 +94,11 @@ def decorated_function(*args: Any, **kwargs: Any) -> Any: # Create an instance of the class instance = self.cls(*self.args, **decorator_kwargs) # If function has **kwargs, we can put the instance there - if 'kwargs' in kwargs: - kwargs['kwargs'] = instance + if "kwargs" in kwargs: + kwargs["kwargs"] = instance # Otherwise, add it to positional arguments else: args = (*args, instance) return f(*args, **kwargs) + return decorated_function diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py index 942d69cb5751a7..891e0327e45a07 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py @@ -39,6 +39,7 @@ def click_track_command(f): """ Decorator to track CLI commands with segment.io """ + def wrapper(*args, **kwargs): ctx = get_current_context() top_level_command = ctx.command_path diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 4cf262b4f3ae91..27c3725c36dd4c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -2,16 +2,15 @@ from typing import Any, Callable, Optional, Union import dagger - -# TODO ben set up for async -# from asyncclick import Context, get_current_context - from click import Context, get_current_context from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr from ..singleton import Singleton +# TODO ben set up for async +# from asyncclick import Context, get_current_context + # this is a bit of a hack to get around how prefect resolves parameters # basically without this, prefect will attempt to access the context @@ -20,6 +19,7 @@ def get_context() -> Context: return get_current_context() + class ClickPipelineContext(BaseModel, Singleton): dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) @@ -41,7 +41,7 @@ def params(self): return {**click_obj, **click_params} class Config: - arbitrary_types_allowed=True + arbitrary_types_allowed = True def __init__(self, **data: dict[str, Any]): """ @@ -67,8 +67,7 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name async with self._dagger_client_lock: if not self._dagger_client: connection = dagger.Connection(dagger.Config(log_output=sys.stdout)) - self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore + self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore client = self._dagger_client assert client, "Error initializing Dagger client" return client.pipeline(pipeline_name) if pipeline_name else client - diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 3fb7f62aedbdaf..061307f1d01663 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -11,10 +11,11 @@ class Singleton: The __new__ method ensures that only one instance of each subclass is created. The _initialized dictionary is used to control when the initialization logic of each subclass is executed. """ - _instances: dict[Type['Singleton'], Any] = {} - _initialized: dict[Type['Singleton'], bool] = {} - def __new__(cls: Type['Singleton'], *args: Any, **kwargs: Any) -> Any: + _instances: dict[Type["Singleton"], Any] = {} + _initialized: dict[Type["Singleton"], bool] = {} + + def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: if cls not in cls._instances: cls._instances[cls] = super().__new__(cls) From 27043ae2544608314beb929553fc739434216dc5 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 24 Oct 2023 11:54:00 -0700 Subject: [PATCH 10/30] Bring up to date with async click --- airbyte-ci/connectors/pipelines/README.md | 9 +- .../airbyte_ci/playground/commands.py | 88 +++++++++++-------- .../pipelines/pipelines/cli/airbyte_ci.py | 11 ++- .../pipelines/cli/click_decorators.py | 2 +- .../pipelines/pipelines/cli/telemetry.py | 2 +- .../models/contexts/click_pipeline_context.py | 18 +--- .../connectors/pipelines/pyproject.toml | 2 +- 7 files changed, 71 insertions(+), 61 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index f833ba7e8a2d93..54c47ca8a895b8 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -52,8 +52,8 @@ If you face any installation problems feel free to reach out the Airbyte Connect ### Setting up connector secrets access If you plan to use Airbyte CI to run CAT (Connector Acceptance Tests), we recommend setting up GSM -access so that Airbyte CI can pull remote secrets from GSM. For setup instructions, see the -CI Credentials package (which Airbyte CI uses under the hood) README's +access so that Airbyte CI can pull remote secrets from GSM. For setup instructions, see the +CI Credentials package (which Airbyte CI uses under the hood) README's [Get GSM Access](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md#get-gsm-access) instructions. @@ -155,7 +155,7 @@ Available commands: | `--enable-dependency-scanning / --disable-dependency-scanning` | False | ` --disable-dependency-scanning` | | When enabled the dependency scanning will be performed to detect the connectors to select according to a dependency change. | | `--docker-hub-username` | | | DOCKER_HUB_USERNAME | Your username to connect to DockerHub. Required for the publish subcommand. | | `--docker-hub-password` | | | DOCKER_HUB_PASSWORD | Your password to connect to DockerHub. Required for the publish subcommand. | - + ### `connectors list` command Retrieve the list of connectors satisfying the provided filters. @@ -408,7 +408,8 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 2.4.0 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. +| 2.4.1 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | +| 2.4.0 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. | | 2.3.1 | [#31748](https://github.com/airbytehq/airbyte/pull/31748) | Use AsyncClick library instead of base Click. | | 2.3.0 | [#31699](https://github.com/airbytehq/airbyte/pull/31699) | Support optional concurrent CAT execution. | | 2.2.6 | [#31752](https://github.com/airbytehq/airbyte/pull/31752) | Only authenticate when secrets are available. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py index 647a50cf6a20db..9ec67f6bd0224b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py @@ -2,20 +2,25 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import click +import logging +import asyncclick as click from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext +from pipelines.helpers.utils import sh_dash_c +from pipelines.consts import DOCKER_VERSION pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() -@click.argument("hold") -@click.option("--opt", default="default_value") +@click.argument("poetry_package_path") +@click.option("--test-directory", default="tests", help="The directory containing the tests to run.") @pass_pipeline_context @click_ignore_unused_kwargs -def playground( +async def playground( ctx: ClickPipelineContext, + poetry_package_path: str, + test_directory: str, ): """ TODO @@ -25,40 +30,47 @@ def playground( Blockers: 1. Need asyncio to run dagger pipeline """ + print(f"params: {ctx.params}") - # dagger_client = await ctx.get_dagger_client(pipeline_name="format_ci") - # pytest_container = await ( - # dagger_client.container() - # .from_("python:3.10.12") - # .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - # .with_exec( - # sh_dash_c( - # [ - # "apt-get update", - # "apt-get install -y bash git curl", - # "pip install pipx", - # "pipx ensurepath", - # "pipx install poetry", - # ] - # ) - # ) - # .with_env_variable("VERSION", DOCKER_VERSION) - # .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) - # .with_mounted_directory( - # "/airbyte", - # dagger_client.host().directory( - # ".", - # exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], - # include=directories_to_mount, - # ), - # ) - # .with_workdir(f"/airbyte/{poetry_package_path}") - # .with_exec(["poetry", "install"]) - # .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) - # .with_exec(["poetry", "run", "pytest", test_directory]) - # ) + logger = logging.getLogger(f"{poetry_package_path}.tests") + logger.info(f"Running tests for {poetry_package_path}") + # The following directories are always mounted because a lot of tests rely on them + directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"] + directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) - # await pytest_container - # return True + dagger_client = await ctx.get_dagger_client(pipeline_name="format_ci") + pytest_container = await ( + dagger_client.container() + .from_("python:3.10.12") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] + ) + ) + .with_env_variable("VERSION", DOCKER_VERSION) + .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) + .with_mounted_directory( + "/airbyte", + dagger_client.host().directory( + ".", + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], + include=directories_to_mount, + ), + ) + .with_workdir(f"/airbyte/{poetry_package_path}") + .with_exec(["poetry", "install"]) + .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + .with_exec(["poetry", "run", "pytest", test_directory]) + ) + + success = await pytest_container + if not success: + click.Abort() - print(f"params: {ctx.params}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 12f0c9521cef79..77a18f6bf96525 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -239,7 +239,6 @@ def _get_modified_files(ctx: click.Context) -> List[Path]: @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_id) @click_append_to_context_object("pull_request", _get_pull_request) -@click_append_to_context_object("modified_files", _get_modified_files) @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, @@ -248,6 +247,16 @@ async def airbyte_ci( display_welcome_message() check_up_to_date() + modified_files = await get_modified_files( + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["diffed_branch"], + ctx.obj["is_local"], + ctx.obj["ci_context"], + ctx.obj["pull_request"], + ) + ctx.obj["modified_files"] = transform_strs_to_paths(modified_files) + if not is_local: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 47538c7a99de01..d4e849d9708734 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -3,7 +3,7 @@ from functools import wraps from typing import Any, Callable, Type -import click +import asyncclick as click def _contains_var_kwarg(f): diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py index 891e0327e45a07..7ad4bea8daaa1a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py @@ -9,7 +9,7 @@ import sys import segment.analytics as analytics -from click import get_current_context +from asyncclick import get_current_context analytics.write_key = "G6G7whgro81g9xM00kN2buclGKvcOjFd" analytics.send = True diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 27c3725c36dd4c..2b6046cc9165c8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -1,29 +1,17 @@ import sys -from typing import Any, Callable, Optional, Union +from typing import Any, Callable, Optional import dagger -from click import Context, get_current_context +from asyncclick import Context, get_current_context from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr from ..singleton import Singleton -# TODO ben set up for async -# from asyncclick import Context, get_current_context - - -# this is a bit of a hack to get around how prefect resolves parameters -# basically without this, prefect will attempt to access the context -# before we create it in main.py in order to resolve it as a parameter -# wrapping it in a function like this prevents that from happening -def get_context() -> Context: - return get_current_context() - - class ClickPipelineContext(BaseModel, Singleton): dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) - _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_context) + _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) @property def params(self): diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index f0e28631d5623d..f09e10c53d96c9 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.4.0" +version = "2.4.1" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From c53310a85d4379feffdc2328c0bad6b62ca95907 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 24 Oct 2023 12:20:41 -0700 Subject: [PATCH 11/30] Replace test with playground impl --- .../airbyte_ci/playground/__init__.py | 3 - .../airbyte_ci/playground/commands.py | 76 ------------------- .../pipelines/airbyte_ci/test/commands.py | 55 +++++++++++++- .../pipelines/airbyte_ci/test/pipeline.py | 71 ----------------- .../pipelines/pipelines/cli/airbyte_ci.py | 21 +++-- .../models/contexts/click_pipeline_context.py | 8 ++ 6 files changed, 75 insertions(+), 159 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py deleted file mode 100644 index c941b30457953b..00000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py deleted file mode 100644 index 9ec67f6bd0224b..00000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/playground/commands.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -import asyncclick as click -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext -from pipelines.helpers.utils import sh_dash_c -from pipelines.consts import DOCKER_VERSION - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) - - -@click.command() -@click.argument("poetry_package_path") -@click.option("--test-directory", default="tests", help="The directory containing the tests to run.") -@pass_pipeline_context -@click_ignore_unused_kwargs -async def playground( - ctx: ClickPipelineContext, - poetry_package_path: str, - test_directory: str, -): - """ - TODO - 1. Make async - 1. Call a dagger pipeline - - Blockers: - 1. Need asyncio to run dagger pipeline - """ - print(f"params: {ctx.params}") - - logger = logging.getLogger(f"{poetry_package_path}.tests") - logger.info(f"Running tests for {poetry_package_path}") - # The following directories are always mounted because a lot of tests rely on them - directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"] - directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) - - dagger_client = await ctx.get_dagger_client(pipeline_name="format_ci") - pytest_container = await ( - dagger_client.container() - .from_("python:3.10.12") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_env_variable("VERSION", DOCKER_VERSION) - .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) - .with_mounted_directory( - "/airbyte", - dagger_client.host().directory( - ".", - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], - include=directories_to_mount, - ), - ) - .with_workdir(f"/airbyte/{poetry_package_path}") - .with_exec(["poetry", "install"]) - .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) - .with_exec(["poetry", "run", "pytest", test_directory]) - ) - - success = await pytest_container - if not success: - click.Abort() - diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index cf47f4168b5d10..426dca55108078 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -2,23 +2,74 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging import asyncclick as click -from pipelines.airbyte_ci.test.pipeline import run_test +from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext +from pipelines.helpers.utils import sh_dash_c +from pipelines.consts import DOCKER_VERSION +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) @click.command() @click.argument("poetry_package_path") @click.option("--test-directory", default="tests", help="The directory containing the tests to run.") +@pass_pipeline_context +@click_ignore_unused_kwargs async def test( + ctx: ClickPipelineContext, poetry_package_path: str, test_directory: str, ): """Runs the tests for the given airbyte-ci package. Args: + ctx (ClickPipelineContext): The context object. poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. test_directory (str): The directory containing the tests to run. """ - success = await run_test(poetry_package_path, test_directory) + + logger = logging.getLogger(f"{poetry_package_path}.tests") + logger.info(f"Running tests for {poetry_package_path}") + + # The following directories are always mounted because a lot of tests rely on them + directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"] + directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) + + pipeline_name = f"Unit tests for {poetry_package_path}" + dagger_client = await ctx.get_dagger_client(pipeline_name=pipeline_name) + pytest_container = await ( + dagger_client.container() + .from_("python:3.10.12") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] + ) + ) + .with_env_variable("VERSION", DOCKER_VERSION) + .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) + .with_mounted_directory( + "/airbyte", + dagger_client.host().directory( + ".", + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], + include=directories_to_mount, + ), + ) + .with_workdir(f"/airbyte/{poetry_package_path}") + .with_exec(["poetry", "install"]) + .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + .with_exec(["poetry", "run", "pytest", test_directory]) + ) + + success = await pytest_container if not success: click.Abort() + diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py deleted file mode 100644 index 16a8e322d7c6d9..00000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -import os -import sys - -import dagger -from pipelines.consts import DOCKER_VERSION -from pipelines.helpers.utils import sh_dash_c - - -async def run_test(poetry_package_path: str, test_directory: str) -> bool: - """Runs the tests for the given airbyte-ci package in a Dagger container. - - Args: - airbyte_ci_package_path (str): Path to the airbyte-ci package to test, relative to airbyte-ci directory. - Returns: - bool: True if the tests passed, False otherwise. - """ - logger = logging.getLogger(f"{poetry_package_path}.tests") - logger.info(f"Running tests for {poetry_package_path}") - # The following directories are always mounted because a lot of tests rely on them - directories_to_always_mount = [".git", ".github", "docs", "airbyte-integrations", "airbyte-ci", "airbyte-cdk", "pyproject.toml"] - directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - docker_host_socket = dagger_client.host().unix_socket("/var/run/buildkit/buildkitd.sock") - pytest_container = await ( - dagger_client.container() - .from_("python:3.10.12") - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec( - sh_dash_c( - [ - "apt-get update", - "apt-get install -y bash git curl", - "pip install pipx", - "pipx ensurepath", - "pipx install poetry", - ] - ) - ) - .with_env_variable("VERSION", DOCKER_VERSION) - .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) - .with_mounted_directory( - "/airbyte", - dagger_client.host().directory( - ".", - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], - include=directories_to_mount, - ), - ) - .with_workdir(f"/airbyte/{poetry_package_path}") - .with_exec(["poetry", "install"]) - .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) - .with_exec(["poetry", "run", "pytest", test_directory]) - ) - if "_EXPERIMENTAL_DAGGER_RUNNER_HOST" in os.environ: - logger.info("Using experimental dagger runner host to run CAT with dagger-in-dagger") - pytest_container = pytest_container.with_env_variable( - "_EXPERIMENTAL_DAGGER_RUNNER_HOST", "unix:///var/run/buildkit/buildkitd.sock" - ).with_unix_socket("/var/run/buildkit/buildkitd.sock", docker_host_socket) - - await pytest_container - return True - except dagger.ExecError as e: - logger.error("Tests failed") - logger.error(e.stderr) - sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 77a18f6bf96525..4186d18c766975 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -36,12 +36,20 @@ def display_welcome_message() -> None: print( """ - █████╗ ██╗██████╗ ██████╗ ██╗ ██╗████████╗███████╗ - ██╔══██╗██║██╔══██╗██╔══██╗╚██╗ ██╔╝╚══██╔══╝██╔════╝ - ███████║██║██████╔╝██████╔╝ ╚████╔╝ ██║ █████╗ - ██╔══██║██║██╔══██╗██╔══██╗ ╚██╔╝ ██║ ██╔══╝ - ██║ ██║██║██║ ██║██████╔╝ ██║ ██║ ███████╗ - ╚═╝ ╚═╝╚═╝╚═╝ ╚═╝╚═════╝ ╚═╝ ╚═╝ ╚══════╝ + ╔─────────────────────────────────────────────────────────────────────────────────────────────────╗ + │ │ + │ │ + │ /$$$$$$ /$$$$$$ /$$$$$$$ /$$$$$$$ /$$ /$$ /$$$$$$$$ /$$$$$$$$ /$$$$$$ /$$$$$$ │ + │ /$$__ $$|_ $$_/| $$__ $$| $$__ $$| $$ /$$/|__ $$__/| $$_____/ /$$__ $$|_ $$_/ │ + │ | $$ \ $$ | $$ | $$ \ $$| $$ \ $$ \ $$ /$$/ | $$ | $$ | $$ \__/ | $$ │ + │ | $$$$$$$$ | $$ | $$$$$$$/| $$$$$$$ \ $$$$/ | $$ | $$$$$ /$$$$$$| $$ | $$ │ + │ | $$__ $$ | $$ | $$__ $$| $$__ $$ \ $$/ | $$ | $$__/|______/| $$ | $$ │ + │ | $$ | $$ | $$ | $$ \ $$| $$ \ $$ | $$ | $$ | $$ | $$ $$ | $$ │ + │ | $$ | $$ /$$$$$$| $$ | $$| $$$$$$$/ | $$ | $$ | $$$$$$$$ | $$$$$$/ /$$$$$$ │ + │ |__/ |__/|______/|__/ |__/|_______/ |__/ |__/ |________/ \______/ |______/ │ + │ │ + │ │ + ╚─────────────────────────────────────────────────────────────────────────────────────────────────╝ """ ) @@ -203,7 +211,6 @@ def _get_modified_files(ctx: click.Context) -> List[Path]: "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", - "playground": "pipelines.airbyte_ci.playground.commands.playground", }, ) @click.version_option(__installed_version__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 2b6046cc9165c8..1502f40ce13b37 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -55,6 +55,14 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name async with self._dagger_client_lock: if not self._dagger_client: connection = dagger.Connection(dagger.Config(log_output=sys.stdout)) + + """ + Sets up the '_dagger_client' attribute, intended for single-threaded use within connectors. + + Caution: + Avoid using this client across multiple thread pools, as it can lead to errors. + Cross-thread pool calls are generally considered an anti-pattern. + """ self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore client = self._dagger_client assert client, "Error initializing Dagger client" From b07da000b60004ca7db0d1204d5cc118dea4e133 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 24 Oct 2023 12:26:42 -0700 Subject: [PATCH 12/30] Add singleton test --- .../connectors/test/steps/common.py | 2 +- .../pipelines/pipelines/models/singleton.py | 1 - .../pipelines/tests/test_models/__init__.py | 3 +++ .../tests/test_models/test_singleton.py | 25 +++++++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/tests/test_models/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index 9995e7f047a811..13b52e67d51d89 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -20,7 +20,7 @@ from pipelines.dagger.actions import secrets from pipelines.dagger.containers import internal_tools from pipelines.helpers.utils import METADATA_FILE_NAME -from pipelines.models.contexts import PipelineContext +from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 061307f1d01663..6c0ec3c7421933 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -16,7 +16,6 @@ class Singleton: _initialized: dict[Type["Singleton"], bool] = {} def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: - if cls not in cls._instances: cls._instances[cls] = super().__new__(cls) cls._initialized[cls] = False diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py new file mode 100644 index 00000000000000..9bdb95e01dde06 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py @@ -0,0 +1,25 @@ +from pipelines.models.singleton import Singleton + +class SingletonChild(Singleton): + def __init__(self): + if not self._initialized[self.__class__]: + self.value = "initialized" + self._initialized[self.__class__] = True + +def test_singleton_instance(): + instance1 = SingletonChild() + instance2 = SingletonChild() + assert instance1 is instance2 + +def test_singleton_unique_per_subclass(): + class AnotherSingletonChild(Singleton): + pass + + instance1 = SingletonChild() + instance2 = AnotherSingletonChild() + assert instance1 is not instance2 + +def test_singleton_initialized(): + instance = SingletonChild() + instance.value # This should initialize the instance + assert instance._initialized[SingletonChild] From 714d2019a35be134ed4bbafccf43a83514f571cd Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 30 Oct 2023 17:56:45 -0700 Subject: [PATCH 13/30] Fix low hanging comments --- .../airbyte_ci/connectors/commands.py | 26 ++++++---------- .../pipelines/pipelines/cli/airbyte_ci.py | 30 +++++++------------ .../pipelines/cli/click_decorators.py | 22 ++++++++++---- .../models/contexts/click_pipeline_context.py | 6 ++++ .../pipelines/pipelines/stolen/__init__.py | 3 -- 5 files changed, 41 insertions(+), 46 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 79c17ce6ecf945..dd798119ea2903 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -9,7 +9,7 @@ import asyncclick as click from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo from pipelines import main_logger -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj, click_append_to_context_object from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors @@ -227,30 +227,22 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: envvar="DOCKER_HUB_PASSWORD", ) @click_merge_args_into_context_obj +@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"]) ) @click_ignore_unused_kwargs async def connectors( ctx: click.Context, - use_remote_secrets: Optional[bool], - names: Tuple[str], - languages: Tuple[ConnectorLanguage], - support_levels: Tuple[str], - modified: bool, - metadata_changes_only: bool, - metadata_query: str, - enable_dependency_scanning: bool, ): """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"]) - ctx.obj["use_remote_secrets"] = should_use_remote_secrets(use_remote_secrets) ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( - names, - support_levels, - languages, - modified, - metadata_changes_only, - metadata_query, + ctx.obj["names"], + ctx.obj["support_levels"], + ctx.obj["languages"], + ctx.obj["modified"], + ctx.obj["metadata_changes_only"], + ctx.obj["metadata_query"], ctx.obj["modified_files"], - enable_dependency_scanning, + ctx.obj["enable_dependency_scanning"], ) log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"]) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 9723580f614a77..48c98545e96346 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -173,13 +173,15 @@ def log_git_info(ctx: click.Context): main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") -def _get_gha_workflow_run_id(ctx: click.Context) -> Optional[str]: - if ctx.obj["is_local"]: - return "TESTEST" - return ctx.obj["gha_workflow_run_id"] +def _get_gha_workflow_run_url(ctx: click.Context) -> Optional[str]: + gha_workflow_run_id = ctx.obj["gha_workflow_run_id"] + if not gha_workflow_run_id: + return None + + return f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" -def _get_pull_request(ctx: click.Context): +def _get_pull_request(ctx: click.Context) -> PullRequest or None: pull_request_number = ctx.obj["pull_request_number"] ci_github_access_token = ctx.obj["ci_github_access_token"] @@ -190,17 +192,6 @@ def _get_pull_request(ctx: click.Context): return github.get_pull_request(pull_request_number, ci_github_access_token) -def _get_modified_files(ctx: click.Context) -> List[Path]: - return transform_strs_to_paths( - get_modified_files( - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj["diffed_branch"], - ctx.obj["is_local"], - ctx.obj["ci_context"], - ctx.obj["pull_request"], - ) - ) def check_local_docker_configuration(): try: docker_client = docker.from_env() @@ -258,15 +249,14 @@ def check_local_docker_configuration(): @click_track_command @click_merge_args_into_context_obj @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) -@click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_id) +@click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, - is_local: bool, ): # noqa D103 display_welcome_message() - if is_local: + if ctx.obj["is_local"]: # This check is meaningful only when running locally # In our CI the docker host used by the Dagger Engine is different from the one used by the runner. check_local_docker_configuration() @@ -283,7 +273,7 @@ async def airbyte_ci( ) ctx.obj["modified_files"] = transform_strs_to_paths(modified_files) - if not is_local: + if not ctx.obj["is_local"]: log_git_info(ctx) set_working_directory_to_root() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index d4e849d9708734..0857e5559b4a3d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -6,16 +6,16 @@ import asyncclick as click -def _contains_var_kwarg(f): - return any(param.kind == inspect.Parameter.VAR_KEYWORD for param in inspect.signature(f).parameters.values()) +def _contains_var_kwarg(f: Callable) -> bool: + return any(param.kind is inspect.Parameter.VAR_KEYWORD for param in inspect.signature(f).parameters.values()) -def _is_kwarg_of(key, f): +def _is_kwarg_of(key: str, f: Callable) -> bool: param = inspect.signature(f).parameters.get(key, False) return param and (param.kind is inspect.Parameter.KEYWORD_ONLY or param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD) -def click_ignore_unused_kwargs(f): +def click_ignore_unused_kwargs(f: Callable) -> Callable: """Make function ignore unmatched kwargs. If the function already has the catch all **kwargs, do nothing. @@ -34,7 +34,7 @@ def inner(*args, **kwargs): return inner -def click_merge_args_into_context_obj(f): +def click_merge_args_into_context_obj(f: Callable) -> Callable: """ Decorator to pass click context and args to children commands. """ @@ -59,7 +59,7 @@ def wrapper(*args, **kwargs): return wrapper -def click_append_to_context_object(key: str, value: Callable | Any): +def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ Decorator to append a value to the click context object. """ @@ -81,12 +81,22 @@ def wrapper(*args, **kwargs): class LazyPassDecorator: + """ + Used to create a decorator that will pass an instance of the given class to the decorated function. + """ def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: + """ + Initialize the decorator with the given source class + """ self.cls = cls self.args = args self.kwargs = kwargs def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: + """ + Create a decorator that will pass an instance of the given class to the decorated function. + """ + @wraps(f) def decorated_function(*args: Any, **kwargs: Any) -> Any: # Check if the kwargs already contain the arguments being passed by the decorator diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 1502f40ce13b37..7e6dec0400505f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -9,6 +9,12 @@ from ..singleton import Singleton class ClickPipelineContext(BaseModel, Singleton): + """ + A replacement class for the Click context object passed to click functions. + + This class is meant to serve as a singleton object that initializes and holds onto a single instance of the + Dagger client, which is used to create containers for running pipelines. + """ dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) diff --git a/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py deleted file mode 100644 index c941b30457953b..00000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/stolen/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# From 29087ea62f42e781ddabb72d8f92a8963cd87832 Mon Sep 17 00:00:00 2001 From: bnchrch Date: Tue, 31 Oct 2023 01:09:24 +0000 Subject: [PATCH 14/30] Automated Commit - Formatting Changes --- .../pipelines/pipelines/airbyte_ci/connectors/commands.py | 4 ++-- .../pipelines/pipelines/airbyte_ci/test/commands.py | 7 ++++--- .../connectors/pipelines/pipelines/cli/airbyte_ci.py | 1 + .../connectors/pipelines/pipelines/cli/click_decorators.py | 1 + .../pipelines/models/contexts/click_pipeline_context.py | 2 ++ .../pipelines/tests/test_models/test_singleton.py | 4 ++++ 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index dd798119ea2903..27c7724796e960 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -9,7 +9,7 @@ import asyncclick as click from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo from pipelines import main_logger -from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj, click_append_to_context_object +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors @@ -227,7 +227,7 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: envvar="DOCKER_HUB_PASSWORD", ) @click_merge_args_into_context_obj -@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"]) ) +@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"])) @click_ignore_unused_kwargs async def connectors( ctx: click.Context, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 426dca55108078..f96330a27587b2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -3,14 +3,16 @@ # import logging + import asyncclick as click from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext -from pipelines.helpers.utils import sh_dash_c from pipelines.consts import DOCKER_VERSION +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) + @click.command() @click.argument("poetry_package_path") @click.option("--test-directory", default="tests", help="The directory containing the tests to run.") @@ -72,4 +74,3 @@ async def test( success = await pytest_container if not success: click.Abort() - diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 48c98545e96346..a5f0d23a606761 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -276,6 +276,7 @@ async def airbyte_ci( if not ctx.obj["is_local"]: log_git_info(ctx) + set_working_directory_to_root() if __name__ == "__main__": diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 0857e5559b4a3d..88cb0019714ac7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -84,6 +84,7 @@ class LazyPassDecorator: """ Used to create a decorator that will pass an instance of the given class to the decorated function. """ + def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: """ Initialize the decorator with the given source class diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 7e6dec0400505f..68c26c39fb9133 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -8,6 +8,7 @@ from ..singleton import Singleton + class ClickPipelineContext(BaseModel, Singleton): """ A replacement class for the Click context object passed to click functions. @@ -15,6 +16,7 @@ class ClickPipelineContext(BaseModel, Singleton): This class is meant to serve as a singleton object that initializes and holds onto a single instance of the Dagger client, which is used to create containers for running pipelines. """ + dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py index 9bdb95e01dde06..2a5b27998ff1e4 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py @@ -1,16 +1,19 @@ from pipelines.models.singleton import Singleton + class SingletonChild(Singleton): def __init__(self): if not self._initialized[self.__class__]: self.value = "initialized" self._initialized[self.__class__] = True + def test_singleton_instance(): instance1 = SingletonChild() instance2 = SingletonChild() assert instance1 is instance2 + def test_singleton_unique_per_subclass(): class AnotherSingletonChild(Singleton): pass @@ -19,6 +22,7 @@ class AnotherSingletonChild(Singleton): instance2 = AnotherSingletonChild() assert instance1 is not instance2 + def test_singleton_initialized(): instance = SingletonChild() instance.value # This should initialize the instance From fb43cfb97e7154c0fa3cfcdbeb5f290645864cf7 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 31 Oct 2023 16:49:29 -0700 Subject: [PATCH 15/30] Move decorator to file --- .../pipelines/pipelines/airbyte_ci/test/commands.py | 10 +++------- airbyte-ci/connectors/pipelines/pipelines/consts.py | 1 + .../models/contexts/click_pipeline_context.py | 6 ++++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index f96330a27587b2..be40f0cace8038 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -5,12 +5,10 @@ import logging import asyncclick as click -from pipelines.cli.click_decorators import LazyPassDecorator, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.consts import DOCKER_VERSION from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext - -pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() @@ -71,6 +69,4 @@ async def test( .with_exec(["poetry", "run", "pytest", test_directory]) ) - success = await pytest_container - if not success: - click.Abort() + await pytest_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 7b4805eabb511f..5d4a879ba4fea5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -27,6 +27,7 @@ PLATFORM_MACHINE_TO_DAGGER_PLATFORM = { "x86_64": Platform("linux/amd64"), + "aarch64": Platform("linux/amd64"), "arm64": Platform("linux/arm64"), "amd64": Platform("linux/amd64"), } diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 68c26c39fb9133..69d45157d946f4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -6,9 +6,12 @@ from dagger.api.gen import Client, Container from pydantic import BaseModel, Field, PrivateAttr +from pipelines.cli.click_decorators import LazyPassDecorator + from ..singleton import Singleton + class ClickPipelineContext(BaseModel, Singleton): """ A replacement class for the Click context object passed to click functions. @@ -75,3 +78,6 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name client = self._dagger_client assert client, "Error initializing Dagger client" return client.pipeline(pipeline_name) if pipeline_name else client + +# Create @pass_pipeline_context decorator for use in click commands +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) From 323139d81765ebc2872a2eb880baeb01665627a3 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 31 Oct 2023 18:07:02 -0700 Subject: [PATCH 16/30] Add tests --- .../pipelines/models/contexts/click_pipeline_context.py | 5 ++--- airbyte-ci/connectors/pipelines/tests/conftest.py | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 69d45157d946f4..ad1ca92cf866df 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -10,6 +10,7 @@ from ..singleton import Singleton +import anyio class ClickPipelineContext(BaseModel, Singleton): @@ -57,9 +58,7 @@ def __init__(self, **data: dict[str, Any]): super().__init__(**data) Singleton._initialized[ClickPipelineContext] = True - import asyncio - - _dagger_client_lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock) + _dagger_client_lock: anyio.Lock = PrivateAttr(default_factory=anyio.Lock) async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name: Optional[str] = None) -> Client: if not self._dagger_client: diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index 318e05aa5f8662..17a4e1625189e8 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -23,8 +23,12 @@ def anyio_backend(): @pytest.fixture(scope="module") -async def dagger_client(): - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: +def dagger_connection(): + return dagger.Connection(dagger.Config(log_output=sys.stderr)) + +@pytest.fixture(scope="module") +async def dagger_client(dagger_connection): + async with dagger_connection as client: yield client From f624fa340ba6c218d563547de0ad99bad1149211 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 13:10:13 -0700 Subject: [PATCH 17/30] Add forgotten test --- .../test_click_pipeline_context.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py new file mode 100644 index 00000000000000..8d9e3cb1884886 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -0,0 +1,64 @@ +import asyncclick as click +import dagger +import pytest +from unittest.mock import patch +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +@pytest.mark.anyio +async def test_get_dagger_client_singleton(dagger_connection): + @click.command() + def cli(): + pass + + ctx = click.Context(cli) + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + client1 = await click_pipeline_context.get_dagger_client() + client2 = await click_pipeline_context.get_dagger_client() + client3 = await click_pipeline_context.get_dagger_client(pipeline_name="pipeline_name") + assert isinstance(client1, dagger.Client) + assert isinstance(client2, dagger.Client) + assert isinstance(client3, dagger.Client) + + assert client1 == client2 + assert client1 != client3 + +@pytest.mark.anyio +async def test_get_dagger_client_click_params(dagger_connection): + @click.command() + def cli(): + pass + + given_click_obj = {"foo": "bar"} + given_click_params = {"baz": "qux"} + + ctx = click.Context(cli, obj=given_click_obj) + ctx.params = given_click_params + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + pipeline_context_params = click_pipeline_context.params + assert pipeline_context_params == {**given_click_obj, **given_click_params} + +@pytest.mark.anyio +async def test_get_dagger_client_click_params_duplicate(dagger_connection): + @click.command() + def cli(): + pass + + given_click_obj = {"foo": "bar"} + given_click_params = {"foo": "qux"} + + ctx = click.Context(cli, obj=given_click_obj) + ctx.params = given_click_params + ctx.command.params = [click.Option(["--foo"])] + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + with pytest.raises(ValueError): + click_pipeline_context.params + From 24297dff8968c715e4c042de5e49800fabbadc9b Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 13:51:16 -0700 Subject: [PATCH 18/30] Update ctx to click pipeline context --- .../pipelines/pipelines/airbyte_ci/test/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index be40f0cace8038..e73d3baf695f1e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -17,14 +17,14 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test( - ctx: ClickPipelineContext, + pipeline_context: ClickPipelineContext, poetry_package_path: str, test_directory: str, ): """Runs the tests for the given airbyte-ci package. Args: - ctx (ClickPipelineContext): The context object. + pipeline_context (ClickPipelineContext): The context object. poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. test_directory (str): The directory containing the tests to run. """ @@ -37,7 +37,7 @@ async def test( directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) pipeline_name = f"Unit tests for {poetry_package_path}" - dagger_client = await ctx.get_dagger_client(pipeline_name=pipeline_name) + dagger_client = await pipeline_context.get_dagger_client(pipeline_name=pipeline_name) pytest_container = await ( dagger_client.container() .from_("python:3.10.12") From 58b28af893e9e71b35d699f46dd5a9aa689c1a9d Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 19:23:57 -0600 Subject: [PATCH 19/30] Add cli tests --- .../pipelines/pipelines/cli/airbyte_ci.py | 22 ++++--- .../pipelines/cli/click_decorators.py | 4 +- .../pipelines/tests/test_cli/__init__.py | 3 + .../tests/test_cli/test_click_decorators.py | 64 +++++++++++++++++++ 4 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index a5f0d23a606761..bb4eacddacb37d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -206,6 +206,17 @@ def check_local_docker_configuration(): ) +async def get_modified_files_str(ctx: click.Context): + modified_files = await get_modified_files( + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["diffed_branch"], + ctx.obj["is_local"], + ctx.obj["ci_context"], + ctx.obj["pull_request"], + ) + return transform_strs_to_paths(modified_files) + # COMMANDS @@ -251,6 +262,7 @@ def check_local_docker_configuration(): @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) +@click_append_to_context_object("modified_files", get_modified_files_str) @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, @@ -263,16 +275,6 @@ async def airbyte_ci( check_up_to_date() - modified_files = await get_modified_files( - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj["diffed_branch"], - ctx.obj["is_local"], - ctx.obj["ci_context"], - ctx.obj["pull_request"], - ) - ctx.obj["modified_files"] = transform_strs_to_paths(modified_files) - if not ctx.obj["is_local"]: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 88cb0019714ac7..ab1dac128c6632 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -65,12 +65,14 @@ def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ def decorator(f): - def wrapper(*args, **kwargs): + async def wrapper(*args, **kwargs): ctx = click.get_current_context() ctx.ensure_object(dict) if callable(value): ctx.obj[key] = value(ctx) + elif inspect.iscoroutinefunction(value): + ctx.obj[key] = await value(ctx) else: ctx.obj[key] = value return f(*args, **kwargs) diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py new file mode 100644 index 00000000000000..c941b30457953b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py new file mode 100644 index 00000000000000..6ce6252ee9f235 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -0,0 +1,64 @@ +import pytest +import asyncclick as click +from asyncclick.testing import CliRunner +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj + +@pytest.mark.anyio +async def test_click_append_to_context_object(): + runner = CliRunner() + + def get_value(ctx): + return "got" + + async def get_async_value(ctx): + return "async_got" + + @click.command(name='test-command') + @click_append_to_context_object('foo', 'bar') + @click_append_to_context_object('baz', lambda _ctx: "qux") + @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + @click_append_to_context_object('get', get_value) + @click_append_to_context_object('async_get', get_async_value) + + def test_command(): + ctx = click.get_current_context() + assert ctx.obj['foo'] == 'bar' + assert ctx.obj['baz'] == 'qux' + assert ctx.obj['foo2'] == 'bar2' + assert ctx.obj['get'] == 'got' + assert ctx.obj['async_get'] == 'async_got' + + + result = await runner.invoke(test_command) + assert result.exit_code == 0 + +@pytest.mark.anyio +async def test_click_ignore_unused_kwargs(): + @click_ignore_unused_kwargs + def decorated_function(a, b): + return a + b + + # Test that the decorated function works as expected with matching kwargs + assert decorated_function(a=1, b=2) == 3 + + # Test that the decorated function ignores unmatched kwargs + assert decorated_function(a=1, b=2, c=3) == 3 + + +@pytest.mark.anyio +async def test_click_merge_args_into_context_obj(): + runner = CliRunner() + + @click.command(name='test-command') + @click.option('--foo', help='foo option') + @click.option('--bar', help='bar option') + @click_merge_args_into_context_obj + @click_ignore_unused_kwargs + def test_command(ctx, foo, bar): + ctx = click.get_current_context() + assert ctx.obj['foo'] == foo + assert ctx.obj['bar'] == bar + + result = await runner.invoke(test_command, ['--foo', 'hello', '--bar', 'world']) + assert result.exit_code == 0 + From a4e1efb3818a2804fca7c380ac0b6b8fbfc6c19e Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 19:41:14 -0600 Subject: [PATCH 20/30] Remove async --- .../pipelines/pipelines/airbyte_ci/connectors/commands.py | 2 +- .../pipelines/airbyte_ci/connectors/list/commands.py | 1 + airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py | 4 +++- .../connectors/pipelines/pipelines/cli/click_decorators.py | 4 +--- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 27c7724796e960..be478a16b1800a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -234,7 +234,7 @@ async def connectors( ): """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"]) - + import pdb; pdb.set_trace() ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( ctx.obj["names"], ctx.obj["support_levels"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index 4b0f54e6c46282..c9945af1ef527f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -14,6 +14,7 @@ async def list( ctx: click.Context, ): + import pdb; pdb.set_trace() selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) table = Table(title=f"{len(selected_connectors)} selected connectors") table.add_column("Modified") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index bb4eacddacb37d..5c5c4e336c7b99 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -262,7 +262,7 @@ async def get_modified_files_str(ctx: click.Context): @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) -@click_append_to_context_object("modified_files", get_modified_files_str) +@click_append_to_context_object("testme", "OUTTT") @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, @@ -275,6 +275,8 @@ async def airbyte_ci( check_up_to_date() + ctx.obj["modified_files"] = await get_modified_files_str(ctx) + if not ctx.obj["is_local"]: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index ab1dac128c6632..88cb0019714ac7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -65,14 +65,12 @@ def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ def decorator(f): - async def wrapper(*args, **kwargs): + def wrapper(*args, **kwargs): ctx = click.get_current_context() ctx.ensure_object(dict) if callable(value): ctx.obj[key] = value(ctx) - elif inspect.iscoroutinefunction(value): - ctx.obj[key] = await value(ctx) else: ctx.obj[key] = value return f(*args, **kwargs) From b1e479b3706322587a5dd68148e2d892efdf0c66 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 1 Nov 2023 19:58:42 -0600 Subject: [PATCH 21/30] Add async back --- .../airbyte_ci/connectors/commands.py | 2 +- .../airbyte_ci/connectors/list/commands.py | 1 - .../pipelines/pipelines/cli/airbyte_ci.py | 4 ++-- .../pipelines/cli/click_decorators.py | 11 ++++++---- .../tests/test_cli/test_click_decorators.py | 20 ++++++++++++++++--- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index be478a16b1800a..27c7724796e960 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -234,7 +234,7 @@ async def connectors( ): """Group all the connectors-ci command.""" validate_environment(ctx.obj["is_local"]) - import pdb; pdb.set_trace() + ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( ctx.obj["names"], ctx.obj["support_levels"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index c9945af1ef527f..4b0f54e6c46282 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -14,7 +14,6 @@ async def list( ctx: click.Context, ): - import pdb; pdb.set_trace() selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) table = Table(title=f"{len(selected_connectors)} selected connectors") table.add_column("Modified") diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 5c5c4e336c7b99..c6ef0b663ad1ae 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -262,7 +262,7 @@ async def get_modified_files_str(ctx: click.Context): @click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) -@click_append_to_context_object("testme", "OUTTT") +@click_append_to_context_object("modified_files", get_modified_files_str) @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, @@ -275,7 +275,7 @@ async def airbyte_ci( check_up_to_date() - ctx.obj["modified_files"] = await get_modified_files_str(ctx) + # ctx.obj["modified_files"] = await get_modified_files_str(ctx) if not ctx.obj["is_local"]: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 88cb0019714ac7..cb1cd7da6e9e55 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -63,17 +63,20 @@ def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ Decorator to append a value to the click context object. """ - def decorator(f): - def wrapper(*args, **kwargs): + async def wrapper(*args, **kwargs): ctx = click.get_current_context() ctx.ensure_object(dict) - if callable(value): + + # if async, get the value, cannot use await + if inspect.iscoroutinefunction(value): + ctx.obj[key] = await value(ctx) + elif callable(value): ctx.obj[key] = value(ctx) else: ctx.obj[key] = value - return f(*args, **kwargs) + return await f(*args, **kwargs) return wrapper diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index 6ce6252ee9f235..64cea33cced6a1 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -14,14 +14,25 @@ async def get_async_value(ctx): return "async_got" @click.command(name='test-command') + @click_append_to_context_object('get', get_value) + @click_append_to_context_object('async_get', get_async_value) @click_append_to_context_object('foo', 'bar') @click_append_to_context_object('baz', lambda _ctx: "qux") @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + def test_command(ctx): + assert ctx.obj['foo'] == 'bar' + assert ctx.obj['baz'] == 'qux' + assert ctx.obj['foo2'] == 'bar2' + assert ctx.obj['get'] == 'got' + assert ctx.obj['async_get'] == 'async_got' + + @click.command(name='test-command') @click_append_to_context_object('get', get_value) @click_append_to_context_object('async_get', get_async_value) - - def test_command(): - ctx = click.get_current_context() + @click_append_to_context_object('foo', 'bar') + @click_append_to_context_object('baz', lambda _ctx: "qux") + @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + async def test_command_async(ctx): assert ctx.obj['foo'] == 'bar' assert ctx.obj['baz'] == 'qux' assert ctx.obj['foo2'] == 'bar2' @@ -32,6 +43,9 @@ def test_command(): result = await runner.invoke(test_command) assert result.exit_code == 0 + result_async = await runner.invoke(test_command_async) + assert result_async.exit_code == 0 + @pytest.mark.anyio async def test_click_ignore_unused_kwargs(): @click_ignore_unused_kwargs From d89f7f38c225d75d83dfd01603793db83b2a2f1b Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 2 Nov 2023 09:15:06 -0600 Subject: [PATCH 22/30] Add doc string --- .../pipelines/models/contexts/click_pipeline_context.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index ad1ca92cf866df..a8abda48fb89af 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -27,6 +27,11 @@ class ClickPipelineContext(BaseModel, Singleton): @property def params(self): + """ + Returns a combination of the click context object and the click context params. + + This means that any arguments or options defined in the parent command will be available to the child command. + """ ctx = self._click_context() click_obj = ctx.obj click_params = ctx.params From 7ca02fcfaf5f9ae9d048abf7e07ae14986e001a8 Mon Sep 17 00:00:00 2001 From: bnchrch Date: Thu, 2 Nov 2023 15:31:46 +0000 Subject: [PATCH 23/30] Automated Commit - Formatting Changes --- .../pipelines/pipelines/cli/airbyte_ci.py | 1 + .../pipelines/cli/click_decorators.py | 2 +- .../models/contexts/click_pipeline_context.py | 7 +- .../connectors/pipelines/tests/conftest.py | 1 + .../tests/test_cli/test_click_decorators.py | 64 +++++++++---------- .../test_click_pipeline_context.py | 13 ++-- 6 files changed, 46 insertions(+), 42 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index c6ef0b663ad1ae..a4a9147e9ac89d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -217,6 +217,7 @@ async def get_modified_files_str(ctx: click.Context): ) return transform_strs_to_paths(modified_files) + # COMMANDS diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index cb1cd7da6e9e55..213c28cb5bbcca 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -63,12 +63,12 @@ def click_append_to_context_object(key: str, value: Callable | Any) -> Callable: """ Decorator to append a value to the click context object. """ + def decorator(f): async def wrapper(*args, **kwargs): ctx = click.get_current_context() ctx.ensure_object(dict) - # if async, get the value, cannot use await if inspect.iscoroutinefunction(value): ctx.obj[key] = await value(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index a8abda48fb89af..6895aacc166b53 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -1,17 +1,15 @@ import sys from typing import Any, Callable, Optional +import anyio import dagger from asyncclick import Context, get_current_context from dagger.api.gen import Client, Container -from pydantic import BaseModel, Field, PrivateAttr - from pipelines.cli.click_decorators import LazyPassDecorator +from pydantic import BaseModel, Field, PrivateAttr from ..singleton import Singleton -import anyio - class ClickPipelineContext(BaseModel, Singleton): """ @@ -83,5 +81,6 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name assert client, "Error initializing Dagger client" return client.pipeline(pipeline_name) if pipeline_name else client + # Create @pass_pipeline_context decorator for use in click commands pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index 17a4e1625189e8..fe7314bbebe7ab 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -26,6 +26,7 @@ def anyio_backend(): def dagger_connection(): return dagger.Connection(dagger.Config(log_output=sys.stderr)) + @pytest.fixture(scope="module") async def dagger_client(dagger_connection): async with dagger_connection as client: diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index 64cea33cced6a1..0aaf7d85cd4820 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -1,8 +1,9 @@ -import pytest import asyncclick as click +import pytest from asyncclick.testing import CliRunner from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj + @pytest.mark.anyio async def test_click_append_to_context_object(): runner = CliRunner() @@ -13,32 +14,31 @@ def get_value(ctx): async def get_async_value(ctx): return "async_got" - @click.command(name='test-command') - @click_append_to_context_object('get', get_value) - @click_append_to_context_object('async_get', get_async_value) - @click_append_to_context_object('foo', 'bar') - @click_append_to_context_object('baz', lambda _ctx: "qux") - @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + @click.command(name="test-command") + @click_append_to_context_object("get", get_value) + @click_append_to_context_object("async_get", get_async_value) + @click_append_to_context_object("foo", "bar") + @click_append_to_context_object("baz", lambda _ctx: "qux") + @click_append_to_context_object("foo2", lambda ctx: ctx.obj.get("foo") + "2") def test_command(ctx): - assert ctx.obj['foo'] == 'bar' - assert ctx.obj['baz'] == 'qux' - assert ctx.obj['foo2'] == 'bar2' - assert ctx.obj['get'] == 'got' - assert ctx.obj['async_get'] == 'async_got' - - @click.command(name='test-command') - @click_append_to_context_object('get', get_value) - @click_append_to_context_object('async_get', get_async_value) - @click_append_to_context_object('foo', 'bar') - @click_append_to_context_object('baz', lambda _ctx: "qux") - @click_append_to_context_object('foo2', lambda ctx: ctx.obj.get('foo') + "2") + assert ctx.obj["foo"] == "bar" + assert ctx.obj["baz"] == "qux" + assert ctx.obj["foo2"] == "bar2" + assert ctx.obj["get"] == "got" + assert ctx.obj["async_get"] == "async_got" + + @click.command(name="test-command") + @click_append_to_context_object("get", get_value) + @click_append_to_context_object("async_get", get_async_value) + @click_append_to_context_object("foo", "bar") + @click_append_to_context_object("baz", lambda _ctx: "qux") + @click_append_to_context_object("foo2", lambda ctx: ctx.obj.get("foo") + "2") async def test_command_async(ctx): - assert ctx.obj['foo'] == 'bar' - assert ctx.obj['baz'] == 'qux' - assert ctx.obj['foo2'] == 'bar2' - assert ctx.obj['get'] == 'got' - assert ctx.obj['async_get'] == 'async_got' - + assert ctx.obj["foo"] == "bar" + assert ctx.obj["baz"] == "qux" + assert ctx.obj["foo2"] == "bar2" + assert ctx.obj["get"] == "got" + assert ctx.obj["async_get"] == "async_got" result = await runner.invoke(test_command) assert result.exit_code == 0 @@ -46,6 +46,7 @@ async def test_command_async(ctx): result_async = await runner.invoke(test_command_async) assert result_async.exit_code == 0 + @pytest.mark.anyio async def test_click_ignore_unused_kwargs(): @click_ignore_unused_kwargs @@ -63,16 +64,15 @@ def decorated_function(a, b): async def test_click_merge_args_into_context_obj(): runner = CliRunner() - @click.command(name='test-command') - @click.option('--foo', help='foo option') - @click.option('--bar', help='bar option') + @click.command(name="test-command") + @click.option("--foo", help="foo option") + @click.option("--bar", help="bar option") @click_merge_args_into_context_obj @click_ignore_unused_kwargs def test_command(ctx, foo, bar): ctx = click.get_current_context() - assert ctx.obj['foo'] == foo - assert ctx.obj['bar'] == bar + assert ctx.obj["foo"] == foo + assert ctx.obj["bar"] == bar - result = await runner.invoke(test_command, ['--foo', 'hello', '--bar', 'world']) + result = await runner.invoke(test_command, ["--foo", "hello", "--bar", "world"]) assert result.exit_code == 0 - diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py index 8d9e3cb1884886..c1b8959ad0177a 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -1,9 +1,11 @@ +from unittest.mock import patch + import asyncclick as click import dagger import pytest -from unittest.mock import patch from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + @pytest.mark.anyio async def test_get_dagger_client_singleton(dagger_connection): @click.command() @@ -14,7 +16,7 @@ def cli(): async with ctx.scope(): click_pipeline_context = ClickPipelineContext() - with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): client1 = await click_pipeline_context.get_dagger_client() client2 = await click_pipeline_context.get_dagger_client() client3 = await click_pipeline_context.get_dagger_client(pipeline_name="pipeline_name") @@ -25,6 +27,7 @@ def cli(): assert client1 == client2 assert client1 != client3 + @pytest.mark.anyio async def test_get_dagger_client_click_params(dagger_connection): @click.command() @@ -39,10 +42,11 @@ def cli(): async with ctx.scope(): click_pipeline_context = ClickPipelineContext() - with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): pipeline_context_params = click_pipeline_context.params assert pipeline_context_params == {**given_click_obj, **given_click_params} + @pytest.mark.anyio async def test_get_dagger_client_click_params_duplicate(dagger_connection): @click.command() @@ -58,7 +62,6 @@ def cli(): async with ctx.scope(): click_pipeline_context = ClickPipelineContext() - with patch('pipelines.models.contexts.click_pipeline_context.dagger.Connection', lambda _x: dagger_connection): + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): with pytest.raises(ValueError): click_pipeline_context.params - From bcee91fd77fc79a11f25ee01871550007ec44a13 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Mon, 6 Nov 2023 10:13:35 -0600 Subject: [PATCH 24/30] Run license check on repo (#32154) --- airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py | 2 ++ .../connectors/pipelines/pipelines/cli/click_decorators.py | 2 ++ airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py | 2 ++ .../pipelines/models/contexts/click_pipeline_context.py | 2 ++ airbyte-ci/connectors/pipelines/pipelines/models/singleton.py | 2 ++ .../pipelines/tests/test_cli/test_click_decorators.py | 2 ++ .../pipelines/tests/test_models/test_click_pipeline_context.py | 2 ++ .../connectors/pipelines/tests/test_models/test_singleton.py | 2 ++ .../source_azure_blob_storage/stream_reader.py | 2 ++ .../source-azure-blob-storage/unit_tests/unit_tests.py | 2 ++ .../connectors/source-github/unit_tests/conftest.py | 2 ++ .../connectors/source-linkedin-ads/unit_tests/conftest.py | 2 ++ .../connectors/source-s3/source_s3/v4/zip_reader.py | 2 ++ .../connectors/source-s3/unit_tests/v4/test_zip_reader.py | 2 ++ .../connectors/source-tiktok-marketing/unit_tests/conftest.py | 2 ++ .../connectors/source-zendesk-support/unit_tests/conftest.py | 2 ++ 16 files changed, 32 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py b/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py index 7a22c221d5794e..54c3e984f93cea 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import time from typing import Any, Optional diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 213c28cb5bbcca..2c4843b973ba75 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import functools import inspect from functools import wraps diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py index c76c0a7c547718..b7214a26f7f9c0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + # Source: https://click.palletsprojects.com/en/8.1.x/complex/ import importlib diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 6895aacc166b53..1d56ce91e421aa 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import sys from typing import Any, Callable, Optional diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index 6c0ec3c7421933..e9d759d401f6f5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from typing import Any, Type diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index 0aaf7d85cd4820..c70b2350147b84 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import asyncclick as click import pytest from asyncclick.testing import CliRunner diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py index c1b8959ad0177a..9a685372c6cdc8 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from unittest.mock import patch import asyncclick as click diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py index 2a5b27998ff1e4..87648f432a127e 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from pipelines.models.singleton import Singleton diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py index 0f6ff208117469..47a235c00f1451 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging from contextlib import contextmanager from io import IOBase diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py index 17b5f7fc591365..88d003f81475b1 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from source_azure_blob_storage.legacy_config_transformer import LegacyConfigTransformer diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py index 3604b32db597ed..c3d9c1c98188f9 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py index 3604b32db597ed..c3d9c1c98188f9 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py index 8168c53508c64b..4f475b80c79757 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import io import struct import zipfile diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py index 8120fe52e43479..c97e468cba6dea 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import datetime import io import struct diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py index a2c4975260d2c1..969410ee24e188 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os import pytest diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py index 3604b32db597ed..c3d9c1c98188f9 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" From fd5e49c1c3fea359fbdc31c04f1f04be05257422 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 6 Nov 2023 18:53:22 -0700 Subject: [PATCH 25/30] Use ctx params instead of args --- .../pipelines/pipelines/airbyte_ci/test/commands.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index e73d3baf695f1e..8a6776c7e47813 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -8,26 +8,23 @@ from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.consts import DOCKER_VERSION from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context, click_merge_args_into_context_obj @click.command() @click.argument("poetry_package_path") @click.option("--test-directory", default="tests", help="The directory containing the tests to run.") +@click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs -async def test( - pipeline_context: ClickPipelineContext, - poetry_package_path: str, - test_directory: str, -): +async def test(pipeline_context: ClickPipelineContext): """Runs the tests for the given airbyte-ci package. Args: pipeline_context (ClickPipelineContext): The context object. - poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. - test_directory (str): The directory containing the tests to run. """ + poetry_package_path = pipeline_context.params["poetry_package_path"] + test_directory = pipeline_context.params["test_directory"] logger = logging.getLogger(f"{poetry_package_path}.tests") logger.info(f"Running tests for {poetry_package_path}") From c290dd84cde2e4526434b51e6286793b78ab1494 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 6 Nov 2023 18:55:52 -0700 Subject: [PATCH 26/30] Revert additional formating --- airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py | 2 -- .../source_azure_blob_storage/stream_reader.py | 2 -- .../source-azure-blob-storage/unit_tests/unit_tests.py | 2 -- .../connectors/source-github/unit_tests/conftest.py | 2 -- .../connectors/source-linkedin-ads/unit_tests/conftest.py | 2 -- .../connectors/source-s3/source_s3/v4/zip_reader.py | 2 -- .../connectors/source-s3/unit_tests/v4/test_zip_reader.py | 2 -- .../connectors/source-tiktok-marketing/unit_tests/conftest.py | 2 -- .../connectors/source-zendesk-support/unit_tests/conftest.py | 2 -- 9 files changed, 18 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py b/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py index 54c3e984f93cea..7a22c221d5794e 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import time from typing import Any, Optional diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py index 47a235c00f1451..0f6ff208117469 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import logging from contextlib import contextmanager from io import IOBase diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py index 88d003f81475b1..17b5f7fc591365 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - from source_azure_blob_storage.legacy_config_transformer import LegacyConfigTransformer diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py index c3d9c1c98188f9..3604b32db597ed 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py index c3d9c1c98188f9..3604b32db597ed 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py index 4f475b80c79757..8168c53508c64b 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import io import struct import zipfile diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py index c97e468cba6dea..8120fe52e43479 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import datetime import io import struct diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py index 969410ee24e188..a2c4975260d2c1 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import os import pytest diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py index c3d9c1c98188f9..3604b32db597ed 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py @@ -1,5 +1,3 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" From 3dc985340d48e6abd222a377f1367d0786df618f Mon Sep 17 00:00:00 2001 From: bnchrch Date: Tue, 7 Nov 2023 02:08:18 +0000 Subject: [PATCH 27/30] Automated Commit - Formatting Changes --- .../connectors/pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 8a6776c7e47813..169cd9c44c3aaf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -8,7 +8,7 @@ from pipelines.cli.click_decorators import click_ignore_unused_kwargs from pipelines.consts import DOCKER_VERSION from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context, click_merge_args_into_context_obj +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, click_merge_args_into_context_obj, pass_pipeline_context @click.command() From 1aeb5773bc084ca72b471a9cafdbb75bcab2c8f5 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 6 Nov 2023 19:33:42 -0700 Subject: [PATCH 28/30] Fix tests --- .../pipelines/airbyte_ci/connectors/commands.py | 1 + .../pipelines/pipelines/airbyte_ci/test/commands.py | 4 ++-- .../connectors/pipelines/pipelines/cli/airbyte_ci.py | 1 + .../pipelines/pipelines/cli/click_decorators.py | 4 +--- .../models/contexts/click_pipeline_context.py | 11 +++++++---- .../pipelines/tests/test_cli/test_click_decorators.py | 8 +++++--- 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 27c7724796e960..539c839f94fb02 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -228,6 +228,7 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: ) @click_merge_args_into_context_obj @click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"])) +@click.pass_context @click_ignore_unused_kwargs async def connectors( ctx: click.Context, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 169cd9c44c3aaf..205498c331dad0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -5,10 +5,10 @@ import logging import asyncclick as click -from pipelines.cli.click_decorators import click_ignore_unused_kwargs +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.consts import DOCKER_VERSION from pipelines.helpers.utils import sh_dash_c -from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, click_merge_args_into_context_obj, pass_pipeline_context +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context @click.command() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index a4a9147e9ac89d..5aebfe7d1fa8eb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -264,6 +264,7 @@ async def get_modified_files_str(ctx: click.Context): @click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) @click_append_to_context_object("pull_request", _get_pull_request) @click_append_to_context_object("modified_files", get_modified_files_str) +@click.pass_context @click_ignore_unused_kwargs async def airbyte_ci( ctx: click.Context, diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 2c4843b973ba75..6aa64a618e313c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -41,9 +41,8 @@ def click_merge_args_into_context_obj(f: Callable) -> Callable: Decorator to pass click context and args to children commands. """ - @click.pass_context def wrapper(*args, **kwargs): - ctx = args[0] + ctx = click.get_current_context() ctx.ensure_object(dict) click_obj = ctx.obj click_params = ctx.params @@ -55,7 +54,6 @@ def wrapper(*args, **kwargs): raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") ctx.obj = {**click_obj, **click_params} - return f(*args, **kwargs) return wrapper diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 1d56ce91e421aa..0f7c343ea212b3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -37,11 +37,14 @@ def params(self): click_params = ctx.params command_name = ctx.command.name - # Error if click_obj and click_params have the same key - all_click_params_keys = [p.name for p in ctx.command.params] - intersection = set(click_obj.keys()) & set(all_click_params_keys) + # Error if click_obj and click_params have the same key, and not the same value + intersection = set(click_obj.keys()) & set(click_params.keys()) if intersection: - raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") + for key in intersection: + if click_obj[key] != click_params[key]: + raise ValueError( + f"Your command '{command_name}' has defined options/arguments with the same key as its parent, but with different values: {intersection}" + ) return {**click_obj, **click_params} diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index c70b2350147b84..28ff2f9da8fc6e 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -22,7 +22,8 @@ async def get_async_value(ctx): @click_append_to_context_object("foo", "bar") @click_append_to_context_object("baz", lambda _ctx: "qux") @click_append_to_context_object("foo2", lambda ctx: ctx.obj.get("foo") + "2") - def test_command(ctx): + async def test_command(): + ctx = click.get_current_context() assert ctx.obj["foo"] == "bar" assert ctx.obj["baz"] == "qux" assert ctx.obj["foo2"] == "bar2" @@ -35,7 +36,8 @@ def test_command(ctx): @click_append_to_context_object("foo", "bar") @click_append_to_context_object("baz", lambda _ctx: "qux") @click_append_to_context_object("foo2", lambda ctx: ctx.obj.get("foo") + "2") - async def test_command_async(ctx): + async def test_command_async(): + ctx = click.get_current_context() assert ctx.obj["foo"] == "bar" assert ctx.obj["baz"] == "qux" assert ctx.obj["foo2"] == "bar2" @@ -71,7 +73,7 @@ async def test_click_merge_args_into_context_obj(): @click.option("--bar", help="bar option") @click_merge_args_into_context_obj @click_ignore_unused_kwargs - def test_command(ctx, foo, bar): + async def test_command(foo, bar): ctx = click.get_current_context() assert ctx.obj["foo"] == foo assert ctx.obj["bar"] == bar From cf50807cd7ee6058ef230798bc1158f20d24af2e Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 6 Nov 2023 19:42:41 -0700 Subject: [PATCH 29/30] Remove comment --- airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 5aebfe7d1fa8eb..e063efd83245d5 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -266,9 +266,7 @@ async def get_modified_files_str(ctx: click.Context): @click_append_to_context_object("modified_files", get_modified_files_str) @click.pass_context @click_ignore_unused_kwargs -async def airbyte_ci( - ctx: click.Context, -): # noqa D103 +async def airbyte_ci(ctx: click.Context): # noqa D103 display_welcome_message() if ctx.obj["is_local"]: # This check is meaningful only when running locally @@ -277,8 +275,6 @@ async def airbyte_ci( check_up_to_date() - # ctx.obj["modified_files"] = await get_modified_files_str(ctx) - if not ctx.obj["is_local"]: log_git_info(ctx) From 7ff56fb5e98df261c75da54488be1b133b909f93 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 7 Nov 2023 11:10:31 -0700 Subject: [PATCH 30/30] Remove unused command --- .../models/contexts/click_pipeline_context.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 0f7c343ea212b3..60d64a75237033 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -68,7 +68,10 @@ def __init__(self, **data: dict[str, Any]): _dagger_client_lock: anyio.Lock = PrivateAttr(default_factory=anyio.Lock) - async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name: Optional[str] = None) -> Client: + async def get_dagger_client(self, pipeline_name: Optional[str] = None) -> Client: + """ + Get (or initialize) the Dagger Client instance. + """ if not self._dagger_client: async with self._dagger_client_lock: if not self._dagger_client: @@ -82,9 +85,9 @@ async def get_dagger_client(self, client: Optional[Client] = None, pipeline_name Cross-thread pool calls are generally considered an anti-pattern. """ self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore - client = self._dagger_client - assert client, "Error initializing Dagger client" - return client.pipeline(pipeline_name) if pipeline_name else client + + assert self._dagger_client, "Error initializing Dagger client" + return self._dagger_client.pipeline(pipeline_name) if pipeline_name else self._dagger_client # Create @pass_pipeline_context decorator for use in click commands