From b03d785c41a197c276e034767117ac79f68a8079 Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 16 Jan 2024 23:15:49 +0100 Subject: [PATCH] airbyte-ci: pass extra options after gradle tasks (#34301) --- airbyte-ci/connectors/pipelines/README.md | 1 + .../pipelines/airbyte_ci/steps/gradle.py | 22 ++++++++----------- .../connectors/pipelines/pyproject.toml | 2 +- .../connectors/pipelines/tests/test_gradle.py | 12 +--------- 4 files changed, 12 insertions(+), 25 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index ede98ed3dd51a..4c8a85289eaa4 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -543,6 +543,7 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| 3.4.2 | [#34301](https://github.com/airbytehq/airbyte/pull/34301) | Pass extra params after Gradle tasks. | | 3.4.1 | [#34067](https://github.com/airbytehq/airbyte/pull/34067) | Use dagster-cloud 1.5.7 for deploy | | 3.4.0 | [#34276](https://github.com/airbytehq/airbyte/pull/34276) | Introduce `--only-step` option for connector tests. | | 3.3.0 | [#34218](https://github.com/airbytehq/airbyte/pull/34218) | Introduce `--ci-requirements` option for client defined CI runners. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index ae44de953449c..08b110dabc0ad 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -3,7 +3,7 @@ # from abc import ABC -from typing import Any, ClassVar, List +from typing import Any, ClassVar, List, Optional, Tuple import pipelines.dagger.actions.system.docker from dagger import CacheSharingMode, CacheVolume @@ -11,7 +11,7 @@ from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions import secrets from pipelines.helpers.utils import sh_dash_c -from pipelines.models.steps import STEP_PARAMS, Step, StepResult +from pipelines.models.steps import Step, StepResult class GradleTask(Step, ABC): @@ -30,20 +30,15 @@ class GradleTask(Step, ABC): LOCAL_MAVEN_REPOSITORY_PATH = "/root/.m2" GRADLE_DEP_CACHE_PATH = "/root/gradle-cache" GRADLE_HOME_PATH = "/root/.gradle" - STATIC_GRADLE_TASK_OPTIONS = ("--no-daemon", "--no-watch-fs") + STATIC_GRADLE_OPTIONS = ("--no-daemon", "--no-watch-fs", "--build-cache", "--scan", "--console=plain") gradle_task_name: ClassVar[str] bind_to_docker_host: ClassVar[bool] = False mount_connector_secrets: ClassVar[bool] = False accept_extra_params = True @property - def default_params(self) -> STEP_PARAMS: - return super().default_params | { - "-Ds3BuildCachePrefix": [self.context.connector.technical_name], # Set the S3 build cache prefix. - "--build-cache": [], # Enable the gradle build cache. - "--scan": [], # Enable the gradle build scan. - "--console": ["plain"], # Disable the gradle rich console. - } + def gradle_task_options(self) -> Tuple[str, ...]: + return self.STATIC_GRADLE_OPTIONS + (f"-Ds3BuildCachePrefix={self.context.connector.technical_name}",) @property def dependency_cache_volume(self) -> CacheVolume: @@ -64,8 +59,9 @@ def build_include(self) -> List[str]: for dependency_directory in self.context.connector.get_local_dependency_paths(with_test_dependencies=True) ] - def _get_gradle_command(self, task: str, *args: Any) -> str: - return f"./gradlew {' '.join(self.STATIC_GRADLE_TASK_OPTIONS + args)} {task}" + def _get_gradle_command(self, task: str, *args: Any, task_options: Optional[List[str]] = None) -> str: + task_options = task_options or [] + return f"./gradlew {' '.join(self.gradle_task_options + args)} {task} {' '.join(task_options)}" async def _run(self, *args: Any, **kwargs: Any) -> StepResult: include = [ @@ -200,7 +196,7 @@ async def _run(self, *args: Any, **kwargs: Any) -> StepResult: # Warm the gradle cache. f"(rsync -a --stats --mkpath {self.GRADLE_DEP_CACHE_PATH}/ {self.GRADLE_HOME_PATH} || true)", # Run the gradle task. - self._get_gradle_command(connector_task, *self.params_as_cli_options), + self._get_gradle_command(connector_task, task_options=self.params_as_cli_options), ] ) ) diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index ab036fe10fa4e..79f37a9efd606 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.4.1" +version = "3.4.2" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_gradle.py b/airbyte-ci/connectors/pipelines/tests/test_gradle.py index 5e867c3582ba7..34312ec1d0d34 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_gradle.py +++ b/airbyte-ci/connectors/pipelines/tests/test_gradle.py @@ -39,18 +39,8 @@ async def test_build_include(self, test_context): def test_params(self, test_context): step = self.DummyStep(test_context) + step.extra_params = {"-x": ["dummyTask", "dummyTask2"]} assert set(step.params_as_cli_options) == { - f"-Ds3BuildCachePrefix={test_context.connector.technical_name}", - "--build-cache", - "--scan", - "--console=plain", - } - step.extra_params = {"-x": ["dummyTask", "dummyTask2"], "--console": ["rich"]} - assert set(step.params_as_cli_options) == { - f"-Ds3BuildCachePrefix={test_context.connector.technical_name}", - "--build-cache", - "--scan", - "--console=rich", "-x=dummyTask", "-x=dummyTask2", }