airbyte-ci: pass extra options after gradle tasks (#34301)

airbyte-ci/connectors/pipelines/README.md

@@ -543,6 +543,7 @@ E.G.: running `pytest` on a specific test folder:
 
 | Version | PR                                                         | Description                                                                                                       |
 | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| 3.4.2   | [#34301](https://github.com/airbytehq/airbyte/pull/34301)  | Pass extra params after Gradle tasks.                                                                             |
 | 3.4.1   | [#34067](https://github.com/airbytehq/airbyte/pull/34067)  | Use dagster-cloud 1.5.7 for deploy |
 | 3.4.0   | [#34276](https://github.com/airbytehq/airbyte/pull/34276)  | Introduce `--only-step` option for connector tests.                                                               |
 | 3.3.0   | [#34218](https://github.com/airbytehq/airbyte/pull/34218)  | Introduce `--ci-requirements` option for client defined CI runners.                                               |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py

@@ -3,15 +3,15 @@
 #
 
 from abc import ABC
-from typing import Any, ClassVar, List
+from typing import Any, ClassVar, List, Optional, Tuple
 
 import pipelines.dagger.actions.system.docker
 from dagger import CacheSharingMode, CacheVolume
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.consts import AMAZONCORRETTO_IMAGE
 from pipelines.dagger.actions import secrets
 from pipelines.helpers.utils import sh_dash_c
-from pipelines.models.steps import STEP_PARAMS, Step, StepResult
+from pipelines.models.steps import Step, StepResult
 
 
 class GradleTask(Step, ABC):
@@ -30,20 +30,15 @@ class GradleTask(Step, ABC):
     LOCAL_MAVEN_REPOSITORY_PATH = "/root/.m2"
     GRADLE_DEP_CACHE_PATH = "/root/gradle-cache"
     GRADLE_HOME_PATH = "/root/.gradle"
-    STATIC_GRADLE_TASK_OPTIONS = ("--no-daemon", "--no-watch-fs")
+    STATIC_GRADLE_OPTIONS = ("--no-daemon", "--no-watch-fs", "--build-cache", "--scan", "--console=plain")
     gradle_task_name: ClassVar[str]
     bind_to_docker_host: ClassVar[bool] = False
     mount_connector_secrets: ClassVar[bool] = False
     accept_extra_params = True
 
     @property
-    def default_params(self) -> STEP_PARAMS:
-        return super().default_params | {
-            "-Ds3BuildCachePrefix": [self.context.connector.technical_name],  # Set the S3 build cache prefix.
-            "--build-cache": [],  # Enable the gradle build cache.
-            "--scan": [],  # Enable the gradle build scan.
-            "--console": ["plain"],  # Disable the gradle rich console.
-        }
+    def gradle_task_options(self) -> Tuple[str, ...]:
+        return self.STATIC_GRADLE_OPTIONS + (f"-Ds3BuildCachePrefix={self.context.connector.technical_name}",)
 
     @property
     def dependency_cache_volume(self) -> CacheVolume:
@@ -64,8 +59,9 @@ def build_include(self) -> List[str]:
             for dependency_directory in self.context.connector.get_local_dependency_paths(with_test_dependencies=True)
         ]
 
-    def _get_gradle_command(self, task: str, *args: Any) -> str:
-        return f"./gradlew {' '.join(self.STATIC_GRADLE_TASK_OPTIONS + args)} {task}"
+    def _get_gradle_command(self, task: str, *args: Any, task_options: Optional[List[str]] = None) -> str:
+        task_options = task_options or []
+        return f"./gradlew {' '.join(self.gradle_task_options + args)} {task} {' '.join(task_options)}"
 
     async def _run(self, *args: Any, **kwargs: Any) -> StepResult:
         include = [
@@ -200,7 +196,7 @@ async def _run(self, *args: Any, **kwargs: Any) -> StepResult:
                     # Warm the gradle cache.
                     f"(rsync -a --stats --mkpath {self.GRADLE_DEP_CACHE_PATH}/ {self.GRADLE_HOME_PATH} || true)",
                     # Run the gradle task.
-                    self._get_gradle_command(connector_task, *self.params_as_cli_options),
+                    self._get_gradle_command(connector_task, task_options=self.params_as_cli_options),
                 ]
             )
         )

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.4.1"
+version = "3.4.2"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-ci/connectors/pipelines/tests/test_gradle.py

@@ -39,18 +39,8 @@ async def test_build_include(self, test_context):
 
     def test_params(self, test_context):
         step = self.DummyStep(test_context)
+        step.extra_params = {"-x": ["dummyTask", "dummyTask2"]}
         assert set(step.params_as_cli_options) == {
-            f"-Ds3BuildCachePrefix={test_context.connector.technical_name}",
-            "--build-cache",
-            "--scan",
-            "--console=plain",
-        }
-        step.extra_params = {"-x": ["dummyTask", "dummyTask2"], "--console": ["rich"]}
-        assert set(step.params_as_cli_options) == {
-            f"-Ds3BuildCachePrefix={test_context.connector.technical_name}",
-            "--build-cache",
-            "--scan",
-            "--console=rich",
             "-x=dummyTask",
             "-x=dummyTask2",
         }