airbyte-ci: simplify gradle task execution (#34560)

airbytehq · Jan 27, 2024 · 02c0ec7 · 02c0ec7
1 parent 9190517
commit 02c0ec7
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 12 deletions.
diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -578,6 +578,7 @@ E.G.: running `pytest` on a specific test folder:
 
 | Version | PR                                                         | Description                                                                                                       |
 | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| 3.7.3   | [#34560](https://github.com/airbytehq/airbyte/pull/34560)  | Simplify Gradle task execution framework by removing local maven repo support.                                    |
 | 3.7.2   | [#34555](https://github.com/airbytehq/airbyte/pull/34555)  | Override secret masking in some very specific special cases.                                                      |
 | 3.7.1   | [#34441](https://github.com/airbytehq/airbyte/pull/34441)  | Support masked secret scrubbing for java CDK v0.15+                                                               |
 | 3.7.0   | [#34343](https://github.com/airbytehq/airbyte/pull/34343)  | allow running connector upgrade_cdk for java connectors                                                           |

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py
@@ -27,7 +27,6 @@ class GradleTask(Step, ABC):
 
     context: ConnectorContext
 
-    LOCAL_MAVEN_REPOSITORY_PATH = "/root/.m2"
     GRADLE_DEP_CACHE_PATH = "/root/gradle-cache"
     GRADLE_HOME_PATH = "/root/.gradle"
     STATIC_GRADLE_OPTIONS = ("--no-daemon", "--no-watch-fs", "--build-cache", "--scan", "--console=plain")
@@ -143,23 +142,19 @@ async def _run(self, *args: Any, **kwargs: Any) -> StepResult:
             # When running locally, this dependency update is slower and less useful than within a CI runner. Skip it.
             warm_dependency_cache_args = ["--dry-run"]
 
-        # Mount the whole git repo to update the cache volume contents and build the CDK.
+        # Mount the whole git repo to update the cache volume contents.
         with_whole_git_repo = (
             gradle_container_base
             # Mount the whole repo.
             .with_directory("/airbyte", self.context.get_repo_dir("."))
-            # Update the cache in place by executing a gradle task which will update all dependencies and build the CDK.
+            # Update the cache in place by executing a gradle task which will update all dependencies.
             .with_exec(
                 sh_dash_c(
                     [
-                        # Ensure that the .m2 directory exists.
-                        f"mkdir -p {self.LOCAL_MAVEN_REPOSITORY_PATH}",
                         # Load from the cache volume.
                         f"(rsync -a --stats --mkpath {self.GRADLE_DEP_CACHE_PATH}/ {self.GRADLE_HOME_PATH} || true)",
                         # Resolve all dependencies and write their checksums to './gradle/verification-metadata.dryrun.xml'.
                         self._get_gradle_command("help", *warm_dependency_cache_args),
-                        # Build the CDK and publish it to the local maven repository.
-                        self._get_gradle_command(":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"),
                         # Store to the cache volume.
                         f"(rsync -a --stats {self.GRADLE_HOME_PATH}/ {self.GRADLE_DEP_CACHE_PATH} || true)",
                     ]
@@ -170,8 +165,8 @@ async def _run(self, *args: Any, **kwargs: Any) -> StepResult:
         # Mount only the code needed to build the connector.
         gradle_container = (
             gradle_container_base
-            # Copy the local maven repository and force evaluation of `with_whole_git_repo` container.
-            .with_directory(self.LOCAL_MAVEN_REPOSITORY_PATH, await with_whole_git_repo.directory(self.LOCAL_MAVEN_REPOSITORY_PATH))
+            # Copy the gradle home directory and force evaluation of `with_whole_git_repo` container.
+            .with_directory(self.GRADLE_HOME_PATH, await with_whole_git_repo.directory(self.GRADLE_HOME_PATH))
             # Mount the connector-agnostic whitelisted files in the git repo.
             .with_mounted_directory("/airbyte", self.context.get_repo_dir(".", include=include))
             # Mount the sources for the connector and its dependencies in the git repo.
@@ -193,8 +188,6 @@ async def _run(self, *args: Any, **kwargs: Any) -> StepResult:
         gradle_container = gradle_container.with_exec(
             sh_dash_c(
                 [
-                    # Warm the gradle cache.
-                    f"(rsync -a --stats --mkpath {self.GRADLE_DEP_CACHE_PATH}/ {self.GRADLE_HOME_PATH} || true)",
                     # Run the gradle task.
                     self._get_gradle_command(connector_task, task_options=self.params_as_cli_options),
                 ]

diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.7.2"
+version = "3.7.3"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]