connectors-ci: improve Java connector build logic and Gradle caching …

…strategy (#26438)

.github/workflows/connector_integration_test_single_dagger.yml

@@ -64,8 +64,6 @@ jobs:
           CI_GIT_REVISION: ${{ github.sha }}
           CI_CONTEXT: "manual"
           CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
-          S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
       - name: Run airbyte-ci connectors test [PULL REQUESTS]
         if: github.event_name == 'pull_request'
         run: |
@@ -90,5 +88,3 @@ jobs:
           CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }}
           CI_CONTEXT: "pull_request"
           CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
-          S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}

.github/workflows/connector_nightly_builds_dagger.yml

@@ -68,5 +68,3 @@ jobs:
           CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }}
           CI_CONTEXT: "nightly_builds"
           CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
-          S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}

.github/workflows/publish_connectors.yml

@@ -33,8 +33,6 @@ jobs:
       SPEC_CACHE_GCS_CREDENTIALS: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
       TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status"
       SLACK_WEBHOOK: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
-      S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-      S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2

tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py

@@ -6,10 +6,10 @@
 
 from __future__ import annotations
 
-import os
 import uuid
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
+from ci_connector_ops.pipelines import consts
 from ci_connector_ops.pipelines.consts import (
     CI_CONNECTOR_OPS_SOURCE_PATH,
     CI_CREDENTIALS_SOURCE_PATH,
@@ -255,7 +255,7 @@ def with_dockerd_service(
         docker_lib_volume_name = f"{docker_lib_volume_name}-{slugify(docker_service_name)}"
     dind = (
         context.dagger_client.container()
-        .from_("docker:23.0.1-dind")
+        .from_(consts.DOCKER_DIND_IMAGE)
         .with_mounted_cache(
             "/var/lib/docker",
             context.dagger_client.cache_volume(docker_lib_volume_name),
@@ -309,7 +309,7 @@ def with_docker_cli(
     Returns:
         Container: A docker cli container bound to a docker host.
     """
-    docker_cli = context.dagger_client.container().from_("docker:23.0.1-cli")
+    docker_cli = context.dagger_client.container().from_(consts.DOCKER_CLI_IMAGE)
     return with_bound_docker_host(context, docker_cli, shared_volume, docker_service_name)
 
 
@@ -361,8 +361,6 @@ def with_gradle(
     Returns:
         Container: A container with Gradle installed and Java sources from the repository.
     """
-    airbyte_gradle_cache: CacheVolume = context.dagger_client.cache_volume(f"{context.connector.technical_name}_airbyte_gradle_cache")
-    root_gradle_cache: CacheVolume = context.dagger_client.cache_volume(f"{context.connector.technical_name}_root_gradle_cache")
 
     include = [
         ".root",
@@ -386,34 +384,27 @@ def with_gradle(
     if sources_to_include:
         include += sources_to_include
 
+    gradle_dependency_cache: CacheVolume = context.dagger_client.cache_volume("gradle-dependencies-caching")
+    gradle_build_cache: CacheVolume = context.dagger_client.cache_volume(f"{context.connector.technical_name}-gradle-build-cache")
+
     shared_tmp_volume = ("/tmp", context.dagger_client.cache_volume("share-tmp-gradle"))
 
     openjdk_with_docker = (
         context.dagger_client.container()
-        # Use openjdk image because it's based on Debian. Alpine with Gradle and Python causes filesystem crash.
         .from_("openjdk:17.0.1-jdk-slim")
         .with_exec(["apt-get", "update"])
-        .with_exec(["apt-get", "install", "-y", "curl", "jq"])
-        .with_env_variable("VERSION", "23.0.1")
+        .with_exec(["apt-get", "install", "-y", "curl", "jq", "rsync"])
+        .with_env_variable("VERSION", consts.DOCKER_VERSION)
         .with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"])
-        .with_exec(["mkdir", "/root/.gradle"])
+        .with_env_variable("GRADLE_HOME", "/root/.gradle")
         .with_exec(["mkdir", "/airbyte"])
-        .with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include))
-        .with_mounted_cache("/airbyte/.gradle", airbyte_gradle_cache, sharing=CacheSharingMode.LOCKED)
         .with_workdir("/airbyte")
+        .with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include))
+        .with_exec(["mkdir", "-p", consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH])
+        .with_mounted_cache(consts.GRADLE_BUILD_CACHE_PATH, gradle_build_cache, sharing=CacheSharingMode.LOCKED)
+        .with_mounted_cache(consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH, gradle_dependency_cache)
+        .with_env_variable("GRADLE_RO_DEP_CACHE", consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH)
     )
-    if context.is_ci and "S3_BUILD_CACHE_ACCESS_KEY_ID" in os.environ and "S3_BUILD_CACHE_SECRET_KEY" in os.environ:
-        openjdk_with_docker = (
-            openjdk_with_docker.with_env_variable("CI", "true")
-            .with_secret_variable(
-                "S3_BUILD_CACHE_ACCESS_KEY_ID", context.dagger_client.host().env_variable("S3_BUILD_CACHE_ACCESS_KEY_ID").secret()
-            )
-            .with_secret_variable(
-                "S3_BUILD_CACHE_SECRET_KEY", context.dagger_client.host().env_variable("S3_BUILD_CACHE_SECRET_KEY").secret()
-            )
-        )
-    else:
-        openjdk_with_docker = openjdk_with_docker.with_mounted_cache("/root/.gradle", root_gradle_cache, sharing=CacheSharingMode.LOCKED)
 
     if bind_to_docker_host:
         return with_bound_docker_host(context, openjdk_with_docker, shared_tmp_volume, docker_service_name=docker_service_name)

tools/ci_connector_ops/ci_connector_ops/pipelines/builds/__init__.py

@@ -6,52 +6,34 @@
 from __future__ import annotations
 
 import platform
-from typing import TYPE_CHECKING, Optional, Tuple
 
 import anyio
-from ci_connector_ops.pipelines.bases import ConnectorReport
+from ci_connector_ops.pipelines.bases import ConnectorReport, StepResult
 from ci_connector_ops.pipelines.builds import common, java_connectors, python_connectors
 from ci_connector_ops.pipelines.contexts import ConnectorContext
 from ci_connector_ops.utils import ConnectorLanguage
-from dagger import Container, Platform
-
-BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")]
-LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}")
-
-if TYPE_CHECKING:
-    from ci_connector_ops.pipelines.bases import StepResult
+from dagger import Platform
 
 
 class NoBuildStepForLanguageError(Exception):
     pass
 
 
 LANGUAGE_BUILD_CONNECTOR_MAPPING = {
-    ConnectorLanguage.PYTHON: python_connectors.BuildConnectorImage,
-    ConnectorLanguage.LOW_CODE: python_connectors.BuildConnectorImage,
-    ConnectorLanguage.JAVA: java_connectors.BuildConnectorImage,
+    ConnectorLanguage.PYTHON: python_connectors.run_connector_build,
+    ConnectorLanguage.LOW_CODE: python_connectors.run_connector_build,
+    ConnectorLanguage.JAVA: java_connectors.run_connector_build,
 }
 
+BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")]
+LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}")
 
-async def run_connector_build(context: ConnectorContext) -> dict[str, Tuple[StepResult, Optional[Container]]]:
-    """Build a connector according to its language and return the build result and the built container.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        dict[str, Tuple[StepResult, Optional[Container]]]: A dictionary with platform as key and a tuple of step result and built container as value.
-    """
-    try:
-        BuildConnectorImage = LANGUAGE_BUILD_CONNECTOR_MAPPING[context.connector.language]
-    except KeyError:
-        raise NoBuildStepForLanguageError(f"No step to build a {context.connector.language} connector was found.")
-
-    per_platform_containers = {}
-    for build_platform in BUILD_PLATFORMS:
-        per_platform_containers[build_platform] = await BuildConnectorImage(context, build_platform).run()
 
-    return per_platform_containers
+async def run_connector_build(context: ConnectorContext) -> StepResult:
+    """Run a build pipeline for a single connector."""
+    if context.connector.language not in LANGUAGE_BUILD_CONNECTOR_MAPPING:
+        raise NoBuildStepForLanguageError(f"No build step for connector language {context.connector.language}.")
+    return await LANGUAGE_BUILD_CONNECTOR_MAPPING[context.connector.language](context)
 
 
 async def run_connector_build_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport:
@@ -63,15 +45,14 @@ async def run_connector_build_pipeline(context: ConnectorContext, semaphore: any
     Returns:
         ConnectorReport: The reports holding builds results.
     """
+    step_results = []
     async with semaphore:
         async with context:
-            build_results_per_platform = await run_connector_build(context)
-            step_results = list(build_results_per_platform.values())
-            if context.is_local:
-                build_result_for_local_platform = build_results_per_platform[LOCAL_BUILD_PLATFORM]
-                load_image_result = await common.LoadContainerToLocalDockerHost(
-                    context, build_result_for_local_platform.output_artifact
-                ).run()
+            build_result = await run_connector_build(context)
+            step_results.append(build_result)
+            if context.is_local and build_result.status is common.StepStatus.SUCCESS:
+                connector_to_load_to_local_docker_host = build_result.output_artifact[LOCAL_BUILD_PLATFORM]
+                load_image_result = await common.LoadContainerToLocalDockerHost(context, connector_to_load_to_local_docker_host).run()
                 step_results.append(load_image_result)
             context.report = ConnectorReport(context, step_results, name="BUILD RESULTS")
         return context.report

tools/ci_connector_ops/ci_connector_ops/pipelines/builds/common.py

@@ -6,6 +6,7 @@
 
 import docker
 from ci_connector_ops.pipelines.bases import Step, StepResult, StepStatus
+from ci_connector_ops.pipelines.consts import BUILD_PLATFORMS
 from ci_connector_ops.pipelines.contexts import ConnectorContext
 from ci_connector_ops.pipelines.utils import export_container_to_tarball
 from dagger import Container, Platform
@@ -21,6 +22,21 @@ def __init__(self, context: ConnectorContext, build_platform: Platform) -> None:
         super().__init__(context)
 
 
+class BuildConnectorImageForAllPlatformsBase(Step, ABC):
+
+    ALL_PLATFORMS = BUILD_PLATFORMS
+
+    title = f"Build connector image for {BUILD_PLATFORMS}"
+
+    def get_success_result(self, build_results_per_platform: dict[Platform, Container]) -> StepResult:
+        return StepResult(
+            self,
+            StepStatus.SUCCESS,
+            stdout="The connector image was successfully built for all platforms.",
+            output_artifact=build_results_per_platform,
+        )
+
+
 class LoadContainerToLocalDockerHost(Step):
     IMAGE_TAG = "dev"
 

tools/ci_connector_ops/ci_connector_ops/pipelines/builds/java_connectors.py

@@ -2,46 +2,87 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-
 from ci_connector_ops.pipelines.actions import environments
 from ci_connector_ops.pipelines.bases import StepResult, StepStatus
-from ci_connector_ops.pipelines.builds.common import BuildConnectorImageBase
+from ci_connector_ops.pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase
+from ci_connector_ops.pipelines.contexts import ConnectorContext
 from ci_connector_ops.pipelines.gradle import GradleTask
+from ci_connector_ops.pipelines.utils import with_exit_code
 from dagger import File, QueryError
 
 
-class BuildConnectorImage(BuildConnectorImageBase, GradleTask):
-    """
-    A step to build a Java connector image using the distTar Gradle task.
-    """
+class BuildConnectorDistributionTar(GradleTask):
 
+    title = "Build connector tar"
     gradle_task_name = "distTar"
 
-    async def build_tar(self) -> File:
-        distTar = (
+    async def _run(self) -> StepResult:
+        with_built_tar = (
             environments.with_gradle(
                 self.context,
                 self.build_include,
                 docker_service_name=self.docker_service_name,
-                bind_to_docker_host=self.BIND_TO_DOCKER_HOST,
             )
             .with_mounted_directory(str(self.context.connector.code_directory), await self._get_patched_connector_dir())
             .with_exec(self._get_gradle_command())
             .with_workdir(f"{self.context.connector.code_directory}/build/distributions")
         )
-
-        distributions = await distTar.directory(".").entries()
+        distributions = await with_built_tar.directory(".").entries()
         tar_files = [f for f in distributions if f.endswith(".tar")]
-        if len(tar_files) > 1:
-            raise Exception(
-                "The distributions directory contains multiple connector tar files. We can't infer which one should be used. Please review and delete any unnecessary tar files."
+        await self._export_gradle_dependency_cache(with_built_tar)
+        if len(tar_files) == 1:
+            return StepResult(
+                self,
+                StepStatus.SUCCESS,
+                stdout="The tar file for the current connector was successfully built.",
+                output_artifact=with_built_tar.file(tar_files[0]),
+            )
+        else:
+            return StepResult(
+                self,
+                StepStatus.FAILURE,
+                stderr="The distributions directory contains multiple connector tar files. We can't infer which one should be used. Please review and delete any unnecessary tar files.",
             )
-        return distTar.file(tar_files[0])
 
-    async def _run(self) -> StepResult:
+
+class BuildConnectorImage(BuildConnectorImageBase):
+    """
+    A step to build a Java connector image using the distTar Gradle task.
+    """
+
+    async def _run(self, distribution_tar: File) -> StepResult:
         try:
-            tar_file = await self.build_tar()
-            java_connector = await environments.with_airbyte_java_connector(self.context, tar_file, self.build_platform)
-            return await self.get_step_result(java_connector.with_exec(["spec"]))
+            java_connector = await environments.with_airbyte_java_connector(self.context, distribution_tar, self.build_platform)
+            spec_exit_code = await with_exit_code(java_connector.with_exec(["spec"]))
+            if spec_exit_code != 0:
+                return StepResult(
+                    self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {self.build_platform}."
+                )
+            return StepResult(
+                self, StepStatus.SUCCESS, stdout="The connector image was successfully built.", output_artifact=java_connector
+            )
         except QueryError as e:
             return StepResult(self, StepStatus.FAILURE, stderr=str(e))
+
+
+class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase):
+    """Build a Java connector image for all platforms."""
+
+    async def _run(self, distribution_tar: File) -> StepResult:
+        build_results_per_platform = {}
+        for platform in self.ALL_PLATFORMS:
+            build_connector_step_result = await BuildConnectorImage(self.context, platform).run(distribution_tar)
+            if build_connector_step_result.status is not StepStatus.SUCCESS:
+                return build_connector_step_result
+            build_results_per_platform[platform] = build_connector_step_result.output_artifact
+        return self.get_success_result(build_results_per_platform)
+
+
+async def run_connector_build(context: ConnectorContext) -> StepResult:
+    """Create the java connector distribution tar file and build the connector image."""
+
+    build_connector_tar_result = await BuildConnectorDistributionTar(context).run()
+    if build_connector_tar_result.status is not StepStatus.SUCCESS:
+        return build_connector_tar_result
+
+    return await BuildConnectorImageForAllPlatforms(context).run(build_connector_tar_result.output_artifact)

tools/ci_connector_ops/ci_connector_ops/pipelines/builds/python_connectors.py

@@ -4,7 +4,8 @@
 
 from ci_connector_ops.pipelines.actions.environments import with_airbyte_python_connector
 from ci_connector_ops.pipelines.bases import StepResult, StepStatus
-from ci_connector_ops.pipelines.builds.common import BuildConnectorImageBase
+from ci_connector_ops.pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase
+from ci_connector_ops.pipelines.contexts import ConnectorContext
 from dagger import QueryError
 
 
@@ -20,3 +21,20 @@ async def _run(self) -> StepResult:
             return await self.get_step_result(connector.with_exec(["spec"]))
         except QueryError as e:
             return StepResult(self, StepStatus.FAILURE, stderr=str(e))
+
+
+class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase):
+    """Build a Python connector image for all platforms."""
+
+    async def _run(self) -> StepResult:
+        build_results_per_platform = {}
+        for platform in self.ALL_PLATFORMS:
+            build_connector_step_result = await BuildConnectorImage(self.context, platform).run()
+            if build_connector_step_result.status is not StepStatus.SUCCESS:
+                return build_connector_step_result
+            build_results_per_platform[platform] = build_connector_step_result.output_artifact
+        return self.get_success_result(build_results_per_platform)
+
+
+async def run_connector_build(context: ConnectorContext) -> StepResult:
+    return await BuildConnectorImageForAllPlatforms(context).run()

tools/ci_connector_ops/ci_connector_ops/pipelines/consts.py

@@ -1,6 +1,11 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
+
+import platform
+
+from dagger import Platform
+
 PYPROJECT_TOML_FILE_PATH = "pyproject.toml"
 
 CONNECTOR_TESTING_REQUIREMENTS = [
@@ -18,3 +23,11 @@
 DEFAULT_PYTHON_EXCLUDE = ["**/.venv", "**/__pycache__"]
 CI_CREDENTIALS_SOURCE_PATH = "tools/ci_credentials"
 CI_CONNECTOR_OPS_SOURCE_PATH = "tools/ci_connector_ops"
+BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")]
+LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}")
+DOCKER_VERSION = "20.10.23"
+DOCKER_DIND_IMAGE = "docker:20-dind"
+DOCKER_CLI_IMAGE = "docker:20-cli"
+GRADLE_CACHE_PATH = "/root/.gradle/caches"
+GRADLE_BUILD_CACHE_PATH = f"{GRADLE_CACHE_PATH}/build-cache-1"
+GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH = "/root/gradle_dependency_cache"