connectors-ci: handle strict encrypt connectors (airbytehq#25864)

tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py

@@ -16,7 +16,7 @@
 import asyncer
 from ci_connector_ops.pipelines.actions import environments
 from ci_connector_ops.pipelines.utils import check_path_in_workdir, slugify, with_exit_code, with_stderr, with_stdout
-from ci_connector_ops.utils import console
+from ci_connector_ops.utils import Connector, console
 from dagger import CacheVolume, Container, Directory, QueryError
 from rich.console import Group
 from rich.panel import Panel
@@ -421,6 +421,19 @@ def docker_service_name(self) -> str:
     def connector_java_build_cache(self) -> CacheVolume:
         return self.context.dagger_client.cache_volume("connector_java_build_cache")
 
+    def get_related_connectors(self) -> List[Connector]:
+        """Retrieve the list of related connectors.
+        This is used to include source code of non strict-encrypt connectors when running build for a strict-encrypt connector.
+
+        Returns:
+            List[Connector]: The list of related connectors.
+        """
+        if self.context.connector.technical_name.endswith("-strict-encrypt"):
+            return [Connector(self.context.connector.technical_name.replace("-strict-encrypt", ""))]
+        if self.context.connector.technical_name == "source-file-secure":
+            return [Connector("source-file")]
+        return []
+
     @property
     def build_include(self) -> List[str]:
         """Retrieve the list of source code directory required to run a Java connector Gradle task.
@@ -430,13 +443,18 @@ def build_include(self) -> List[str]:
         Returns:
             List[str]: List of directories or files to be mounted to the container to run a Java connector Gradle task.
         """
+        to_include = [self.JAVA_BUILD_INCLUDE]
+
         if self.context.connector.connector_type == "source":
-            return self.JAVA_BUILD_INCLUDE + self.SOURCE_BUILD_INCLUDE
+            to_include.append(self.SOURCE_BUILD_INCLUDE)
         elif self.context.connector.connector_type == "destination":
-            return self.JAVA_BUILD_INCLUDE + self.DESTINATION_BUILD_INCLUDE
+            to_include.append(self.DESTINATION_BUILD_INCLUDE)
         else:
             raise ValueError(f"{self.context.connector.connector_type} is not supported")
 
+        with_related_connectors_source_code = to_include + [connector.code_directory for connector in self.get_related_connectors()]
+        return with_related_connectors_source_code
+
     async def _get_patched_connector_dir(self) -> Directory:
         """Patch the build.gradle file of the connector under test by removing the lines declared in LINES_TO_REMOVE_FROM_GRADLE_FILE.
 

tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py

@@ -16,7 +16,7 @@
 from ci_connector_ops.pipelines.bases import PytestStep, Step, StepResult, StepStatus
 from ci_connector_ops.pipelines.contexts import CIContext
 from ci_connector_ops.pipelines.utils import METADATA_FILE_NAME
-from ci_connector_ops.utils import DESTINATION_DEFINITIONS_FILE_PATH, SOURCE_DEFINITIONS_FILE_PATH
+from ci_connector_ops.utils import DESTINATION_DEFINITIONS_FILE_PATH, SOURCE_DEFINITIONS_FILE_PATH, Connector
 from dagger import File
 
 
@@ -139,15 +139,28 @@ async def _run(self) -> StepResult:
             StepResult: Failure or success of the QA checks with stdout and stderr.
         """
         ci_connector_ops = await environments.with_ci_connector_ops(self.context)
+        include = [
+            str(self.context.connector.code_directory),
+            str(self.context.connector.documentation_file_path),
+            str(self.context.connector.icon_path),
+            SOURCE_DEFINITIONS_FILE_PATH,
+            DESTINATION_DEFINITIONS_FILE_PATH,
+        ]
+        if (
+            self.context.connector.technical_name.endswith("strict-encrypt")
+            or self.context.connector.technical_name == "source-file-secure"
+        ):
+            original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", ""))
+            include += [
+                str(original_connector.code_directory),
+                str(original_connector.documentation_file_path),
+                str(original_connector.icon_path),
+            ]
+
         filtered_repo = self.context.get_repo_dir(
-            include=[
-                str(self.context.connector.code_directory),
-                str(self.context.connector.documentation_file_path),
-                str(self.context.connector.icon_path),
-                SOURCE_DEFINITIONS_FILE_PATH,
-                DESTINATION_DEFINITIONS_FILE_PATH,
-            ],
+            include=include,
         )
+
         qa_checks = (
             ci_connector_ops.with_mounted_directory("/airbyte", filtered_repo)
             .with_workdir("/airbyte")

tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py

@@ -73,7 +73,8 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]:
     context.secrets_dir = await secrets.get_connector_secret_dir(context)
 
     step_results = await run_steps([BuildConnectorImage(context, LOCAL_BUILD_PLATFORM), UnitTests(context)])
-
+    if any([result.status is StepStatus.FAILURE for result in step_results]):
+        return step_results
     if context.connector.supports_normalization:
         normalization_image = f"{context.connector.normalization_repository}:dev"
         context.logger.info(f"This connector supports normalization: will build {normalization_image}.")

tools/ci_connector_ops/ci_connector_ops/utils.py

@@ -7,6 +7,7 @@
 from dataclasses import dataclass
 from enum import Enum
 from functools import cached_property
+from glob import glob
 from pathlib import Path
 from typing import Dict, List, Optional, Set, Tuple
 
@@ -241,5 +242,8 @@ def get_changed_connectors() -> Set[Connector]:
 
 
 def get_all_released_connectors() -> Set:
-    all_definitions = OSS_CATALOG["sources"] + OSS_CATALOG["destinations"]
-    return {Connector(definition["dockerRepository"].replace("airbyte/", "")) for definition in all_definitions}
+    return {
+        Connector(Path(metadata_file).parent.name)
+        for metadata_file in glob("airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
+        if "-scaffold-" not in metadata_file
+    }