diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py index a62086fde1531..4d67b1ded6d8f 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py @@ -16,7 +16,7 @@ import asyncer from ci_connector_ops.pipelines.actions import environments from ci_connector_ops.pipelines.utils import check_path_in_workdir, slugify, with_exit_code, with_stderr, with_stdout -from ci_connector_ops.utils import console +from ci_connector_ops.utils import Connector, console from dagger import CacheVolume, Container, Directory, QueryError from rich.console import Group from rich.panel import Panel @@ -421,6 +421,19 @@ def docker_service_name(self) -> str: def connector_java_build_cache(self) -> CacheVolume: return self.context.dagger_client.cache_volume("connector_java_build_cache") + def get_related_connectors(self) -> List[Connector]: + """Retrieve the list of related connectors. + This is used to include source code of non strict-encrypt connectors when running build for a strict-encrypt connector. + + Returns: + List[Connector]: The list of related connectors. + """ + if self.context.connector.technical_name.endswith("-strict-encrypt"): + return [Connector(self.context.connector.technical_name.replace("-strict-encrypt", ""))] + if self.context.connector.technical_name == "source-file-secure": + return [Connector("source-file")] + return [] + @property def build_include(self) -> List[str]: """Retrieve the list of source code directory required to run a Java connector Gradle task. @@ -430,13 +443,18 @@ def build_include(self) -> List[str]: Returns: List[str]: List of directories or files to be mounted to the container to run a Java connector Gradle task. """ + to_include = [self.JAVA_BUILD_INCLUDE] + if self.context.connector.connector_type == "source": - return self.JAVA_BUILD_INCLUDE + self.SOURCE_BUILD_INCLUDE + to_include.append(self.SOURCE_BUILD_INCLUDE) elif self.context.connector.connector_type == "destination": - return self.JAVA_BUILD_INCLUDE + self.DESTINATION_BUILD_INCLUDE + to_include.append(self.DESTINATION_BUILD_INCLUDE) else: raise ValueError(f"{self.context.connector.connector_type} is not supported") + with_related_connectors_source_code = to_include + [connector.code_directory for connector in self.get_related_connectors()] + return with_related_connectors_source_code + async def _get_patched_connector_dir(self) -> Directory: """Patch the build.gradle file of the connector under test by removing the lines declared in LINES_TO_REMOVE_FROM_GRADLE_FILE. diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py index 40b15055f4989..6916ef4745caf 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py @@ -16,7 +16,7 @@ from ci_connector_ops.pipelines.bases import PytestStep, Step, StepResult, StepStatus from ci_connector_ops.pipelines.contexts import CIContext from ci_connector_ops.pipelines.utils import METADATA_FILE_NAME -from ci_connector_ops.utils import DESTINATION_DEFINITIONS_FILE_PATH, SOURCE_DEFINITIONS_FILE_PATH +from ci_connector_ops.utils import DESTINATION_DEFINITIONS_FILE_PATH, SOURCE_DEFINITIONS_FILE_PATH, Connector from dagger import File @@ -139,15 +139,28 @@ async def _run(self) -> StepResult: StepResult: Failure or success of the QA checks with stdout and stderr. """ ci_connector_ops = await environments.with_ci_connector_ops(self.context) + include = [ + str(self.context.connector.code_directory), + str(self.context.connector.documentation_file_path), + str(self.context.connector.icon_path), + SOURCE_DEFINITIONS_FILE_PATH, + DESTINATION_DEFINITIONS_FILE_PATH, + ] + if ( + self.context.connector.technical_name.endswith("strict-encrypt") + or self.context.connector.technical_name == "source-file-secure" + ): + original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", "")) + include += [ + str(original_connector.code_directory), + str(original_connector.documentation_file_path), + str(original_connector.icon_path), + ] + filtered_repo = self.context.get_repo_dir( - include=[ - str(self.context.connector.code_directory), - str(self.context.connector.documentation_file_path), - str(self.context.connector.icon_path), - SOURCE_DEFINITIONS_FILE_PATH, - DESTINATION_DEFINITIONS_FILE_PATH, - ], + include=include, ) + qa_checks = ( ci_connector_ops.with_mounted_directory("/airbyte", filtered_repo) .with_workdir("/airbyte") diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py index 114af7982de47..6499dec0baaaf 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py +++ b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py @@ -73,7 +73,8 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]: context.secrets_dir = await secrets.get_connector_secret_dir(context) step_results = await run_steps([BuildConnectorImage(context, LOCAL_BUILD_PLATFORM), UnitTests(context)]) - + if any([result.status is StepStatus.FAILURE for result in step_results]): + return step_results if context.connector.supports_normalization: normalization_image = f"{context.connector.normalization_repository}:dev" context.logger.info(f"This connector supports normalization: will build {normalization_image}.") diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py index 26e5d6d503c0a..7b7a4af4a41a4 100644 --- a/tools/ci_connector_ops/ci_connector_ops/utils.py +++ b/tools/ci_connector_ops/ci_connector_ops/utils.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from enum import Enum from functools import cached_property +from glob import glob from pathlib import Path from typing import Dict, List, Optional, Set, Tuple @@ -241,5 +242,8 @@ def get_changed_connectors() -> Set[Connector]: def get_all_released_connectors() -> Set: - all_definitions = OSS_CATALOG["sources"] + OSS_CATALOG["destinations"] - return {Connector(definition["dockerRepository"].replace("airbyte/", "")) for definition in all_definitions} + return { + Connector(Path(metadata_file).parent.name) + for metadata_file in glob("airbyte-integrations/connectors/**/metadata.yaml", recursive=True) + if "-scaffold-" not in metadata_file + }