Merge branch 'master' into artem1205/source-github-add-docs

airbyte-cdk/java/airbyte-cdk/archive/base-standard-source-test-file/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id 'application'
-    id 'airbyte-docker'
+    id 'airbyte-docker-legacy'
 }
 
 dependencies {

airbyte-cdk/java/airbyte-cdk/archive/bases-destination-jdbc/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id 'application'
-    id 'airbyte-docker'
+    id 'airbyte-docker-legacy'
     id 'airbyte-integration-test-java'
 }
 

airbyte-cdk/java/airbyte-cdk/archive/source-jdbc/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id 'application'
-    id 'airbyte-docker'
+    id 'airbyte-docker-legacy'
     id 'airbyte-integration-test-java'
     id "java-library"
     // https://docs.gradle.org/current/userguide/java_testing.html#sec:java_test_fixtures

airbyte-cdk/java/airbyte-cdk/archive/standard-source-test/build.gradle

@@ -6,7 +6,7 @@ buildscript {
 
 plugins {
     id 'application'
-    id 'airbyte-docker'
+    id 'airbyte-docker-legacy'
 }
 
 import org.jsoup.Jsoup;

airbyte-cdk/java/airbyte-cdk/build.gradle

@@ -1,7 +1,6 @@
 allprojects {
     apply plugin: 'java-library'
     apply plugin: 'maven-publish'
-    apply plugin: 'airbyte-docker'
     apply plugin: 'airbyte-java-cdk'
     apply plugin: 'airbyte-integration-test-java'
     apply plugin: 'airbyte-performance-test-java'

airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtilsTest.java

@@ -27,6 +27,7 @@
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 class DebeziumEventUtilsTest {
@@ -60,6 +61,7 @@ void testConvertRelationalDbChangeEvent() throws IOException {
     deepCompare(expectedDelete, actualDelete);
   }
 
+  @Disabled("See https://github.com/airbytehq/airbyte/issues/31032")
   @Test
   void testConvertMongoDbChangeEvent() throws IOException {
     final String objectId = "64f24244f95155351c4185b1";

airbyte-cdk/python/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id 'airbyte-python'
-    id 'airbyte-docker'
+    id 'airbyte-docker-legacy'
 }
 
 def generateComponentManifestClassFiles = tasks.register('generateComponentManifestClassFiles', Exec) {

airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 from typing import Iterable, Optional, Set, Tuple
 
-from connector_ops.utils import Connector
+from connector_ops.utils import Connector, ConnectorLanguage
 from pydash.objects import get
 
 
@@ -235,7 +235,11 @@ def check_connector_has_no_critical_vulnerabilities(connector: Connector) -> boo
 
 
 def check_metadata_version_matches_dockerfile_label(connector: Connector) -> bool:
-    return connector.version_in_dockerfile_label == connector.version
+    version_in_dockerfile = connector.version_in_dockerfile_label
+    if version_in_dockerfile is None:
+        # Java connectors don't have Dockerfiles.
+        return connector.language == ConnectorLanguage.JAVA
+    return version_in_dockerfile == connector.version
 
 
 QA_CHECKS = [

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -314,12 +314,8 @@ def language(self) -> ConnectorLanguage:
             return ConnectorLanguage.LOW_CODE
         if Path(self.code_directory / "setup.py").is_file() or Path(self.code_directory / "pyproject.toml").is_file():
             return ConnectorLanguage.PYTHON
-        try:
-            with open(self.code_directory / "Dockerfile") as dockerfile:
-                if "FROM airbyte/integration-base-java" in dockerfile.read():
-                    return ConnectorLanguage.JAVA
-        except FileNotFoundError:
-            pass
+        if Path(self.code_directory / "src" / "main" / "java").exists():
+            return ConnectorLanguage.JAVA
         return None
 
     @property
@@ -329,11 +325,14 @@ def version(self) -> str:
         return self.metadata["dockerImageTag"]
 
     @property
-    def version_in_dockerfile_label(self) -> str:
-        with open(self.code_directory / "Dockerfile") as f:
-            for line in f:
-                if "io.airbyte.version" in line:
-                    return line.split("=")[1].strip()
+    def version_in_dockerfile_label(self) -> Optional[str]:
+        try:
+            with open(self.code_directory / "Dockerfile") as f:
+                for line in f:
+                    if "io.airbyte.version" in line:
+                        return line.split("=")[1].strip()
+        except FileNotFoundError as e:
+            return None
         raise ConnectorVersionNotFound(
             """
             Could not find the connector version from its Dockerfile.

airbyte-ci/connectors/connector_ops/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "connector_ops"
-version = "0.2.3"
+version = "0.2.4"
 description = "Packaged maintained by the connector operations team to perform CI for connectors"
 authors = ["Airbyte <contact@airbyte.io>"]
 

airbyte-ci/connectors/pipelines/README.md

@@ -398,6 +398,7 @@ This command runs the Python tests for a airbyte-ci poetry package.
 ## Changelog
 | Version | PR                                                        | Description                                                                                               |
 |---------| --------------------------------------------------------- |-----------------------------------------------------------------------------------------------------------|
+| 1.4.4   | [#30743](https://github.com/airbytehq/airbyte/pull/30743) | Add `--disable-report-auto-open` and `--use-host-gradle-dist-tar` to allow gradle integration.            |
 | 1.4.3   | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Add --version and version check                                                                           |
 | 1.4.2   | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Remove directory name requirement                                                                         |
 | 1.4.1   | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Load base migration guide into QA Test container for strict encrypt variants                              |
@@ -486,3 +487,4 @@ and when in doubt, you can reinstall the CLI with the `--force` option:
 ```bash
 $ pipx reinstall pipelines --force
 ```
+

airbyte-ci/connectors/pipelines/pipelines/bases.py

@@ -624,8 +624,9 @@ async def save(self) -> None:
         absolute_path = await local_html_path.resolve()
         if self.pipeline_context.is_local:
             self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}")
-            self.pipeline_context.logger.info("Opening HTML report in browser.")
-            webbrowser.open(absolute_path.as_uri())
+            if self.pipeline_context.open_report_in_browser:
+                self.pipeline_context.logger.info("Opening HTML report in browser.")
+                webbrowser.open(absolute_path.as_uri())
         if self.remote_storage_enabled:
             await self.save_remote(local_html_path, self.html_report_remote_storage_key, "text/html")
             self.pipeline_context.logger.info(f"HTML report uploaded to {self.html_report_url}")

airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py

@@ -9,9 +9,10 @@
 
 import anyio
 from connector_ops.utils import ConnectorLanguage
-from dagger import Platform
 from pipelines.bases import ConnectorReport, StepResult
-from pipelines.builds import common, java_connectors, python_connectors
+from pipelines.builds import java_connectors, python_connectors
+from pipelines.builds.common import LoadContainerToLocalDockerHost, StepStatus
+from pipelines.consts import LOCAL_BUILD_PLATFORM
 from pipelines.contexts import ConnectorContext
 
 
@@ -25,9 +26,6 @@ class NoBuildStepForLanguageError(Exception):
     ConnectorLanguage.JAVA: java_connectors.run_connector_build,
 }
 
-BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")]
-LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}")
-
 
 async def run_connector_build(context: ConnectorContext) -> StepResult:
     """Run a build pipeline for a single connector."""
@@ -50,9 +48,8 @@ async def run_connector_build_pipeline(context: ConnectorContext, semaphore: any
         async with context:
             build_result = await run_connector_build(context)
             step_results.append(build_result)
-            if context.is_local and build_result.status is common.StepStatus.SUCCESS:
-                connector_to_load_to_local_docker_host = build_result.output_artifact[LOCAL_BUILD_PLATFORM]
-                load_image_result = await common.LoadContainerToLocalDockerHost(context, connector_to_load_to_local_docker_host).run()
+            if context.is_local and build_result.status is StepStatus.SUCCESS:
+                load_image_result = await LoadContainerToLocalDockerHost(context, LOCAL_BUILD_PLATFORM, build_result.output_artifact).run()
                 step_results.append(load_image_result)
             context.report = ConnectorReport(context, step_results, name="BUILD RESULTS")
         return context.report

airbyte-ci/connectors/pipelines/pipelines/builds/common.py

@@ -3,51 +3,69 @@
 #
 
 from abc import ABC
-from typing import Tuple
+from typing import List, Tuple
 
 import docker
-from dagger import Container, Platform
+from dagger import Container, ExecError, Platform, QueryError
 from pipelines.bases import Step, StepResult, StepStatus
 from pipelines.consts import BUILD_PLATFORMS
 from pipelines.contexts import ConnectorContext
 from pipelines.utils import export_container_to_tarball
 
 
-class BuildConnectorImageBase(Step, ABC):
+class BuildConnectorImagesBase(Step, ABC):
+    """
+    A step to build connector images for a set of platforms.
+    """
+
     @property
     def title(self):
-        return f"Build {self.context.connector.technical_name} docker image for platform {self.build_platform}"
+        return f"Build {self.context.connector.technical_name} docker image for platform(s) {', '.join(self.build_platforms)}"
 
-    def __init__(self, context: ConnectorContext, build_platform: Platform) -> None:
-        self.build_platform = build_platform
+    def __init__(self, context: ConnectorContext, *build_platforms: List[Platform]) -> None:
+        self.build_platforms = build_platforms if build_platforms else BUILD_PLATFORMS
         super().__init__(context)
 
+    async def _run(self, *args) -> StepResult:
+        build_results_per_platform = {}
+        for platform in self.build_platforms:
+            try:
+                connector = await self._build_connector(platform, *args)
+                try:
+                    await connector.with_exec(["spec"])
+                except ExecError:
+                    return StepResult(
+                        self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {platform}."
+                    )
+                build_results_per_platform[platform] = connector
+            except QueryError as e:
+                return StepResult(self, StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}")
+        success_message = (
+            f"The {self.context.connector.technical_name} docker image "
+            f"was successfully built for platform(s) {', '.join(self.build_platforms)}"
+        )
+        return StepResult(self, StepStatus.SUCCESS, stdout=success_message, output_artifact=build_results_per_platform)
 
-class BuildConnectorImageForAllPlatformsBase(Step, ABC):
-
-    ALL_PLATFORMS = BUILD_PLATFORMS
-
-    title = f"Build connector image for {BUILD_PLATFORMS}"
+    async def _build_connector(self, platform: Platform, *args) -> Container:
+        """Implement the generation of the image for the platform and return the corresponding container.
 
-    def get_success_result(self, build_results_per_platform: dict[Platform, Container]) -> StepResult:
-        return StepResult(
-            self,
-            StepStatus.SUCCESS,
-            stdout="The connector image was successfully built for all platforms.",
-            output_artifact=build_results_per_platform,
-        )
+        Returns:
+            Container: The container to package as a docker image for this platform.
+        """
+        raise NotImplementedError("`BuildConnectorImagesBase`s must define a '_build_connector' attribute.")
 
 
 class LoadContainerToLocalDockerHost(Step):
     IMAGE_TAG = "dev"
 
-    def __init__(self, context: ConnectorContext, container: Container) -> None:
+    def __init__(self, context: ConnectorContext, platform: Platform, containers: dict[Platform, Container]) -> None:
         super().__init__(context)
-        self.container = container
+        self.platform = platform
+        self.container = containers[platform]
 
     @property
     def title(self):
-        return f"Load {self.image_name}:{self.IMAGE_TAG} to the local docker host."
+        return f"Load {self.image_name}:{self.IMAGE_TAG} for platform {self.platform} to the local docker host."
 
     @property
     def image_name(self) -> Tuple: