connectors-ci: build multi platform connector images (#25286)

.github/workflows/connector_integration_test_single_dagger.yml

@@ -51,7 +51,7 @@ jobs:
             mkdir -p "$DAGGER_TMP_BINDIR"
             curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors test ${{ github.event.inputs.test-connectors-options }}
+          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ github.event.inputs.test-connectors-options }} test
         env:
           _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
@@ -75,7 +75,7 @@ jobs:
             mkdir -p "$DAGGER_TMP_BINDIR"
             curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors test --modified
+          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --modified test
         env:
           _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}

.github/workflows/connector_nightly_builds_dagger.yml

@@ -56,8 +56,7 @@ jobs:
             mkdir -p "$DAGGER_TMP_BINDIR"
             curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR"
           fi
-          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors test ${{ inputs.test-connectors-options || '--concurrency=5 --release-stage=generally_available --release-stage=beta' }}
-        env:
+          airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ inputs.test-connectors-options || '--concurrency=20 --release-stage=generally_available --release-stage=beta' }} test        env:
           _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k"
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
           AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}

tools/ci_connector_ops/.gitignore

@@ -1 +1 @@
-test_reports
+pipeline_reports

tools/ci_connector_ops/ci_connector_ops/pipelines/README.md

@@ -39,6 +39,8 @@ If you don't want to use the remote secrets please call airbyte-ci connectors-ci
 airbyte-ci connectors-ci --use-remote-secrets=False
 ```
 
+
+
 ### Environment variables required for CI run:
 
 - `GCP_GSM_CREDENTIALS`: the credentials to connect to GSM
@@ -52,26 +54,26 @@ airbyte-ci connectors-ci --use-remote-secrets=False
 (source-pokeapi does not require GSM access)
 
 ```bash
-airbyte-ci connectors test --name=source-pokeapi
+airbyte-ci connectors --name=source-pokeapi test
 ```
 
 ### **Run the pipeline for multiple connectors**
 
 ```bash
-airbyte-ci connectors test --name=source-pokeapi --name=source-openweather
+airbyte-ci connectors --name=source-pokeapi --name=source-openweather test
 ```
 
 ### **Run the pipeline for generally available connectors**
 
 ```bash
-airbyte-ci connectors test --release-stage=generally_available
+airbyte-ci connectors --release-stage=generally_available test
 ```
 
 ### **Run the pipeline for the connectors you changed on the branch**
 
 ```bash
 touch airbyte-integrations/connectors/source-pokeapi/random_file_addition.txt
-airbyte-ci connectors test --modified #the source-pokeapi pipeline should run
+airbyte-ci connectors --modified test #the source-pokeapi pipeline should run
 ```
 
 ### Local VS. CI
@@ -88,7 +90,7 @@ The main differences are that:
 - The pipeline will pull the branch under test from Airbyte's GitHub repo
 - The pipeline will upload per connector test reports to S3
 
-## What does a connector pipeline run
+### What does a connector test pipeline run
 
 ```mermaid
 flowchart TB
@@ -113,12 +115,35 @@ This is the DAG we expect for every connector for which the pipeline is triggere
 The Airbyte git repo will be the local one if you use `--is-local=True` command line option.
 The connector secrets won't be downloaded nor uploaded if you use the `--use-remote-secrets=False` command line option.
 
+## Running the Connector build pipeline
+You can build connector images that will be available on your docker host.
+Both linux/arm64 and linux/amd64 will be built but only the image corresponding to your architecture will be loaded to your host.
+
+```bash
+airbyte-ci connectors --name=source-postgres build
+```
+
+**You can build multiple connectors in a single command:**
 
-### Performance benchmarks
+Build all the modified connectors on your branch (given you committed the changes):
+```bash
+airbyte-ci connectors --modified
+```
 
-| Connector      | Run integration test GHA duration                                      | Dagger POC duration (CI no cache)                                      |
-| -------------- | ---------------------------------------------------------------------- | ---------------------------------------------------------------------- |
-| source-pokeapi | [7mn22s](https://github.com/airbytehq/airbyte/actions/runs/4395453220) | [5mn26s](https://github.com/airbytehq/airbyte/actions/runs/4403595746) |
+Build only source-postgres and source-pokeapi:
+```bash
+airbyte-ci connectors --name=source-postgres --name=source-pokeapi build
+```
+
+Build all GA connectors:
+```bash
+airbyte-ci connectors --release-stage=generally_available build
+```
+
+Build all GA java connectors:
+```bash
+airbyte-ci connectors --release-stage=generally_available --language=java build
+```
 
 ## Running the Metadata pipelines
 The new metadata service also uses dagger for its reproducible CI pipeline.

tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py

@@ -13,7 +13,7 @@
 from dagger import CacheSharingMode, CacheVolume, Container, Directory, File, Secret
 
 if TYPE_CHECKING:
-    from ci_connector_ops.pipelines.contexts import ConnectorTestContext, PipelineContext
+    from ci_connector_ops.pipelines.contexts import ConnectorContext, PipelineContext
 
 
 PYPROJECT_TOML_FILE_PATH = "pyproject.toml"
@@ -150,11 +150,11 @@ async def with_installed_python_package(
     return container
 
 
-def with_airbyte_connector(context: ConnectorTestContext) -> Container:
+def with_airbyte_connector(context: ConnectorContext) -> Container:
     """Load an airbyte connector source code in a testing environment.
 
     Args:
-        context (ConnectorTestContext): The current test context, providing the repository directory from which the connector sources will be pulled.
+        context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled.
     Returns:
         Container: A python environment container (with the connector source code).
     """
@@ -163,11 +163,11 @@ def with_airbyte_connector(context: ConnectorTestContext) -> Container:
     return with_python_package(context, testing_environment, connector_source_path, exclude=["secrets"])
 
 
-async def with_installed_airbyte_connector(context: ConnectorTestContext) -> Container:
+async def with_installed_airbyte_connector(context: ConnectorContext) -> Container:
     """Install an airbyte connector python package in a testing environment.
 
     Args:
-        context (ConnectorTestContext): The current test context, providing the repository directory from which the connector sources will be pulled.
+        context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled.
     Returns:
         Container: A python environment container (with the connector installed).
     """
@@ -249,12 +249,12 @@ async def with_ci_connector_ops(context: PipelineContext) -> Container:
 
 
 def with_dockerd_service(
-    context: ConnectorTestContext, shared_volume: Optional(Tuple[str, CacheVolume]) = None, docker_service_name: Optional[str] = None
+    context: ConnectorContext, shared_volume: Optional(Tuple[str, CacheVolume]) = None, docker_service_name: Optional[str] = None
 ) -> Container:
     """Create a container running dockerd, exposing its 2375 port, can be used as the docker host for docker-in-docker use cases.
 
     Args:
-        context (ConnectorTestContext): The current connector test context.
+        context (ConnectorContext): The current connector context.
         shared_volume (Optional, optional): A tuple in the form of (mounted path, cache volume) that will be mounted to the dockerd container. Defaults to None.
         docker_service_name (Optional[str], optional): The name of the docker service, appended to volume name, useful context isolation. Defaults to None.
 
@@ -281,15 +281,15 @@ def with_dockerd_service(
 
 
 def with_bound_docker_host(
-    context: ConnectorTestContext,
+    context: ConnectorContext,
     container: Container,
     shared_volume: Optional(Tuple[str, CacheVolume]) = None,
     docker_service_name: Optional[str] = None,
 ) -> Container:
     """Bind a container to a docker host. It will use the dockerd service as a docker host.
 
     Args:
-        context (ConnectorTestContext): The current connector test context.
+        context (ConnectorContext): The current connector context.
         container (Container): The container to bind to the docker host.
         shared_volume (Optional, optional): A tuple in the form of (mounted path, cache volume) that will be both mounted to the container and the dockerd container. Defaults to None.
         docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to None.
@@ -308,12 +308,12 @@ def with_bound_docker_host(
 
 
 def with_docker_cli(
-    context: ConnectorTestContext, shared_volume: Optional(Tuple[str, CacheVolume]) = None, docker_service_name: Optional[str] = None
+    context: ConnectorContext, shared_volume: Optional(Tuple[str, CacheVolume]) = None, docker_service_name: Optional[str] = None
 ) -> Container:
     """Create a container with the docker CLI installed and bound to a persistent docker host.
 
     Args:
-        context (ConnectorTestContext): The current connector test context.
+        context (ConnectorContext): The current connector context.
         shared_volume (Optional, optional): A tuple in the form of (mounted path, cache volume) that will be both mounted to the container and the dockerd container. Defaults to None.
         docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to None.
 
@@ -324,11 +324,11 @@ def with_docker_cli(
     return with_bound_docker_host(context, docker_cli, shared_volume, docker_service_name)
 
 
-async def with_connector_acceptance_test(context: ConnectorTestContext, connector_under_test_image_tar: File) -> Container:
+async def with_connector_acceptance_test(context: ConnectorContext, connector_under_test_image_tar: File) -> Container:
     """Create a container to run connector acceptance tests, bound to a persistent docker host.
 
     Args:
-        context (ConnectorTestContext): The current connector test context.
+        context (ConnectorContext): The current connector context.
         connector_under_test_image_tar (File): The file containing the tar archive the image of the connector under test.
     Returns:
         Container: A container with connector acceptance tests installed.
@@ -355,15 +355,15 @@ async def with_connector_acceptance_test(context: ConnectorTestContext, connecto
 
 
 def with_gradle(
-    context: ConnectorTestContext,
+    context: ConnectorContext,
     sources_to_include: List[str] = None,
     bind_to_docker_host: bool = True,
     docker_service_name: Optional[str] = "gradle",
 ) -> Container:
     """Create a container with Gradle installed and bound to a persistent docker host.
 
     Args:
-        context (ConnectorTestContext): The current connector test context.
+        context (ConnectorContext): The current connector context.
         sources_to_include (List[str], optional): List of additional source path to mount to the container. Defaults to None.
         bind_to_docker_host (bool): Whether to bind the gradle container to a docker host.
         docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to "gradle".
@@ -419,13 +419,11 @@ def with_gradle(
         return openjdk_with_docker
 
 
-async def load_image_to_docker_host(
-    context: ConnectorTestContext, tar_file: File, image_tag: str, docker_service_name: Optional[str] = None
-):
+async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str, docker_service_name: Optional[str] = None):
     """Load a docker image tar archive to the docker host.
 
     Args:
-        context (ConnectorTestContext): The current connector test context.
+        context (ConnectorContext): The current connector context.
         tar_file (File): The file object holding the docker image tar archive.
         image_tag (str): The tag to create on the image if it has no tag.
         docker_service_name (str): Name of the docker service, useful for context isolation.
@@ -476,15 +474,15 @@ def with_poetry_module(context: PipelineContext, parent_dir: Directory, module_p
     )
 
 
-def with_integration_base(context: PipelineContext, jdk_version: str = "17.0.4") -> Container:
+def with_integration_base(context: PipelineContext, build_platform: str, jdk_version: str = "17.0.4") -> Container:
     """Create an integration base container.
 
     Reproduce with Dagger the Dockerfile defined here: airbyte-integrations/bases/base/Dockerfile
     """
     base_sh = context.get_repo_dir("airbyte-integrations/bases/base", include=["base.sh"]).file("base.sh")
 
     return (
-        context.dagger_client.container()
+        context.dagger_client.container(platform=build_platform)
         .from_(f"amazoncorretto:{jdk_version}")
         .with_workdir("/airbyte")
         .with_file("base.sh", base_sh)
@@ -494,7 +492,7 @@ def with_integration_base(context: PipelineContext, jdk_version: str = "17.0.4")
     )
 
 
-async def with_java_base(context: PipelineContext, jdk_version: str = "17.0.4") -> Container:
+async def with_java_base(context: PipelineContext, build_platform: str, jdk_version: str = "17.0.4") -> Container:
     """Create a java base container.
 
     Reproduce with Dagger the Dockerfile defined here: airbyte-integrations/bases/base-java/Dockerfile_
@@ -505,7 +503,7 @@ async def with_java_base(context: PipelineContext, jdk_version: str = "17.0.4")
     java_base_version = await get_version_from_dockerfile(dockerfile)
 
     return (
-        with_integration_base(context, jdk_version)
+        with_integration_base(context, build_platform, jdk_version)
         .with_exec(["yum", "install", "-y", "tar", "openssl"])
         .with_exec(["yum", "clean", "all"])
         .with_workdir("/airbyte")
@@ -522,11 +520,11 @@ async def with_java_base(context: PipelineContext, jdk_version: str = "17.0.4")
     )
 
 
-async def with_airbyte_java_connector(context: ConnectorTestContext, connector_java_tar_file: File):
+async def with_airbyte_java_connector(context: ConnectorContext, connector_java_tar_file: File, build_platform: str):
     dockerfile = context.get_connector_dir(include=["Dockerfile"]).file("Dockerfile")
     connector_version = await get_version_from_dockerfile(dockerfile)
     application = context.connector.technical_name
-    java_base = await with_java_base(context)
+    java_base = await with_java_base(context, build_platform)
     enable_sentry = await should_enable_sentry(dockerfile)
 
     return (

tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py

@@ -13,14 +13,14 @@
 from dagger import Directory
 
 if TYPE_CHECKING:
-    from ci_connector_ops.pipelines.contexts import ConnectorTestContext
+    from ci_connector_ops.pipelines.contexts import ConnectorContext
 
 
-async def download(context: ConnectorTestContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Directory:
+async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Directory:
     """Use the ci-credentials tool to download the secrets stored for a specific connector to a Directory.
 
     Args:
-        context (ConnectorTestContext): The context providing a connector object.
+        context (ConnectorContext): The context providing a connector object.
         gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS".
 
     Returns:
@@ -40,11 +40,11 @@ async def download(context: ConnectorTestContext, gcp_gsm_env_variable_name: str
     )
 
 
-async def upload(context: ConnectorTestContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> int:
+async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> int:
     """Use the ci-credentials tool to upload the secrets stored in the context's updated_secrets-dir.
 
     Args:
-        context (ConnectorTestContext): The context providing a connector object and the update secrets dir.
+        context (ConnectorContext): The context providing a connector object and the update secrets dir.
         gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS".
 
     Returns:
@@ -62,11 +62,11 @@ async def upload(context: ConnectorTestContext, gcp_gsm_env_variable_name: str =
     )
 
 
-async def get_connector_secret_dir(context: ConnectorTestContext) -> Directory:
+async def get_connector_secret_dir(context: ConnectorContext) -> Directory:
     """Download the secrets from GSM or use the local secrets directory for a connector.
 
     Args:
-        context (ConnectorTestContext): The context providing the connector directory and the use_remote_secrets flag.
+        context (ConnectorContext): The context providing the connector directory and the use_remote_secrets flag.
 
     Returns:
         Directory: A directory with the downloaded connector secrets.