diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml
index 16b84b191cc3e3..ded1f9fb11ae1e 100644
--- a/.github/workflows/publish_connectors.yml
+++ b/.github/workflows/publish_connectors.yml
@@ -18,10 +18,6 @@ on:
type: string
default: ci-runner-connector-publish-large-dagger-0-6-4
required: true
- airbyte-ci-binary-url:
- description: "URL to airbyte-ci binary"
- required: false
- default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci
jobs:
publish_connectors:
name: Publish connectors
@@ -66,7 +62,6 @@ jobs:
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}"
- airbyte_ci_binary_url: ${{ github.event.inputs.airbyte-ci-binary-url }}
set-instatus-incident-on-failure:
name: Create Instatus Incident on Failure
diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
index abae5c2fb1b535..a0adab57d120c2 100644
--- a/airbyte-ci/connectors/pipelines/README.md
+++ b/airbyte-ci/connectors/pipelines/README.md
@@ -124,17 +124,18 @@ At this point you can run `airbyte-ci` commands.
#### Options
-| Option | Default value | Mapped environment variable | Description |
-| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- |
-| `--enable-dagger-run/--disable-dagger-run` | `--enable-dagger-run`` | | Disables the Dagger terminal UI. | | | | |
-| `--is-local/--is-ci` | `--is-local` | | Determines the environment in which the CLI runs: local environment or CI environment. |
-| `--git-branch` | The checked out git branch name | `CI_GIT_BRANCH` | The git branch on which the pipelines will run. |
-| `--git-revision` | The current branch head | `CI_GIT_REVISION` | The commit hash on which the pipelines will run. |
-| `--diffed-branch` | `origin/master` | | Branch to which the git diff will happen to detect new or modified files. |
-| `--gha-workflow-run-id` | | | GHA CI only - The run id of the GitHub action workflow |
-| `--ci-context` | `manual` | | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` |
-| `--pipeline-start-timestamp` | Current epoch time | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation. |
-| `--show-dagger-logs/--hide-dagger-logs` | `--hide-dagger-logs` | | Flag to show or hide the dagger logs. |
+| Option | Default value | Mapped environment variable | Description |
+| ------------------------------------------ | ---------------------------------------------------------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- |
+| `--enable-dagger-run/--disable-dagger-run` | `--enable-dagger-run`` | | Disables the Dagger terminal UI. | | |
+| `--enable-auto-update/--disable-auto-update` | `--enable-auto-update`` | | Disables the auto update prompt | | |
+| `--is-local/--is-ci` | `--is-local` | | Determines the environment in which the CLI runs: local environment or CI environment. |
+| `--git-branch` | The checked out git branch name | `CI_GIT_BRANCH` | The git branch on which the pipelines will run. |
+| `--git-revision` | The current branch head | `CI_GIT_REVISION` | The commit hash on which the pipelines will run. |
+| `--diffed-branch` | `origin/master` | | Branch to which the git diff will happen to detect new or modified files. |
+| `--gha-workflow-run-id` | | | GHA CI only - The run id of the GitHub action workflow |
+| `--ci-context` | `manual` | | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` |
+| `--pipeline-start-timestamp` | Current epoch time | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation. |
+| `--show-dagger-logs/--hide-dagger-logs` | `--hide-dagger-logs` | | Flag to show or hide the dagger logs. |
### `connectors` command subgroup
@@ -254,9 +255,6 @@ It's mainly purposed for local use.
Build a single connector:
`airbyte-ci connectors --name=source-pokeapi build`
-Build a single connector for multiple architectures:
-`airbyte-ci connectors --name=source-pokeapi build --architecture=linux/amd64 --architecture=linux/arm64`
-
Build multiple connectors:
`airbyte-ci connectors --name=source-pokeapi --name=source-bigquery build`
@@ -293,17 +291,11 @@ flowchart TD
distTar-->connector
normalization--"if supports normalization"-->connector
- load[Load to docker host with :dev tag]
+ load[Load to docker host with :dev tag, current platform]
spec[Get spec]
connector-->spec--"if success"-->load
```
-### Options
-
-| Option | Multiple | Default value | Description |
-| --------------------- | -------- | -------------- | ----------------------------------------------------------------- |
-| `--architecture`/`-a` | True | Local platform | Defines for which architecture the connector image will be built. |
-
### `connectors publish` command
Run a publish pipeline for one or multiple connectors.
It's mainly purposed for CI use to release a connector update.
@@ -442,7 +434,6 @@ This command runs the Python tests for a airbyte-ci poetry package.
## Changelog
| Version | PR | Description |
| ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
-| 2.9.0 | [#32816](https://github.com/airbytehq/airbyte/pull/32816) | Add `--architecture` option to connector build. |
| 2.8.0 | [#31930](https://github.com/airbytehq/airbyte/pull/31930) | Move pipx install to `airbyte-ci-dev`, and add auto-update feature targeting binary |
| 2.7.3 | [#32847](https://github.com/airbytehq/airbyte/pull/32847) | Improve --modified behaviour for pull requests. |
| 2.7.2 | [#32839](https://github.com/airbytehq/airbyte/pull/32839) | Revert changes in v2.7.1. |
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py
index 3c4e04c6028bbf..c152604893920e 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py
@@ -2,16 +2,11 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
-from typing import List
-
import asyncclick as click
-import dagger
-from pipelines import main_logger
from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build_pipeline
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
-from pipelines.consts import BUILD_PLATFORMS, LOCAL_BUILD_PLATFORM
@click.command(cls=DaggerPipelineCommand, help="Build all images for the selected connectors.")
@@ -22,20 +17,10 @@
default=False,
type=bool,
)
-@click.option(
- "-a",
- "--architecture",
- "build_architectures",
- help="Architecture for which to build the connector image. If not specified, the image will be built for the local architecture.",
- multiple=True,
- default=[LOCAL_BUILD_PLATFORM],
- type=click.Choice(BUILD_PLATFORMS, case_sensitive=True),
-)
@click.pass_context
-async def build(ctx: click.Context, use_host_gradle_dist_tar: bool, build_architectures: List[str]) -> bool:
+async def build(ctx: click.Context, use_host_gradle_dist_tar: bool) -> bool:
"""Runs a build pipeline for the selected connectors."""
- build_platforms = [dagger.Platform(architecture) for architecture in build_architectures]
- main_logger.info(f"Building connectors for {build_platforms}, use --architecture to change this.")
+
connectors_contexts = [
ConnectorContext(
pipeline_name=f"Build connector {connector.technical_name}",
@@ -56,7 +41,6 @@ async def build(ctx: click.Context, use_host_gradle_dist_tar: bool, build_archit
use_host_gradle_dist_tar=use_host_gradle_dist_tar,
s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),
- targeted_platforms=build_platforms,
)
for connector in ctx.obj["selected_connectors_with_modified_files"]
]
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py
index e18a6e3d38f6d0..5bbc035fe1bd32 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py
@@ -5,13 +5,17 @@
from __future__ import annotations
+import platform
+
import anyio
from connector_ops.utils import ConnectorLanguage
-from pipelines.airbyte_ci.connectors.build_image.steps import java_connectors, python_connectors
+from pipelines.models.steps import StepResult
+from pipelines.airbyte_ci.connectors.build_image.steps import python_connectors
from pipelines.airbyte_ci.connectors.build_image.steps.common import LoadContainerToLocalDockerHost, StepStatus
+from pipelines.consts import LOCAL_BUILD_PLATFORM
+from pipelines.airbyte_ci.connectors.build_image.steps import java_connectors
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.airbyte_ci.connectors.reports import ConnectorReport
-from pipelines.models.steps import StepResult
class NoBuildStepForLanguageError(Exception):
@@ -37,7 +41,7 @@ async def run_connector_build_pipeline(context: ConnectorContext, semaphore: any
Args:
context (ConnectorContext): The initialized connector context.
- semaphore (anyio.Semaphore): The semaphore to use to limit the number of concurrent builds.
+
Returns:
ConnectorReport: The reports holding builds results.
"""
@@ -45,10 +49,9 @@ async def run_connector_build_pipeline(context: ConnectorContext, semaphore: any
async with semaphore:
async with context:
build_result = await run_connector_build(context)
- per_platform_built_containers = build_result.output_artifact
step_results.append(build_result)
if context.is_local and build_result.status is StepStatus.SUCCESS:
- load_image_result = await LoadContainerToLocalDockerHost(context, per_platform_built_containers).run()
+ load_image_result = await LoadContainerToLocalDockerHost(context, LOCAL_BUILD_PLATFORM, build_result.output_artifact).run()
step_results.append(load_image_result)
context.report = ConnectorReport(context, step_results, name="BUILD RESULTS")
return context.report
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py
index eed5ed96c2921a..267238ee23478a 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py
@@ -2,14 +2,14 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
-import json
from abc import ABC
from typing import List, Tuple
import docker
from dagger import Container, ExecError, Platform, QueryError
from pipelines.airbyte_ci.connectors.context import ConnectorContext
-from pipelines.helpers.utils import export_containers_to_tarball
+from pipelines.consts import BUILD_PLATFORMS
+from pipelines.helpers.utils import export_container_to_tarball
from pipelines.models.steps import Step, StepResult, StepStatus
@@ -22,8 +22,8 @@ class BuildConnectorImagesBase(Step, ABC):
def title(self):
return f"Build {self.context.connector.technical_name} docker image for platform(s) {', '.join(self.build_platforms)}"
- def __init__(self, context: ConnectorContext) -> None:
- self.build_platforms: List[Platform] = context.targeted_platforms
+ def __init__(self, context: ConnectorContext, *build_platforms: List[Platform]) -> None:
+ self.build_platforms = build_platforms if build_platforms else BUILD_PLATFORMS
super().__init__(context)
async def _run(self, *args) -> StepResult:
@@ -58,40 +58,26 @@ async def _build_connector(self, platform: Platform, *args) -> Container:
class LoadContainerToLocalDockerHost(Step):
IMAGE_TAG = "dev"
- def __init__(self, context: ConnectorContext, containers: dict[Platform, Container]) -> None:
+ def __init__(self, context: ConnectorContext, platform: Platform, containers: dict[Platform, Container]) -> None:
super().__init__(context)
- self.containers = containers
+ self.platform = platform
+ self.container = containers[platform]
@property
def title(self):
- return f"Load {self.image_name}:{self.IMAGE_TAG} to the local docker host."
+ return f"Load {self.image_name}:{self.IMAGE_TAG} for platform {self.platform} to the local docker host."
@property
def image_name(self) -> Tuple:
return f"airbyte/{self.context.connector.technical_name}"
async def _run(self) -> StepResult:
- container_variants = list(self.containers.values())
- _, exported_tar_path = await export_containers_to_tarball(self.context, container_variants)
- if not exported_tar_path:
- return StepResult(
- self,
- StepStatus.FAILURE,
- stderr=f"Failed to export the connector image {self.image_name}:{self.IMAGE_TAG} to a tarball.",
- )
+ _, exported_tarball_path = await export_container_to_tarball(self.context, self.container)
+ client = docker.from_env()
try:
- client = docker.from_env()
- response = client.api.import_image_from_file(str(exported_tar_path), repository=self.image_name, tag=self.IMAGE_TAG)
- try:
- image_sha = json.loads(response)["status"]
- except (json.JSONDecodeError, KeyError):
- return StepResult(
- self,
- StepStatus.FAILURE,
- stderr=f"Failed to import the connector image {self.image_name}:{self.IMAGE_TAG} to your Docker host: {response}",
- )
- return StepResult(
- self, StepStatus.SUCCESS, stdout=f"Loaded image {self.image_name}:{self.IMAGE_TAG} to your Docker host ({image_sha})."
- )
- except docker.errors.DockerException as e:
- return StepResult(self, StepStatus.FAILURE, stderr=f"Something went wrong while interacting with the local docker client: {e}")
+ with open(exported_tarball_path, "rb") as tarball_content:
+ new_image = client.images.load(tarball_content.read())[0]
+ new_image.tag(self.image_name, tag=self.IMAGE_TAG)
+ return StepResult(self, StepStatus.SUCCESS)
+ except ConnectionError:
+ return StepResult(self, StepStatus.FAILURE, stderr="The connection to the local docker host failed.")
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py
index f8a4c7ed0d61a3..704ce6fa3849d4 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py
@@ -2,10 +2,13 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
-from dagger import Container, Directory, File, Platform, QueryError
+from typing import List, Optional, Tuple, Union
+
+from dagger import Container, Directory, ExecError, File, Host, Platform, QueryError
from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.airbyte_ci.steps.gradle import GradleTask
+from pipelines.consts import LOCAL_BUILD_PLATFORM
from pipelines.dagger.containers import java
from pipelines.models.steps import StepResult, StepStatus
@@ -53,7 +56,7 @@ async def run_connector_build(context: ConnectorContext) -> StepResult:
# Special case: use a local dist tar to speed up local development.
dist_dir = await context.dagger_client.host().directory(dist_tar_directory_path(context), include=["*.tar"])
# Speed things up by only building for the local platform.
- return await BuildConnectorImages(context).run(dist_dir)
+ return await BuildConnectorImages(context, LOCAL_BUILD_PLATFORM).run(dist_dir)
# Default case: distribution tar is built by the dagger pipeline.
build_connector_tar_result = await BuildConnectorDistributionTar(context).run()
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py
index 3553cafa8ac1d8..8a2115a41135dc 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py
@@ -6,15 +6,14 @@
from datetime import datetime
from types import TracebackType
-from typing import Iterable, Optional
+from typing import Optional
import yaml
from anyio import Path
from asyncer import asyncify
-from dagger import Directory, Platform, Secret
+from dagger import Directory, Secret
from github import PullRequest
from pipelines.airbyte_ci.connectors.reports import ConnectorReport
-from pipelines.consts import BUILD_PLATFORMS
from pipelines.dagger.actions import secrets
from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles
from pipelines.helpers.github import update_commit_status_check
@@ -61,7 +60,6 @@ def __init__(
s3_build_cache_access_key_id: Optional[str] = None,
s3_build_cache_secret_key: Optional[str] = None,
concurrent_cat: Optional[bool] = False,
- targeted_platforms: Optional[Iterable[Platform]] = BUILD_PLATFORMS,
):
"""Initialize a connector context.
@@ -90,7 +88,6 @@ def __init__(
s3_build_cache_access_key_id (Optional[str], optional): Gradle S3 Build Cache credentials. Defaults to None.
s3_build_cache_secret_key (Optional[str], optional): Gradle S3 Build Cache credentials. Defaults to None.
concurrent_cat (bool, optional): Whether to run the CAT tests in parallel. Defaults to False.
- targeted_platforms (Optional[Iterable[Platform]], optional): The platforms to build the connector image for. Defaults to BUILD_PLATFORMS.
"""
self.pipeline_name = pipeline_name
@@ -113,7 +110,6 @@ def __init__(
self.s3_build_cache_access_key_id = s3_build_cache_access_key_id
self.s3_build_cache_secret_key = s3_build_cache_secret_key
self.concurrent_cat = concurrent_cat
- self.targeted_platforms = targeted_platforms
super().__init__(
pipeline_name=pipeline_name,
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py
index 2a5908bdb15032..ffb754cf47f7e5 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py
@@ -205,7 +205,7 @@ async def _run(self, built_connector: Container) -> StepResult:
return StepResult(self, status=StepStatus.SUCCESS, stdout="Uploaded connector spec to spec cache bucket.")
-# Pipeline
+## Pipeline
async def run_connector_publish_pipeline(context: PublishConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport:
diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py
index f3393e0b18f0ed..ca334022e10f3c 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py
@@ -21,7 +21,7 @@
from pipelines.consts import LOCAL_BUILD_PLATFORM
from pipelines.dagger.actions import secrets
from pipelines.dagger.actions.system import docker
-from pipelines.helpers.utils import export_containers_to_tarball
+from pipelines.helpers.utils import export_container_to_tarball
from pipelines.models.steps import StepResult, StepStatus
@@ -101,15 +101,15 @@ async def run_docker_build_dependent_steps(dist_tar_dir: Directory) -> List[Step
context.logger.info(f"This connector supports normalization: will build {normalization_image}.")
build_normalization_results = await BuildOrPullNormalization(context, normalization_image, LOCAL_BUILD_PLATFORM).run()
normalization_container = build_normalization_results.output_artifact
- normalization_tar_file, _ = await export_containers_to_tarball(
- context, [normalization_container], tar_file_name=f"{context.connector.normalization_repository}_{context.git_revision}.tar"
+ normalization_tar_file, _ = await export_container_to_tarball(
+ context, normalization_container, tar_file_name=f"{context.connector.normalization_repository}_{context.git_revision}.tar"
)
step_results.append(build_normalization_results)
else:
normalization_tar_file = None
connector_container = build_connector_image_results.output_artifact[LOCAL_BUILD_PLATFORM]
- connector_image_tar_file, _ = await export_containers_to_tarball(context, [connector_container])
+ connector_image_tar_file, _ = await export_container_to_tarball(context, connector_container)
async with asyncer.create_task_group() as docker_build_dependent_group:
soon_integration_tests_results = docker_build_dependent_group.soonify(IntegrationTests(context).run)(
diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py
index ba1db767c0dd96..4ccaf7522644bc 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/consts.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py
@@ -2,6 +2,7 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
+import os
import platform
from enum import Enum
@@ -19,7 +20,7 @@
"pytest-custom_exit_code",
]
-BUILD_PLATFORMS = (Platform("linux/amd64"), Platform("linux/arm64"))
+BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")]
PLATFORM_MACHINE_TO_DAGGER_PLATFORM = {
"x86_64": Platform("linux/amd64"),
@@ -27,8 +28,7 @@
"aarch64": Platform("linux/amd64"),
"amd64": Platform("linux/amd64"),
}
-LOCAL_MACHINE_TYPE = platform.machine()
-LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[LOCAL_MACHINE_TYPE]
+LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[platform.machine()]
AMAZONCORRETTO_IMAGE = "amazoncorretto:17.0.8-al2023"
NODE_IMAGE = "node:18.18.0-slim"
GO_IMAGE = "golang:1.17"
diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py
index 509ff15e423cba..d2709257e44906 100644
--- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py
+++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py
@@ -267,31 +267,31 @@ async def execute_concurrently(steps: List[Callable], concurrency=5):
return [task.value for task in tasks]
-async def export_containers_to_tarball(
- context: ConnectorContext, container_variants: List[Container], tar_file_name: Optional[str] = None
+async def export_container_to_tarball(
+ context: ConnectorContext, container: Container, tar_file_name: Optional[str] = None
) -> Tuple[Optional[File], Optional[Path]]:
"""Save the container image to the host filesystem as a tar archive.
- Exports a list of container variants to a tarball file.
- The list of container variants should be platform/os specific variants of the same container image.
- The tarball file is saved to the host filesystem in the directory specified by the host_image_export_dir_path attribute of the context.
-
- Args:
- context (ConnectorContext): The current connector context.
- container_variants (List[Container]): The list of container variants to export.
- tar_file_name (Optional[str], optional): The name of the tar archive file. Defaults to None.
+ Exporting a container image as a tar archive allows user to have a dagger built container image available on their host filesystem.
+ They can load this tar file to their main docker host with 'docker load'.
+ This mechanism is also used to share dagger built containers with other steps like AcceptanceTest that have their own dockerd service.
+ We 'docker load' this tar file to AcceptanceTest's docker host to make sure the container under test image is available for testing.
Returns:
Tuple[Optional[File], Optional[Path]]: A tuple with the file object holding the tar archive on the host and its path.
"""
- tar_file_name = f"{slugify(context.connector.technical_name)}_{context.git_revision}.tar" if tar_file_name is None else tar_file_name
+ if tar_file_name is None:
+ tar_file_name = f"{context.connector.technical_name}_{context.git_revision}.tar"
+ tar_file_name = slugify(tar_file_name)
local_path = Path(f"{context.host_image_export_dir_path}/{tar_file_name}")
- export_success = await context.dagger_client.container().export(
- str(local_path), platform_variants=container_variants, forced_compression=ImageLayerCompression.Gzip
- )
+ export_success = await container.export(str(local_path), forced_compression=ImageLayerCompression.Gzip)
if export_success:
- return context.dagger_client.host().file(str(local_path)), local_path
- return None, None
+ exported_file = (
+ context.dagger_client.host().directory(context.host_image_export_dir_path, include=[tar_file_name]).file(tar_file_name)
+ )
+ return exported_file, local_path
+ else:
+ return None, None
def format_duration(time_delta: datetime.timedelta) -> str:
diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
index e15e04b45a1d24..13c7814009a986 100644
--- a/airbyte-ci/connectors/pipelines/pyproject.toml
+++ b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "pipelines"
-version = "2.9.0"
+version = "2.8.0"
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
authors = ["Airbyte "]
diff --git a/airbyte-ci/connectors/pipelines/pytest.ini b/airbyte-ci/connectors/pipelines/pytest.ini
index b228671b5fa2b4..0bd08b038c23b4 100644
--- a/airbyte-ci/connectors/pipelines/pytest.ini
+++ b/airbyte-ci/connectors/pipelines/pytest.ini
@@ -1,4 +1,2 @@
[pytest]
addopts = --cov=pipelines
-markers =
- slow: marks tests as slow (deselect with '-m "not slow"')
diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/__init__.py
deleted file mode 100644
index c941b30457953b..00000000000000
--- a/airbyte-ci/connectors/pipelines/tests/test_build_image/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py
deleted file mode 100644
index aabef380337471..00000000000000
--- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import os
-from typing import Dict
-
-import dagger
-import docker
-import pytest
-from pipelines.airbyte_ci.connectors.build_image.steps import common
-from pipelines.consts import BUILD_PLATFORMS
-from pipelines.models.steps import StepStatus
-
-pytestmark = [
- pytest.mark.anyio,
-]
-
-
-@pytest.mark.slow
-class TestLoadContainerToLocalDockerHost:
- @pytest.fixture(scope="class")
- def certified_connector(self, all_connectors):
- for connector in all_connectors:
- if connector.support_level == "certified":
- return connector
- pytest.skip("No certified connector found")
-
- @pytest.fixture
- def built_containers(self, dagger_client, certified_connector) -> Dict[dagger.Platform, dagger.Container]:
- return {
- platform: dagger_client.container(platform=platform).from_(f'{certified_connector.metadata["dockerRepository"]}:latest')
- for platform in BUILD_PLATFORMS
- }
-
- @pytest.fixture
- def test_context(self, mocker, dagger_client, certified_connector, tmp_path):
- return mocker.Mock(
- secrets_to_mask=[], dagger_client=dagger_client, connector=certified_connector, host_image_export_dir_path=tmp_path
- )
-
- @pytest.fixture
- def step(self, test_context, built_containers):
- return common.LoadContainerToLocalDockerHost(test_context, built_containers)
-
- @pytest.fixture
- def bad_docker_host(self):
- original_docker_host = os.environ.get("DOCKER_HOST")
- yield "tcp://localhost:9999"
- if original_docker_host:
- os.environ["DOCKER_HOST"] = original_docker_host
- else:
- del os.environ["DOCKER_HOST"]
-
- async def test_run(self, test_context, step):
- """Test that the step runs successfully and that the image is loaded in the local docker host."""
- docker_client = docker.from_env()
- step.IMAGE_TAG = "test-load-container"
- try:
- docker_client.images.remove(f"{test_context.connector.metadata['dockerRepository']}:{step.IMAGE_TAG}")
- except docker.errors.ImageNotFound:
- pass
- result = await step.run()
- assert result.status is StepStatus.SUCCESS
- docker_client.images.get(f"{test_context.connector.metadata['dockerRepository']}:{step.IMAGE_TAG}")
- docker_client.images.remove(f"{test_context.connector.metadata['dockerRepository']}:{step.IMAGE_TAG}")
-
- async def test_run_export_failure(self, step, mocker):
- """Test that the step fails if the export of the container fails."""
- mocker.patch.object(common, "export_containers_to_tarball", return_value=(None, None))
- result = await step.run()
- assert result.status is StepStatus.FAILURE
- assert "Failed to export the connector image" in result.stderr
-
- async def test_run_connection_error(self, step, bad_docker_host):
- """Test that the step fails if the connection to the docker host fails."""
- os.environ["DOCKER_HOST"] = bad_docker_host
- result = await step.run()
- assert result.status is StepStatus.FAILURE
- assert "Something went wrong while interacting with the local docker client" in result.stderr
-
- async def test_run_import_failure(self, step, mocker):
- """Test that the step fails if the docker import of the tar fails."""
- mock_docker_client = mocker.MagicMock()
- mock_docker_client.api.import_image_from_file.return_value = "bad response"
- mocker.patch.object(common.docker, "from_env", return_value=mock_docker_client)
- result = await step.run()
- assert result.status is StepStatus.FAILURE
- assert "Failed to import the connector image" in result.stderr
diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/dummy_build_customization.py b/airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py
similarity index 100%
rename from airbyte-ci/connectors/pipelines/tests/test_build_image/dummy_build_customization.py
rename to airbyte-ci/connectors/pipelines/tests/test_builds/dummy_build_customization.py
diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py
similarity index 80%
rename from airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py
rename to airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py
index bb8ac23a10ea0b..96d6fc79807ef9 100644
--- a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py
+++ b/airbyte-ci/connectors/pipelines/tests/test_builds/test_python_connectors.py
@@ -7,7 +7,6 @@
import pytest
from pipelines.airbyte_ci.connectors.build_image.steps import build_customization, python_connectors
from pipelines.airbyte_ci.connectors.context import ConnectorContext
-from pipelines.consts import BUILD_PLATFORMS
from pipelines.models.steps import StepStatus
pytestmark = [
@@ -16,13 +15,9 @@
class TestBuildConnectorImage:
- @pytest.fixture
- def all_platforms(self):
- return BUILD_PLATFORMS
-
@pytest.fixture
def test_context(self, mocker):
- return mocker.Mock(secrets_to_mask=[], targeted_platforms=BUILD_PLATFORMS)
+ return mocker.Mock(secrets_to_mask=[])
@pytest.fixture
def test_context_with_connector_with_base_image(self, test_context):
@@ -50,9 +45,7 @@ def connector_with_base_image_with_build_customization(self, connector_with_base
(connector_with_base_image_no_build_customization.code_directory / "build_customization.py").unlink()
@pytest.fixture
- def test_context_with_real_connector_using_base_image(
- self, connector_with_base_image_no_build_customization, dagger_client, current_platform
- ):
+ def test_context_with_real_connector_using_base_image(self, connector_with_base_image_no_build_customization, dagger_client):
context = ConnectorContext(
pipeline_name="test build",
connector=connector_with_base_image_no_build_customization,
@@ -61,14 +54,13 @@ def test_context_with_real_connector_using_base_image(
report_output_prefix="test",
is_local=True,
use_remote_secrets=True,
- targeted_platforms=[current_platform],
)
context.dagger_client = dagger_client
return context
@pytest.fixture
def test_context_with_real_connector_using_base_image_with_build_customization(
- self, connector_with_base_image_with_build_customization, dagger_client, current_platform
+ self, connector_with_base_image_with_build_customization, dagger_client
):
context = ConnectorContext(
pipeline_name="test build",
@@ -78,7 +70,6 @@ def test_context_with_real_connector_using_base_image_with_build_customization(
report_output_prefix="test",
is_local=True,
use_remote_secrets=True,
- targeted_platforms=[current_platform],
)
context.dagger_client = dagger_client
return context
@@ -91,7 +82,7 @@ def connector_without_base_image(self, all_connectors):
pytest.skip("No connector without a connectorBuildOptions.baseImage metadata found")
@pytest.fixture
- def test_context_with_real_connector_without_base_image(self, connector_without_base_image, dagger_client, current_platform):
+ def test_context_with_real_connector_without_base_image(self, connector_without_base_image, dagger_client):
context = ConnectorContext(
pipeline_name="test build",
connector=connector_without_base_image,
@@ -100,28 +91,25 @@ def test_context_with_real_connector_without_base_image(self, connector_without_
report_output_prefix="test",
is_local=True,
use_remote_secrets=True,
- targeted_platforms=[current_platform],
)
context.dagger_client = dagger_client
return context
- async def test__run_using_base_image_with_mocks(self, mocker, test_context_with_connector_with_base_image, all_platforms):
+ async def test__run_using_base_image_with_mocks(self, mocker, test_context_with_connector_with_base_image, current_platform):
container_built_from_base = mocker.AsyncMock()
mocker.patch.object(
python_connectors.BuildConnectorImages, "_build_from_base_image", mocker.AsyncMock(return_value=container_built_from_base)
)
mocker.patch.object(python_connectors.BuildConnectorImages, "get_step_result", mocker.AsyncMock())
- step = python_connectors.BuildConnectorImages(test_context_with_connector_with_base_image)
+ step = python_connectors.BuildConnectorImages(test_context_with_connector_with_base_image, current_platform)
step_result = await step._run()
- assert step._build_from_base_image.call_count == len(all_platforms)
- container_built_from_base.with_exec.assert_called_with(["spec"])
+ step._build_from_base_image.assert_called_once()
+ container_built_from_base.with_exec.assert_called_once_with(["spec"])
assert step_result.status is StepStatus.SUCCESS
- for platform in all_platforms:
- assert step_result.output_artifact[platform] == container_built_from_base
+ assert step_result.output_artifact[current_platform] == container_built_from_base
- @pytest.mark.slow
async def test_building_from_base_image_for_real(self, test_context_with_real_connector_using_base_image, current_platform):
- step = python_connectors.BuildConnectorImages(test_context_with_real_connector_using_base_image)
+ step = python_connectors.BuildConnectorImages(test_context_with_real_connector_using_base_image, current_platform)
step_result = await step._run()
step_result.status is StepStatus.SUCCESS
built_container = step_result.output_artifact[current_platform]
@@ -139,31 +127,31 @@ async def test_building_from_base_image_for_real(self, test_context_with_real_co
== test_context_with_real_connector_using_base_image.connector.metadata["dockerRepository"]
)
- @pytest.mark.slow
async def test_building_from_base_image_with_customization_for_real(
self, test_context_with_real_connector_using_base_image_with_build_customization, current_platform
):
- step = python_connectors.BuildConnectorImages(test_context_with_real_connector_using_base_image_with_build_customization)
+ step = python_connectors.BuildConnectorImages(
+ test_context_with_real_connector_using_base_image_with_build_customization, current_platform
+ )
step_result = await step._run()
step_result.status is StepStatus.SUCCESS
built_container = step_result.output_artifact[current_platform]
assert await built_container.env_variable("MY_PRE_BUILD_ENV_VAR") == "my_pre_build_env_var_value"
assert await built_container.env_variable("MY_POST_BUILD_ENV_VAR") == "my_post_build_env_var_value"
- async def test__run_using_base_dockerfile_with_mocks(self, mocker, test_context_with_connector_without_base_image, all_platforms):
+ async def test__run_using_base_dockerfile_with_mocks(self, mocker, test_context_with_connector_without_base_image, current_platform):
container_built_from_dockerfile = mocker.AsyncMock()
mocker.patch.object(
python_connectors.BuildConnectorImages, "_build_from_dockerfile", mocker.AsyncMock(return_value=container_built_from_dockerfile)
)
- step = python_connectors.BuildConnectorImages(test_context_with_connector_without_base_image)
+ step = python_connectors.BuildConnectorImages(test_context_with_connector_without_base_image, current_platform)
step_result = await step._run()
- assert step._build_from_dockerfile.call_count == len(all_platforms)
- container_built_from_dockerfile.with_exec.assert_called_with(["spec"])
+ step._build_from_dockerfile.assert_called_once()
+ container_built_from_dockerfile.with_exec.assert_called_once_with(["spec"])
assert step_result.status is StepStatus.SUCCESS
- for platform in all_platforms:
- assert step_result.output_artifact[platform] == container_built_from_dockerfile
+ assert step_result.output_artifact[current_platform] == container_built_from_dockerfile
- async def test_building_from_dockerfile_for_real(self, test_context_with_real_connector_without_base_image):
- step = python_connectors.BuildConnectorImages(test_context_with_real_connector_without_base_image)
+ async def test_building_from_dockerfile_for_real(self, test_context_with_real_connector_without_base_image, current_platform):
+ step = python_connectors.BuildConnectorImages(test_context_with_real_connector_without_base_image, current_platform)
step_result = await step._run()
step_result.status is StepStatus.SUCCESS
diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/__init__.py
deleted file mode 100644
index c941b30457953b..00000000000000
--- a/airbyte-ci/connectors/pipelines/tests/test_helpers/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py
index da63c33fb01cda..8bcde17e715e05 100644
--- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py
+++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py
@@ -28,7 +28,7 @@ def certified_connector_with_setup(self, all_connectors):
pytest.skip("No certified connector with setup.py found.")
@pytest.fixture
- def context_for_certified_connector_with_setup(self, certified_connector_with_setup, dagger_client, current_platform):
+ def context_for_certified_connector_with_setup(self, certified_connector_with_setup, dagger_client):
context = ConnectorContext(
pipeline_name="test unit tests",
connector=certified_connector_with_setup,
@@ -37,7 +37,6 @@ def context_for_certified_connector_with_setup(self, certified_connector_with_se
report_output_prefix="test",
is_local=True,
use_remote_secrets=True,
- targeted_platforms=[current_platform],
)
context.dagger_client = dagger_client
context.connector_secrets = {}
@@ -45,11 +44,11 @@ def context_for_certified_connector_with_setup(self, certified_connector_with_se
@pytest.fixture
async def certified_container_with_setup(self, context_for_certified_connector_with_setup, current_platform):
- result = await BuildConnectorImages(context_for_certified_connector_with_setup).run()
+ result = await BuildConnectorImages(context_for_certified_connector_with_setup, current_platform).run()
return result.output_artifact[current_platform]
@pytest.fixture
- def context_for_connector_with_poetry(self, connector_with_poetry, dagger_client, current_platform):
+ def context_for_connector_with_poetry(self, connector_with_poetry, dagger_client):
context = ConnectorContext(
pipeline_name="test unit tests",
connector=connector_with_poetry,
@@ -58,7 +57,6 @@ def context_for_connector_with_poetry(self, connector_with_poetry, dagger_client
report_output_prefix="test",
is_local=True,
use_remote_secrets=True,
- targeted_platforms=[current_platform],
)
context.dagger_client = dagger_client
context.connector_secrets = {}
@@ -66,7 +64,7 @@ def context_for_connector_with_poetry(self, connector_with_poetry, dagger_client
@pytest.fixture
async def container_with_poetry(self, context_for_connector_with_poetry, current_platform):
- result = await BuildConnectorImages(context_for_connector_with_poetry).run()
+ result = await BuildConnectorImages(context_for_connector_with_poetry, current_platform).run()
return result.output_artifact[current_platform]
async def test__run_for_setup_py(self, context_for_certified_connector_with_setup, certified_container_with_setup):
diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_utils.py
similarity index 80%
rename from airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py
rename to airbyte-ci/connectors/pipelines/tests/test_utils.py
index 31c6434da79705..9d9328f384178b 100644
--- a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py
+++ b/airbyte-ci/connectors/pipelines/tests/test_utils.py
@@ -5,7 +5,6 @@
from pathlib import Path
from unittest import mock
-import dagger
import pytest
from connector_ops.utils import Connector, ConnectorLanguage
from pipelines import consts
@@ -191,48 +190,3 @@ def test_sh_dash_c():
assert utils.sh_dash_c(["foo", "bar"]) == ["sh", "-c", "set -o xtrace && foo && bar"]
assert utils.sh_dash_c(["foo"]) == ["sh", "-c", "set -o xtrace && foo"]
assert utils.sh_dash_c([]) == ["sh", "-c", "set -o xtrace"]
-
-
-@pytest.mark.anyio
-@pytest.mark.parametrize("tar_file_name", [None, "custom_tar_name.tar"])
-async def test_export_containers_to_tarball(mocker, dagger_client, tmp_path, tar_file_name):
- context = mocker.Mock(
- dagger_client=dagger_client,
- connector=mocker.Mock(technical_name="my_connector"),
- host_image_export_dir_path=tmp_path,
- git_revision="my_git_revision",
- )
- container_variants = [
- dagger_client.container(platform=dagger.Platform("linux/arm64")).from_("bash:latest"),
- dagger_client.container(platform=dagger.Platform("linux/amd64")).from_("bash:latest"),
- ]
- expected_tar_file_path = tmp_path / "my_connector_my_git_revision.tar" if tar_file_name is None else tmp_path / tar_file_name
- exported_tar_file, exported_tar_file_path = await utils.export_containers_to_tarball(
- context, container_variants, tar_file_name=tar_file_name
- )
- assert exported_tar_file_path == expected_tar_file_path
- assert await exported_tar_file.size() == expected_tar_file_path.stat().st_size
-
-
-@pytest.mark.anyio
-async def test_export_containers_to_tarball_failure(mocker, tmp_path):
- mock_dagger_client = mocker.Mock()
- mock_export = mocker.AsyncMock(return_value=False)
- mock_dagger_client.container.return_value.export = mock_export
-
- context = mocker.Mock(
- dagger_client=mock_dagger_client,
- connector=mocker.Mock(technical_name="my_connector"),
- host_image_export_dir_path=tmp_path,
- git_revision="my_git_revision",
- )
-
- container_variants = mocker.Mock()
- exported_tar_file, exported_tar_file_path = await utils.export_containers_to_tarball(context, container_variants)
- mock_export.assert_called_once_with(
- str(tmp_path / "my_connector_my_git_revision.tar"),
- platform_variants=container_variants,
- forced_compression=dagger.ImageLayerCompression.Gzip,
- )
- assert exported_tar_file is None
- assert exported_tar_file_path is None