airbyte-lib: Enable tests (#34381)

Co-authored-by: Aaron ("AJ") Steers <aj@airbyte.io>
Co-authored-by: alafanechere <augustin.lafanechere@gmail.com>

.github/workflows/airbyte-ci-tests.yml

@@ -165,5 +165,5 @@ jobs:
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
-          subcommand: "test airbyte-lib --pass-env-var=GCP_GSM_CREDENTIALS --poetry-run-command='pytest'"
+          subcommand: "test airbyte-lib --side-car-docker-engine --pass-env-var=GCP_GSM_CREDENTIALS --poetry-run-command='pytest'"
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}

airbyte-ci/connectors/ci_credentials/pyproject.toml

@@ -17,7 +17,7 @@ click = "^8.1.3"
 pyyaml = "^6.0"
 common_utils = { path = "../common_utils", develop = true }
 
-[tool.poetry.group.test.dependencies]
+[tool.poetry.group.dev.dependencies]
 requests-mock = "^1.10.0"
 pytest = "^7.3.1"
 

airbyte-ci/connectors/common_utils/pyproject.toml

@@ -14,11 +14,10 @@ cryptography = "^3.4.7"
 requests = "^2.28.2"
 pyjwt = "^2.1.0"
 
-[tool.poetry.group.test.dependencies]
-pytest = "^7.2.2"
 
 [tool.poetry.group.dev.dependencies]
 requests-mock = "^1.9.3"
+pytest = "^7.2.2"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

airbyte-ci/connectors/connector_ops/pyproject.toml

@@ -23,7 +23,7 @@ ci-credentials = {path = "../ci_credentials"}
 pandas = "^2.0.3"
 simpleeval = "^0.9.13"
 
-[tool.poetry.group.test.dependencies]
+[tool.poetry.group.dev.dependencies]
 pytest = "^7.4.0"
 pytest-mock = "^3.10.0"
 freezegun = "^1.1.0"

airbyte-ci/connectors/metadata_service/lib/pyproject.toml

@@ -18,11 +18,8 @@ pydash = "^6.0.2"
 semver = "^3.0.1"
 
 
-[tool.poetry.group.test.dependencies]
-pytest = "^7.2.2"
-
-
 [tool.poetry.group.dev.dependencies]
+pytest = "^7.2.2"
 datamodel-code-generator = "^0.17.1"
 pytest-mock = "^3.10.0"
 poethepoet = "^0.20.0"

airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml

@@ -34,14 +34,10 @@ pendulum = "<3.0.0"
 
 
 [tool.poetry.group.dev.dependencies]
+pytest = "^7.2.2"
 ptpython = "^3.0.23"
 pdbpp = "^0.10.3"
 
-
-[tool.poetry.group.test.dependencies]
-pytest = "^7.2.2"
-
-
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

airbyte-ci/connectors/pipelines/README.md

@@ -547,6 +547,7 @@ E.G.: running `pytest` on a specific test folder:
 
 | Version | PR                                                         | Description                                                                                                       |
 | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| 3.5.2   | [#34381](https://github.com/airbytehq/airbyte/pull/34381)  | Bind a sidecar docker host for `airbyte-ci test`                                                                  |
 | 3.5.1   | [#34321](https://github.com/airbytehq/airbyte/pull/34321)  | Upgrade to Dagger 0.9.6 .                                                                                         |
 | 3.5.0   | [#33313](https://github.com/airbytehq/airbyte/pull/33313)  | Pass extra params after Gradle tasks.                                                                             |
 | 3.4.2   | [#34301](https://github.com/airbytehq/airbyte/pull/34301)  | Pass extra params after Gradle tasks.                                                                             |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py

@@ -11,7 +11,8 @@
 import asyncclick as click
 import asyncer
 from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj
-from pipelines.consts import DOCKER_VERSION
+from pipelines.consts import DOCKER_HOST_NAME, DOCKER_HOST_PORT, DOCKER_VERSION
+from pipelines.dagger.actions.system import docker
 from pipelines.helpers.utils import sh_dash_c
 from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context
 
@@ -61,6 +62,9 @@ def validate_env_vars_exist(_ctx: dict, _param: dict, value: List[str]) -> List[
     required=False,
     callback=validate_env_vars_exist,
 )
+@click.option(
+    "--side-car-docker-engine", help="Run a docker engine side car bound to poetry container.", default=False, type=bool, is_flag=True
+)
 @click_merge_args_into_context_obj
 @pass_pipeline_context
 @click_ignore_unused_kwargs
@@ -96,7 +100,15 @@ async def test(pipeline_context: ClickPipelineContext) -> None:
     directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount]))
 
     pipeline_name = f"Unit tests for {poetry_package_path}"
+
     dagger_client = await pipeline_context.get_dagger_client(pipeline_name=pipeline_name)
+
+    dockerd_service = None
+    if pipeline_context.params["side_car_docker_engine"]:
+        dockerd_service = docker.with_global_dockerd_service(dagger_client)
+
+        await dockerd_service.start()
+
     test_container = await (
         dagger_client.container()
         .from_("python:3.10.12")
@@ -124,11 +136,19 @@ async def test(pipeline_context: ClickPipelineContext) -> None:
         )
         .with_workdir(f"/airbyte/{poetry_package_path}")
         .with_exec(["poetry", "install", "--with=dev"])
-        .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock"))
         .with_env_variable("CI", str(pipeline_context.params["is_ci"]))
         .with_workdir(f"/airbyte/{poetry_package_path}")
     )
 
+    if dockerd_service:
+        test_container = (
+            test_container.with_env_variable("DOCKER_HOST", f"tcp://{DOCKER_HOST_NAME}:{DOCKER_HOST_PORT}")
+            .with_env_variable("DOCKER_HOST_NAME", DOCKER_HOST_NAME)
+            .with_service_binding(DOCKER_HOST_NAME, dockerd_service)
+        )
+    else:
+        test_container = test_container.with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock"))
+
     # register passed env vars as secrets and add them to the container
     for var in pipeline_context.params["passed_env_vars"]:
         secret = dagger_client.set_secret(var, os.environ[var])
@@ -141,6 +161,8 @@ async def test(pipeline_context: ClickPipelineContext) -> None:
             soon_command_execution_result = poetry_commands_task_group.soonify(run_poetry_command)(test_container, command)
             soon_command_executions_results.append(soon_command_execution_result)
 
+    if dockerd_service:
+        await dockerd_service.stop()
     for result in soon_command_executions_results:
         stdout, stderr = result.value
         logger.info(stdout)

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "3.5.1"
+version = "3.5.2"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <contact@airbyte.io>"]
 
@@ -28,11 +28,6 @@ pygit2 = "^1.13.1"
 asyncclick = "^8.1.3.4"
 certifi = "^2023.11.17"
 
-[tool.poetry.group.test.dependencies]
-pytest = "^6.2.5"
-pytest-mock = "^3.10.0"
-
-
 [tool.poetry.group.dev.dependencies]
 freezegun = "^1.2.2"
 pytest-cov = "^4.1.0"

airbyte-ci/connectors/qa-engine/pyproject.toml

@@ -29,12 +29,10 @@ ruamel-yaml = "^0.17.30"
 connector-ops = {path = "../connector_ops"}
 
 [tool.poetry.group.dev.dependencies]
-pyinstrument = "*"
-
-[tool.poetry.group.test.dependencies]
 pytest = "~6.2.5"
 pytest-mock = "~3.10.0"
 freezegun = "*"
+pyinstrument = "*"
 
 [tool.poetry.scripts]
 run-qa-engine = "qa_engine.main:main"

airbyte-lib/airbyte_lib/config.py

@@ -9,5 +9,5 @@
 
 class CacheConfigBase(
     BaseModel
-):  # TODO: meta=EnforceOverrides (Pydantic doesn't like it currently)
+):  # TODO: meta=EnforceOverrides (Pydantic doesn't like it currently.)
     pass

airbyte-lib/tests/conftest.py

@@ -7,13 +7,13 @@
 import os
 import socket
 import time
-from typing import Optional
 from airbyte_lib.caches.snowflake import SnowflakeCacheConfig
 
 import docker
 import psycopg
 import pytest
 from google.cloud import secretmanager
+from pytest_docker.plugin import get_docker_ip
 
 from airbyte_lib.caches import PostgresCacheConfig
 
@@ -47,7 +47,7 @@ def remove_postgres_container():
 def test_pg_connection(host) -> bool:
     pg_url = f"postgresql://postgres:postgres@{host}:{PYTEST_POSTGRES_PORT}/postgres"
 
-    max_attempts = 10
+    max_attempts = 120
     for attempt in range(max_attempts):
         try:
             conn = psycopg.connect(pg_url)
@@ -71,23 +71,44 @@ def pg_dsn():
         # if the image needs to download on-demand.
         client.images.pull(PYTEST_POSTGRES_IMAGE)
 
+    try:
+        previous_container = client.containers.get(PYTEST_POSTGRES_CONTAINER)
+        previous_container.remove()
+    except docker.errors.NotFound:
+        pass
+
+    postgres_is_running = False
     postgres = client.containers.run(
         image=PYTEST_POSTGRES_IMAGE,
         name=PYTEST_POSTGRES_CONTAINER,
         environment={"POSTGRES_USER": "postgres", "POSTGRES_PASSWORD": "postgres", "POSTGRES_DB": "postgres"},
         ports={"5432/tcp": PYTEST_POSTGRES_PORT},
         detach=True,
     )
-    time.sleep(0.5)
+
+    attempts = 10
+    while not postgres_is_running and attempts > 0:
+        try:
+            postgres.reload()
+            postgres_is_running = postgres.status == "running"
+        except docker.errors.NotFound:
+            attempts -= 1
+            time.sleep(3)
+    if not postgres_is_running:
+        raise Exception(f"Failed to start the PostgreSQL container. Status: {postgres.status}.")
 
     final_host = None
-    # Try to connect to the database using localhost and the docker host IP
-    for host in ["localhost", "172.17.0.1"]:
-        if test_pg_connection(host):
-            final_host = host
-            break
+    if host := os.environ.get("DOCKER_HOST_NAME"):
+        final_host = host if test_pg_connection(host) else None
     else:
-        raise Exception("Failed to connect to the PostgreSQL database.")
+    # Try to connect to the database using localhost and the docker host IP
+        for host in ["127.0.0.1", "localhost", "host.docker.internal", "172.17.0.1"]:
+            if test_pg_connection(host):
+                final_host = host
+                break
+
+    if final_host is None:
+        raise Exception(f"Failed to connect to the PostgreSQL database on host {host}.")
 
     yield final_host
     # Stop and remove the container after the tests are done