add AirbyteLib devcontainer config

.devcontainer/airbyte-lib/devcontainer.json

@@ -0,0 +1,73 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+  "name": "AirbyteLib DevContainer (Python)",
+
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "image": "mcr.microsoft.com/devcontainers/python:0-3.10",
+
+  // Features to add to the dev container. More info: https://containers.dev/features.
+  "features": {
+    "ghcr.io/devcontainers-contrib/features/poetry:2": {},
+    "ghcr.io/devcontainers/features/docker-in-docker": {},
+    "ghcr.io/devcontainers/features/python:1": {
+      "installJupyterlab": true
+    }
+  },
+
+  "overrideFeatureInstallOrder": [
+    // Deterministic order maximizes cache reuse
+    "ghcr.io/devcontainers-contrib/features/poetry",
+    "ghcr.io/devcontainers/features/docker-in-docker",
+    "ghcr.io/devcontainers/features/python"
+  ],
+  // "workspaceFolder": "/workspaces/airbyte/airbyte-lib",
+
+  // Configure tool-specific properties.
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        // Python extensions:
+        "charliermarsh.ruff",
+        "matangover.mypy",
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        // Toml support
+        "tamasfe.even-better-toml",
+        // Yaml and JSON Schema support:
+        "redhat.vscode-yaml",
+        // Contributing:
+        "GitHub.vscode-pull-request-github"
+      ],
+      "settings": {
+        "extensions.ignoreRecommendations": true,
+        "git.autofetch": true,
+        "git.openRepositoryInParentFolders": "always",
+        "python.defaultInterpreterPath": "/workspaces/airbyte/airbyte-lib/.venv/bin/python",
+        "python.interpreter.infoVisibility": "always",
+        "python.terminal.activateEnvironment": true,
+        "python.testing.pytestEnabled": true,
+        "python.testing.cwd": "/workspaces/airbyte/airbyte-lib",
+        "python.testing.pytestArgs": [
+          "--rootdir=/workspaces/airbyte/airbyte-lib",
+          "."
+        ],
+        "[python]": {
+          "editor.defaultFormatter": "charliermarsh.ruff"
+        }
+      }
+    }
+  },
+  "containerEnv": {
+    "POETRY_VIRTUALENVS_IN_PROJECT": "true"
+  },
+
+  // Mark the root directory as 'safe' for git.
+  "initializeCommand": "git config --add safe.directory /workspaces/airbyte",
+  // Use 'postCreateCommand' to run commands after the container is created.
+  "postCreateCommand": "python -m pip install jupyter notebook -U && cd airbyte-lib && poetry install"
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+  // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "root"
+}

.github/workflows/airbyte-ci-tests.yml

@@ -61,6 +61,9 @@ jobs:
               - 'airbyte-ci/connectors/metadata_service/lib/**'
               - 'airbyte-ci/connectors/metadata_service/orchestrator/**'
               - '!**/*.md'
+            airbyte_lib:
+              - 'airbyte_lib/**'
+              - '!**/*.md'
 
       - name: Run airbyte-ci/connectors/connector_ops tests
         if: steps.changes.outputs.ops_any_changed == 'true'
@@ -132,3 +135,18 @@ jobs:
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
           tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
+
+      - name: Run airbyte-lib tests
+        if: steps.changes.outputs.airbyte_lib_any_changed == 'true'
+        id: run-airbyte-lib-tests
+        uses: ./.github/actions/run-dagger-pipeline
+        with:
+          context: "pull_request"
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+          subcommand: "test airbyte-lib"
+          airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url || 'https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci' }}
+          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}

airbyte-lib/.gitignore

@@ -0,0 +1 @@
+.venv*

airbyte-lib/README.md

@@ -0,0 +1,10 @@
+# airbyte-lib
+
+airbyte-lib is a library that allows to run Airbyte syncs embedded into any Python application, without the need to run Airbyte server.
+
+## Development
+
+* Make sure [Poetry is installed](https://python-poetry.org/docs/#).
+* Run `poetry install`
+* For examples, check out the `examples` folder. They can be run via `poetry run python examples/<example file>`
+* Unit tests and type checks can be run via `poetry run pytest`

airbyte-lib/airbyte_lib/__init__.py

@@ -0,0 +1,13 @@
+
+from .factories import (get_connector, get_default_cache, new_local_cache)
+from .sync_results import (Dataset, SyncResult)
+from .source import (Source)
+
+__all__ = [
+    "get_connector",
+    "get_default_cache",
+    "new_local_cache",
+    "Dataset",
+    "SyncResult",
+    "Source",
+]

airbyte-lib/airbyte_lib/_util.py

@@ -0,0 +1,66 @@
+"""Internal utility functions, especially for dealing with Airbyte Protocol."""
+
+import datetime
+from functools import lru_cache
+from typing import Any, cast
+from collections.abc import Iterable
+
+from airbyte_protocol.models import (
+    AirbyteMessage,
+    AirbyteRecordMessage,
+    ConfiguredAirbyteCatalog,
+    ConfiguredAirbyteStream,
+    Type,
+)
+
+
+def airbyte_messages_to_record_dicts(messages: Iterable[AirbyteMessage]) -> Iterable[dict[str, Any]]:
+    """Convert an AirbyteMessage to a dictionary."""
+    yield from (
+        cast(dict[str, Any], airbyte_message_to_record_dict(message))
+        for message in messages
+        if message is not None
+    )
+
+
+def airbyte_message_to_record_dict(message: AirbyteMessage) -> dict[str, Any] | None:
+    """Convert an AirbyteMessage to a dictionary.
+
+    Return None if the message is not a record message.
+    """
+    if message.type != Type.RECORD:
+        return None
+
+    return airbyte_record_message_to_dict(message.record)
+
+def airbyte_record_message_to_dict(record_message: AirbyteRecordMessage) -> dict[str, Any] | None:
+    """Convert an AirbyteMessage to a dictionary.
+
+    Return None if the message is not a record message.
+    """
+    result = record_message.data
+
+    # TODO: Add the metadata columns (this breaks tests)
+    # result["_airbyte_extracted_at"] = datetime.datetime.fromtimestamp(
+    #     record_message.emitted_at
+    # )
+
+    return result
+
+
+def get_primary_keys_from_stream(
+    stream_name: str, configured_catalog: ConfiguredAirbyteCatalog
+) -> set[str]:
+    """Get the primary keys from a stream in the configured catalog."""
+    stream = next(
+        (
+            stream
+            for stream in configured_catalog.streams
+            if stream.stream.name == stream_name
+        ),
+        None,
+    )
+    if stream is None:
+        raise ValueError(f"Stream {stream_name} not found in catalog.")
+
+    return set(stream.stream.source_defined_primary_key or [])

airbyte-lib/airbyte_lib/caches/__init__.py

@@ -0,0 +1,20 @@
+"""Base module for all caches."""
+
+from airbyte_lib.caches.base import SQLCacheBase
+from airbyte_lib.caches.duckdb import DuckDBCache, DuckDBCacheConfig
+from airbyte_lib.caches.memory import InMemoryCache, InMemoryCacheConfig
+from airbyte_lib.caches.postgres import PostgresCache, PostgresCacheConfig
+from airbyte_lib.types import SQLTypeConverter
+
+
+# We export these classes for easy access: `airbyte_lib.caches...`
+__all__ = [
+    "DuckDBCache",
+    "DuckDBCacheConfig",
+    "InMemoryCache",
+    "InMemoryCacheConfig",
+    "PostgresCache",
+    "PostgresCacheConfig",
+    "SQLCacheBase",
+    "SQLTypeConverter",
+]