From 5d22a5bd8bdf2478250b88d1a812611daf1d2428 Mon Sep 17 00:00:00 2001 From: Harrison O'Connor-Hoover Date: Tue, 4 Aug 2026 06:32:45 -0400 Subject: [PATCH] Add Odoo JSON-2 connector slice --- HANDOFF.md | 42 ++--- README.md | 5 + connectors/odoo.example.yaml | 40 +++++ docs/decisions.md | 10 ++ docs/odoo.md | 42 +++++ models/staging/stg_odoo__partners.sql | 29 +++ models/staging/stg_odoo__partners.yml | 56 ++++++ src/dander/cli/main.py | 3 + src/dander/ingestion/__init__.py | 2 + src/dander/ingestion/enterprise.py | 204 ++++++++++++++++++---- src/dander/ingestion/source.py | 1 + tests/cli/test_cli.py | 38 +++- tests/ingestion/test_enterprise_source.py | 130 ++++++++++++++ 13 files changed, 549 insertions(+), 53 deletions(-) create mode 100644 connectors/odoo.example.yaml create mode 100644 docs/odoo.md create mode 100644 models/staging/stg_odoo__partners.sql create mode 100644 models/staging/stg_odoo__partners.yml diff --git a/HANDOFF.md b/HANDOFF.md index 6e32933..755adac 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,43 +2,43 @@ ## Finished -- Added a hand-rolled `netsuite_suiteql` customer source using the existing OAuth1 TBA, retry, raw-schema, watermark, and SCD1 runtime. -- Added a stateful six-operation FastAPI simulator with invented customers, real HMAC-SHA256 signature verification, offset paging, and named auth/throttle/permission/malformed failures. -- Proved stateful create, update, duplicate-free replay, monotonic watermarks, transport-link removal, and proof-fixture cleanup over loopback HTTP. -- Added the tracked OpenAPI contract, customer staging model, and explicit “simulator-validated, not NetSuite-validated” documentation. -- Preserved completed Odoo work unchanged on `codex/odoo-json2` at `14244cd`. +- Merged the simulator-validated NetSuite SuiteQL foundation through protected PR #52. +- Added a read-only Odoo 19+ JSON-2 `res.partner` source, connector template, staging model, and operator guide. +- Proved Odoo against ephemeral official Odoo/PostgreSQL containers, then removed them. +- Rebased the Odoo work onto the NetSuite-enabled `main` while preserving both enterprise sources. +- Preserved the original Odoo commit on `backup/odoo-json2-pre-netsuite`. ## Try It ```bash -uv sync --extra dev -uv run python -m dander.dev.netsuite_simulator -uv run pytest tests/integration/test_netsuite_simulator.py +cp connectors/odoo.example.yaml connectors/odoo.yaml +export ODOO_API_KEY='YOUR_API_KEY' +uv run dander run odoo --dry-run --sandbox --project YOUR_PROJECT ``` ## Checks -- Ruff lint and format passed; strict mypy passed for all 150 source files. -- All 664 tests passed. -- Wheel/sdist build and inspection passed; NetSuite contract/docs/fixtures are packaged. -- A source-free wheel installed with the documented `dev` extra and exposed all six simulator operations. -- No NetSuite tenant, GCP resource, Terraform state, public package, or remote branch was changed. +- Ruff lint/format and strict mypy passed; all 672 tests passed after the rebase. +- NetSuite, Odoo, Workday, and generic dlt source routing pass together through one adapter helper. +- The earlier Odoo live JSON-2 acceptance passed with bounded two-row pages, bearer auth, database routing, null normalization, and watermark replay. +- No Odoo deployment, retained-project change, version bump, or package publication occurred. ## Decisions -- SuiteQL replaces the old record-list example because the latter returns only IDs and links; the query is uniquely ordered by customer ID. -- The first slice is a full read with bounded pages and idempotent SCD1 replay; 100,000-row and concurrent-offset limits remain explicit. -- OAuth1 TBA is compatibility coverage only. Current OAuth2 and one real-tenant proof gate any supported future release. +- Odoo targets the current JSON-2 API; deprecated XML-RPC/JSON-RPC is not supported. +- The first slice is read-only `res.partner`; no retained-project Odoo pipeline was added. +- Salesforce remains memory-bounded but still rereads Accounts through synchronous QueryAll. ## Remaining -- Push `codex/netsuite-simulator` and open a focused PR only when requested. +- Push the rebased Odoo branch and open the focused PR. - Let protected CI repeat Linux tests, packaging, scans, and Terraform validation. -- Obtain an authorized NetSuite/SDN sandbox for the narrow acceptance in `docs/netsuite-simulator.md`. -- Keep the connector out of the supported `0.2.0` surface; consider `0.3.0` only after tenant acceptance. +- Treat Odoo offset paging during concurrent source mutation as a documented first-slice limit. +- Scope Salesforce Bulk API 2.0 plus server-filtered SOQL as separate scale work. +- Continue reviewing the daily operator soak in issue #26. ## Review First - `src/dander/ingestion/enterprise.py` -- `src/dander/dev/netsuite_simulator.py` -- `tests/integration/test_netsuite_simulator.py` +- `connectors/odoo.example.yaml` +- `docs/odoo.md` diff --git a/README.md b/README.md index f453d57..fca0152 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,11 @@ client credentials, primitive internal values, stable offset paging, and a decla The first slice performs a full read and idempotent SCD1 publication; it does not claim unsafe timestamp-plus-offset incrementality. See [`docs/servicenow.md`](docs/servicenow.md). +`connectors/odoo.example.yaml` reads Odoo 19+ contacts and companies through the current JSON-2 +API using a bearer API key, bounded pages, and an inclusive `write_date` watermark. Odoo Online +requires a Custom plan for external API access; the official Odoo Community Docker image provides +a free local development target. See [`docs/odoo.md`](docs/odoo.md). + `connectors/netsuite.example.yaml` is a **simulator-validated, not NetSuite-validated** customer SuiteQL slice. It uses bounded offset paging, stable ordering, declared schemas, and the existing OAuth1 TBA signer. It is not part of the public `0.2.0` support surface; real-tenant acceptance and diff --git a/connectors/odoo.example.yaml b/connectors/odoo.example.yaml new file mode 100644 index 0000000..f6fc0e2 --- /dev/null +++ b/connectors/odoo.example.yaml @@ -0,0 +1,40 @@ +# Read-only Odoo 19+ JSON-2 connector. The API key remains a secret reference; for an Odoo host +# that serves only one database, remove auth_options.database and the X-Odoo-Database header is +# omitted. Odoo Online requires a Custom plan for external API access. Odoo Community can be run +# locally with the official Odoo and PostgreSQL images for no-license-cost development. +name: odoo +engine: odoo_json2 +base_url: http://localhost:8069 +auth_strategy: api_key_bearer +auth_ref: ODOO_API_KEY +auth_options: + database: dander_odoo +rate_limit: + requests_per_second: 4 + burst: 1 + backoff: exponential + max_retries: 4 + +endpoints: + - name: partners + path: /json/2/res.partner/search_read + pagination: + kind: offset + page_size: 100 + primary_key: [id] + incremental_cursor: write_date + field_types: + id: INT64 + active: BOOL + is_company: BOOL + raw_schema: + - {name: id, type: INT64, mode: REQUIRED} + - {name: name, type: STRING} + - {name: email, type: STRING} + - {name: phone, type: STRING} + - {name: city, type: STRING} + - {name: country_code, type: STRING} + - {name: is_company, type: BOOL, mode: REQUIRED} + - {name: active, type: BOOL, mode: REQUIRED} + - {name: create_date, type: STRING} + - {name: write_date, type: STRING, mode: REQUIRED} diff --git a/docs/decisions.md b/docs/decisions.md index f419806..77991a8 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -393,3 +393,13 @@ - OAuth1 TBA is retained only to exercise Dander's existing compatibility strategy. Oracle's announced 2027.1 restriction on new TBA REST integrations makes current OAuth2 acceptance a release gate; until a real tenant passes, the connector is not NetSuite-validated or supported. + +## 2026-08-04 — Odoo starts on JSON-2 against Community + +- Odoo 19+'s JSON-2 API is the connector contract. New work does not depend on the deprecated + XML-RPC/JSON-RPC endpoints. +- The first vertical slice reads only `res.partner` through the existing Source/runtime/writer + boundary, using API-key bearer auth, bounded offset pages, and declared raw fields. +- Official Odoo Community and PostgreSQL containers are the free acceptance target because Odoo + Online exposes its external API only on the Custom plan. Concurrently mutating large tables + need later snapshot/keyset paging before this slice is described as scale-ready. diff --git a/docs/odoo.md b/docs/odoo.md new file mode 100644 index 0000000..dcfc8e8 --- /dev/null +++ b/docs/odoo.md @@ -0,0 +1,42 @@ +# Odoo JSON-2 connector + +Dander's first Odoo slice reads contacts and companies (`res.partner`) through the Odoo 19+ +JSON-2 API. It uses bearer API-key authentication, bounded offset pages, declared raw fields, +idempotent SCD1 publication, and an inclusive `write_date` watermark boundary. + +## Free development environment + +Odoo Online's free and Standard plans do not expose the external API. For development without an +Odoo license fee, run Odoo 19 Community and PostgreSQL locally using the official Docker images. +The official image instructions show the required PostgreSQL service and Odoo port `8069`. + +After creating a database and admin user, create an API key under **Preferences → Account +Security → New API Key**. Copy `connectors/odoo.example.yaml` to `connectors/odoo.yaml`, set the +database name, and expose only the key to Dander: + +```bash +cp connectors/odoo.example.yaml connectors/odoo.yaml +read -rs ODOO_API_KEY && printf '\n' +export ODOO_API_KEY +uv run dander run odoo --dry-run --sandbox --project YOUR_NO_BILLING_GCP_PROJECT +uv run dander run odoo --sandbox --project YOUR_NO_BILLING_GCP_PROJECT \ + --build-models --select-model stg_odoo__partners +``` + +For hosted execution, store the key in Secret Manager and map `ODOO_API_KEY` to that secret in the +pipeline manifest. Never commit the key or a populated local connector. + +## Current boundary + +This is intentionally one read-only model. It does not write to Odoo, discover arbitrary models, +or support Odoo 18's deprecated XML-RPC/JSON-RPC APIs. Offset pages are ordered by immutable Odoo +record ID. The inclusive `write_date` boundary may replay tied rows; Dander's SCD1 writer makes +that safe. Concurrent source mutations can still shift membership between offset pages, so this +first slice is intended for evaluation and modest tables until snapshot/keyset paging is added. +API keys expire and must be rotated according to the Odoo account's policy. + +References: + +- [Odoo 19 external JSON-2 API](https://www.odoo.com/documentation/19.0/developer/reference/external_api.html) +- [Official Odoo Docker image](https://hub.docker.com/_/odoo) +- [Odoo pricing](https://www.odoo.com/pricing) diff --git a/models/staging/stg_odoo__partners.sql b/models/staging/stg_odoo__partners.sql new file mode 100644 index 0000000..cf97ec1 --- /dev/null +++ b/models/staging/stg_odoo__partners.sql @@ -0,0 +1,29 @@ +-- Purpose: normalize Odoo contacts and companies for CRM analysis. +-- Grain: one row per Odoo res.partner id. +WITH source AS ( + SELECT + id AS partner_id, + name AS partner_name, + email, + phone, + city, + country_code, + is_company, + active AS is_active, + SAFE.PARSE_TIMESTAMP('%F %H:%M:%S', create_date) AS created_at, + PARSE_TIMESTAMP('%F %H:%M:%S', write_date) AS updated_at + FROM {{ ref('raw_odoo_partners') }} +) + +SELECT + partner_id, + partner_name, + email, + phone, + city, + country_code, + is_company, + is_active, + created_at, + updated_at +FROM source diff --git a/models/staging/stg_odoo__partners.yml b/models/staging/stg_odoo__partners.yml new file mode 100644 index 0000000..78a05cb --- /dev/null +++ b/models/staging/stg_odoo__partners.yml @@ -0,0 +1,56 @@ +model: stg_odoo__partners +description: Odoo contacts and companies normalized from the read-only JSON-2 contract. +owner: data-eng +materialization: incremental +dataset: staging +source_system: odoo +sensitivity: confidential +unique_key: [partner_id] +incremental_cursor: updated_at + +columns: + - name: partner_id + type: INT64 + description: Stable Odoo res.partner identifier. + - name: partner_name + type: STRING + description: Contact or company display name. + - name: email + type: STRING + description: Primary email address when present. + - name: phone + type: STRING + description: Primary telephone number when present. + - name: city + type: STRING + description: Contact city when present. + - name: country_code + type: STRING + description: Contact country code when present. + - name: is_company + type: BOOL + description: Whether this partner represents a company. + - name: is_active + type: BOOL + description: Whether this partner is active. + - name: created_at + type: TIMESTAMP + description: UTC record creation time when available. + - name: updated_at + type: TIMESTAMP + description: UTC record modification watermark. + +tests: + - column: partner_id + not_null: true + unique: true + - column: is_company + not_null: true + - column: is_active + not_null: true + +metrics: + - name: partner_count + description: Number of Odoo contacts and companies represented. + aggregation: count_distinct + field: partner_id diff --git a/src/dander/cli/main.py b/src/dander/cli/main.py index fed9b22..5705f25 100644 --- a/src/dander/cli/main.py +++ b/src/dander/cli/main.py @@ -47,6 +47,7 @@ Endpoint, IngestionEngine, NetSuiteSuiteQLSource, + OdooJson2Source, Source, SourceConfig, WorkdayRaasSource, @@ -1675,6 +1676,8 @@ def _build_source_adapter(config: SourceConfig, auth: AuthStrategy) -> Source: return WorkdayRaasSource(config, auth) if config.engine is IngestionEngine.NETSUITE_SUITEQL: return NetSuiteSuiteQLSource(config, auth) + if config.engine is IngestionEngine.ODOO_JSON2: + return OdooJson2Source(config, auth) return DltRestSource(config, auth) diff --git a/src/dander/ingestion/__init__.py b/src/dander/ingestion/__init__.py index fc04650..79a8c52 100644 --- a/src/dander/ingestion/__init__.py +++ b/src/dander/ingestion/__init__.py @@ -9,6 +9,7 @@ EnterpriseSource, EnterpriseSourceError, NetSuiteSuiteQLSource, + OdooJson2Source, WorkdayRaasSource, ) from dander.ingestion.pagination import ( @@ -37,6 +38,7 @@ "NoPagination", "NetSuiteSuiteQLSource", "OffsetPagination", + "OdooJson2Source", "PageNumberPagination", "PaginationKind", "PaginationStrategy", diff --git a/src/dander/ingestion/enterprise.py b/src/dander/ingestion/enterprise.py index 3cc2858..8a48ef7 100644 --- a/src/dander/ingestion/enterprise.py +++ b/src/dander/ingestion/enterprise.py @@ -10,7 +10,7 @@ import httpx from dander.ingestion.pagination import NoPagination, OffsetPagination, PageNumberPagination -from dander.ingestion.source import BackoffKind, Endpoint, Source +from dander.ingestion.source import BackoffKind, Endpoint, RawField, Source if TYPE_CHECKING: from collections.abc import Callable, Iterator, Mapping @@ -39,7 +39,7 @@ def send(self, request: httpx.Request) -> _Response: class EnterpriseSource(Source): - """Shared authenticated transport for bespoke enterprise sources.""" + """Shared authenticated transport for sources that fully control the request cycle.""" def __init__( self, @@ -263,6 +263,49 @@ def _suiteql_page( return [dict(row) for row in cast("list[dict[str, Any]]", items)], has_more +class OdooJson2Source(EnterpriseSource): + """Read Odoo 19+ models through the JSON-2 ``search_read`` API.""" + + def discover(self) -> Mapping[str, Any]: + """Return declared model schemas without reading business records.""" + return { + endpoint.name: { + "path": endpoint.path, + "primary_key": list(endpoint.primary_key), + "incremental_cursor": endpoint.incremental_cursor, + "field_types": dict(endpoint.field_types), + } + for endpoint in self.config.endpoints + } + + def extract(self, endpoint: str, *, since: str | None = None) -> Iterator[Mapping[str, Any]]: + """Yield one normalized Odoo record at a time using bounded offset pages.""" + declaration = self._endpoint(endpoint) + pagination = _validate_odoo_endpoint(declaration) + domain = _odoo_domain(declaration, since) + + offset = 0 + while True: + request = _build_odoo_request( + self.config, + declaration, + pagination, + domain=domain, + offset=offset, + ) + response = self._send(request, endpoint) + rows = _select_odoo_rows(response.json(), endpoint) + for row in rows: + normalized = _normalize_odoo_row(row, declaration.raw_schema) + yield _cast_row(normalized, declaration) + + if len(rows) < pagination.page_size: + return + if self.config.rate_limit is not None: + self._sleep(1 / self.config.rate_limit.requests_per_second) + offset += pagination.page_size + + def _select_rows(payload: object, endpoint: Endpoint) -> list[object]: selected = payload if endpoint.data_selector is not None: @@ -277,6 +320,75 @@ def _select_rows(payload: object, endpoint: Endpoint) -> list[object]: return selected +def _validate_odoo_endpoint(endpoint: Endpoint) -> OffsetPagination: + pagination = endpoint.pagination + if not isinstance(pagination, OffsetPagination): + raise EnterpriseSourceError( + f"Odoo JSON-2 endpoint {endpoint.name!r} requires offset pagination" + ) + if not endpoint.raw_schema: + raise EnterpriseSourceError( + f"Odoo JSON-2 endpoint {endpoint.name!r} requires a declared raw schema" + ) + if not endpoint.path.startswith("/json/2/") or not endpoint.path.endswith("/search_read"): + raise EnterpriseSourceError( + f"Odoo JSON-2 endpoint {endpoint.name!r} must target /json/2//search_read" + ) + return pagination + + +def _odoo_domain(endpoint: Endpoint, since: str | None) -> list[list[object]]: + if since is None or endpoint.incremental_cursor is None: + return [] + # The inclusive boundary deliberately replays tied watermark values; SCD1 publication makes + # that replay idempotent and avoids dropping rows that share a timestamp. + return [[endpoint.incremental_cursor, ">=", since]] + + +def _build_odoo_request( + config: SourceConfig, + endpoint: Endpoint, + pagination: OffsetPagination, + *, + domain: list[list[object]], + offset: int, +) -> httpx.Request: + headers = {"Content-Type": "application/json; charset=utf-8"} + database = config.auth_options.get("database") + if database is not None: + if not isinstance(database, str) or not database.strip(): + raise EnterpriseSourceError( + "Odoo auth_options.database must be a non-empty string when set" + ) + headers["X-Odoo-Database"] = database + body: dict[str, object] = { + "domain": domain, + "fields": [field.name for field in endpoint.raw_schema], + "limit": pagination.page_size, + "offset": offset, + "order": "id asc", + } + return httpx.Request( + "POST", + f"{config.base_url.rstrip('/')}/{endpoint.path.lstrip('/')}", + headers=headers, + json=body, + ) + + +def _select_odoo_rows(payload: object, endpoint: str) -> list[dict[str, Any]]: + if not isinstance(payload, list): + raise EnterpriseSourceError(f"Endpoint {endpoint!r} response data must be a list") + rows: list[dict[str, Any]] = [] + for index, row in enumerate(payload): + if not isinstance(row, dict): + raise EnterpriseSourceError( + f"Endpoint {endpoint!r} returned a non-mapping row at index {index}" + ) + rows.append(row) + return rows + + def _cast_row(row: dict[str, Any], endpoint: Endpoint) -> dict[str, Any]: cast = dict(row) for field, data_type in endpoint.field_types.items(): @@ -291,33 +403,63 @@ def _cast_row(row: dict[str, Any], endpoint: Endpoint) -> dict[str, Any]: return cast +def _normalize_odoo_row(row: dict[str, Any], schema: list[RawField]) -> dict[str, Any]: + """Convert Odoo's ``false`` sentinel to null for non-boolean scalar fields.""" + normalized = dict(row) + for field in schema: + if normalized.get(field.name) is False and field.data_type != "BOOL": + normalized[field.name] = None + return normalized + + +def _cast_integer(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, (str, int, float, Decimal)): + raise TypeError + return int(value) + + +def _cast_float(value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (str, int, float, Decimal)): + raise TypeError + return float(value) + + +def _cast_numeric(value: object) -> Decimal: + if isinstance(value, bool): + raise TypeError + return Decimal(str(value)) + + +def _cast_boolean(value: object) -> bool: + if isinstance(value, bool): + return value + normalized = str(value).lower() + if normalized not in {"true", "false"}: + raise ValueError + return normalized == "true" + + +def _cast_timestamp(value: object) -> datetime: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError + return parsed + + +_SCALAR_CASTERS: dict[str, Callable[[object], object]] = { + "BOOL": _cast_boolean, + "DATE": lambda value: date.fromisoformat(str(value)), + "FLOAT64": _cast_float, + "INT64": _cast_integer, + "NUMERIC": _cast_numeric, + "STRING": str, + "TIMESTAMP": _cast_timestamp, +} + + def _cast_value(value: object, data_type: str) -> object: - if data_type == "STRING": - return str(value) - if data_type == "INT64": - if isinstance(value, bool) or not isinstance(value, (str, int, float, Decimal)): - raise TypeError - return int(value) - if data_type == "FLOAT64": - if isinstance(value, bool) or not isinstance(value, (str, int, float, Decimal)): - raise TypeError - return float(value) - if data_type == "NUMERIC": - if isinstance(value, bool): - raise TypeError - return Decimal(str(value)) - if data_type == "BOOL": - if isinstance(value, bool): - return value - normalized = str(value).lower() - if normalized not in {"true", "false"}: - raise ValueError - return normalized == "true" - if data_type == "DATE": - return date.fromisoformat(str(value)) - if data_type == "TIMESTAMP": - parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) - if parsed.tzinfo is None: - raise ValueError - return parsed - raise AssertionError(f"Unhandled declared data type: {data_type}") + try: + caster = _SCALAR_CASTERS[data_type] + except KeyError as error: + raise AssertionError(f"Unhandled declared data type: {data_type}") from error + return caster(value) diff --git a/src/dander/ingestion/source.py b/src/dander/ingestion/source.py index 2e6b266..17631c6 100644 --- a/src/dander/ingestion/source.py +++ b/src/dander/ingestion/source.py @@ -229,6 +229,7 @@ class IngestionEngine(StrEnum): DLT = "dlt" NETSUITE_SUITEQL = "netsuite_suiteql" + ODOO_JSON2 = "odoo_json2" WORKDAY_RAAS = "workday_raas" diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 7256610..a62f61e 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -6,7 +6,9 @@ from typer.testing import CliRunner -from dander.cli.main import app +from dander.cli.main import _build_source_adapter, app +from dander.ingestion import OdooJson2Source, load_source_config +from dander.security import NoAuth _REPO_ROOT = Path(__file__).parents[2] @@ -132,6 +134,40 @@ def test_harvest_v3_dry_run_validates_without_credentials() -> None: assert "greenhouse_candidates" in result.output +def test_odoo_json2_dry_run_validates_without_credentials(tmp_path: Path) -> None: + connectors = tmp_path / "connectors" + connectors.mkdir() + (connectors / "odoo.yaml").write_text( + (_REPO_ROOT / "connectors" / "odoo.example.yaml").read_text(encoding="utf-8"), + encoding="utf-8", + ) + + result = CliRunner().invoke( + app, + [ + "run", + "odoo", + "--dry-run", + "--project", + "unit-project", + "--connectors-dir", + str(connectors), + ], + ) + + assert result.exit_code == 0, result.output + assert "odoo_partners" in result.output + assert "SCD1" in result.output + + +def test_odoo_json2_engine_selects_enterprise_source() -> None: + config = load_source_config(_REPO_ROOT / "connectors" / "odoo.example.yaml") + + source = _build_source_adapter(config, NoAuth()) + + assert isinstance(source, OdooJson2Source) + + def test_sandbox_dry_run_declares_replace_mode_without_network() -> None: result = CliRunner().invoke( app, diff --git a/tests/ingestion/test_enterprise_source.py b/tests/ingestion/test_enterprise_source.py index 4f5e68f..eaacffb 100644 --- a/tests/ingestion/test_enterprise_source.py +++ b/tests/ingestion/test_enterprise_source.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from datetime import UTC, date, datetime from decimal import Decimal @@ -12,6 +13,7 @@ CursorPagination, EnterpriseSource, EnterpriseSourceError, + OdooJson2Source, WorkdayRaasSource, ) from dander.ingestion.source import Endpoint, RateLimitConfig, SourceConfig @@ -84,6 +86,35 @@ def _config(*, page_size: int = 2) -> SourceConfig: ) +def _odoo_config(*, page_size: int = 2) -> SourceConfig: + return SourceConfig( + name="odoo", + engine="odoo_json2", + base_url="https://odoo.example.test", + auth_strategy="api_key_bearer", + auth_ref="ODOO_API_KEY", + auth_options={"database": "dander_test"}, + endpoints=[ + Endpoint( + name="partners", + path="/json/2/res.partner/search_read", + pagination={"kind": "offset", "page_size": page_size}, + incremental_cursor="write_date", + primary_key=["id"], + field_types={"id": "INT64", "active": "BOOL", "is_company": "BOOL"}, + raw_schema=[ + {"name": "id", "type": "INT64", "mode": "REQUIRED"}, + {"name": "name", "type": "STRING"}, + {"name": "email", "type": "STRING"}, + {"name": "active", "type": "BOOL"}, + {"name": "is_company", "type": "BOOL"}, + {"name": "write_date", "type": "STRING"}, + ], + ) + ], + ) + + def test_workday_source_pages_authenticates_casts_and_passes_cursor() -> None: client = _Client( [ @@ -232,3 +263,102 @@ def test_workday_source_retries_with_bounded_declared_backoff() -> None: assert list(source.extract("workers")) == [] assert delays == [0.5, 1.0] + + +def test_odoo_source_posts_bounded_pages_and_replays_cursor_boundary() -> None: + client = _Client( + [ + [ + { + "id": 1, + "name": "Acme", + "email": False, + "active": True, + "is_company": True, + "write_date": "2026-08-04 10:00:00", + }, + { + "id": 2, + "name": "Example", + "email": "hello@example.test", + "active": True, + "is_company": False, + "write_date": "2026-08-04 11:00:00", + }, + ], + [], + ] + ) + auth = _Auth() + source = OdooJson2Source(_odoo_config(), auth, client=client) + + rows = list(source.extract("partners", since="2026-08-04 09:00:00")) + + assert rows[0]["id"] == 1 + assert rows[0]["email"] is None + assert rows[0]["is_company"] is True + assert auth.requests == 2 + first_body = json.loads(client.requests[0].content) + second_body = json.loads(client.requests[1].content) + assert first_body == { + "domain": [["write_date", ">=", "2026-08-04 09:00:00"]], + "fields": ["id", "name", "email", "active", "is_company", "write_date"], + "limit": 2, + "offset": 0, + "order": "id asc", + } + assert second_body["offset"] == 2 + assert all(request.method == "POST" for request in client.requests) + assert all(request.headers["Authorization"] == "Basic synthetic" for request in client.requests) + assert all(request.headers["X-Odoo-Database"] == "dander_test" for request in client.requests) + + +def test_odoo_discovery_uses_declarations_without_network() -> None: + client = _Client([]) + source = OdooJson2Source(_odoo_config(), _Auth(), client=client) + + discovered = source.discover() + + assert discovered["partners"]["primary_key"] == ["id"] + assert discovered["partners"]["incremental_cursor"] == "write_date" + assert client.requests == [] + + +@pytest.mark.parametrize( + ("config", "payloads", "message"), + [ + (_odoo_config(), [{"records": []}], "must be a list"), + (_odoo_config(), [["not-a-row"]], "non-mapping row"), + ( + _odoo_config().model_copy( + update={ + "endpoints": [_odoo_config().endpoints[0].model_copy(update={"raw_schema": []})] + } + ), + [], + "requires a declared raw schema", + ), + ( + _odoo_config().model_copy( + update={ + "endpoints": [ + _odoo_config() + .endpoints[0] + .model_copy(update={"path": "/json/2/res.partner/read"}) + ] + } + ), + [], + "must target", + ), + ], +) +def test_odoo_source_rejects_invalid_contracts( + config: SourceConfig, + payloads: list[object], + message: str, +) -> None: + source = OdooJson2Source(config, _Auth(), client=_Client(payloads)) + + with pytest.raises(EnterpriseSourceError, match=message): + list(source.extract("partners"))