diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 000000000..8342cdc3f --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,14 @@ +# Copilot Instructions + +Follow the repository's canonical engineering skills under +`docs/engineering/skills/`. + +For API v2 migration contract, route-group metadata, generated migration docs, +or migration guard changes, read +`docs/engineering/skills/migration_contracts.md`. + +For tests, read `docs/engineering/skills/testing.md` before adding, moving, or +reviewing test files. + +For pull requests, read `docs/engineering/skills/github-prs.md` before opening, +replacing, or sharing a PR. diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3fb2e694e..72269106e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -18,6 +18,18 @@ jobs: run: pip install ruff>=0.9.0 - name: Format check with ruff run: ruff format --check . + quality-guards: + name: Quality guards + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Run quality guards + run: python scripts/run_quality_guards.py check-changelog: name: Check changelog fragment runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..3113e7459 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,28 @@ +# Agent Instructions + +These instructions apply repository-wide. + +## Skills System + +Canonical AI-facing engineering skills live under `docs/engineering/skills/`. +Use those files as the source of truth across Codex, Claude, Copilot, and other +AI tools. + +When changing API v2 migration contracts, route-group migration metadata, PR +cutover plans, generated migration docs, or migration guard scripts, read +`docs/engineering/skills/migration_contracts.md`. + +When adding, moving, or reviewing tests, read +`docs/engineering/skills/testing.md`. + +## GitHub PRs + +Read `docs/engineering/skills/github-prs.md` before opening, replacing, or +sharing any pull request. + +Before creating or sharing a migration PR, run the focused migration guards: + +```bash +python scripts/run_quality_guards.py +python scripts/export_migration_contracts.py +``` diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..bb654f145 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,30 @@ +# Claude Instructions + +These instructions apply repository-wide. + +## Canonical Guidance + +Repository-wide AI-facing engineering guidance lives in `AGENTS.md`. +Canonical skills live under `docs/engineering/skills/`. + +Use those files as the source of truth. This file is a Claude adapter and should +stay thin; do not duplicate detailed testing, CI, formatting, or architecture +rules here. + +## Required Skill Lookup + +Before opening, replacing, or sharing a PR, read +`docs/engineering/skills/github-prs.md`. + +When changing API v2 migration contracts, route-group migration metadata, +generated migration docs, or migration guard scripts, read +`docs/engineering/skills/migration_contracts.md`. + +When adding, moving, or reviewing tests, read +`docs/engineering/skills/testing.md`. + +## Safety Boundaries + +Do not claim a route, database table, compute path, or deployment surface has +migrated unless the relevant contract tests, migration flags, and generated +migration docs identify that state. diff --git a/Makefile b/Makefile index 4fe1df11c..bafbcf161 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,11 @@ test-env-vars: pytest tests/env_variables test: - MAX_HOUSEHOLDS=1000 coverage run -a --branch -m pytest tests/to_refactor tests/unit tests/integration/test_budget_window_in_flight_dedupe.py --disable-pytest-warnings - coverage xml -i + MAX_HOUSEHOLDS=1000 python -m coverage run -a --branch -m pytest tests/to_refactor tests/unit tests/contract tests/integration/test_budget_window_in_flight_dedupe.py --disable-pytest-warnings + python -m coverage xml -i + +quality-guards: + python scripts/run_quality_guards.py debug-test: MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 pytest -vv --durations=0 tests diff --git a/changelog.d/migration-pr1-contract-harness.added.md b/changelog.d/migration-pr1-contract-harness.added.md new file mode 100644 index 000000000..20bb1aa7a --- /dev/null +++ b/changelog.d/migration-pr1-contract-harness.added.md @@ -0,0 +1 @@ +Added migration contract tests, typed migration registries, no-op source flags, migration-context logging, baseline capture tooling, and model-agnostic AI harness docs for the API v2 migration. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md new file mode 100644 index 000000000..01a9e9e27 --- /dev/null +++ b/docs/engineering/migration-contracts.md @@ -0,0 +1,100 @@ +# Migration Contracts + +Generated from `policyengine_api/migration_registry.py` and `tests/contract/registry.py`. + +## Summary + +| Metric | Count | +| --- | ---: | +| route group count | 10 | +| workflow count | 7 | +| request count | 14 | +| db entity count | 6 | +| sim flow count | 3 | + +## Route Groups + +| Route group | Path segments | DB entity | Simulation flow | +| --- | --- | --- | --- | +| `metadata` | `metadata` | `metadata` | `none` | +| `policy` | `policy`, `policies`, `user-policy` | `policy` | `none` | +| `household` | `household`, `calculate`, `calculate-full` | `household` | `household` | +| `economy` | `economy` | `simulation` | `economy` | +| `simulation` | `simulation`, `simulations` | `simulation` | `economy` | +| `report` | `report` | `report` | `report` | +| `user_profile` | `user-profile` | `user` | `none` | +| `simulation_analysis` | `simulation-analysis` | `none` | `none` | +| `tracer_analysis` | `tracer-analysis` | `none` | `none` | +| `ai` | `ai-prompts` | `none` | `none` | + +## App V2 Workflow Contracts + +### `policy_save_search` + +- Current contract: `api_v1_compatible` +- Future owner: PR 10: Policy Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/us/policy` | 201 | `policy` | `status`, `message`, `result.policy_id` | +| `GET` | `/us/policy/{policy_id}` | 200 | `policy` | `status`, `message`, `result` | +| `GET` | `/us/policies` | 200 | `policy` | `result` | + +### `household_save_edit_read` + +- Current contract: `api_v1_compatible` +- Future owner: PR 11: Household Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/us/household` | 201 | `household` | `status`, `message`, `result.household_id` | +| `PUT` | `/us/household/{household_id}` | 200 | `household` | `status`, `message`, `result.household_id` | +| `GET` | `/us/household/{household_id}` | 200 | `household` | `status`, `message`, `result` | + +### `household_calculate` + +- Current contract: `api_v1_compatible` +- Future owner: PR 13: Household Calculation Compute Cutover + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/us/calculate` | 200 | `household` | `status`, `message`, `result` | + +### `region_selection` + +- Current contract: `api_v1_compatible` +- Future owner: PR 9: v2 Metadata, Regions, Datasets, Parameters, and Variables + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `GET` | `/us/metadata` | 200 | `metadata` | `status`, `result.current_law_id`, `result.economy_options.region`, `result.economy_options.time_period` | +| `GET` | `/uk/metadata` | 200 | `metadata` | `status`, `result.current_law_id`, `result.economy_options.region`, `result.economy_options.time_period` | + +### `simulation_submit_poll` + +- Current contract: `api_v1_compatible` +- Future owner: PR 13: Household Calculation Compute Cutover + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/us/simulation` | 201 | `simulation` | `status`, `message`, `result.id`, `result.status` | +| `GET` | `/us/simulation/{simulation_id}` | 200 | `simulation` | `status`, `message`, `result` | + +### `report_create_poll` + +- Current contract: `api_v1_compatible` +- Future owner: PR 14: Economy Simulation and Economic Impact Compute Cutover + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `POST` | `/us/report` | 201 | `report` | `status`, `message`, `result.id`, `result.status` | +| `GET` | `/us/report/{report_id}` | 200 | `report` | `status`, `message`, `result` | + +### `budget_window_submit_poll` + +- Current contract: `api_v1_compatible` +- Future owner: PR 15: Budget-Window and Remaining Simulation API Migration + +| Method | Path | Status | Route group | Stable response fields | +| --- | --- | ---: | --- | --- | +| `GET` | `/us/economy/{policy_id}/over/{baseline_policy_id}/budget-window?region=us&start_year=2026&window_size=1` | 200 | `economy` | `status`, `result.kind`, `progress`, `completed_years`, `computing_years`, `queued_years`, `error` | diff --git a/docs/engineering/skills/README.md b/docs/engineering/skills/README.md new file mode 100644 index 000000000..2f3fea74b --- /dev/null +++ b/docs/engineering/skills/README.md @@ -0,0 +1,16 @@ +# Engineering Skills + +This directory is the canonical source for AI-facing engineering rules. + +Tool-specific instruction files such as `AGENTS.md`, `CLAUDE.md`, and +`.github/copilot-instructions.md` should point here instead of duplicating +implementation-specific guidance. When a rule changes, update the skill here +first, then keep adapters thin. + +Current skills: + +- `github-prs.md`: PR workflow and migration PR handoff expectations. +- `migration_contracts.md`: API v2 migration route contracts, route-group + metadata, generated migration artifacts, and quality guards. +- `testing.md`: focused test commands and dependency boundaries for migration + work. diff --git a/docs/engineering/skills/github-prs.md b/docs/engineering/skills/github-prs.md new file mode 100644 index 000000000..94b6cce60 --- /dev/null +++ b/docs/engineering/skills/github-prs.md @@ -0,0 +1,26 @@ +# GitHub PRs + +Use this skill before opening, replacing, or sharing a pull request. + +## Migration PRs + +Migration PRs should keep the user experience stable unless the PR explicitly +declares an API contract change. If a PR changes migration route contracts, +route-group metadata, generated migration artifacts, or the cutover guardrails, +it should: + +1. Include a changelog fragment. +2. Refresh generated migration artifacts with + `python scripts/export_migration_contracts.py`. +3. Run `python scripts/run_quality_guards.py`. +4. Run the focused contract or unit tests that cover the changed surface. + +## PR Descriptions + +For migration work, identify: + +- which route groups or workflows are covered; +- what remains on the Flask/API v1 path; +- what is newly prepared for FastAPI, SQLAlchemy/Alembic, Supabase, Cloud Run, + or Modal migration; +- which user-visible API contract changes are intentionally introduced. diff --git a/docs/engineering/skills/migration_contracts.md b/docs/engineering/skills/migration_contracts.md new file mode 100644 index 000000000..56259efd9 --- /dev/null +++ b/docs/engineering/skills/migration_contracts.md @@ -0,0 +1,46 @@ +# Migration Contracts + +Use this skill when changing API v2 migration contracts, route-group migration +metadata, generated migration docs, or guard scripts. + +## Sources Of Truth + +- `policyengine_api/migration_registry.py` defines route groups, path segments, + database entities, and simulation flows used by migration flags and request + logging. +- `tests/contract/registry.py` defines app-v2 user workflows and stable route + response fields that must be preserved while the migration is staged. +- `scripts/export_migration_contracts.py` merges those sources into: + - `docs/generated/migration_contracts.json` + - `docs/engineering/migration-contracts.md` + +Generated migration artifacts are checked-in review material. If a PR changes a +route group, workflow contract, stable response field, or future owner PR, run +the exporter and commit the regenerated files in the same PR. + +## Update Workflow + +```bash +python scripts/export_migration_contracts.py +python scripts/run_quality_guards.py +python -m pytest tests/contract tests/unit/test_migration_contract_artifacts.py -q +``` + +The migration-contracts guard checks that workflow names and route requests are +unique, every contract route group is declared in the migration registry, stable +response fields are present, and generated artifacts match the current +registry. + +## Annotation Rules + +Keep contract metadata explicit and durable: + +- Use stable route-group names that match the migration flag environment + surface. +- Record the current user-facing contract, not an aspirational target. +- Keep `future_owner_pr` at the granularity of the migration plan so reviewers + can tell which later PR owns each workflow. +- Add only response fields that are meaningful user-facing compatibility + anchors. +- If a route intentionally changes its API contract, update the generated docs + and call that out in the PR description. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md new file mode 100644 index 000000000..9a8a2611d --- /dev/null +++ b/docs/engineering/skills/testing.md @@ -0,0 +1,26 @@ +# Testing Skill + +Use this skill whenever adding, moving, or reviewing tests. + +## Migration Test Layout + +- Put API migration compatibility tests under `tests/contract/`. +- Put focused unit tests for migration flags, generated artifacts, guard + scripts, or baseline tools under `tests/unit/`. +- Keep contract tests isolated from live Cloud SQL, Modal, external AI APIs, and + network credentials unless the test is explicitly marked as a live integration + probe. + +## Focused Commands + +For PR 1 migration harness changes, prefer these focused checks before running +the full suite: + +```bash +python scripts/run_quality_guards.py +python scripts/export_migration_contracts.py +python -m pytest tests/contract tests/unit/test_migration_flags.py tests/unit/test_migration_contract_artifacts.py tests/unit/test_capture_migration_baseline.py tests/unit/routes/test_migration_context_logging.py -q +``` + +Run `ruff format --check` and `ruff check` on changed Python files before +handoff. diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json new file mode 100644 index 000000000..e24e03993 --- /dev/null +++ b/docs/generated/migration_contracts.json @@ -0,0 +1,308 @@ +{ + "metadata": { + "db_entity_count": 6, + "request_count": 14, + "route_group_count": 10, + "sim_flow_count": 3, + "workflow_count": 7 + }, + "route_groups": [ + { + "db_entity": "metadata", + "name": "metadata", + "path_segments": [ + "metadata" + ], + "sim_flow": null + }, + { + "db_entity": "policy", + "name": "policy", + "path_segments": [ + "policy", + "policies", + "user-policy" + ], + "sim_flow": null + }, + { + "db_entity": "household", + "name": "household", + "path_segments": [ + "household", + "calculate", + "calculate-full" + ], + "sim_flow": "household" + }, + { + "db_entity": "simulation", + "name": "economy", + "path_segments": [ + "economy" + ], + "sim_flow": "economy" + }, + { + "db_entity": "simulation", + "name": "simulation", + "path_segments": [ + "simulation", + "simulations" + ], + "sim_flow": "economy" + }, + { + "db_entity": "report", + "name": "report", + "path_segments": [ + "report" + ], + "sim_flow": "report" + }, + { + "db_entity": "user", + "name": "user_profile", + "path_segments": [ + "user-profile" + ], + "sim_flow": null + }, + { + "db_entity": null, + "name": "simulation_analysis", + "path_segments": [ + "simulation-analysis" + ], + "sim_flow": null + }, + { + "db_entity": null, + "name": "tracer_analysis", + "path_segments": [ + "tracer-analysis" + ], + "sim_flow": null + }, + { + "db_entity": null, + "name": "ai", + "path_segments": [ + "ai-prompts" + ], + "sim_flow": null + } + ], + "version": 1, + "workflows": [ + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 10: Policy Migration", + "name": "policy_save_search", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/us/policy", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.policy_id" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/us/policy/{policy_id}", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/us/policies", + "route_group": "policy", + "stable_response_fields": [ + "result" + ] + } + ] + }, + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 11: Household Migration", + "name": "household_save_edit_read", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/us/household", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result.household_id" + ] + }, + { + "expected_status": 200, + "method": "PUT", + "path": "/us/household/{household_id}", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result.household_id" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/us/household/{household_id}", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result" + ] + } + ] + }, + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 13: Household Calculation Compute Cutover", + "name": "household_calculate", + "requests": [ + { + "expected_status": 200, + "method": "POST", + "path": "/us/calculate", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result" + ] + } + ] + }, + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 9: v2 Metadata, Regions, Datasets, Parameters, and Variables", + "name": "region_selection", + "requests": [ + { + "expected_status": 200, + "method": "GET", + "path": "/us/metadata", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "result.current_law_id", + "result.economy_options.region", + "result.economy_options.time_period" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/uk/metadata", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "result.current_law_id", + "result.economy_options.region", + "result.economy_options.time_period" + ] + } + ] + }, + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 13: Household Calculation Compute Cutover", + "name": "simulation_submit_poll", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/us/simulation", + "route_group": "simulation", + "stable_response_fields": [ + "status", + "message", + "result.id", + "result.status" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/us/simulation/{simulation_id}", + "route_group": "simulation", + "stable_response_fields": [ + "status", + "message", + "result" + ] + } + ] + }, + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 14: Economy Simulation and Economic Impact Compute Cutover", + "name": "report_create_poll", + "requests": [ + { + "expected_status": 201, + "method": "POST", + "path": "/us/report", + "route_group": "report", + "stable_response_fields": [ + "status", + "message", + "result.id", + "result.status" + ] + }, + { + "expected_status": 200, + "method": "GET", + "path": "/us/report/{report_id}", + "route_group": "report", + "stable_response_fields": [ + "status", + "message", + "result" + ] + } + ] + }, + { + "current_contract": "api_v1_compatible", + "future_owner_pr": "PR 15: Budget-Window and Remaining Simulation API Migration", + "name": "budget_window_submit_poll", + "requests": [ + { + "expected_status": 200, + "method": "GET", + "path": "/us/economy/{policy_id}/over/{baseline_policy_id}/budget-window?region=us&start_year=2026&window_size=1", + "route_group": "economy", + "stable_response_fields": [ + "status", + "result.kind", + "progress", + "completed_years", + "computing_years", + "queued_years", + "error" + ] + } + ] + } + ] +} diff --git a/docs/migration-pr1-baseline-runbook.md b/docs/migration-pr1-baseline-runbook.md new file mode 100644 index 000000000..d9255a221 --- /dev/null +++ b/docs/migration-pr1-baseline-runbook.md @@ -0,0 +1,56 @@ +# PR 1 Migration Baseline Runbook + +PR 1 does not shift traffic or change API behavior. Before PR 2 starts, capture +baseline production or staging metrics so later cutovers can compare against the +same surface. + +## Scope + +- Included: current API v1 route contracts, current simulation gateway contract, + app-v2 workflow contract registry, no-op migration flags, migration-context + logging, and opt-in baseline capture. +- Not included: FastAPI shell, Cloud Run deployment, Supabase, Alembic, route + rewrites, simulation facade, v2-alpha Modal workers, or app-v2 UUID contract + changes. + +## Local Verification + +```bash +ruff format --check . +FLASK_DEBUG=1 pytest tests/contract tests/unit/test_migration_flags.py tests/unit/routes/test_migration_context_logging.py tests/unit/test_capture_migration_baseline.py tests/unit/libs/test_simulation_api_modal.py +make test +``` + +## Baseline Capture + +Run against an explicitly chosen deployed URL: + +```bash +API_BASE_URL=https://example-dot-policyengine-api.appspot.com \ +python scripts/capture_migration_baseline.py --repetitions 5 +``` + +To include the current simulation gateway completion baseline, provide a +representative economy-comparison payload: + +```bash +API_BASE_URL=https://example-dot-policyengine-api.appspot.com \ +SIMULATION_API_URL=https://policyengine--policyengine-simulation-gateway-web-app.modal.run \ +SIMULATION_PAYLOAD_FILE=/path/to/representative-simulation-payload.json \ +python scripts/capture_migration_baseline.py --repetitions 5 +``` + +Record: + +- request count +- status counts +- error rate +- p50 latency +- p95 latency +- simulation completion count +- simulation completion failures +- p50 completion time +- p95 completion time +- any probe errors + +The script is opt-in. Normal CI must not depend on live deployed services. diff --git a/policyengine_api/api.py b/policyengine_api/api.py index 7d6a67b01..6e1e9a98d 100644 --- a/policyengine_api/api.py +++ b/policyengine_api/api.py @@ -26,6 +26,7 @@ def log_timing(message): from flask_caching import Cache from policyengine_api.utils import make_cache_key +from policyengine_api.migration_logging import register_migration_request_logging log_timing("Caching utilities import completed") @@ -102,6 +103,9 @@ def log_timing(message): CORS(app) log_timing("CORS initialised") +register_migration_request_logging(app) +log_timing("Migration request logging initialised") + app.register_blueprint(error_bp) log_timing("Error routes registered") diff --git a/policyengine_api/migration_flags.py b/policyengine_api/migration_flags.py new file mode 100644 index 000000000..c67ddda70 --- /dev/null +++ b/policyengine_api/migration_flags.py @@ -0,0 +1,161 @@ +"""No-op migration flags for the API v2 migration. + +These flags do not change behavior in PR 1. They only give later PRs a stable +configuration surface and give current requests enough context for logging. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +from policyengine_api.migration_registry import ( + ROUTE_GROUP_BY_SEGMENT, + ROUTE_GROUP_CONFIG_BY_NAME, +) + + +API_HOST_BACKENDS = frozenset({"app_engine", "cloud_run"}) +ROUTE_IMPLEMENTATIONS = frozenset({"flask_fallback", "fastapi_native"}) +DB_WRITE_SOURCES = frozenset({"cloud_sql", "dual_write", "supabase"}) +DB_READ_SOURCES = frozenset({"cloud_sql", "read_compare", "supabase"}) +SIM_FRONT_DOORS = frozenset({"old_gateway_direct", "cloud_run_facade"}) +SIM_COMPUTE_BACKENDS = frozenset( + {"old_gateway", "v2_shadow", "v2_percent", "v2_primary"} +) + +DEFAULT_API_HOST_BACKEND = "app_engine" +DEFAULT_ROUTE_IMPLEMENTATION = "flask_fallback" +DEFAULT_DB_SOURCE = "cloud_sql" +DEFAULT_SIM_FRONT_DOOR = "old_gateway_direct" +DEFAULT_SIM_COMPUTE_BACKEND = "old_gateway" + + +@dataclass(frozen=True) +class MigrationContext: + api_host_backend: str + route_group: str + route_impl: str + db_entity: str | None + db_write: str | None + db_read: str | None + sim_flow: str | None + sim_front_door: str + sim_compute: str | None + + def to_log_dict(self) -> dict: + return { + "api_host_backend": self.api_host_backend, + "route_group": self.route_group, + "route_impl": self.route_impl, + "db_entity": self.db_entity, + "db_write": self.db_write, + "db_read": self.db_read, + "sim_flow": self.sim_flow, + "sim_front_door": self.sim_front_door, + "sim_compute": self.sim_compute, + } + + +def _read_choice(env_name: str, default: str, valid_values: frozenset[str]) -> str: + value = os.environ.get(env_name, default) + if value not in valid_values: + choices = ", ".join(sorted(valid_values)) + raise ValueError(f"{env_name}={value!r} is invalid; expected one of: {choices}") + return value + + +def infer_route_group(path: str) -> str: + """Infer a migration route group from a request path.""" + if path in {"/", ""}: + return "home" + if path in {"/liveness-check", "/readiness-check"}: + return "health" + if path == "/specification": + return "specification" + + segments = [segment for segment in path.strip("/").split("/") if segment] + if not segments: + return "home" + + first = segments[0] + if first in ROUTE_GROUP_BY_SEGMENT: + return ROUTE_GROUP_BY_SEGMENT[first] + + if len(segments) >= 2 and segments[1] in ROUTE_GROUP_BY_SEGMENT: + return ROUTE_GROUP_BY_SEGMENT[segments[1]] + + return "unknown" + + +def get_route_impl(route_group: str) -> str: + env_name = f"ROUTE_IMPL_{route_group.upper()}" + return _read_choice( + env_name, + DEFAULT_ROUTE_IMPLEMENTATION, + ROUTE_IMPLEMENTATIONS, + ) + + +def get_db_write(entity: str) -> str: + env_name = f"DB_WRITE_{entity.upper()}" + return _read_choice(env_name, DEFAULT_DB_SOURCE, DB_WRITE_SOURCES) + + +def get_db_read(entity: str) -> str: + env_name = f"DB_READ_{entity.upper()}" + return _read_choice(env_name, DEFAULT_DB_SOURCE, DB_READ_SOURCES) + + +def get_sim_compute(flow: str) -> str: + env_name = f"SIM_COMPUTE_{flow.upper()}" + return _read_choice( + env_name, + DEFAULT_SIM_COMPUTE_BACKEND, + SIM_COMPUTE_BACKENDS, + ) + + +def get_migration_context( + route_group: str, + *, + db_entity: str | None = None, + sim_flow: str | None = None, +) -> MigrationContext: + """Return current migration flag values for a request or route group.""" + route_config = ROUTE_GROUP_CONFIG_BY_NAME.get(route_group) + if db_entity is None and route_config is not None: + db_entity = route_config.db_entity + if sim_flow is None and route_config is not None: + sim_flow = route_config.sim_flow + + return MigrationContext( + api_host_backend=_read_choice( + "API_HOST_BACKEND", + DEFAULT_API_HOST_BACKEND, + API_HOST_BACKENDS, + ), + route_group=route_group, + route_impl=get_route_impl(route_group), + db_entity=db_entity, + db_write=get_db_write(db_entity) if db_entity else None, + db_read=get_db_read(db_entity) if db_entity else None, + sim_flow=sim_flow, + sim_front_door=_read_choice( + "SIM_FRONT_DOOR", + DEFAULT_SIM_FRONT_DOOR, + SIM_FRONT_DOORS, + ), + sim_compute=get_sim_compute(sim_flow) if sim_flow else None, + ) + + +def get_migration_log_context(route_group: str) -> dict: + """Best-effort logging context; never raises on invalid flag settings.""" + try: + return get_migration_context(route_group).to_log_dict() + except ValueError as error: + return { + "route_group": route_group, + "migration_flag_error": str(error), + } diff --git a/policyengine_api/migration_logging.py b/policyengine_api/migration_logging.py new file mode 100644 index 000000000..8bbb31dc5 --- /dev/null +++ b/policyengine_api/migration_logging.py @@ -0,0 +1,57 @@ +"""Request logging helpers for migration observability.""" + +from __future__ import annotations + +import time +import uuid + +import flask + +from policyengine_api.gcp_logging import logger +from policyengine_api.migration_flags import ( + get_migration_log_context, + infer_route_group, +) + + +def register_migration_request_logging(app: flask.Flask) -> None: + """Register no-op migration context logging on a Flask app.""" + + @app.before_request + def set_request_migration_context(): + flask.g.request_started_at = time.time() + flask.g.request_id = ( + flask.request.headers.get("X-Request-ID") or uuid.uuid4().hex + ) + + @app.after_request + def log_request_migration_context(response): + try: + route_group = infer_route_group(flask.request.path) + migration_context = get_migration_log_context(route_group) + elapsed_ms = None + started_at = getattr(flask.g, "request_started_at", None) + if started_at is not None: + elapsed_ms = round((time.time() - started_at) * 1000, 2) + + logger.log_struct( + { + "message": "API request served", + "request_id": getattr(flask.g, "request_id", None), + "method": flask.request.method, + "path": flask.request.path, + "status_code": response.status_code, + "latency_ms": elapsed_ms, + "country_id": flask.request.view_args.get("country_id") + if flask.request.view_args + else None, + "migration": migration_context, + }, + severity="INFO" if response.status_code < 500 else "ERROR", + ) + except Exception: + try: + app.logger.exception("Failed to log migration request context") + except Exception: + pass + return response diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py new file mode 100644 index 000000000..fd2c60de1 --- /dev/null +++ b/policyengine_api/migration_registry.py @@ -0,0 +1,74 @@ +"""Typed migration registry for route-group metadata.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RouteGroupConfig: + name: str + path_segments: tuple[str, ...] + db_entity: str | None = None + sim_flow: str | None = None + + +ROUTE_GROUPS: tuple[RouteGroupConfig, ...] = ( + RouteGroupConfig( + name="metadata", + path_segments=("metadata",), + db_entity="metadata", + ), + RouteGroupConfig( + name="policy", + path_segments=("policy", "policies", "user-policy"), + db_entity="policy", + ), + RouteGroupConfig( + name="household", + path_segments=("household", "calculate", "calculate-full"), + db_entity="household", + sim_flow="household", + ), + RouteGroupConfig( + name="economy", + path_segments=("economy",), + db_entity="simulation", + sim_flow="economy", + ), + RouteGroupConfig( + name="simulation", + path_segments=("simulation", "simulations"), + db_entity="simulation", + sim_flow="economy", + ), + RouteGroupConfig( + name="report", + path_segments=("report",), + db_entity="report", + sim_flow="report", + ), + RouteGroupConfig( + name="user_profile", + path_segments=("user-profile",), + db_entity="user", + ), + RouteGroupConfig( + name="simulation_analysis", + path_segments=("simulation-analysis",), + ), + RouteGroupConfig( + name="tracer_analysis", + path_segments=("tracer-analysis",), + ), + RouteGroupConfig( + name="ai", + path_segments=("ai-prompts",), + ), +) + +ROUTE_GROUP_BY_SEGMENT = { + segment: group.name for group in ROUTE_GROUPS for segment in group.path_segments +} + +ROUTE_GROUP_CONFIG_BY_NAME = {group.name: group for group in ROUTE_GROUPS} diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 000000000..56e9643cc --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Importable script helpers for tests.""" diff --git a/scripts/capture_migration_baseline.py b/scripts/capture_migration_baseline.py new file mode 100644 index 000000000..3f2e44387 --- /dev/null +++ b/scripts/capture_migration_baseline.py @@ -0,0 +1,291 @@ +"""Capture opt-in baseline metrics for migration cutover planning. + +This script is intentionally not part of normal CI. It requires API_BASE_URL and +only performs lightweight smoke requests unless callers provide a deployed API +that can run the existing integration probes. +""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +import httpx + + +@dataclass(frozen=True) +class ProbeResult: + name: str + method: str + path: str + status_code: int + latency_ms: float + ok: bool + error: str | None = None + completion_ms: float | None = None + completed: bool | None = None + + +DEFAULT_PROBES = ( + ("liveness", "GET", "/liveness-check", (200,)), + ("readiness", "GET", "/readiness-check", (200,)), + ("us_metadata", "GET", "/us/metadata", (200,)), +) + +SIMULATION_SUCCESS_STATUSES = frozenset({"complete", "completed", "success", "ok"}) +SIMULATION_FAILURE_STATUSES = frozenset({"failed", "failure", "error", "errored"}) + + +def _percentile(values: list[float], percentile: float) -> float | None: + if not values: + return None + if len(values) == 1: + return round(values[0], 2) + index = round((len(values) - 1) * percentile) + return round(sorted(values)[index], 2) + + +def _request_probe( + client: httpx.Client, + *, + name: str, + method: str, + path: str, + expected_statuses: tuple[int, ...], + json_payload: dict | None = None, +) -> tuple[ProbeResult, httpx.Response | None]: + started_at = time.perf_counter() + try: + response = client.request(method, path, json=json_payload) + latency_ms = (time.perf_counter() - started_at) * 1000 + return ( + ProbeResult( + name=name, + method=method, + path=path, + status_code=response.status_code, + latency_ms=round(latency_ms, 2), + ok=response.status_code in expected_statuses, + ), + response, + ) + except httpx.HTTPError as error: + latency_ms = (time.perf_counter() - started_at) * 1000 + return ( + ProbeResult( + name=name, + method=method, + path=path, + status_code=0, + latency_ms=round(latency_ms, 2), + ok=False, + error=str(error), + ), + None, + ) + + +def run_probes(base_url: str, repetitions: int) -> list[ProbeResult]: + results: list[ProbeResult] = [] + with httpx.Client(base_url=base_url.rstrip("/"), timeout=90.0) as client: + for _ in range(repetitions): + for name, method, path, expected_statuses in DEFAULT_PROBES: + result, _ = _request_probe( + client, + name=name, + method=method, + path=path, + expected_statuses=expected_statuses, + ) + results.append(result) + return results + + +def run_simulation_gateway_probe( + gateway_url: str, + payload: dict, + *, + poll_timeout_seconds: float, + poll_interval_seconds: float, +) -> list[ProbeResult]: + results: list[ProbeResult] = [] + started_at = time.perf_counter() + + with httpx.Client(base_url=gateway_url.rstrip("/"), timeout=90.0) as client: + submit_result, submit_response = _request_probe( + client, + name="simulation_gateway_submit", + method="POST", + path="/simulate/economy/comparison", + expected_statuses=(202,), + json_payload=payload, + ) + results.append(submit_result) + if submit_response is None or not submit_result.ok: + return results + + try: + job_id = submit_response.json()["job_id"] + except (KeyError, ValueError, TypeError) as error: + results.append( + ProbeResult( + name="simulation_gateway_completion", + method="GET", + path="/jobs/", + status_code=0, + latency_ms=0, + ok=False, + error=f"Could not parse simulation job_id: {error}", + completed=False, + ) + ) + return results + + deadline = started_at + poll_timeout_seconds + last_result: ProbeResult | None = None + last_payload: dict | None = None + while time.perf_counter() < deadline: + poll_result, poll_response = _request_probe( + client, + name="simulation_gateway_poll", + method="GET", + path=f"/jobs/{job_id}", + expected_statuses=(200, 202), + ) + last_result = poll_result + results.append(poll_result) + + if poll_response is None: + return results + + try: + last_payload = poll_response.json() + except ValueError: + last_payload = {} + + status = str(last_payload.get("status", "")).lower() + if status in SIMULATION_SUCCESS_STATUSES | SIMULATION_FAILURE_STATUSES: + completion_ms = (time.perf_counter() - started_at) * 1000 + results.append( + ProbeResult( + name="simulation_gateway_completion", + method="GET", + path=f"/jobs/{job_id}", + status_code=poll_response.status_code, + latency_ms=poll_result.latency_ms, + ok=status in SIMULATION_SUCCESS_STATUSES, + completion_ms=round(completion_ms, 2), + completed=status in SIMULATION_SUCCESS_STATUSES, + error=last_payload.get("error"), + ) + ) + return results + + time.sleep(poll_interval_seconds) + + completion_ms = (time.perf_counter() - started_at) * 1000 + results.append( + ProbeResult( + name="simulation_gateway_completion", + method="GET", + path=last_result.path if last_result else "/jobs/", + status_code=last_result.status_code if last_result else 0, + latency_ms=last_result.latency_ms if last_result else 0, + ok=False, + completion_ms=round(completion_ms, 2), + completed=False, + error=f"Simulation did not complete within {poll_timeout_seconds}s", + ) + ) + return results + + +def summarize(results: Iterable[ProbeResult]) -> dict: + rows = list(results) + request_rows = [row for row in rows if row.completed is None] + latencies = [row.latency_ms for row in request_rows if row.ok] + failures = [row for row in request_rows if not row.ok] + completions = [ + row.completion_ms + for row in rows + if row.completed is True and row.completion_ms is not None + ] + completion_failures = [row for row in rows if row.completed is False] + status_counts: dict[str, int] = {} + for row in request_rows: + key = str(row.status_code) + status_counts[key] = status_counts.get(key, 0) + 1 + + return { + "request_count": len(request_rows), + "failure_count": len(failures), + "error_rate": round(len(failures) / len(request_rows), 4) + if request_rows + else None, + "p50_latency_ms": round(statistics.median(latencies), 2) if latencies else None, + "p95_latency_ms": _percentile(latencies, 0.95), + "completion_count": len(completions), + "completion_failure_count": len(completion_failures), + "p50_completion_ms": round(statistics.median(completions), 2) + if completions + else None, + "p95_completion_ms": _percentile(completions, 0.95), + "status_counts": status_counts, + "probes": [row.__dict__ for row in rows], + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default=os.environ.get("API_BASE_URL")) + parser.add_argument( + "--simulation-gateway-url", + default=os.environ.get("SIMULATION_API_URL"), + ) + parser.add_argument( + "--simulation-payload-file", + default=os.environ.get("SIMULATION_PAYLOAD_FILE"), + ) + parser.add_argument("--repetitions", type=int, default=3) + parser.add_argument("--poll-timeout-seconds", type=float, default=900.0) + parser.add_argument("--poll-interval-seconds", type=float, default=5.0) + args = parser.parse_args(argv) + + results: list[ProbeResult] = [] + if not args.base_url: + print("API_BASE_URL is not set; skipping API route baseline capture.") + else: + results.extend(run_probes(args.base_url, args.repetitions)) + + if args.simulation_gateway_url and args.simulation_payload_file: + payload = json.loads(Path(args.simulation_payload_file).read_text()) + results.extend( + run_simulation_gateway_probe( + args.simulation_gateway_url, + payload, + poll_timeout_seconds=args.poll_timeout_seconds, + poll_interval_seconds=args.poll_interval_seconds, + ) + ) + elif args.simulation_gateway_url or args.simulation_payload_file: + print( + "SIMULATION_API_URL and SIMULATION_PAYLOAD_FILE are both required; " + "skipping simulation gateway baseline capture." + ) + + if not results: + print("No baseline probes ran.") + return 0 + + print(json.dumps(summarize(results), indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export_migration_contracts.py b/scripts/export_migration_contracts.py new file mode 100644 index 000000000..218fe3f4c --- /dev/null +++ b/scripts/export_migration_contracts.py @@ -0,0 +1,176 @@ +"""Export API v2 migration contracts into model-neutral review artifacts.""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import asdict +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from policyengine_api.migration_registry import ROUTE_GROUPS # noqa: E402 +from tests.contract.registry import APP_V2_WORKFLOW_CONTRACTS # noqa: E402 + + +DEFAULT_JSON = REPO_ROOT / "docs" / "generated" / "migration_contracts.json" +DEFAULT_MARKDOWN = REPO_ROOT / "docs" / "engineering" / "migration-contracts.md" + + +def build_payload() -> dict[str, Any]: + """Build the migration contract payload from typed registries.""" + + route_groups = [] + for group in ROUTE_GROUPS: + row = asdict(group) + row["path_segments"] = list(row["path_segments"]) + route_groups.append(row) + workflows = [] + request_count = 0 + + for workflow in APP_V2_WORKFLOW_CONTRACTS: + requests = [] + for request in workflow.requests: + row = asdict(request) + row["stable_response_fields"] = list(row["stable_response_fields"]) + requests.append(row) + request_count += len(requests) + workflows.append( + { + "name": workflow.name, + "current_contract": workflow.current_contract, + "future_owner_pr": workflow.future_owner_pr, + "requests": requests, + } + ) + + return { + "version": 1, + "metadata": { + "route_group_count": len(route_groups), + "workflow_count": len(workflows), + "request_count": request_count, + "db_entity_count": len( + {group["db_entity"] for group in route_groups if group["db_entity"]} + ), + "sim_flow_count": len( + {group["sim_flow"] for group in route_groups if group["sim_flow"]} + ), + }, + "route_groups": route_groups, + "workflows": workflows, + } + + +def json_text(payload: dict[str, Any]) -> str: + """Return the canonical JSON text for a migration contract payload.""" + + return json.dumps(payload, indent=2, sort_keys=True) + "\n" + + +def render_markdown(payload: dict[str, Any]) -> str: + """Render migration contracts as reviewer- and agent-readable Markdown.""" + + lines = [ + "# Migration Contracts", + "", + "Generated from `policyengine_api/migration_registry.py` and " + "`tests/contract/registry.py`.", + "", + "## Summary", + "", + "| Metric | Count |", + "| --- | ---: |", + ] + + for key, value in payload["metadata"].items(): + label = key.replace("_", " ") + lines.append(f"| {label} | {value} |") + + lines.extend( + [ + "", + "## Route Groups", + "", + "| Route group | Path segments | DB entity | Simulation flow |", + "| --- | --- | --- | --- |", + ] + ) + + for group in payload["route_groups"]: + path_segments = ", ".join(f"`{segment}`" for segment in group["path_segments"]) + lines.append( + f"| `{group['name']}` | {path_segments} | " + f"`{group['db_entity'] or 'none'}` | " + f"`{group['sim_flow'] or 'none'}` |" + ) + + lines.extend(["", "## App V2 Workflow Contracts", ""]) + for workflow in payload["workflows"]: + lines.extend( + [ + f"### `{workflow['name']}`", + "", + f"- Current contract: `{workflow['current_contract']}`", + f"- Future owner: {workflow['future_owner_pr']}", + "", + "| Method | Path | Status | Route group | Stable response fields |", + "| --- | --- | ---: | --- | --- |", + ] + ) + for request in workflow["requests"]: + fields = ", ".join( + f"`{field}`" for field in request["stable_response_fields"] + ) + lines.append( + f"| `{request['method']}` | `{request['path']}` | " + f"{request['expected_status']} | `{request['route_group']}` | " + f"{fields} |" + ) + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + +def write_outputs( + payload: dict[str, Any], + *, + json_path: Path = DEFAULT_JSON, + markdown_path: Path = DEFAULT_MARKDOWN, +) -> None: + """Write generated migration contract artifacts.""" + + json_path.parent.mkdir(parents=True, exist_ok=True) + markdown_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text(json_text(payload)) + markdown_path.write_text(render_markdown(payload)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--json", default=str(DEFAULT_JSON)) + parser.add_argument("--markdown", default=str(DEFAULT_MARKDOWN)) + args = parser.parse_args(argv) + + payload = build_payload() + write_outputs( + payload, + json_path=Path(args.json), + markdown_path=Path(args.markdown), + ) + print( + "Exported " + f"{payload['metadata']['workflow_count']} workflows and " + f"{payload['metadata']['request_count']} requests to " + f"{args.json} and {args.markdown}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/guards/__init__.py b/scripts/guards/__init__.py new file mode 100644 index 000000000..2187bd52a --- /dev/null +++ b/scripts/guards/__init__.py @@ -0,0 +1 @@ +"""Repository quality guards.""" diff --git a/scripts/guards/migration_contracts.py b/scripts/guards/migration_contracts.py new file mode 100644 index 000000000..2cdec9d31 --- /dev/null +++ b/scripts/guards/migration_contracts.py @@ -0,0 +1,144 @@ +"""Guardrails for API v2 migration contract metadata.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from scripts import export_migration_contracts + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _check_unique_values( + values: list[str], + *, + label: str, +) -> list[str]: + violations = [] + seen: set[str] = set() + duplicates: set[str] = set() + for value in values: + if value in seen: + duplicates.add(value) + seen.add(value) + for value in sorted(duplicates): + violations.append(f"duplicate {label}: {value!r}") + return violations + + +def _check_route_groups(payload: dict[str, Any]) -> list[str]: + violations = [] + route_groups = payload["route_groups"] + names = [group["name"] for group in route_groups] + violations.extend(_check_unique_values(names, label="route group")) + + route_group_by_segment: dict[str, str] = {} + for group in route_groups: + if not group["path_segments"]: + violations.append(f"{group['name']}: path_segments must not be empty") + for segment in group["path_segments"]: + existing_group = route_group_by_segment.get(segment) + if existing_group is not None: + violations.append( + f"duplicate route path segment {segment!r} appears in " + f"{existing_group!r} and {group['name']!r}" + ) + continue + route_group_by_segment[segment] = group["name"] + if group["db_entity"] == "": + violations.append(f"{group['name']}: db_entity should be null, not empty") + if group["sim_flow"] == "": + violations.append(f"{group['name']}: sim_flow should be null, not empty") + + return violations + + +def _check_workflows(payload: dict[str, Any]) -> list[str]: + violations = [] + declared_route_groups = {group["name"] for group in payload["route_groups"]} + workflows = payload["workflows"] + violations.extend( + _check_unique_values( + [workflow["name"] for workflow in workflows], + label="workflow", + ) + ) + + request_keys = [] + for workflow in workflows: + if workflow["current_contract"] != "api_v1_compatible": + violations.append( + f"{workflow['name']}: current_contract should be api_v1_compatible" + ) + if not workflow["future_owner_pr"]: + violations.append(f"{workflow['name']}: future_owner_pr is required") + if not workflow["requests"]: + violations.append(f"{workflow['name']}: at least one request is required") + + for request in workflow["requests"]: + context = f"{workflow['name']} {request['method']} {request['path']}" + request_keys.append(f"{request['method']} {request['path']}") + if request["route_group"] not in declared_route_groups: + violations.append( + f"{context}: unknown route_group {request['route_group']!r}" + ) + if not request["stable_response_fields"]: + violations.append(f"{context}: stable_response_fields is required") + if not request["path"].startswith("/"): + violations.append(f"{context}: path must start with /") + if request["expected_status"] not in {200, 201, 202}: + violations.append( + f"{context}: unexpected status {request['expected_status']}" + ) + + violations.extend(_check_unique_values(request_keys, label="contract request")) + return violations + + +def _check_generated_artifacts(payload: dict[str, Any]) -> list[str]: + violations = [] + expected_json = export_migration_contracts.json_text(payload) + expected_markdown = export_migration_contracts.render_markdown(payload) + + artifact_expectations = ( + (export_migration_contracts.DEFAULT_JSON, expected_json), + (export_migration_contracts.DEFAULT_MARKDOWN, expected_markdown), + ) + for path, expected in artifact_expectations: + if not path.exists(): + violations.append(f"{path.relative_to(REPO_ROOT)} is missing") + continue + if path.read_text() != expected: + violations.append( + f"{path.relative_to(REPO_ROOT)} is stale; run " + "python scripts/export_migration_contracts.py" + ) + + return violations + + +def check() -> list[str]: + payload = export_migration_contracts.build_payload() + return [ + *_check_route_groups(payload), + *_check_workflows(payload), + *_check_generated_artifacts(payload), + ] + + +def main() -> int: + violations = check() + if not violations: + print("migration-contracts guard passed") + return 0 + + print("migration-contracts guard failed:") + for violation in violations: + print(f" - {violation}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_quality_guards.py b/scripts/run_quality_guards.py new file mode 100644 index 000000000..b51a3dd5d --- /dev/null +++ b/scripts/run_quality_guards.py @@ -0,0 +1,39 @@ +"""Run repository quality guards.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.guards import migration_contracts # noqa: E402 + + +Guard = tuple[str, Callable[[], list[str]]] + +GUARDS: tuple[Guard, ...] = (("migration-contracts", migration_contracts.check),) + + +def main() -> int: + failed = False + for name, check in GUARDS: + violations = check() + if not violations: + print(f"{name}: passed") + continue + + failed = True + print(f"{name}: failed") + for violation in violations: + print(f" - {violation}") + + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/contract/__init__.py b/tests/contract/__init__.py new file mode 100644 index 000000000..9323cd415 --- /dev/null +++ b/tests/contract/__init__.py @@ -0,0 +1 @@ +"""Contract tests for current API v1 migration behavior.""" diff --git a/tests/contract/conftest.py b/tests/contract/conftest.py new file mode 100644 index 000000000..45a720dc3 --- /dev/null +++ b/tests/contract/conftest.py @@ -0,0 +1,4 @@ +import os + + +os.environ.setdefault("FLASK_DEBUG", "1") diff --git a/tests/contract/helpers.py b/tests/contract/helpers.py new file mode 100644 index 000000000..285450e80 --- /dev/null +++ b/tests/contract/helpers.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import json +from typing import Any + + +VOLATILE_STRING = "" + + +def response_json(response) -> dict[str, Any]: + return json.loads(response.data.decode("utf-8")) + + +def assert_subset(actual: Any, expected: Any) -> None: + if expected == VOLATILE_STRING: + assert actual is not None + return + + if isinstance(expected, dict): + assert isinstance(actual, dict) + for key, expected_value in expected.items(): + assert key in actual + assert_subset(actual[key], expected_value) + return + + if isinstance(expected, list): + assert isinstance(actual, list) + assert len(actual) >= len(expected) + for actual_value, expected_value in zip(actual, expected): + assert_subset(actual_value, expected_value) + return + + assert actual == expected + + +def assert_field_path_exists(payload: dict[str, Any], field_path: str) -> None: + current: Any = payload + for segment in field_path.split("."): + if isinstance(current, list): + assert current, f"{field_path} resolved through an empty list" + current = current[0] + assert isinstance(current, dict), ( + f"{field_path} segment {segment} hit {current}" + ) + assert segment in current + current = current[segment] diff --git a/tests/contract/registry.py b/tests/contract/registry.py new file mode 100644 index 000000000..1ba70280d --- /dev/null +++ b/tests/contract/registry.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ContractRequest: + method: str + path: str + expected_status: int + stable_response_fields: tuple[str, ...] + route_group: str + + +@dataclass(frozen=True) +class WorkflowContract: + name: str + current_contract: str + future_owner_pr: str + requests: tuple[ContractRequest, ...] + + +APP_V2_WORKFLOW_CONTRACTS: tuple[WorkflowContract, ...] = ( + WorkflowContract( + name="policy_save_search", + current_contract="api_v1_compatible", + future_owner_pr="PR 10: Policy Migration", + requests=( + ContractRequest( + method="POST", + path="/us/policy", + expected_status=201, + stable_response_fields=("status", "message", "result.policy_id"), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/us/policy/{policy_id}", + expected_status=200, + stable_response_fields=("status", "message", "result"), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/us/policies", + expected_status=200, + stable_response_fields=("result",), + route_group="policy", + ), + ), + ), + WorkflowContract( + name="household_save_edit_read", + current_contract="api_v1_compatible", + future_owner_pr="PR 11: Household Migration", + requests=( + ContractRequest( + method="POST", + path="/us/household", + expected_status=201, + stable_response_fields=("status", "message", "result.household_id"), + route_group="household", + ), + ContractRequest( + method="PUT", + path="/us/household/{household_id}", + expected_status=200, + stable_response_fields=("status", "message", "result.household_id"), + route_group="household", + ), + ContractRequest( + method="GET", + path="/us/household/{household_id}", + expected_status=200, + stable_response_fields=("status", "message", "result"), + route_group="household", + ), + ), + ), + WorkflowContract( + name="household_calculate", + current_contract="api_v1_compatible", + future_owner_pr="PR 13: Household Calculation Compute Cutover", + requests=( + ContractRequest( + method="POST", + path="/us/calculate", + expected_status=200, + stable_response_fields=("status", "message", "result"), + route_group="household", + ), + ), + ), + WorkflowContract( + name="region_selection", + current_contract="api_v1_compatible", + future_owner_pr="PR 9: v2 Metadata, Regions, Datasets, Parameters, and Variables", + requests=( + ContractRequest( + method="GET", + path="/us/metadata", + expected_status=200, + stable_response_fields=( + "status", + "result.current_law_id", + "result.economy_options.region", + "result.economy_options.time_period", + ), + route_group="metadata", + ), + ContractRequest( + method="GET", + path="/uk/metadata", + expected_status=200, + stable_response_fields=( + "status", + "result.current_law_id", + "result.economy_options.region", + "result.economy_options.time_period", + ), + route_group="metadata", + ), + ), + ), + WorkflowContract( + name="simulation_submit_poll", + current_contract="api_v1_compatible", + future_owner_pr="PR 13: Household Calculation Compute Cutover", + requests=( + ContractRequest( + method="POST", + path="/us/simulation", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.id", + "result.status", + ), + route_group="simulation", + ), + ContractRequest( + method="GET", + path="/us/simulation/{simulation_id}", + expected_status=200, + stable_response_fields=("status", "message", "result"), + route_group="simulation", + ), + ), + ), + WorkflowContract( + name="report_create_poll", + current_contract="api_v1_compatible", + future_owner_pr="PR 14: Economy Simulation and Economic Impact Compute Cutover", + requests=( + ContractRequest( + method="POST", + path="/us/report", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.id", + "result.status", + ), + route_group="report", + ), + ContractRequest( + method="GET", + path="/us/report/{report_id}", + expected_status=200, + stable_response_fields=("status", "message", "result"), + route_group="report", + ), + ), + ), + WorkflowContract( + name="budget_window_submit_poll", + current_contract="api_v1_compatible", + future_owner_pr="PR 15: Budget-Window and Remaining Simulation API Migration", + requests=( + ContractRequest( + method="GET", + path="/us/economy/{policy_id}/over/{baseline_policy_id}/budget-window?region=us&start_year=2026&window_size=1", + expected_status=200, + stable_response_fields=( + "status", + "result.kind", + "progress", + "completed_years", + "computing_years", + "queued_years", + "error", + ), + route_group="economy", + ), + ), + ), +) + +APP_V2_ROUTE_CONTRACTS = tuple( + request for workflow in APP_V2_WORKFLOW_CONTRACTS for request in workflow.requests +) diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py new file mode 100644 index 000000000..742287943 --- /dev/null +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -0,0 +1,26 @@ +from policyengine_api.migration_registry import ROUTE_GROUP_CONFIG_BY_NAME +from tests.contract.registry import APP_V2_ROUTE_CONTRACTS, APP_V2_WORKFLOW_CONTRACTS + + +def test_app_v2_workflow_contract_registry_is_complete(): + assert {workflow.name for workflow in APP_V2_WORKFLOW_CONTRACTS} == { + "policy_save_search", + "household_save_edit_read", + "household_calculate", + "region_selection", + "simulation_submit_poll", + "report_create_poll", + "budget_window_submit_poll", + } + + for workflow in APP_V2_WORKFLOW_CONTRACTS: + assert workflow.current_contract == "api_v1_compatible" + assert workflow.future_owner_pr + assert workflow.requests + + for request in APP_V2_ROUTE_CONTRACTS: + assert request.method in {"GET", "POST", "PUT", "PATCH"} + assert request.path.startswith("/") + assert request.expected_status in {200, 201, 202} + assert request.stable_response_fields + assert request.route_group in ROUTE_GROUP_CONFIG_BY_NAME diff --git a/tests/contract/test_simulation_gateway_contract.py b/tests/contract/test_simulation_gateway_contract.py new file mode 100644 index 000000000..1716577af --- /dev/null +++ b/tests/contract/test_simulation_gateway_contract.py @@ -0,0 +1,143 @@ +from unittest.mock import MagicMock + +import httpx +import pytest + +import policyengine_api.libs.simulation_api_modal as simulation_api_modal +from policyengine_api.libs.simulation_api_modal import SimulationAPIModal +from tests.fixtures.libs.simulation_api_modal import ( + MOCK_BATCH_JOB_ID, + MOCK_BATCH_POLL_RESPONSE_COMPLETE, + MOCK_BATCH_SUBMIT_RESPONSE_SUCCESS, + MOCK_HEALTH_RESPONSE, + MOCK_MODAL_JOB_ID, + MOCK_POLL_RESPONSE_COMPLETE, + MOCK_RESOLVED_APP_NAME, + MOCK_SIMULATION_PAYLOAD_WITH_TELEMETRY, + MOCK_SUBMIT_RESPONSE_SUCCESS, +) + + +@pytest.fixture(autouse=True) +def disable_modal_logging(monkeypatch): + monkeypatch.setattr(simulation_api_modal, "logger", MagicMock()) + + +def _client_for(responses: dict[tuple[str, str], httpx.Response]) -> SimulationAPIModal: + transport = httpx.MockTransport( + lambda request: responses[(request.method, request.url.path)] + ) + client = SimulationAPIModal() + client.client = httpx.Client( + base_url=client.base_url, + transport=transport, + ) + return client + + +def _response(status_code: int, json_data: dict) -> httpx.Response: + return httpx.Response(status_code=status_code, json=json_data) + + +def _clear_gateway_auth_env(monkeypatch): + for key in ( + "GATEWAY_AUTH_ISSUER", + "GATEWAY_AUTH_AUDIENCE", + "GATEWAY_AUTH_CLIENT_ID", + "GATEWAY_AUTH_CLIENT_SECRET", + "GATEWAY_AUTH_CLIENT_SECRET_RESOURCE", + "GATEWAY_AUTH_REQUIRED", + ): + monkeypatch.delenv(key, raising=False) + + +def test_gateway_comparison_submit_and_poll_contract(monkeypatch): + _clear_gateway_auth_env(monkeypatch) + monkeypatch.setenv("SIMULATION_API_URL", "https://simulation.test") + client = _client_for( + { + ("POST", "/simulate/economy/comparison"): _response( + status_code=202, + json_data=MOCK_SUBMIT_RESPONSE_SUCCESS, + ), + ("GET", f"/jobs/{MOCK_MODAL_JOB_ID}"): _response( + status_code=200, + json_data=MOCK_POLL_RESPONSE_COMPLETE, + ), + } + ) + + execution = client.run( + { + **MOCK_SIMULATION_PAYLOAD_WITH_TELEMETRY, + "model_version": "1.702.0", + "data_version": "ignored-by-gateway", + } + ) + completed = client.get_execution_by_id(execution.job_id) + + assert execution.job_id == MOCK_MODAL_JOB_ID + assert execution.status == "submitted" + assert completed.status == "complete" + assert completed.result == MOCK_POLL_RESPONSE_COMPLETE["result"] + assert completed.resolved_app_name == MOCK_RESOLVED_APP_NAME + + +def test_gateway_budget_window_submit_and_poll_contract(monkeypatch): + _clear_gateway_auth_env(monkeypatch) + monkeypatch.setenv("SIMULATION_API_URL", "https://simulation.test") + client = _client_for( + { + ( + "POST", + "/simulate/economy/budget-window", + ): _response( + status_code=202, + json_data=MOCK_BATCH_SUBMIT_RESPONSE_SUCCESS, + ), + ( + "GET", + f"/budget-window-jobs/{MOCK_BATCH_JOB_ID}", + ): _response( + status_code=200, + json_data=MOCK_BATCH_POLL_RESPONSE_COMPLETE, + ), + } + ) + + execution = client.run_budget_window_batch( + { + **MOCK_SIMULATION_PAYLOAD_WITH_TELEMETRY, + "model_version": "1.702.0", + "data_version": "ignored-by-gateway", + } + ) + completed = client.get_budget_window_batch_by_id(execution.batch_job_id) + + assert execution.batch_job_id == MOCK_BATCH_JOB_ID + assert execution.status == "submitted" + assert completed.status == "complete" + assert completed.result == MOCK_BATCH_POLL_RESPONSE_COMPLETE["result"] + + +def test_gateway_versions_and_health_contract(monkeypatch): + _clear_gateway_auth_env(monkeypatch) + monkeypatch.setenv("SIMULATION_API_URL", "https://simulation.test") + client = _client_for( + { + ("GET", "/versions/us"): _response( + status_code=200, + json_data={"latest": "1.702.0", "1.702.0": MOCK_RESOLVED_APP_NAME}, + ), + ("GET", "/health"): _response( + status_code=200, + json_data=MOCK_HEALTH_RESPONSE, + ), + } + ) + + app_name, version = client.resolve_app_name("us") + + assert app_name == MOCK_RESOLVED_APP_NAME + assert version == "1.702.0" + assert client.health_check() is True diff --git a/tests/contract/test_v1_route_contracts.py b/tests/contract/test_v1_route_contracts.py new file mode 100644 index 000000000..a3fec206a --- /dev/null +++ b/tests/contract/test_v1_route_contracts.py @@ -0,0 +1,416 @@ +from contextlib import ExitStack +import importlib +import sys +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from flask import Flask, Response + +from policyengine_api.endpoints.household import get_calculate +from policyengine_api.endpoints.policy import get_policy_search +from policyengine_api.routes.household_routes import household_bp +from policyengine_api.routes.policy_routes import policy_bp +from policyengine_api.routes.report_output_routes import report_output_bp +from policyengine_api.routes.simulation_routes import simulation_bp +from tests.contract.helpers import ( + assert_field_path_exists, + assert_subset, + response_json, +) +from tests.contract.registry import APP_V2_ROUTE_CONTRACTS, ContractRequest + + +class _BudgetWindowEconomicImpactResult: + def __init__( + self, + *, + data: dict | None, + progress: int | None = None, + completed_years: list[str] | None = None, + computing_years: list[str] | None = None, + queued_years: list[str] | None = None, + message: str | None = None, + error: str | None = None, + cache_status: str | None = None, + ): + self.data = data + self.progress = progress + self.completed_years = completed_years or [] + self.computing_years = computing_years or [] + self.queued_years = queued_years or [] + self.message = message + self.error = error + self.cache_status = cache_status + + @classmethod + def completed(cls, data: dict): + return cls(data=data, progress=100) + + def to_dict(self): + return { + "status": "ok", + "data": self.data, + "progress": self.progress, + "completed_years": self.completed_years, + "computing_years": self.computing_years, + "queued_years": self.queued_years, + "message": self.message, + "error": self.error, + } + + +class _EconomyService: + def get_budget_window_economic_impact(self, **kwargs): + return _BudgetWindowEconomicImpactResult.completed( + {"kind": "budgetWindow", "windowSize": 1} + ) + + +class _MetadataService: + def get_metadata(self, country_id: str): + return { + "current_law_id": 2 if country_id == "us" else 1, + "economy_options": { + "region": [{"name": country_id, "label": f"the {country_id}"}], + "time_period": [{"name": 2026, "label": "2026"}], + }, + } + + +def _load_blueprint_with_fake_service( + *, + service_module_name: str, + route_module_name: str, + fake_service_module, + blueprint_name: str, +): + sentinel = object() + original_service_module = sys.modules.get(service_module_name, sentinel) + sys.modules.pop(route_module_name, None) + sys.modules[service_module_name] = fake_service_module + try: + return getattr(importlib.import_module(route_module_name), blueprint_name) + finally: + sys.modules.pop(route_module_name, None) + if original_service_module is sentinel: + sys.modules.pop(service_module_name, None) + else: + sys.modules[service_module_name] = original_service_module + + +def _load_contract_metadata_blueprint(): + return _load_blueprint_with_fake_service( + service_module_name="policyengine_api.services.metadata_service", + route_module_name="policyengine_api.routes.metadata_routes", + fake_service_module=SimpleNamespace(MetadataService=_MetadataService), + blueprint_name="metadata_bp", + ) + + +def _load_contract_economy_blueprint(): + return _load_blueprint_with_fake_service( + service_module_name="policyengine_api.services.economy_service", + route_module_name="policyengine_api.routes.economy_routes", + fake_service_module=SimpleNamespace( + EconomyService=_EconomyService, + EconomicImpactResult=object, + BudgetWindowEconomicImpactResult=_BudgetWindowEconomicImpactResult, + ), + blueprint_name="economy_bp", + ) + + +def _client(): + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(_load_contract_metadata_blueprint()) + app.register_blueprint(policy_bp) + app.register_blueprint(household_bp) + app.register_blueprint(_load_contract_economy_blueprint()) + app.register_blueprint(simulation_bp) + app.register_blueprint(report_output_bp) + app.route("//policies", methods=["GET"])(get_policy_search) + app.route("//calculate", methods=["POST"])(get_calculate) + + @app.route("/liveness-check") + def liveness_check(): + return Response("OK", status=200, mimetype="text/plain") + + @app.route("/readiness-check") + def readiness_check(): + return Response("OK", status=200, mimetype="text/plain") + + return app.test_client() + + +def _resolved_path(path: str) -> str: + return ( + path.replace("{policy_id}", "22") + .replace("{baseline_policy_id}", "2") + .replace("{household_id}", "456") + .replace("{simulation_id}", "11") + .replace("{report_id}", "33") + ) + + +def _json_payload(contract: ContractRequest) -> dict | None: + if contract.path == "/us/policy": + return {"label": "Utah reform", "data": {"gov.example.parameter": 1}} + if contract.path == "/us/household": + return {"label": "Empty household", "data": {}} + if contract.path == "/us/household/{household_id}": + return {"label": "Updated household", "data": {"people": {"you": {}}}} + if contract.path == "/us/calculate": + return {"household": {"people": {"you": {}}}, "policy": {}} + if contract.path == "/us/simulation": + return { + "population_id": "household-1", + "population_type": "household", + "policy_id": 22, + } + if contract.path == "/us/report": + return {"simulation_1_id": 11, "simulation_2_id": None, "year": "2026"} + return None + + +def _policy_search_rows(): + return SimpleNamespace( + fetchall=lambda: [ + {"id": 123, "label": "Tax reform", "policy_hash": "hash-1"}, + {"id": 124, "label": "Tax reform", "policy_hash": "hash-1"}, + ] + ) + + +def _fake_country(): + return SimpleNamespace( + metadata={}, + calculate=lambda household, policy: { + "people": {"you": {"age": {"2026": 40}}}, + "policy": policy, + }, + ) + + +def _patched_route_dependencies(): + stack = ExitStack() + stack.enter_context( + patch( + "policyengine_api.routes.policy_routes.policy_service.get_policy", + return_value={"id": 22, "label": "Current law", "policy_json": {}}, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.policy_routes.policy_service.set_policy", + return_value=(123, "Policy created successfully", False), + ) + ) + stack.enter_context( + patch( + "policyengine_api.endpoints.policy.database.query", + return_value=_policy_search_rows(), + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.household_routes.household_service.create_household", + return_value=456, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.household_routes.household_service.get_household", + return_value={"id": 456, "label": "Empty household", "household_json": {}}, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.household_routes.household_service.update_household", + return_value={"household_json": {"people": {"you": {}}}}, + ) + ) + stack.enter_context( + patch( + "policyengine_api.endpoints.household.get_countries", + return_value={"us": _fake_country()}, + ) + ) + stack.enter_context( + patch( + "policyengine_api.endpoints.household.get_invalid_inputs_response", + return_value=None, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.simulation_routes.simulation_service.find_existing_simulation", + return_value=None, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.simulation_routes.simulation_service.create_simulation", + return_value={ + "id": 11, + "country_id": "us", + "population_id": "household-1", + "population_type": "household", + "policy_id": 22, + "status": "pending", + }, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.simulation_routes.simulation_service.get_simulation", + return_value={"id": 11, "status": "pending", "country_id": "us"}, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.report_output_routes.report_output_service.find_existing_report_output", + return_value=None, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.report_output_routes.report_output_service.create_report_output", + return_value={ + "id": 33, + "country_id": "us", + "simulation_1_id": 11, + "simulation_2_id": None, + "status": "pending", + "year": "2026", + }, + ) + ) + stack.enter_context( + patch( + "policyengine_api.routes.report_output_routes.report_output_service.get_report_output", + return_value={"id": 33, "status": "pending", "country_id": "us"}, + ) + ) + return stack + + +def _expected_subset(contract: ContractRequest) -> dict: + if contract.path == "/us/policy": + return { + "status": "ok", + "message": "Policy created successfully", + "result": {"policy_id": 123}, + } + if contract.path == "/us/policy/{policy_id}": + return {"status": "ok", "message": None, "result": {"label": "Current law"}} + if contract.path == "/us/policies": + return { + "status": "ok", + "message": "Policies found", + "result": [{"id": 123, "label": "Tax reform"}], + } + if contract.path == "/us/household": + return {"status": "ok", "message": None, "result": {"household_id": 456}} + if contract.path == "/us/household/{household_id}" and contract.method == "PUT": + return { + "status": "ok", + "message": None, + "result": {"household_id": 456, "household_json": {"people": {"you": {}}}}, + } + if contract.path == "/us/household/{household_id}": + return {"status": "ok", "result": {"id": 456, "label": "Empty household"}} + if contract.path == "/us/calculate": + return { + "status": "ok", + "message": None, + "result": {"people": {"you": {"age": {"2026": 40}}}}, + } + if contract.path in {"/us/metadata", "/uk/metadata"}: + country_id = contract.path.strip("/").split("/")[0] + return { + "status": "ok", + "message": None, + "result": { + "current_law_id": 2 if country_id == "us" else 1, + "economy_options": { + "region": [{"name": country_id}], + "time_period": [{"name": 2026}], + }, + }, + } + if contract.path == "/us/simulation": + return { + "status": "ok", + "message": "Simulation created successfully", + "result": {"id": 11, "status": "pending"}, + } + if contract.path == "/us/simulation/{simulation_id}": + return {"status": "ok", "message": None, "result": {"id": 11}} + if contract.path == "/us/report": + return { + "status": "ok", + "message": "Report output created successfully", + "result": {"id": 33, "status": "pending"}, + } + if contract.path == "/us/report/{report_id}": + return {"status": "ok", "message": None, "result": {"id": 33}} + if "budget-window" in contract.path: + return { + "status": "ok", + "message": None, + "result": {"kind": "budgetWindow", "windowSize": 1}, + "progress": 100, + "completed_years": [], + "computing_years": [], + "queued_years": [], + "error": None, + } + raise AssertionError(f"Missing expected subset for {contract}") + + +@pytest.mark.parametrize( + "contract", + APP_V2_ROUTE_CONTRACTS, + ids=lambda contract: f"{contract.method} {contract.path}", +) +def test_app_v2_api_v1_route_contract(contract): + with _patched_route_dependencies(): + response = _client().open( + _resolved_path(contract.path), + method=contract.method, + json=_json_payload(contract), + ) + + assert response.status_code == contract.expected_status + payload = response_json(response) + assert_subset(payload, _expected_subset(contract)) + for field_path in contract.stable_response_fields: + assert_field_path_exists(payload, field_path) + + +def test_health_routes_contract(): + client = _client() + liveness = client.get("/liveness-check") + readiness = client.get("/readiness-check") + + assert liveness.status_code == 200 + assert liveness.data == b"OK" + assert "text/plain" in liveness.content_type + assert readiness.status_code == 200 + assert readiness.data == b"OK" + assert "text/plain" in readiness.content_type + + +def test_invalid_country_contract(): + response = _client().get("/zz/metadata") + + assert response.status_code == 400 + assert_subset( + response_json(response), + { + "status": "error", + "message": "Country zz not found. Available countries are: uk, us, ca, ng, il", + }, + ) diff --git a/tests/unit/routes/test_migration_context_logging.py b/tests/unit/routes/test_migration_context_logging.py new file mode 100644 index 000000000..e0798d70e --- /dev/null +++ b/tests/unit/routes/test_migration_context_logging.py @@ -0,0 +1,42 @@ +from unittest.mock import patch + +from flask import Flask, Response + +from policyengine_api.migration_logging import register_migration_request_logging + + +def _app(): + app = Flask(__name__) + app.config["TESTING"] = True + + @app.route("/readiness-check") + def readiness_check(): + return Response("OK", status=200, mimetype="text/plain") + + register_migration_request_logging(app) + return app + + +def test_request_logging_includes_migration_context(): + with patch("policyengine_api.migration_logging.logger") as mock_logger: + response = _app().test_client().get("/readiness-check") + + assert response.status_code == 200 + log_payload = mock_logger.log_struct.call_args.args[0] + assert log_payload["message"] == "API request served" + assert log_payload["path"] == "/readiness-check" + assert log_payload["status_code"] == 200 + assert log_payload["migration"]["route_group"] == "health" + assert log_payload["migration"]["api_host_backend"] == "app_engine" + assert log_payload["migration"]["route_impl"] == "flask_fallback" + + +def test_request_logging_failure_does_not_change_response(): + with patch( + "policyengine_api.migration_logging.logger.log_struct", + side_effect=RuntimeError("logging failed"), + ): + response = _app().test_client().get("/readiness-check") + + assert response.status_code == 200 + assert response.data == b"OK" diff --git a/tests/unit/test_capture_migration_baseline.py b/tests/unit/test_capture_migration_baseline.py new file mode 100644 index 000000000..bf85ef1a8 --- /dev/null +++ b/tests/unit/test_capture_migration_baseline.py @@ -0,0 +1,155 @@ +import json + +import httpx + +from scripts import capture_migration_baseline + + +def test_baseline_summary_computes_error_rate_and_latency(): + results = [ + capture_migration_baseline.ProbeResult( + name="health", + method="GET", + path="/readiness-check", + status_code=200, + latency_ms=10.0, + ok=True, + ), + capture_migration_baseline.ProbeResult( + name="metadata", + method="GET", + path="/us/metadata", + status_code=500, + latency_ms=30.0, + ok=False, + error="boom", + ), + capture_migration_baseline.ProbeResult( + name="simulation_gateway_completion", + method="GET", + path="/jobs/job-1", + status_code=200, + latency_ms=5.0, + ok=True, + completion_ms=100.0, + completed=True, + ), + ] + + summary = capture_migration_baseline.summarize(results) + + assert summary["request_count"] == 2 + assert summary["failure_count"] == 1 + assert summary["error_rate"] == 0.5 + assert summary["p50_latency_ms"] == 10.0 + assert summary["p95_latency_ms"] == 10.0 + assert summary["completion_count"] == 1 + assert summary["completion_failure_count"] == 0 + assert summary["p50_completion_ms"] == 100.0 + assert summary["p95_completion_ms"] == 100.0 + assert summary["status_counts"] == {"200": 1, "500": 1} + + +def test_baseline_script_skips_without_base_url(capsys, monkeypatch): + monkeypatch.delenv("API_BASE_URL", raising=False) + exit_code = capture_migration_baseline.main([]) + + assert exit_code == 0 + assert "No baseline probes ran" in capsys.readouterr().out + + +def test_run_probes_uses_lightweight_baseline_routes(monkeypatch): + requests = [] + real_client = httpx.Client + + def handler(request): + requests.append((request.method, request.url.path)) + return httpx.Response(200, json={"status": "ok"}) + + monkeypatch.setattr( + capture_migration_baseline.httpx, + "Client", + lambda **kwargs: real_client( + transport=httpx.MockTransport(handler), + base_url=kwargs["base_url"], + timeout=kwargs["timeout"], + ), + ) + + results = capture_migration_baseline.run_probes("https://api.test", 1) + + assert [(result.method, result.path) for result in results] == requests + assert requests == [ + ("GET", "/liveness-check"), + ("GET", "/readiness-check"), + ("GET", "/us/metadata"), + ] + assert json.loads(json.dumps(capture_migration_baseline.summarize(results))) + + +def test_run_probes_treats_unexpected_4xx_as_failure(monkeypatch): + real_client = httpx.Client + + monkeypatch.setattr( + capture_migration_baseline.httpx, + "Client", + lambda **kwargs: real_client( + transport=httpx.MockTransport( + lambda request: httpx.Response(404, json={"status": "missing"}) + ), + base_url=kwargs["base_url"], + timeout=kwargs["timeout"], + ), + ) + + summary = capture_migration_baseline.summarize( + capture_migration_baseline.run_probes("https://api.test", 1) + ) + + assert summary["request_count"] == 3 + assert summary["failure_count"] == 3 + assert summary["status_counts"] == {"404": 3} + + +def test_run_simulation_gateway_probe_records_completion(monkeypatch): + requests = [] + real_client = httpx.Client + + def handler(request): + requests.append((request.method, request.url.path)) + if request.method == "POST": + return httpx.Response(202, json={"job_id": "job-1", "status": "submitted"}) + return httpx.Response( + 200, + json={"job_id": "job-1", "status": "complete", "result": {"ok": True}}, + ) + + monkeypatch.setattr( + capture_migration_baseline.httpx, + "Client", + lambda **kwargs: real_client( + transport=httpx.MockTransport(handler), + base_url=kwargs["base_url"], + timeout=kwargs["timeout"], + ), + ) + + results = capture_migration_baseline.run_simulation_gateway_probe( + "https://simulation.test", + {"country": "us"}, + poll_timeout_seconds=1, + poll_interval_seconds=0, + ) + summary = capture_migration_baseline.summarize(results) + + assert requests == [ + ("POST", "/simulate/economy/comparison"), + ("GET", "/jobs/job-1"), + ] + assert [result.name for result in results] == [ + "simulation_gateway_submit", + "simulation_gateway_poll", + "simulation_gateway_completion", + ] + assert summary["completion_count"] == 1 + assert summary["completion_failure_count"] == 0 diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py new file mode 100644 index 000000000..f8c3142d3 --- /dev/null +++ b/tests/unit/test_migration_contract_artifacts.py @@ -0,0 +1,67 @@ +import json + +from scripts import export_migration_contracts +from scripts.guards import migration_contracts + + +def test_migration_contract_payload_summarizes_route_contracts(): + payload = export_migration_contracts.build_payload() + + assert payload["version"] == 1 + assert payload["metadata"] == { + "route_group_count": 10, + "workflow_count": 7, + "request_count": 14, + "db_entity_count": 6, + "sim_flow_count": 3, + } + assert {workflow["name"] for workflow in payload["workflows"]} == { + "policy_save_search", + "household_save_edit_read", + "household_calculate", + "region_selection", + "simulation_submit_poll", + "report_create_poll", + "budget_window_submit_poll", + } + + +def test_generated_migration_contract_json_is_current(): + payload = export_migration_contracts.build_payload() + + assert json.loads(export_migration_contracts.DEFAULT_JSON.read_text()) == payload + + +def test_generated_migration_contract_markdown_is_current(): + payload = export_migration_contracts.build_payload() + + assert export_migration_contracts.DEFAULT_MARKDOWN.read_text() == ( + export_migration_contracts.render_markdown(payload) + ) + + +def test_migration_contract_quality_guard_passes(): + assert migration_contracts.check() == [] + + +def test_migration_contract_quality_guard_rejects_duplicate_path_segments( + monkeypatch, +): + payload = export_migration_contracts.build_payload() + duplicate_group = { + **payload["route_groups"][0], + "name": "duplicate_metadata", + } + monkeypatch.setattr( + export_migration_contracts, + "build_payload", + lambda: { + **payload, + "route_groups": [*payload["route_groups"], duplicate_group], + }, + ) + + assert any( + "duplicate route path segment" in violation + for violation in migration_contracts.check() + ) diff --git a/tests/unit/test_migration_flags.py b/tests/unit/test_migration_flags.py new file mode 100644 index 000000000..b44a4edf0 --- /dev/null +++ b/tests/unit/test_migration_flags.py @@ -0,0 +1,75 @@ +import pytest + +from policyengine_api.migration_flags import ( + get_migration_context, + infer_route_group, +) + + +def test_default_migration_context_preserves_current_behavior(monkeypatch): + for key in ( + "API_HOST_BACKEND", + "ROUTE_IMPL_POLICY", + "DB_WRITE_POLICY", + "DB_READ_POLICY", + "SIM_FRONT_DOOR", + ): + monkeypatch.delenv(key, raising=False) + + context = get_migration_context("policy") + + assert context.api_host_backend == "app_engine" + assert context.route_impl == "flask_fallback" + assert context.db_entity == "policy" + assert context.db_write == "cloud_sql" + assert context.db_read == "cloud_sql" + assert context.sim_front_door == "old_gateway_direct" + assert context.sim_compute is None + + +def test_explicit_valid_migration_context_values(monkeypatch): + monkeypatch.setenv("API_HOST_BACKEND", "cloud_run") + monkeypatch.setenv("ROUTE_IMPL_ECONOMY", "fastapi_native") + monkeypatch.setenv("DB_WRITE_SIMULATION", "dual_write") + monkeypatch.setenv("DB_READ_SIMULATION", "read_compare") + monkeypatch.setenv("SIM_FRONT_DOOR", "cloud_run_facade") + monkeypatch.setenv("SIM_COMPUTE_ECONOMY", "v2_shadow") + + context = get_migration_context("economy") + + assert context.api_host_backend == "cloud_run" + assert context.route_impl == "fastapi_native" + assert context.db_entity == "simulation" + assert context.db_write == "dual_write" + assert context.db_read == "read_compare" + assert context.sim_front_door == "cloud_run_facade" + assert context.sim_compute == "v2_shadow" + + +def test_invalid_migration_flag_raises(monkeypatch): + monkeypatch.setenv("DB_READ_POLICY", "spreadsheets") + + with pytest.raises(ValueError, match="DB_READ_POLICY"): + get_migration_context("policy") + + +@pytest.mark.parametrize( + ("path", "expected_group"), + [ + ("/", "home"), + ("/readiness-check", "health"), + ("/us/metadata", "metadata"), + ("/us/policy/1", "policy"), + ("/us/policies", "policy"), + ("/us/household/1", "household"), + ("/us/calculate", "household"), + ("/us/economy/1/over/2", "economy"), + ("/us/economy/1/over/2/budget-window", "economy"), + ("/us/simulation/1", "simulation"), + ("/simulations", "simulation"), + ("/us/report/1", "report"), + ("/us/ai-prompts/simulation_analysis", "ai"), + ], +) +def test_infer_route_group(path, expected_group): + assert infer_route_group(path) == expected_group