diff --git a/.github/scripts/combine-test-reports.sh b/.github/scripts/combine-test-reports.sh
index e1ebb6f20d..2d6d9225d7 100755
--- a/.github/scripts/combine-test-reports.sh
+++ b/.github/scripts/combine-test-reports.sh
@@ -1,171 +1,27 @@
#!/bin/bash
+# Thin wrapper around `python3 -m tests.rig report combine`.
+#
+# Kept for backward compatibility with any external script or local workflow
+# that still invokes this path. Prefer calling the rig directly.
set -euo pipefail
-# Script to combine multiple test reports into a single markdown file
-# Usage: ./combine-test-reports.sh
-
-OUTPUT_FILE="combined-test-report.md"
-REPORTS=()
-
-# Find all test report files
-for report in runner-report.md sdk1-report.md; do
- if [ -f "$report" ]; then
- REPORTS+=("$report")
- fi
-done
-
-# Exit if no reports found
-if [ ${#REPORTS[@]} -eq 0 ]; then
- echo "No test reports found. Skipping report generation."
- exit 0
+REPORTS_DIR="${REPORTS_DIR:-reports}"
+
+# Stock Ubuntu CI runners only ship `python3`, not `python`. Pick whichever is
+# on PATH; bail loudly if neither.
+if command -v python3 >/dev/null 2>&1; then
+ PY=python3
+elif command -v python >/dev/null 2>&1; then
+ PY=python
+else
+ echo "combine-test-reports.sh: no python interpreter on PATH" >&2
+ exit 1
fi
-# Function to strip LaTeX formatting from pytest-md-report output
-# Converts $$\textcolor{...}{\tt{VALUE}}$$ to just VALUE
-strip_latex() {
- local text="$1"
- # Extract content between \tt{ and }}
- if [[ "$text" =~ \\tt\{([^}]+)\} ]]; then
- echo "${BASH_REMATCH[1]}"
- else
- echo "$text"
- fi
-}
-
-# Function to extract test counts from pytest-md-report markdown table
-extract_test_counts() {
- local report_file=$1
- local passed=0
- local failed=0
- local total=0
-
- # Find the header row to determine column positions
- local header_line=$(grep -E '^\|.*filepath' "$report_file" | head -1)
-
- if [ -z "$header_line" ]; then
- echo "0:0:0"
- return
- fi
-
- # Extract column names and find positions (strip LaTeX from headers)
- IFS='|' read -ra headers <<< "$header_line"
- local passed_col=-1
- local failed_col=-1
- local subtotal_col=-1
-
- for i in "${!headers[@]}"; do
- local col=$(strip_latex "${headers[$i]}" | tr -d ' ' | tr '[:upper:]' '[:lower:]')
- case "$col" in
- passed) passed_col=$i ;;
- failed) failed_col=$i ;;
- subtotal|sub) subtotal_col=$i ;;
- esac
- done
-
- # Find the TOTAL row (TOTAL appears in first column, not as SUBTOTAL in header)
- local total_line=$(grep -E '^\|.*\\tt\{TOTAL\}' "$report_file" | head -1)
-
- if [ -z "$total_line" ]; then
- echo "0:0:0"
- return
- fi
-
- # Parse the TOTAL row values
- IFS='|' read -ra values <<< "$total_line"
-
- # Extract passed count (strip LaTeX and get number)
- if [ "$passed_col" -ge 0 ] && [ "$passed_col" -lt "${#values[@]}" ]; then
- local clean_value=$(strip_latex "${values[$passed_col]}")
- passed=$(echo "$clean_value" | tr -d ' ' | grep -oE '[0-9]+' | head -1 || echo "0")
- fi
-
- # Extract failed count (strip LaTeX and get number)
- if [ "$failed_col" -ge 0 ] && [ "$failed_col" -lt "${#values[@]}" ]; then
- local clean_value=$(strip_latex "${values[$failed_col]}")
- failed=$(echo "$clean_value" | tr -d ' ' | grep -oE '[0-9]+' | head -1 || echo "0")
- fi
-
- # Extract total from SUBTOTAL column (strip LaTeX and get number)
- if [ "$subtotal_col" -ge 0 ] && [ "$subtotal_col" -lt "${#values[@]}" ]; then
- local clean_value=$(strip_latex "${values[$subtotal_col]}")
- total=$(echo "$clean_value" | tr -d ' ' | grep -oE '[0-9]+' | head -1 || echo "0")
- fi
-
- # If total is still 0, calculate from passed + failed
- if [ "$total" -eq 0 ]; then
- total=$((passed + failed))
- fi
-
- echo "${total}:${passed}:${failed}"
-}
+"$PY" -m tests.rig report combine --reports-dir "$REPORTS_DIR"
-# Initialize the combined report with collapsed summary
-cat > "$OUTPUT_FILE" << 'EOF'
-# Test Results
-
-
-Summary
-
-EOF
-
-# Extract and display summary for each report
-for report in "${REPORTS[@]}"; do
- report_name=$(basename "$report" .md)
-
- # Convert report name to title case
- if [ "$report_name" = "runner-report" ]; then
- title="Runner Tests"
- elif [ "$report_name" = "sdk1-report" ]; then
- title="SDK1 Tests"
- else
- title="${report_name}"
- fi
-
- # Extract counts
- counts=$(extract_test_counts "$report")
- IFS=':' read -r total passed failed <<< "$counts"
-
- # Determine status icon
- if [ "$failed" -gt 0 ]; then
- status="❌"
- elif [ "$passed" -gt 0 ]; then
- status="✅"
- else
- status="⚠️"
- fi
-
- echo "- ${status} **${title}**: ${passed} passed, ${failed} failed (${total} total)" >> "$OUTPUT_FILE"
-done
-
-cat >> "$OUTPUT_FILE" << 'EOF'
-
-
-
----
-
-EOF
-
-# Combine all reports with collapsible sections
-for report in "${REPORTS[@]}"; do
- report_name=$(basename "$report" .md)
-
- # Convert report name to title case
- if [ "$report_name" = "runner-report" ]; then
- title="Runner Tests"
- elif [ "$report_name" = "sdk1-report" ]; then
- title="SDK1 Tests"
- else
- title="${report_name}"
- fi
-
- echo "" >> "$OUTPUT_FILE"
- echo "${title} - Full Report
" >> "$OUTPUT_FILE"
- echo "" >> "$OUTPUT_FILE"
- cat "$report" >> "$OUTPUT_FILE"
- echo "" >> "$OUTPUT_FILE"
- echo " " >> "$OUTPUT_FILE"
- echo "" >> "$OUTPUT_FILE"
-done
-
-echo "Combined test report created: $OUTPUT_FILE"
-echo "Included reports: ${REPORTS[*]}"
+# Backward-compatible alias for the existing sticky-comment step which uploads
+# combined-test-report.md from the repo root.
+if [ -f "$REPORTS_DIR/combined-test-report.md" ] && [ ! -f combined-test-report.md ]; then
+ cp "$REPORTS_DIR/combined-test-report.md" combined-test-report.md
+fi
diff --git a/.github/workflows/ci-test-e2e.yaml b/.github/workflows/ci-test-e2e.yaml
new file mode 100644
index 0000000000..323a294beb
--- /dev/null
+++ b/.github/workflows/ci-test-e2e.yaml
@@ -0,0 +1,85 @@
+name: Run e2e tests (rig + docker compose)
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ types: [labeled, synchronize]
+ branches: [main]
+ schedule:
+ # Nightly at 02:00 UTC.
+ - cron: "0 2 * * *"
+ workflow_dispatch:
+
+jobs:
+ e2e:
+ # Only run on PRs that opt in via the `run-e2e` label, plus main + nightly + manual.
+ if: >
+ github.event_name != 'pull_request' ||
+ contains(github.event.pull_request.labels.*.name, 'run-e2e')
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v6
+ with:
+ fetch-depth: 0
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+ with:
+ version: "0.6.14"
+ python-version: 3.12.9
+
+ - name: Install tox with UV
+ run: uv tool install tox --with tox-uv
+
+ - name: Validate test manifests
+ run: tox -e rig -- validate
+
+ - name: Restore main-branch test baseline
+ # See ci-test.yaml for the rationale on namespacing per-workflow.
+ uses: actions/cache@v5
+ with:
+ path: reports/previous-summary.json
+ key: unstract-test-baseline-e2e-main-${{ github.run_id }}
+ restore-keys: |
+ unstract-test-baseline-e2e-main-
+ unstract-test-baseline-e2e-
+
+ - name: Run e2e tier via docker compose
+ env:
+ UNSTRACT_E2E_RUNTIME: compose
+ run: |
+ # Use --tier e2e (not `all`) so this workflow runs only e2e groups.
+ if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+ tox -e e2e -- --fail-on-critical-gap --update-baseline
+ else
+ tox -e e2e
+ fi
+
+ - name: Capture docker compose logs on failure
+ if: failure()
+ run: |
+ mkdir -p reports
+ docker compose -p unstract-test \
+ -f docker/docker-compose.yaml \
+ -f tests/compose/docker-compose.test.yaml \
+ logs --no-color > reports/docker-compose-logs.txt || true
+
+ - name: Output e2e report to job summary
+ if: always() && hashFiles('reports/summary.md') != ''
+ shell: bash
+ run: |
+ cat reports/summary.md >> $GITHUB_STEP_SUMMARY
+
+ - name: Upload e2e reports artifact
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: test-reports-e2e
+ path: reports/
+ if-no-files-found: ignore
+ retention-days: 30
diff --git a/.github/workflows/ci-test.yaml b/.github/workflows/ci-test.yaml
index c3a227ecfd..13de1475f4 100644
--- a/.github/workflows/ci-test.yaml
+++ b/.github/workflows/ci-test.yaml
@@ -1,4 +1,4 @@
-name: Run tox tests with UV
+name: Run unit + integration tests (rig)
on:
push:
@@ -24,11 +24,12 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
+ with:
+ fetch-depth: 0 # rig --changed-only needs git history
- name: Install uv
- uses: astral-sh/setup-uv@v7
+ uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
- # Install a specific version of uv.
version: "0.6.14"
python-version: 3.12.9
@@ -40,29 +41,78 @@ jobs:
restore-keys: |
${{ runner.os }}-tox-uv-
+ - name: Restore main-branch test baseline (for regression detection)
+ # actions/cache only saves on a cache miss. Include the run id in the
+ # key so each main build writes a fresh cache entry; the prefix in
+ # restore-keys pulls the most recent baseline.
+ #
+ # The unit/integration lane keeps a SEPARATE baseline from the e2e
+ # workflow because their `scope_groups` don't overlap — restoring an
+ # e2e-tier baseline here would flag every e2e-covered path as a
+ # regression in this lane (and vice versa). Each workflow is the
+ # source of truth for the paths covered by its own tiers.
+ uses: actions/cache@v5
+ with:
+ path: reports/previous-summary.json
+ key: unstract-test-baseline-ut-main-${{ github.run_id }}
+ restore-keys: |
+ unstract-test-baseline-ut-main-
+ unstract-test-baseline-ut-
+
- name: Install tox with UV
run: uv tool install tox --with tox-uv
- - name: Run tox
- id: tox
+ - name: Validate test manifests
+ # Cheap pre-flight: catches groups.yaml / critical_paths.yaml schema
+ # errors before we spend minutes on tier runs. Also catches malformed
+ # manifests on PRs that only touch paths-ignored files (because this
+ # step always runs).
+ run: tox -e rig -- validate
+
+ - name: Run unit tier
+ # Each tier runs as a separate rig invocation so its results land in
+ # reports// before the next tier starts. --update-baseline (on
+ # main only) merges this tier's covered paths into the cached
+ # previous-summary.json; later tiers union on top.
run: |
- tox
+ if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+ tox -e unit -- --fail-on-critical-gap --update-baseline
+ else
+ tox -e unit
+ fi
- - name: Combine test reports
- if: always() && (hashFiles('runner-report.md') != '' || hashFiles('sdk1-report.md') != '')
+ - name: Run integration tier
+ if: always()
run: |
- bash .github/scripts/combine-test-reports.sh
+ if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+ tox -e integration -- --fail-on-critical-gap --update-baseline
+ else
+ tox -e integration
+ fi
+
+ - name: Re-aggregate reports from both tiers
+ if: always()
+ run: tox -e rig -- report combine
- name: Render combined test report to PR
uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987 # v3.0.2
- if: always() && hashFiles('combined-test-report.md') != '' && github.event.pull_request.head.repo.fork == false
+ if: always() && hashFiles('reports/combined-test-report.md') != '' && github.event.pull_request.head.repo.fork == false
with:
header: test-results
recreate: true
- path: combined-test-report.md
+ path: reports/combined-test-report.md
- name: Output combined report to job summary
- if: always() && hashFiles('combined-test-report.md') != ''
+ if: always() && hashFiles('reports/summary.md') != ''
shell: bash
run: |
- cat combined-test-report.md >> $GITHUB_STEP_SUMMARY
+ cat reports/summary.md >> $GITHUB_STEP_SUMMARY
+
+ - name: Upload reports artifact
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: test-reports-unit-integration
+ path: reports/
+ if-no-files-found: ignore
+ retention-days: 14
diff --git a/.gitignore b/.gitignore
index c655f4acff..427afc934d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -713,3 +713,7 @@ AGENTS.md
# MCP servers
.serena
+
+# Unstract test rig
+reports/
+.test-selection
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e7f17901d1..7e0004b295 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -86,6 +86,19 @@ repos:
files: ^frontend/src/
types_or: [javascript, jsx, ts, tsx, json]
+ - id: unstract-tests
+ name: Run selected unstract tests (.test-selection)
+ # Only fires when the developer has opted in by creating .test-selection
+ # (newline-delimited group names, gitignored). Runs serially without
+ # coverage to stay fast in a commit hook.
+ # Use python3 (stock Ubuntu / many container images ship no `python`
+ # symlink), matching .github/scripts/combine-test-reports.sh.
+ entry: bash -c 'if [ -f .test-selection ]; then "$(command -v python3 || command -v python)" -m tests.rig run --from-file .test-selection --no-coverage --no-parallel; fi'
+ language: system
+ pass_filenames: false
+ always_run: true
+ stages: [pre-commit]
+
- repo: https://github.com/igorshubovych/markdownlint-cli
rev: v0.42.0
hooks:
diff --git a/pyproject.toml b/pyproject.toml
index 543d942abc..04a4845fb3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,6 +26,21 @@ workers = [
"unstract-workers",
]
+test-rig = [
+ # Test runner foundation. Installed by tox into the rig env.
+ "pytest>=8.0.1",
+ "pytest-xdist>=3.5.0",
+ "pytest-cov>=4.1.0",
+ "pytest-md-report>=0.6.2",
+ "pytest-mock>=3.12.0",
+ "pytest-timeout>=2.3.1",
+ "pytest-django>=4.8.0",
+ "coverage[toml]>=7.4.0",
+ "pyyaml>=6.0.1",
+ "testcontainers[postgres,redis,minio,rabbitmq]>=4.7.0",
+ "requests>=2.31.0",
+]
+
hook-check-django-migrations = [
"celery>=5.3.4",
"cron-descriptor==1.4.0",
@@ -192,7 +207,44 @@ testpaths = ["tests"]
markers = [
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
"integration: marks tests as integration (deselect with '-m \"not integration\"')",
+ "unit: marks tests as unit (no external services required)",
+ "e2e: marks tests as end-to-end (require full platform stack)",
+ "critical: marks tests covering a critical path (see tests/critical_paths.yaml)",
+]
+
+[tool.coverage.run]
+branch = false
+relative_files = true
+omit = [
+ # Test modules shouldn't count themselves as covered code; that inflates
+ # the percentage and hides real gaps in the modules under test.
+ # Keep the patterns narrow: target test FILES, not "anything in a tests/
+ # directory" (which would exclude tests/rig/*.py — those ARE the
+ # production modules for the unit-rig group).
+ "**/test_*.py",
+ "**/*_tests.py",
+ "**/tests.py",
+]
+
+[tool.coverage.paths]
+# Each group runs pytest from its own workdir, so per-group .coverage files
+# store paths anchored at that workdir (e.g. "src/unstract/sdk1/foo.py" from
+# unstract/sdk1/; "shared/foo.py" from workers/). `coverage combine` uses
+# these mappings to fold parallel workdir paths back onto a single
+# repo-relative path, otherwise xml/html show duplicate or orphaned files.
+sdk1 = ["unstract/sdk1/src/unstract/sdk1", "*/unstract/sdk1/src/unstract/sdk1"]
+runner = ["runner/src/unstract/runner", "*/runner/src/unstract/runner"]
+prompt_service = [
+ "prompt-service/src/unstract/prompt_service",
+ "*/prompt-service/src/unstract/prompt_service",
]
+platform_service = ["platform-service/src", "*/platform-service/src"]
+workers = ["workers/shared", "*/workers/shared"]
+connectors = ["unstract/connectors/src", "*/unstract/connectors/src"]
+core = ["unstract/core/src", "*/unstract/core/src"]
+tool_registry = ["unstract/tool-registry/src", "*/unstract/tool-registry/src"]
+backend = ["backend", "*/backend"]
+rig = ["tests/rig", "*/tests/rig"]
[tool.mypy]
python_version = "3.12"
diff --git a/runner/src/unstract/runner/clients/test_docker.py b/runner/src/unstract/runner/clients/test_docker.py
index 1c1418b000..6841c4e2e8 100644
--- a/runner/src/unstract/runner/clients/test_docker.py
+++ b/runner/src/unstract/runner/clients/test_docker.py
@@ -34,7 +34,15 @@ def docker_client(mocker):
@pytest.fixture
-def docker_client_with_sidecar():
+def docker_client_with_sidecar(mocker):
+ # Mirror `docker_client`: mock DockerClient.from_env so constructing the
+ # Client doesn't reach for a real Docker daemon (CI runners have one, bare
+ # dev/sandbox environments don't — and the test is about sidecar wiring,
+ # not the daemon connection).
+ mock_client = MagicMock()
+ mocker.patch(f"{DOCKER_MODULE}.DockerClient.from_env", return_value=mock_client)
+ mocker.patch(f"{DOCKER_MODULE}.Client._Client__private_login")
+
image_name = "test-image"
image_tag = "latest"
logger = logging.getLogger("test-logger")
diff --git a/tests/README.md b/tests/README.md
new file mode 100644
index 0000000000..e5677bffc5
--- /dev/null
+++ b/tests/README.md
@@ -0,0 +1,250 @@
+# Unstract test rig
+
+This directory hosts the test foundation for the Unstract platform: cross-service integration + end-to-end tests, plus the rig that orchestrates every test
+suite in the repo (including the per-service unit tests that live alongside their source code).
+
+> Per-service **unit tests stay co-located** with the code they exercise (`backend//tests/`, `workers/tests/`, `unstract/sdk1/tests/`, etc.).
+> Only **e2e** and **cross-service integration** tests live here.
+
+---
+
+## Layout
+
+```
+tests/
+├── groups.yaml # SINGLE source of truth: groups, paths, deps
+├── critical_paths.yaml # Critical user/system flows + their declared coverage
+├── conftest.py # Shared pytest markers for the tests/ tree
+├── rig/ # The rig itself (Python package)
+│ ├── cli.py # `python -m tests.rig `
+│ ├── groups.py # YAML loader + dep-graph expansion
+│ ├── selection.py # CLI / file / `all` / changed-only resolution
+│ ├── runtime.py # docker-compose | testcontainers | local
+│ ├── reporting.py # JUnit + markdown summary writer
+│ ├── coverage.py # Per-group coverage files + combine
+│ └── critical_paths.py # Gap + regression detection
+├── e2e/
+│ ├── conftest.py # Session-scoped `platform` fixture
+│ ├── smoke/ # Login → /health smoke
+│ ├── workflows/ # (future) workflow execution e2e
+│ ├── api_deployment/ # (future) API deployment e2e
+│ ├── prompt_studio/ # (future) Prompt Studio e2e
+│ └── hurl/ # (future) hurl-based HTTP suites
+├── integration/ # Cross-service tests needing infra but not full platform
+├── fixtures/ # Sample PDFs, JSON, adapter configs
+└── compose/
+ └── docker-compose.test.yaml # Test overlay on docker/docker-compose.yaml
+```
+
+---
+
+## Quick start
+
+```bash
+# List every defined group with its tier + dep edges.
+tox -e rig -- list-groups
+
+# Show what would actually run for a selection, expanded over depends_on.
+tox -e rig -- expand e2e-workflow
+
+# Run all unit groups in parallel, with coverage.
+tox -e unit
+
+# Run a single group (positional arg).
+tox -e groups -- unit-sdk1
+
+# Run multiple groups; deps are pulled in automatically.
+tox -e groups -- unit-backend e2e-smoke
+
+# Run everything (unit + integration + e2e).
+tox -e groups -- all
+
+# Pre-commit / fast iteration: read a newline-delimited list of group names.
+echo unit-backend > .test-selection
+tox -e groups -- --from-file .test-selection --no-coverage --no-parallel
+
+# E2E lane (docker-compose by default; testcontainers for local dev).
+tox -e e2e -- e2e-smoke
+UNSTRACT_E2E_RUNTIME=testcontainers tox -e e2e -- e2e-smoke
+```
+
+The rig CLI is also callable directly without tox:
+
+```bash
+python -m tests.rig run --tier unit
+python -m tests.rig validate
+python -m tests.rig platform up --runtime compose
+python -m tests.rig report combine
+```
+
+---
+
+## The two manifests
+
+### `groups.yaml` — the unit of selection
+
+Every test group is declared here. The rig refuses to start if `groups.yaml` has a cycle, an unknown `depends_on` target, or a missing path on a
+non-`optional` group.
+
+Minimum a new group needs:
+
+```yaml
+my-new-group:
+ tier: unit # unit | integration | e2e
+ workdir: backend # where pytest is invoked from
+ paths: [some_app/tests] # passed as pytest args
+```
+
+Optional knobs (see `groups.yaml` for examples):
+
+| Key | Purpose |
+|---|---|
+| `markers` | Forwarded to pytest `-m` (e.g. `"unit and not slow"`). |
+| `pytest_extra` | Extra pytest CLI flags. |
+| `env` | Env vars set for this group's pytest process. |
+| `uv_sync_group` | Runs `uv sync --group ` in the workdir before pytest. |
+| `install_editable` | Runs `uv pip install -e .` in the workdir. |
+| `pip_install` | Explicit deps to install before pytest. |
+| `requires_services` | Infra needed (`postgres`, `redis`, `minio`, ...). |
+| `requires_platform` | Set true for e2e — rig brings the full platform up. |
+| `depends_on` | Other groups that must run first. |
+| `critical` | Marks the group as covering a critical path. |
+| `timeout_seconds` | Override the default 600s. |
+| `optional` | Two effects: (1) skip silently if paths/workdir are missing (placeholders, gitignored cloud-only dirs); (2) **non-blocking** — if the group runs and fails, its red result still shows in the summary but does not gate the overall exit code. Use for groups that need infra CI doesn't provision (e.g. live-DB connector tests) where a red run shouldn't block merge. |
+
+### `critical_paths.yaml` — what we promise not to break
+
+Each entry is a high-value user or system flow with an `id`, an `entry` (HTTP endpoint or internal hop), and a list of `covered_by` groups. The rig reports each path as:
+
+- ✅ **covered** — at least one group in `covered_by` ran green this build.
+- ⚠️ **gap** — no covering group ran green (or `covered_by` is empty).
+- ❌ **regression** — was ✅ on the cached main baseline but isn't now.
+
+The rig itself does not know about PRs or main — it just emits the markers and respects `--fail-on-critical-gap`. The CI workflow at `.github/workflows/ci-test.yaml` is what decides to pass that flag on main and not on PRs, so gaps surface as warnings during review and as errors only when merging. Regressions are **always** errors — the team target is zero.
+
+---
+
+## How selection works
+
+Resolution order, then dep-expansion, then topo-sort:
+
+```
+positional GROUPS ∪ --from-file lines ∪ --tier filter ∪ --changed-only diff
+```
+
+The literal `all` expands to every group. An empty resolved set is treated as an error, not "run everything" — fail loudly rather than surprise.
+
+`--changed-only` runs `git diff ...HEAD` (default base: `origin/main`) and selects every group whose `workdir` or `paths` overlap a changed file.
+Useful for fast feedback on feature branches.
+
+---
+
+## E2E runtime
+
+Three modes behind one protocol, chosen by `--runtime` or `UNSTRACT_E2E_RUNTIME`. CI defaults to `compose`; everywhere else defaults to `testcontainers`.
+
+| Mode | Use when | How it works |
+|---|---|---|
+| `compose` | CI; testing the prod image. | `docker compose -f docker/docker-compose.yaml -f tests/compose/docker-compose.test.yaml up -d --wait`, then HTTP. Teardown wipes volumes. |
+| `testcontainers` | Local iteration on infra-only groups. | Stands up Postgres/Redis/RabbitMQ/MinIO via testcontainers and exposes their handles on `PlatformEndpoints.infra`. **Stub today**: does NOT auto-launch backend/prompt-service/etc. as subprocesses — full-platform e2e on testcontainers will need that wiring added. Use `compose` for now if you need the whole stack. |
+| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. |
+
+The rig brings the platform up **once** per `run` invocation (if any selected group has `requires_platform: true`) and exports its URLs via env vars (`UNSTRACT_BACKEND_URL`, `UNSTRACT_PROMPT_SERVICE_URL`, etc.). The rig uses `env.setdefault(...)` so a pre-set value (e.g. from `local` runtime or a developer override) wins over the runtime's default — useful when iterating against a custom stack, but means stale shell env can mask wiring bugs. The smoke test asserts the fixture's URL matches the env var to catch this.
+
+The `platform` pytest fixture in `tests/e2e/conftest.py` reads those env vars; e2e tests run elsewhere (without the rig) just skip with a clear message.
+
+---
+
+## Reports
+
+After every `run`, the rig writes:
+
+```
+reports/
+├── summary.md # human-readable, used for PR sticky comments
+├── summary.json # machine-readable
+├── combined-test-report.md # alias kept for backward compatibility
+├── coverage.xml # Cobertura (when --coverage)
+├── htmlcov/ # browsable coverage (when --coverage)
+└── /
+ ├── junit.xml # pytest --junitxml
+ ├── report.md # pytest-md-report output
+ └── exit.txt # group's pytest exit code
+```
+
+`reports/summary.md` has two sections:
+
+1. **Per-group results** table (passed/failed/errors/skipped/duration).
+2. **Critical paths**, split into:
+ - ❌ Regressions — must be zero.
+ - ⚠️ Critical paths not yet covered — the gaps backlog.
+ - ✅ Covered critical paths (collapsed) — what's protected.
+
+CI uploads the whole `reports/` directory as an artifact and posts `combined-test-report.md` as a sticky PR comment.
+
+---
+
+## Coverage
+
+Coverage is **on by default** and can be disabled per-run with `--no-coverage` (pre-commit and quick local runs typically disable it).
+
+Each group gets its own `COVERAGE_FILE` so parallel pytest invocations don't trample each other. After all groups complete, the rig runs
+`coverage combine` + `coverage xml` + `coverage html`.
+
+We **do not** chase 100% coverage. The bar is critical-path coverage; the rig's job is to make gaps and regressions visible, not to enforce a number.
+
+---
+
+## Branch policy
+
+The rig itself has **no branch awareness**. Branch behavior is enforced in GitHub Actions, not in the rig:
+
+- On `main`, each tier runs in its own step (`tox -e unit` then `tox -e integration`, then `tox -e e2e` in the slow lane). Each invocation passes `--fail-on-critical-gap --update-baseline`. The rig merges (rather than overwrites) covered paths into `previous-summary.json` so the second tier's run preserves the first tier's coverage.
+- On PRs, the same tiered steps run without `--fail-on-critical-gap`, so gaps are visible but don't block.
+- The e2e workflow only runs on main, on PRs labeled `run-e2e`, on nightly cron, or via manual dispatch.
+
+Developers can scope local runs however they like via positional args, `--from-file .test-selection`, `--tier`, or `--changed-only`.
+
+---
+
+## Adding tests
+
+| Where it goes | What kind of test |
+|---|---|
+| `backend//tests/`, `workers/tests/`, `unstract//tests/`, ... | Unit tests for that service. |
+| `tests/integration//` | Cross-service tests that need real infra but not the full platform. |
+| `tests/e2e//` | HTTP-level tests against a running platform. |
+| `tests/e2e/hurl/` | Hurl-based HTTP suites. |
+
+After adding tests, either:
+1. Reuse an existing group whose `paths` already cover your file, **or**
+2. Add a new group to `groups.yaml` (and, if relevant, a `critical_paths.yaml` entry that lists it in `covered_by`).
+
+Validate with `python -m tests.rig validate` before pushing.
+
+---
+
+## Common commands cheat sheet
+
+```bash
+# Discovery
+python -m tests.rig list-groups
+python -m tests.rig list-critical-paths
+python -m tests.rig expand e2e-workflow
+python -m tests.rig validate
+
+# Running
+tox -e unit # all unit groups
+tox -e e2e -- e2e-smoke # one e2e group
+tox -e groups -- unit-backend unit-workers # multiple groups
+tox -e groups -- --from-file .test-selection # opt-in file
+tox -e groups -- --changed-only # diff vs origin/main
+tox -e groups -- all --no-coverage # everything, fast
+
+# Platform control (manual)
+python -m tests.rig platform up --runtime compose
+python -m tests.rig platform down
+
+# Re-aggregate existing reports
+python -m tests.rig report combine
+```
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/compose/docker-compose.test.yaml b/tests/compose/docker-compose.test.yaml
new file mode 100644
index 0000000000..f99e54de34
--- /dev/null
+++ b/tests/compose/docker-compose.test.yaml
@@ -0,0 +1,33 @@
+# E2E test overlay for docker/docker-compose.yaml.
+#
+# Used by tests/rig/runtime.py's ComposeRuntime:
+# docker compose -f docker/docker-compose.yaml \
+# -f tests/compose/docker-compose.test.yaml up -d --wait
+#
+# Keep this file minimal so e2e tests run against images as close to production
+# as possible. Override only what the test stack actually needs (test creds,
+# pinned image tags, ENVIRONMENT=test sentinel).
+
+services:
+ backend:
+ # Default to `latest` so contributors can run without setting
+ # UNSTRACT_TEST_VERSION; CI sets it to the build SHA so e2e runs against
+ # the freshly built image.
+ image: unstract/backend:${UNSTRACT_TEST_VERSION:-latest}
+ environment:
+ - ENVIRONMENT=test
+
+ prompt-service:
+ image: unstract/prompt-service:${UNSTRACT_TEST_VERSION:-latest}
+ environment:
+ - ENVIRONMENT=test
+
+ platform-service:
+ image: unstract/platform-service:${UNSTRACT_TEST_VERSION:-latest}
+ environment:
+ - ENVIRONMENT=test
+
+ runner:
+ image: unstract/runner:${UNSTRACT_TEST_VERSION:-latest}
+ environment:
+ - ENVIRONMENT=test
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000000..8f0c067ca6
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,20 @@
+"""Top-level conftest for tests/ tree.
+
+Registers shared markers and re-exports e2e fixtures so any test under
+``tests/`` can ``import`` from ``tests.e2e.fixtures`` without path gymnastics.
+"""
+
+from __future__ import annotations
+
+
+def pytest_configure(config) -> None:
+ # Markers are also registered in the root pyproject.toml; declaring them
+ # here too lets ad-hoc invocations (`pytest tests/`) succeed without
+ # importing pyproject ini-options when pytest is run from a sub-workdir.
+ config.addinivalue_line("markers", "unit: marks tests as unit (no external services)")
+ config.addinivalue_line("markers", "integration: marks tests as integration")
+ config.addinivalue_line(
+ "markers", "e2e: marks tests as end-to-end (require platform)"
+ )
+ config.addinivalue_line("markers", "critical: marks tests covering a critical path")
+ config.addinivalue_line("markers", "slow: marks tests as slow")
diff --git a/tests/critical_paths.yaml b/tests/critical_paths.yaml
new file mode 100644
index 0000000000..5ea4512f49
--- /dev/null
+++ b/tests/critical_paths.yaml
@@ -0,0 +1,64 @@
+# Unstract critical paths registry.
+#
+# A "critical path" is an end-to-end user or system flow whose failure would
+# constitute a production incident. The rig reports:
+# ✅ covered — at least one group in `covered_by` ran green this build
+# ⚠️ gap — `covered_by` is empty OR no group covering it ran
+# ❌ regression — a path that was ✅ on the cached main baseline is now not ✅
+#
+# We intentionally do NOT chase 100% coverage. Focus on filling these gaps first.
+version: 1
+
+paths:
+ - id: auth-login
+ description: "User can log in and obtain a session cookie."
+ entry: "POST /api/v1/auth/login"
+ covered_by: [e2e-smoke]
+
+ - id: adapter-register-llm
+ description: "Register and validate an LLM adapter."
+ entry: "POST /api/v1/adapter/"
+ # Honest declaration: unit-backend is currently optional/gated and
+ # e2e-smoke only hits /health/. Track as a gap until a real adapter test
+ # exists (likely under tests/e2e/smoke/ or a new tests/e2e/adapters/ group).
+ covered_by: []
+
+ - id: workflow-create-execute
+ description: "Create a workflow, configure source+destination, execute, poll, fetch result."
+ entry: "POST /api/v1/workflow/{id}/execute/"
+ covered_by: [e2e-workflow]
+
+ - id: api-deployment-run
+ description: "Deploy a workflow as an API, POST a document, receive structured JSON."
+ entry: "POST /deployment/api/{org}/{name}/"
+ covered_by: [e2e-api-deployment]
+
+ - id: prompt-studio-fetch-response
+ description: "Prompt Studio: create project, add prompt, run single-pass, get response."
+ entry: "POST /api/v1/prompt-studio/prompt-studio-tool/{id}/fetch_response/"
+ covered_by: [e2e-prompt-studio]
+
+ - id: pipeline-etl-execute
+ description: "Run an ETL pipeline from source connector to destination."
+ entry: "POST /api/v1/pipeline/{id}/execute/"
+ covered_by: [] # gap
+
+ - id: tool-sandbox-exec
+ description: "Tool image runs in sandbox container and emits structured output."
+ entry: "internal: tool-registry → runner → docker run"
+ covered_by: [unit-runner]
+
+ - id: usage-token-tracking
+ description: "Per-execution token usage is recorded and retrievable."
+ entry: "GET /api/v1/usage/get_token_usage/"
+ covered_by: [] # gap
+
+ - id: workflow-execution-fan-out
+ description: "Multi-file workflow execution fans out to file-processing workers and rejoins."
+ entry: "internal: backend → rabbitmq → workers/file_processing"
+ covered_by: [] # gap
+
+ - id: callback-result-delivery
+ description: "Async results are posted back via the callback worker."
+ entry: "internal: workers/callback → backend /internal endpoints"
+ covered_by: [] # gap
diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py
new file mode 100644
index 0000000000..b184df9641
--- /dev/null
+++ b/tests/e2e/conftest.py
@@ -0,0 +1,26 @@
+"""E2E fixtures: a session-scoped ``platform`` fixture that yields URLs/creds.
+
+The rig brings the platform up at the *rig* level (once per ``run`` invocation,
+not per pytest session) and propagates URLs into pytest via env vars. This
+conftest reads those env vars; if they're missing, e2e tests are skipped with
+a clear message rather than spuriously failing.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from tests.rig.runtime import PlatformEndpoints
+
+
+@pytest.fixture(scope="session")
+def platform() -> PlatformEndpoints:
+ if not os.environ.get("UNSTRACT_BACKEND_URL"):
+ pytest.skip(
+ "platform URLs not set in env; run via "
+ "`python -m tests.rig run --tier e2e ...` or export "
+ "UNSTRACT_BACKEND_URL (etc.) yourself."
+ )
+ return PlatformEndpoints.from_env()
diff --git a/tests/e2e/smoke/__init__.py b/tests/e2e/smoke/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/e2e/smoke/test_smoke.py b/tests/e2e/smoke/test_smoke.py
new file mode 100644
index 0000000000..4e496196fa
--- /dev/null
+++ b/tests/e2e/smoke/test_smoke.py
@@ -0,0 +1,35 @@
+"""Single placeholder e2e smoke test.
+
+Exercises the rig's e2e plumbing end-to-end (platform up → URL injection →
+pytest discovery → result aggregation → platform down) without depending on
+auth state or seed data. Real workflow + API-deployment tests will land in
+adjacent files (tests/e2e/workflows/, tests/e2e/api_deployment/, ...).
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+import requests
+
+pytestmark = [pytest.mark.e2e, pytest.mark.critical]
+
+
+def test_rig_session_sentinel_present() -> None:
+ """The rig stamps ``UNSTRACT_RIG_SESSION_ID`` into every group's env.
+ Its absence means this test ran outside the rig (or with stale shell env
+ that didn't get the sentinel), in which case the backend URL is suspect.
+ """
+ assert os.environ.get("UNSTRACT_RIG_SESSION_ID"), (
+ "UNSTRACT_RIG_SESSION_ID not set — either this test wasn't launched "
+ "via `python -m tests.rig run`, or the rig failed to propagate the "
+ "sentinel. Re-run via the rig; if launched manually, set the env var "
+ "yourself to assert acknowledgement that you've audited the platform "
+ "URLs in your shell."
+ )
+
+
+def test_backend_health(platform) -> None:
+ response = requests.get(f"{platform.backend_url.rstrip('/')}/health/", timeout=10)
+ assert response.status_code < 500, f"backend /health returned {response.status_code}"
diff --git a/tests/fixtures/.gitkeep b/tests/fixtures/.gitkeep
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/groups.yaml b/tests/groups.yaml
new file mode 100644
index 0000000000..298cfdc637
--- /dev/null
+++ b/tests/groups.yaml
@@ -0,0 +1,181 @@
+# Unstract test rig — group definitions.
+#
+# Each group is the unit of selection. A group's `paths` are passed to pytest
+# (or another runner). `depends_on` is expanded transitively before execution.
+#
+# Add new groups by appending to this file. Validate with:
+# python -m tests.rig validate
+# python -m tests.rig list-groups
+# python -m tests.rig expand
+version: 1
+
+defaults:
+ timeout_seconds: 600
+ parallel: true
+ runner: pytest
+
+groups:
+ # ── Unit tier: pure pytest, no external services ───────────────────────────
+ unit-rig:
+ # Self-tests for the rig's pure logic (dep-graph, evaluate, junit parse).
+ # Run from the repo root so the `tests.rig` package import resolves.
+ tier: unit
+ workdir: .
+ paths: [tests/rig/tests]
+ coverage_source: tests/rig
+
+ unit-sdk1:
+ tier: unit
+ workdir: unstract/sdk1
+ paths: [tests]
+ markers: "not slow and not integration"
+ uv_sync_group: test
+ coverage_source: src/unstract/sdk1
+
+ unit-runner:
+ tier: unit
+ workdir: runner
+ # Runner co-locates its tests under src/. Pytest recurses from here.
+ # The project has no `test` uv group, so deps are pip-installed inline.
+ paths: [src]
+ pip_install:
+ - "flask~=3.1.0"
+ - "docker==6.1.3"
+ - "redis~=5.2.1"
+ - "python-dotenv>=1.0.0"
+ - "kubernetes"
+ install_editable: true
+ coverage_source: src/unstract/runner
+
+ unit-prompt-service:
+ tier: unit
+ workdir: prompt-service
+ paths: [src/unstract/prompt_service/tests/unit]
+ uv_sync_group: test
+ # The parent conftest pulls in heavy adapter chains (pinecone etc.); unit
+ # tests must not depend on integration fixtures.
+ pytest_extra: ["--noconftest"]
+ markers: "not slow"
+ coverage_source: src/unstract/prompt_service
+
+ unit-platform-service:
+ tier: unit
+ workdir: platform-service
+ paths: [tests]
+ uv_sync_group: test
+ coverage_source: src
+ optional: true # service has only a couple of tests today
+
+ unit-workers:
+ tier: unit
+ workdir: workers
+ paths: [tests, shared/tests]
+ markers: "unit"
+ uv_sync_group: test
+ coverage_source: shared
+
+ unit-backend:
+ tier: unit
+ workdir: backend
+ # List paths explicitly. `[.]` recurses into every test_*.py in backend/,
+ # including vendored fixtures and pluggable-app tests that don't belong
+ # in the OSS rig — keep this list scoped to the apps actually under test.
+ paths:
+ - account_v2/tests
+ - adapter_processor_v2/tests
+ - api_deployment_v2/tests
+ - connector_v2/tests
+ - dashboard_metrics/tests
+ - file_management/tests
+ - project/tests
+ - prompt_studio/tests
+ - tenant_account_v2/tests
+ - usage_v2/tests
+ - utils/tests
+ - workflow_manager/endpoint_v2/tests
+ uv_sync_group: test
+ env:
+ DJANGO_SETTINGS_MODULE: backend.settings.test_cases
+ # Backend ORM imports require a real Postgres; rig provisions it via
+ # testcontainers or compose when this group is selected.
+ requires_services: [postgres, redis]
+ coverage_source: .
+ optional: true # gated until backend test_cases settings are complete
+
+ unit-connectors:
+ tier: unit
+ workdir: unstract/connectors
+ paths: [tests]
+ uv_sync_group: test
+ coverage_source: src
+ optional: true
+
+ unit-core:
+ tier: unit
+ workdir: unstract/core
+ paths: [tests]
+ # No `test` uv group in unstract/core today; rig still injects pytest plugins.
+ install_editable: true
+ coverage_source: src
+ optional: true
+
+ unit-tool-registry:
+ tier: unit
+ workdir: unstract/tool-registry
+ paths: [tests]
+ uv_sync_group: test
+ coverage_source: src
+ optional: true
+
+ # ── Integration tier: needs infra but not full platform ────────────────────
+ integration-workflow-execution:
+ tier: integration
+ paths: [tests/integration/workflow_execution]
+ requires_services: [postgres, redis, minio]
+ depends_on: [unit-sdk1, unit-workers]
+ optional: true # placeholder until tests are authored
+
+ # ── E2E tier: real HTTP against running platform ───────────────────────────
+ # Every group below has `requires_platform: true` and must (transitively)
+ # depend on `e2e-smoke` — the rig's `_validate_platform_groups_depend_on_gate`
+ # enforces this so smoke failures cleanly skip dependents rather than running
+ # them against a half-up stack.
+ e2e-smoke:
+ tier: e2e
+ paths: [tests/e2e/smoke]
+ requires_platform: true
+ critical: true
+ timeout_seconds: 1200
+ depends_on: [unit-sdk1, unit-workers]
+
+ e2e-workflow:
+ tier: e2e
+ paths: [tests/e2e/workflows]
+ requires_platform: true
+ critical: true
+ depends_on: [e2e-smoke]
+ optional: true
+
+ e2e-api-deployment:
+ tier: e2e
+ paths: [tests/e2e/api_deployment]
+ requires_platform: true
+ critical: true
+ depends_on: [e2e-smoke]
+ optional: true
+
+ e2e-prompt-studio:
+ tier: e2e
+ paths: [tests/e2e/prompt_studio]
+ requires_platform: true
+ critical: true
+ depends_on: [e2e-smoke]
+ optional: true
+
+ e2e-hurl:
+ tier: e2e
+ runner: hurl
+ paths: [tests/e2e/hurl]
+ requires_platform: true
+ depends_on: [e2e-smoke]
+ optional: true
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/rig/__init__.py b/tests/rig/__init__.py
new file mode 100644
index 0000000000..58585086ca
--- /dev/null
+++ b/tests/rig/__init__.py
@@ -0,0 +1,21 @@
+"""Unstract test rig — entry point package.
+
+The rig is invoked as ``python -m tests.rig `` (see :mod:`tests.rig.cli`).
+It is the single dispatcher behind ``tox`` envs, the pre-commit hook, and CI.
+"""
+
+from tests.rig.critical_paths import (
+ CriticalPath,
+ CriticalPathRegistry,
+ load_critical_paths,
+)
+from tests.rig.groups import GroupDefinition, GroupManifest, load_groups
+
+__all__ = [
+ "GroupDefinition",
+ "GroupManifest",
+ "load_groups",
+ "CriticalPath",
+ "CriticalPathRegistry",
+ "load_critical_paths",
+]
diff --git a/tests/rig/__main__.py b/tests/rig/__main__.py
new file mode 100644
index 0000000000..5229fd2e6e
--- /dev/null
+++ b/tests/rig/__main__.py
@@ -0,0 +1,4 @@
+from tests.rig.cli import main
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/rig/cli.py b/tests/rig/cli.py
new file mode 100644
index 0000000000..2866a417f3
--- /dev/null
+++ b/tests/rig/cli.py
@@ -0,0 +1,753 @@
+"""Rig CLI — the single entry point for tox, pre-commit, and CI.
+
+Subcommands:
+ run Execute selected groups (with dep-expansion).
+ list-groups Print known groups, tiers, and dep edges.
+ list-critical-paths Print critical-path coverage status (best-effort).
+ expand Print the topo-sorted set of groups that ``run`` would execute.
+ validate Validate manifests; non-zero on schema errors.
+ platform ``up | down | status`` the e2e platform stack.
+ report ``combine`` — re-aggregate reports/ after the fact.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+import time
+import uuid
+from functools import lru_cache
+from pathlib import Path
+from xml.sax import saxutils
+
+from tests.rig import critical_paths as cp
+from tests.rig.coverage import combine_and_report, coverage_env
+from tests.rig.groups import (
+ REPO_ROOT,
+ TIERS,
+ GroupDefinition,
+ load_groups,
+)
+from tests.rig.reporting import GroupResult, parse_junit, write_summary
+from tests.rig.runtime import PlatformEndpoints, PlatformRuntime, pick_runtime
+from tests.rig.selection import resolve
+
+# Pytest exit codes that the rig treats as non-failure for aggregation:
+# 0 — all tests passed
+# 5 — no tests collected (optional placeholders, empty hurl group, etc.)
+_NON_FAILING_PYTEST_EXIT_CODES = (0, 5)
+
+
+@lru_cache(maxsize=1)
+def _rig_session_id() -> str:
+ """Stable per-invocation sentinel, computed once.
+
+ Stamped into ``UNSTRACT_RIG_SESSION_ID`` for every group's pytest env so
+ e2e tests can prove the rig ran. URL ownership is intentionally cooperative
+ — the rig sets ``UNSTRACT_*_URL`` via ``setdefault``, so a developer's
+ pre-set value wins (see tests/README.md). The session id is the rig's
+ signature, not a claim that the rig owns the URLs.
+ """
+ return uuid.uuid4().hex
+
+
+def _subprocess_env() -> dict[str, str]:
+ """Base environment for every group's ``uv``/pytest subprocess.
+
+ Drops ``VIRTUAL_ENV`` so ``uv run`` does clean per-group project discovery
+ (each group runs against its own workdir's ``.venv``). When the rig runs
+ under tox, tox exports ``VIRTUAL_ENV=.tox/``, which doesn't match any
+ group's project and makes ``uv`` emit a "does not match the project
+ environment path" warning before ignoring it anyway. Stripping it removes
+ the noise and the ambiguity. ``UV_PROJECT_ENVIRONMENT`` is dropped for the
+ same reason.
+ """
+ env = os.environ.copy()
+ env.pop("VIRTUAL_ENV", None)
+ env.pop("UV_PROJECT_ENVIRONMENT", None)
+ return env
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = _build_parser()
+ args = parser.parse_args(argv)
+ return args.func(args)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+ p = argparse.ArgumentParser(prog="tests.rig", description="Unstract unified test rig")
+ sub = p.add_subparsers(dest="command", required=True)
+
+ pr = sub.add_parser("run", help="Run selected groups")
+ pr.add_argument("groups", nargs="*", help="Group names, or 'all'.")
+ pr.add_argument("--from-file", type=Path, help="File with one group name per line.")
+ pr.add_argument("--tier", choices=TIERS)
+ pr.add_argument("--marker", help="Pytest -m marker expression to forward.")
+ pr.add_argument(
+ "--paths", help="Comma-separated pytest paths/nodeids (overrides group paths)."
+ )
+ pr.add_argument("--runtime", choices=["compose", "testcontainers", "local"])
+ pr.add_argument("--coverage", dest="coverage", action="store_true", default=True)
+ pr.add_argument("--no-coverage", dest="coverage", action="store_false")
+ pr.add_argument("--parallel", dest="parallel", action="store_true", default=True)
+ pr.add_argument("--no-parallel", dest="parallel", action="store_false")
+ pr.add_argument(
+ "--workers",
+ default="auto",
+ help="pytest-xdist worker count (default: auto).",
+ )
+ pr.add_argument("--timeout", type=int, help="Per-group timeout override in seconds.")
+ pr.add_argument("--reports-dir", type=Path, default=REPO_ROOT / "reports")
+ pr.add_argument(
+ "--fail-on-critical-gap",
+ action="store_true",
+ help="Treat uncovered critical paths as a build failure.",
+ )
+ pr.add_argument(
+ "--changed-only",
+ action="store_true",
+ help="Auto-select groups overlapping `git diff origin/main...HEAD`.",
+ )
+ pr.add_argument("--changed-base", default="origin/main")
+ pr.add_argument(
+ "--baseline",
+ type=Path,
+ default=REPO_ROOT / "reports" / "previous-summary.json",
+ )
+ pr.add_argument(
+ "--update-baseline",
+ action="store_true",
+ help=(
+ "On green main builds, merge this build's covered paths into the "
+ "baseline. Merging (not overwriting) preserves coverage recorded by "
+ "earlier tier invocations in the same workflow."
+ ),
+ )
+ pr.add_argument("--dry-run", action="store_true", help="Plan only; do not execute.")
+ pr.set_defaults(func=cmd_run)
+
+ pl = sub.add_parser("list-groups", help="List all defined groups.")
+ pl.set_defaults(func=cmd_list_groups)
+
+ pc = sub.add_parser("list-critical-paths", help="Print critical-path status table.")
+ pc.add_argument(
+ "--baseline",
+ type=Path,
+ default=REPO_ROOT / "reports" / "previous-summary.json",
+ )
+ pc.set_defaults(func=cmd_list_critical)
+
+ pe = sub.add_parser("expand", help="Show what `run` would execute, in topo order.")
+ pe.add_argument("groups", nargs="*")
+ pe.add_argument("--from-file", type=Path)
+ pe.add_argument("--tier", choices=TIERS)
+ pe.add_argument("--changed-only", action="store_true")
+ pe.add_argument("--changed-base", default="origin/main")
+ pe.set_defaults(func=cmd_expand)
+
+ pv = sub.add_parser("validate", help="Validate manifests.")
+ pv.set_defaults(func=cmd_validate)
+
+ pp = sub.add_parser("platform", help="Manage the e2e platform stack.")
+ pp.add_argument("action", choices=["up", "down", "status"])
+ pp.add_argument("--runtime", choices=["compose", "testcontainers", "local"])
+ pp.set_defaults(func=cmd_platform)
+
+ pre = sub.add_parser("report", help="Re-aggregate existing reports/.")
+ pre.add_argument("action", choices=["combine"])
+ pre.add_argument("--reports-dir", type=Path, default=REPO_ROOT / "reports")
+ pre.set_defaults(func=cmd_report)
+
+ return p
+
+
+# ── subcommands ───────────────────────────────────────────────────────────────
+
+
+def cmd_list_groups(_args: argparse.Namespace) -> int:
+ manifest = load_groups()
+ for name in manifest.names():
+ g = manifest.get(name)
+ deps = ", ".join(g.depends_on) or "—"
+ flags: list[str] = []
+ if g.critical:
+ flags.append("critical")
+ if g.requires_platform:
+ flags.append("platform")
+ if g.optional:
+ flags.append("optional")
+ if g.requires_services:
+ flags.append("svc:" + "+".join(g.requires_services))
+ print(
+ f" {name:<32} tier={g.tier:<11} runner={g.runner:<7} "
+ f"deps=[{deps}] {' '.join(flags)}"
+ )
+ return 0
+
+
+def cmd_list_critical(args: argparse.Namespace) -> int:
+ manifest = load_groups()
+ registry = cp.load_critical_paths()
+ errors = cp.validate_registry_against_manifest(registry, manifest)
+ for err in errors:
+ print(f"ERROR: {err}", file=sys.stderr)
+ try:
+ baseline = cp.load_baseline(args.baseline)
+ except cp.BaselineCorruptError as exc:
+ print(f"[rig] {exc}", file=sys.stderr)
+ baseline = None
+ statuses = cp.evaluate(registry, groups_run_green=set(), baseline=baseline)
+ icons = {"covered": "✅", "gap": "⚠️", "regression": "❌"}
+ for s in statuses:
+ cov = ", ".join(s.path.covered_by) or "—"
+ print(f" {icons[s.state]} {s.path.id:<28} declared coverage: {cov}")
+ return 1 if errors else 0
+
+
+def cmd_expand(args: argparse.Namespace) -> int:
+ manifest = load_groups()
+ ordered = resolve(
+ manifest,
+ positional=args.groups or [],
+ from_file=args.from_file,
+ tier=args.tier,
+ changed_only=args.changed_only,
+ changed_base=args.changed_base,
+ )
+ if not ordered:
+ print("(no groups selected)", file=sys.stderr)
+ return 1
+ for name in ordered:
+ g = manifest.get(name)
+ print(f" {name} (tier={g.tier}, runner={g.runner})")
+ return 0
+
+
+def cmd_validate(_args: argparse.Namespace) -> int:
+ """Validate both manifests. Schema/path/cycle errors come from
+ ``load_groups`` (raised); cross-manifest errors come from
+ ``validate_registry_against_manifest``.
+ """
+ try:
+ manifest = load_groups()
+ except (ValueError, FileNotFoundError) as exc:
+ print(f"ERROR: {exc}", file=sys.stderr)
+ return 1
+ try:
+ registry = cp.load_critical_paths()
+ except (ValueError, FileNotFoundError) as exc:
+ print(f"ERROR: {exc}", file=sys.stderr)
+ return 1
+ errors = cp.validate_registry_against_manifest(registry, manifest)
+ for err in errors:
+ print(f"ERROR: {err}", file=sys.stderr)
+ if errors:
+ return 1
+ print(f"OK — {len(manifest.names())} groups, {len(registry.paths)} critical paths")
+ return 0
+
+
+def cmd_platform(args: argparse.Namespace) -> int:
+ runtime = pick_runtime(args.runtime)
+ if args.action == "up":
+ try:
+ endpoints = runtime.up()
+ except Exception:
+ # If up() raised mid-way (e.g. one testcontainer started, the next
+ # failed), down() cleans up the partial stack.
+ runtime.down()
+ raise
+ print(f"Platform up via runtime={runtime.name}:")
+ print(f" backend : {endpoints.backend_url}")
+ print(f" prompt-service : {endpoints.prompt_service_url}")
+ print(f" platform-service: {endpoints.platform_service_url}")
+ print(f" runner : {endpoints.runner_url}")
+ return 0
+ if args.action == "down":
+ runtime.down()
+ return 0
+ if args.action == "status":
+ print(
+ f"runtime={runtime.name} (status check is best-effort; "
+ f"see `docker compose ps` for compose)"
+ )
+ return 0
+ return 2
+
+
+def cmd_report(args: argparse.Namespace) -> int:
+ reports_dir: Path = args.reports_dir
+ combine_and_report(reports_dir)
+ manifest = load_groups()
+ registry = cp.load_critical_paths()
+ group_results: list[GroupResult] = []
+ for name in manifest.names():
+ tier = manifest.get(name).tier
+ result = parse_junit(name, tier, reports_dir)
+ if result is not None:
+ group_results.append(result)
+ green = _green_group_names(group_results)
+ baseline_corrupt = False
+ try:
+ baseline = cp.load_baseline(reports_dir / "previous-summary.json")
+ except cp.BaselineCorruptError as exc:
+ print(f"[rig] {exc}", file=sys.stderr)
+ baseline = None
+ baseline_corrupt = True
+ statuses = cp.evaluate(registry, groups_run_green=green, baseline=baseline)
+ write_summary(
+ reports_dir=reports_dir,
+ group_results=group_results,
+ critical_statuses=statuses,
+ baseline_corrupt=baseline_corrupt,
+ )
+ print(f"Wrote {reports_dir / 'summary.md'}")
+ return 0
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+ manifest = load_groups()
+ registry = cp.load_critical_paths()
+ errs = cp.validate_registry_against_manifest(registry, manifest)
+ for err in errs:
+ print(f"ERROR: {err}", file=sys.stderr)
+ if errs:
+ return 2
+
+ ordered = resolve(
+ manifest,
+ positional=args.groups or [],
+ from_file=args.from_file,
+ tier=args.tier,
+ changed_only=args.changed_only,
+ changed_base=args.changed_base,
+ )
+ if not ordered:
+ print(
+ "ERROR: no groups selected. Pass group names, `all`, --tier, or --from-file.",
+ file=sys.stderr,
+ )
+ return 2
+
+ runnable = [n for n in ordered if not _is_missing_placeholder(manifest.get(n))]
+ skipped = [n for n in ordered if n not in runnable]
+ for n in skipped:
+ print(f"SKIP {n} (optional + workdir/paths absent)")
+ # Scope of this invocation = every group we will actually run AFTER
+ # dep-expansion (includes dependencies the user didn't ask for directly).
+ # Skipped optional placeholders are excluded: their critical paths were
+ # never going to be exercised here, so they must classify as gap, not
+ # regression. evaluate() uses this to distinguish "this path's group ran
+ # red" (regression) from "this path belongs to a tier we weren't running
+ # this time" (gap).
+ scope_groups = set(runnable)
+
+ reports_dir: Path = args.reports_dir
+ reports_dir.mkdir(parents=True, exist_ok=True)
+
+ needs_platform = any(manifest.get(n).requires_platform for n in runnable)
+ runtime: PlatformRuntime | None = None
+ endpoints: PlatformEndpoints | None = None
+ group_results: list[GroupResult] = []
+ overall_exit = 0
+
+ try:
+ if needs_platform and not args.dry_run:
+ runtime = pick_runtime(args.runtime)
+ print(f"[rig] bringing platform up via runtime={runtime.name}")
+ # `up()` is inside the try so a failure here still triggers `down()`
+ # in the finally, cleaning up any partial stack.
+ endpoints = runtime.up()
+
+ for name in runnable:
+ group = manifest.get(name)
+ print(
+ f"\n[rig] running group: {name} "
+ f"(tier={group.tier}, runner={group.runner})"
+ )
+ if args.dry_run:
+ continue
+ result, exit_code = _execute_group(
+ group,
+ reports_dir=reports_dir,
+ marker=args.marker,
+ paths_override=args.paths,
+ coverage=args.coverage,
+ parallel=args.parallel and group.parallel,
+ workers=args.workers,
+ timeout=args.timeout or group.timeout_seconds,
+ endpoints=endpoints,
+ )
+ if result is not None:
+ group_results.append(result)
+ # `optional: true` groups run and surface their result in the
+ # summary, but never gate the overall exit. This honors the
+ # developer intent for groups that need infra we don't provision in
+ # CI (live-DB connector tests) or that are pluggable/cloud-only:
+ # red shows in the report, merge isn't blocked. Both exit-folds
+ # below are gated on `not group.optional` so the skip is consistent
+ # whether the failure came via exit code or junit attestation.
+ #
+ # Always fold the exit code into overall_exit, even when junit.xml
+ # was never written (segfault/OOM/missing binary). Otherwise the
+ # rig silently returns 0 for catastrophic group failures.
+ if (
+ exit_code not in _NON_FAILING_PYTEST_EXIT_CODES
+ and not group.optional
+ and overall_exit == 0
+ ):
+ overall_exit = exit_code
+ # Belt-and-braces: if the junit attests to errors/failures the exit
+ # code didn't (truncated junit → errors=1 with exit 0), the report
+ # shows ❌ but exit would otherwise stay 0. Keep them in sync.
+ if (
+ result is not None
+ and (result.errors or result.failed)
+ and not group.optional
+ and overall_exit == 0
+ ):
+ overall_exit = 1
+ finally:
+ if runtime is not None and not args.dry_run:
+ print(f"[rig] tearing platform down (runtime={runtime.name})")
+ # Don't let a teardown failure mask the in-flight exception.
+ # Python re-raises whatever exception we caught here if down()
+ # raises during a `finally`, hiding the real cause upstream.
+ try:
+ runtime.down()
+ except Exception as exc:
+ print(
+ f"[rig] teardown failed (runtime={runtime.name}): {exc}",
+ file=sys.stderr,
+ )
+
+ if args.coverage and not args.dry_run:
+ combine_and_report(reports_dir)
+
+ green = _green_group_names(group_results)
+ # A corrupt baseline can't be silently treated as empty (that would turn
+ # the next build into a regression festival once a one-tier baseline gets
+ # written back). But we must still write the per-group summary so the
+ # developer can see what passed/failed — otherwise the reporting path
+ # silently fails on top of the baseline error.
+ baseline_corrupt = False
+ try:
+ baseline = cp.load_baseline(args.baseline)
+ except cp.BaselineCorruptError as exc:
+ print(f"[rig] ❌ {exc}", file=sys.stderr)
+ baseline = None
+ baseline_corrupt = True
+
+ statuses = cp.evaluate(
+ registry,
+ groups_run_green=green,
+ baseline=baseline,
+ scope_groups=scope_groups,
+ )
+ write_summary(
+ reports_dir=reports_dir,
+ group_results=group_results,
+ critical_statuses=statuses,
+ baseline_corrupt=baseline_corrupt,
+ )
+
+ # Surface baseline corruption regardless of whether the build was
+ # otherwise green or red. A red-then-fixed cycle without this flip would
+ # silently swallow the corrupt cache and disable regression detection on
+ # the next N builds.
+ if baseline_corrupt and overall_exit == 0:
+ overall_exit = 1
+
+ # In a dry run no groups executed, so every covered path looks like a
+ # gap/regression. A dry run is plan-only: report but never fail (and never
+ # write a baseline) on the back of results that didn't happen.
+ if args.dry_run:
+ return overall_exit
+
+ regressions = [s for s in statuses if s.state == "regression"]
+ if regressions:
+ print(
+ f"\n[rig] ❌ {len(regressions)} critical-path regression(s) detected",
+ file=sys.stderr,
+ )
+ if overall_exit == 0:
+ overall_exit = 1
+
+ gaps = [s for s in statuses if s.state == "gap"]
+ if gaps and args.fail_on_critical_gap:
+ print(
+ f"\n[rig] ⚠️ {len(gaps)} critical-path gap(s) detected "
+ f"(fail-on-critical-gap)",
+ file=sys.stderr,
+ )
+ if overall_exit == 0:
+ overall_exit = 1
+
+ if args.update_baseline and overall_exit == 0:
+ try:
+ cp.merge_into_baseline(statuses, args.baseline)
+ print(f"[rig] merged into baseline: {args.baseline}")
+ except cp.BaselineCorruptError as exc:
+ print(f"[rig] ❌ baseline write skipped: {exc}", file=sys.stderr)
+ overall_exit = 1
+
+ return overall_exit
+
+
+# ── execution helpers ─────────────────────────────────────────────────────────
+
+
+def _green_group_names(results: list[GroupResult]) -> set[str]:
+ return {r.name for r in results if r.status in ("pass", "empty")}
+
+
+def _is_missing_placeholder(group: GroupDefinition) -> bool:
+ if not group.optional:
+ return False
+ wd = group.absolute_workdir()
+ if not wd.exists():
+ return True
+ return not all(p.exists() for p in group.absolute_paths())
+
+
+def _execute_group(
+ group: GroupDefinition,
+ *,
+ reports_dir: Path,
+ marker: str | None,
+ paths_override: str | None,
+ coverage: bool,
+ parallel: bool,
+ workers: str,
+ timeout: int,
+ endpoints: PlatformEndpoints | None,
+) -> tuple[GroupResult | None, int]:
+ group_reports = reports_dir / group.name
+ group_reports.mkdir(parents=True, exist_ok=True)
+ junit = group_reports / "junit.xml"
+ md_report = group_reports / "report.md"
+
+ env = _subprocess_env()
+ env.update(group.env)
+ if endpoints is not None:
+ env.setdefault("UNSTRACT_BACKEND_URL", endpoints.backend_url)
+ env.setdefault("UNSTRACT_PROMPT_SERVICE_URL", endpoints.prompt_service_url)
+ env.setdefault("UNSTRACT_PLATFORM_SERVICE_URL", endpoints.platform_service_url)
+ env.setdefault("UNSTRACT_RUNNER_URL", endpoints.runner_url)
+ env.setdefault("UNSTRACT_X2TEXT_URL", endpoints.x2text_url)
+ # Stamp the run with a per-invocation sentinel so e2e tests can
+ # distinguish "rig brought the platform up" from "stale shell env
+ # leaked in". `setdefault` would let a leaked sentinel win, which
+ # defeats the purpose — set unconditionally.
+ env["UNSTRACT_RIG_SESSION_ID"] = _rig_session_id()
+ if coverage and group.coverage_source:
+ env.update(coverage_env(group.name, reports_dir))
+
+ # Best-effort dep prep. Each `uv` call uses check=False so a transient
+ # install failure (e.g. network blip during `uv pip install`) doesn't kill
+ # the whole rig; pytest will surface a real missing-module error if so.
+ # If you're debugging "ModuleNotFoundError" in a group, scroll up for the
+ # uv warnings — they're the smoking gun.
+ _prepare_group_env(group, env=env)
+
+ workdir = group.absolute_workdir()
+
+ if group.runner == "hurl":
+ cmd = _hurl_command(group, workdir)
+ exit_code = _spawn(cmd, env=env, cwd=workdir, timeout=timeout)
+ # Match the exit.txt write's defensive handling: a read-only reports
+ # dir or full disk shouldn't abort the whole run and orphan completed
+ # groups before the summary renders.
+ try:
+ _write_synthetic_junit(junit, group.name, exit_code)
+ except OSError as err:
+ print(
+ f"[rig] could not write synthetic junit for {group.name}: {err}",
+ file=sys.stderr,
+ )
+ else:
+ cmd = _pytest_command(
+ group,
+ workdir=workdir,
+ junit=junit,
+ md_report=md_report,
+ marker=marker,
+ paths_override=paths_override,
+ coverage=coverage,
+ parallel=parallel,
+ workers=workers,
+ timeout=timeout,
+ )
+ exit_code = _spawn(cmd, env=env, cwd=workdir, timeout=timeout)
+
+ try:
+ (group_reports / "exit.txt").write_text(str(exit_code))
+ except OSError as err:
+ print(
+ f"[rig] could not write exit.txt for {group.name}: {err}",
+ file=sys.stderr,
+ )
+
+ return parse_junit(group.name, group.tier, reports_dir), exit_code
+
+
+RIG_PYTEST_PLUGINS = (
+ "pytest>=8.0.1",
+ "pytest-md-report>=0.6.2",
+ "pytest-timeout>=2.3.1",
+ "pytest-xdist>=3.5.0",
+ "pytest-cov>=4.1.0",
+ "pytest-mock>=3.12.0",
+)
+
+
+def _prepare_group_env(group: GroupDefinition, *, env: dict[str, str]) -> None:
+ """Sync deps for a group. Mirrors what the old per-service tox envs did,
+ and ensures the rig's pytest plugins are present in the group's venv.
+ """
+ workdir = group.absolute_workdir()
+ if not shutil.which("uv"):
+ return
+ if group.uv_sync_group:
+ subprocess.run(
+ ["uv", "sync", "--group", group.uv_sync_group],
+ cwd=workdir,
+ env=env,
+ check=False,
+ )
+ if group.install_editable:
+ subprocess.run(
+ ["uv", "pip", "install", "-e", "."],
+ cwd=workdir,
+ env=env,
+ check=False,
+ )
+ if group.pip_install:
+ subprocess.run(
+ ["uv", "pip", "install", *group.pip_install],
+ cwd=workdir,
+ env=env,
+ check=False,
+ )
+ # NOTE: rig pytest plugins (pytest-timeout, pytest-md-report, etc.) are
+ # injected via `uv run --with ...` in _pytest_command, not installed here.
+ # That avoids losing them on the next `uv run` (which re-syncs the venv).
+
+
+def _pytest_command(
+ group: GroupDefinition,
+ *,
+ workdir: Path,
+ junit: Path,
+ md_report: Path,
+ marker: str | None,
+ paths_override: str | None,
+ coverage: bool,
+ parallel: bool,
+ workers: str,
+ timeout: int,
+) -> list[str]:
+ use_uv = shutil.which("uv") is not None
+ if use_uv:
+ # `uv run` re-syncs the project's venv each call, which would wipe any
+ # plugins added via `uv pip install`. `--with` injects them into the
+ # ephemeral run environment, surviving the sync.
+ with_args: list[str] = []
+ for spec in RIG_PYTEST_PLUGINS:
+ with_args += ["--with", spec]
+ base: list[str] = ["uv", "run", *with_args, "pytest"]
+ else:
+ base = [sys.executable, "-m", "pytest"]
+
+ cmd = [
+ *base,
+ "-v",
+ f"--junitxml={junit}",
+ f"--timeout={timeout}",
+ ]
+ # pytest-md-report does not aggregate worker output reliably under xdist.
+ # Emit markdown only on serial runs; junit + reporting.py's _render_markdown
+ # cover the parallel case.
+ if not parallel:
+ cmd += [
+ "--md-report",
+ "--md-report-flavor=gfm",
+ f"--md-report-output={md_report}",
+ ]
+ if coverage and group.coverage_source:
+ cmd += [
+ f"--cov={group.coverage_source}",
+ "--cov-report=",
+ ]
+ if parallel:
+ cmd += ["-n", workers]
+ effective_marker = marker or group.markers
+ if effective_marker:
+ cmd += ["-m", effective_marker]
+ cmd += list(group.pytest_extra)
+
+ if paths_override:
+ cmd += [p.strip() for p in paths_override.split(",") if p.strip()]
+ else:
+ # Paths are relative to workdir so pytest runs as `cd workdir && pytest `.
+ for p in group.paths:
+ cmd.append(p)
+ return cmd
+
+
+def _hurl_command(group: GroupDefinition, workdir: Path) -> list[str]:
+ files: list[str] = []
+ for p in group.paths:
+ target = workdir / p
+ if target.is_dir():
+ files.extend(sorted(str(f) for f in target.rglob("*.hurl")))
+ elif target.is_file():
+ files.append(str(target))
+ if not files:
+ # Surface as exit 5 ("no tests collected") for consistency with pytest.
+ return ["sh", "-c", "exit 5"]
+ return ["hurl", "--test", *files]
+
+
+def _write_synthetic_junit(path: Path, group_name: str, exit_code: int) -> None:
+ """Synthesise a JUnit XML for hurl runs.
+
+ Exit 5 ("no tests collected") must produce failures=0; otherwise an empty
+ hurl group would show ⚪ via :class:`GroupResult` while also being counted
+ as a failure in totals + critical-path evaluation.
+
+ ``group_name`` is XML-escaped: a group key containing ``"``/``&``/``<``
+ would otherwise produce malformed XML, which ``parse_junit`` then reads as
+ a phantom error on a green hurl run.
+ """
+ is_failure = exit_code != 0 and exit_code != 5
+ failures = 1 if is_failure else 0
+ failure_tag = f'' if is_failure else ""
+ name = saxutils.escape(group_name, {'"': """})
+ path.write_text(
+ '\n'
+ f'\n'
+ f' {failure_tag}'
+ f"\n"
+ f"\n"
+ )
+
+
+def _spawn(cmd: list[str], *, env: dict[str, str], cwd: Path, timeout: int) -> int:
+ start = time.monotonic()
+ try:
+ result = subprocess.run(cmd, cwd=cwd, env=env, timeout=timeout + 30)
+ return result.returncode
+ except subprocess.TimeoutExpired:
+ print(
+ f"[rig] TIMEOUT after {time.monotonic() - start:.0f}s: {' '.join(cmd)}",
+ file=sys.stderr,
+ )
+ return 124
+ except FileNotFoundError as exc:
+ print(f"[rig] command not found: {exc}", file=sys.stderr)
+ return 127
diff --git a/tests/rig/coverage.py b/tests/rig/coverage.py
new file mode 100644
index 0000000000..710c4ade20
--- /dev/null
+++ b/tests/rig/coverage.py
@@ -0,0 +1,138 @@
+"""Coverage helpers.
+
+Each group runs with its own ``COVERAGE_FILE`` so parallel pytest invocations
+don't trample each other. After all groups complete, ``combine_and_report``
+merges them into a single ``.coverage`` and emits XML/HTML.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+from tests.rig.groups import REPO_ROOT
+
+log = logging.getLogger(__name__)
+
+# Cap each coverage subprocess. Runs after the per-group loop, so per-group
+# timeouts don't apply; without this a slow `uv run --with` resolve could hang
+# the job to the CI ceiling.
+_COVERAGE_TIMEOUT_SECONDS = 300
+
+
+def _clean_env() -> dict[str, str]:
+ """Env for ``uv run`` that drops a leaked ``VIRTUAL_ENV`` (e.g. tox's
+ ``.tox/``) so uv resolves the project venv without a mismatch warning.
+ """
+ env = os.environ.copy()
+ env.pop("VIRTUAL_ENV", None)
+ env.pop("UV_PROJECT_ENVIRONMENT", None)
+ return env
+
+
+def coverage_env(group_name: str, reports_dir: Path) -> dict[str, str]:
+ """Env vars to scope a group's coverage file under ``reports_dir``."""
+ cov_file = reports_dir / f".coverage.{group_name}"
+ cov_file.parent.mkdir(parents=True, exist_ok=True)
+ return {"COVERAGE_FILE": str(cov_file)}
+
+
+def _coverage_base() -> list[str]:
+ """Pick a runner for ``coverage`` that has the package available.
+
+ Prefers ``uv run --with coverage`` so the same dependency-resolution
+ strategy as the test runs is used, and ``coverage`` doesn't need to be
+ installed in the parent interpreter. Falls back to ``python -m coverage``
+ if uv isn't around.
+ """
+ if shutil.which("uv"):
+ return ["uv", "run", "--with", "coverage[toml]", "coverage"]
+ return [sys.executable, "-m", "coverage"]
+
+
+def combine_and_report(reports_dir: Path) -> None:
+ """Combine all per-group ``.coverage.`` files and emit xml + html.
+
+ Combine runs in ``reports_dir`` so the .coverage files are colocated; xml
+ and html run from ``REPO_ROOT`` so coverage can resolve the original source
+ file paths (stored as repo-relative in ``COVERAGE_FILE`` thanks to
+ ``[tool.coverage.run].relative_files = true``).
+
+ Errors are logged (not raised) because a coverage failure shouldn't drop
+ the test run's exit code.
+ """
+ files = sorted(reports_dir.glob(".coverage.*"))
+ if not files:
+ return
+
+ target = reports_dir / ".coverage"
+ if target.exists():
+ target.unlink()
+
+ base = _coverage_base()
+ clean_env = _clean_env()
+ if not _run_coverage(
+ [*base, "combine", *[str(p) for p in files]],
+ cwd=reports_dir,
+ env=clean_env,
+ ):
+ return
+
+ combined = reports_dir / ".coverage"
+ xml_cmd = [
+ *base,
+ "xml",
+ "--data-file",
+ str(combined),
+ "-o",
+ str(reports_dir / "coverage.xml"),
+ ]
+ html_cmd = [
+ *base,
+ "html",
+ "--data-file",
+ str(combined),
+ "-d",
+ str(reports_dir / "htmlcov"),
+ ]
+ for cmd in (xml_cmd, html_cmd):
+ _run_coverage(cmd, cwd=REPO_ROOT, env=clean_env)
+
+
+def _run_coverage(cmd: list[str], *, cwd: Path, env: dict[str, str]) -> bool:
+ """Run a coverage subprocess with a timeout. Returns True on success.
+
+ A timeout is essential: ``uv run --with coverage[toml]`` can resolve over a
+ slow index, and this runs AFTER the per-group loop so the rig's per-group
+ timeouts don't apply — without a cap it could hang the job to the CI limit.
+ Failures are logged, not raised (coverage is best-effort reporting).
+ """
+ try:
+ result = subprocess.run(
+ cmd,
+ cwd=cwd,
+ capture_output=True,
+ text=True,
+ env=env,
+ timeout=_COVERAGE_TIMEOUT_SECONDS,
+ )
+ except subprocess.TimeoutExpired:
+ log.warning(
+ "coverage step timed out after %ds: %s",
+ _COVERAGE_TIMEOUT_SECONDS,
+ " ".join(cmd),
+ )
+ return False
+ if result.returncode != 0:
+ log.warning(
+ "%s failed (exit %d): %s",
+ " ".join(cmd),
+ result.returncode,
+ (result.stderr or result.stdout).strip(),
+ )
+ return False
+ return True
diff --git a/tests/rig/critical_paths.py b/tests/rig/critical_paths.py
new file mode 100644
index 0000000000..b0cbbd0596
--- /dev/null
+++ b/tests/rig/critical_paths.py
@@ -0,0 +1,216 @@
+"""Load ``tests/critical_paths.yaml`` and compute gaps + regressions."""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Collection
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+
+from tests.rig.groups import REPO_ROOT, GroupManifest
+
+log = logging.getLogger(__name__)
+
+CriticalPathState = Literal["covered", "gap", "regression"]
+
+DEFAULT_REGISTRY = REPO_ROOT / "tests" / "critical_paths.yaml"
+
+
+class BaselineCorruptError(RuntimeError):
+ """Raised when the baseline file exists but cannot be parsed.
+
+ The rig refuses to silently treat a corrupt baseline as empty because that
+ would (a) demote real regressions to gaps and (b) wipe the other tier's
+ coverage on the next merge. The CI workflow should delete the cache and
+ retry, surfacing the corruption explicitly.
+ """
+
+
+@dataclass(frozen=True)
+class CriticalPath:
+ id: str
+ description: str
+ entry: str
+ covered_by: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class CriticalPathRegistry:
+ paths: tuple[CriticalPath, ...]
+ _by_id: dict[str, CriticalPath] = field(default_factory=dict, init=False, repr=False)
+
+ def __post_init__(self) -> None:
+ # Duplicate ids would silently last-wins in the lookup while both rows
+ # still render — fail loudly at load time instead.
+ seen: set[str] = set()
+ dupes = sorted({p.id for p in self.paths if p.id in seen or seen.add(p.id)})
+ if dupes:
+ raise ValueError(f"duplicate critical-path ids: {dupes}")
+ # `frozen=True` blocks direct assignment; route through object.__setattr__.
+ object.__setattr__(self, "_by_id", {p.id: p for p in self.paths})
+
+ def by_id(self, path_id: str) -> CriticalPath:
+ if path_id not in self._by_id:
+ raise KeyError(path_id)
+ return self._by_id[path_id]
+
+
+@dataclass(frozen=True)
+class CriticalPathStatus:
+ path: CriticalPath
+ state: CriticalPathState
+ covering_groups_run: tuple[str, ...]
+ notes: str = ""
+
+ def __post_init__(self) -> None:
+ # Make the contradictory states unrepresentable rather than relying on
+ # evaluate()'s discipline: covered ⇒ at least one covering group ran;
+ # gap/regression ⇒ none did.
+ if self.state == "covered" and not self.covering_groups_run:
+ raise ValueError("state='covered' requires a non-empty covering_groups_run")
+ if self.state in ("gap", "regression") and self.covering_groups_run:
+ raise ValueError(f"state={self.state!r} must have empty covering_groups_run")
+
+
+def load_critical_paths(path: Path | None = None) -> CriticalPathRegistry:
+ raw = yaml.safe_load((path or DEFAULT_REGISTRY).read_text())
+ if not isinstance(raw, dict) or "paths" not in raw:
+ raise ValueError(f"{path or DEFAULT_REGISTRY}: expected top-level `paths:` list")
+ return CriticalPathRegistry(
+ paths=tuple(
+ CriticalPath(
+ id=str(p["id"]),
+ description=str(p.get("description", "")),
+ entry=str(p.get("entry", "")),
+ covered_by=tuple(p.get("covered_by") or ()),
+ )
+ for p in raw["paths"]
+ )
+ )
+
+
+def validate_registry_against_manifest(
+ registry: CriticalPathRegistry, manifest: GroupManifest
+) -> list[str]:
+ """Return human-readable errors for unknown groups referenced in ``covered_by``."""
+ errors: list[str] = []
+ known = set(manifest.names())
+ for path in registry.paths:
+ for g in path.covered_by:
+ if g not in known:
+ errors.append(
+ f"critical path {path.id!r}: "
+ f"covered_by references unknown group {g!r}"
+ )
+ return errors
+
+
+def evaluate(
+ registry: CriticalPathRegistry,
+ *,
+ groups_run_green: Collection[str],
+ baseline: dict[str, Any] | None,
+ scope_groups: Collection[str] | None = None,
+) -> list[CriticalPathStatus]:
+ """Compute the status for each critical path against this build's results.
+
+ Args:
+ registry: parsed critical-paths registry.
+ groups_run_green: names of groups that ran AND passed in this build.
+ baseline: parsed previous-summary.json from the main-branch cache, or None.
+ Expected shape: ``{"covered_paths": ["auth-login", ...]}``.
+ scope_groups: collection of every group the caller considered running
+ this invocation (including dep-expanded deps and skipped
+ optional placeholders). When a critical path's ``covered_by``
+ is fully outside ``scope_groups``, the path is classified as
+ ``gap`` rather than ``regression`` — running only the unit
+ tier shouldn't flag e2e-tier paths as regressed. If ``None``,
+ no scoping is applied (back-compat).
+
+ Returns:
+ Statuses in the original registry order.
+ """
+ previously_covered: set[str] = set((baseline or {}).get("covered_paths") or [])
+ # Convert to sets internally so per-path membership checks stay O(1) even
+ # when callers pass lists/tuples; the public signature accepts Collection
+ # to leave that choice to them.
+ green = set(groups_run_green)
+ scope = None if scope_groups is None else set(scope_groups)
+ statuses: list[CriticalPathStatus] = []
+ for path in registry.paths:
+ covering = tuple(g for g in path.covered_by if g in green)
+ in_scope = scope is None or any(g in scope for g in path.covered_by)
+ state: CriticalPathState
+ if covering:
+ state = "covered"
+ note = ""
+ elif path.id in previously_covered and in_scope:
+ state = "regression"
+ note = "Was covered on the cached baseline; not covered in this build."
+ else:
+ state = "gap"
+ note = (
+ "Out of scope for this invocation."
+ if not in_scope
+ else "No group covering this path ran green in this build."
+ )
+ statuses.append(
+ CriticalPathStatus(
+ path=path,
+ state=state,
+ covering_groups_run=covering,
+ notes=note,
+ )
+ )
+ return statuses
+
+
+def merge_into_baseline(statuses: list[CriticalPathStatus], destination: Path) -> None:
+ """Merge this build's green critical paths into the cached baseline.
+
+ Two tiers run in separate processes (unit, then integration; then e2e in a
+ separate workflow). Each invocation only knows about the paths covered by
+ *its* groups. A naive overwrite would erase the other tier's coverage, so
+ we union with whatever's already on disk.
+
+ A corrupt baseline raises :class:`BaselineCorruptError` rather than being
+ treated as empty: silently dropping previously-covered paths would erase
+ the other tier's contribution and turn the next build into a regression
+ festival. CI should delete the cache and retry on this exception.
+ """
+ existing: set[str] = set()
+ if destination.exists():
+ try:
+ parsed = json.loads(destination.read_text())
+ existing = set(parsed.get("covered_paths") or [])
+ except (json.JSONDecodeError, OSError) as exc:
+ raise BaselineCorruptError(
+ f"refusing to merge into corrupt baseline {destination}: {exc}. "
+ "Delete the cache entry and re-run."
+ ) from exc
+ fresh = {s.path.id for s in statuses if s.state == "covered"}
+ payload = {"covered_paths": sorted(existing | fresh)}
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_text(json.dumps(payload, indent=2))
+
+
+def load_baseline(source: Path) -> dict[str, Any] | None:
+ """Load the cached baseline.
+
+ Returns None if the file doesn't exist (first build / fresh cache).
+ Raises :class:`BaselineCorruptError` if the file exists but is unreadable
+ or unparseable — see :func:`merge_into_baseline` for the rationale.
+ """
+ if not source.exists():
+ return None
+ try:
+ return json.loads(source.read_text())
+ except (json.JSONDecodeError, OSError) as exc:
+ raise BaselineCorruptError(
+ f"baseline at {source} is unreadable: {exc}. "
+ "Delete the cache entry and re-run."
+ ) from exc
diff --git a/tests/rig/groups.py b/tests/rig/groups.py
new file mode 100644
index 0000000000..4c7328a8b5
--- /dev/null
+++ b/tests/rig/groups.py
@@ -0,0 +1,246 @@
+"""Load and validate the ``tests/groups.yaml`` manifest.
+
+The manifest is the single source of truth for what a group is, how to run it,
+and what it depends on. Cycles fail at load time. Path existence is checked
+unless the group is marked ``optional: true`` (which is how we declare
+"placeholder for future tests" without breaking the rig today).
+"""
+
+from __future__ import annotations
+
+import graphlib
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import MappingProxyType
+from typing import Any, Literal, get_args
+
+import yaml
+
+Tier = Literal["unit", "integration", "e2e"]
+Runner = Literal["pytest", "hurl"]
+
+TIERS: tuple[Tier, ...] = get_args(Tier)
+RUNNERS: tuple[Runner, ...] = get_args(Runner)
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_MANIFEST = REPO_ROOT / "tests" / "groups.yaml"
+
+
+@dataclass(frozen=True)
+class GroupDefinition:
+ name: str
+ tier: Tier
+ paths: tuple[str, ...]
+ runner: Runner = "pytest"
+ workdir: str = "."
+ markers: str | None = None
+ pytest_extra: tuple[str, ...] = ()
+ env: Mapping[str, str] = field(default_factory=dict)
+ uv_sync_group: str | None = None
+ pip_install: tuple[str, ...] = ()
+ install_editable: bool = False
+ requires_services: tuple[str, ...] = ()
+ requires_platform: bool = False
+ depends_on: tuple[str, ...] = ()
+ critical: bool = False
+ coverage_source: str | None = None
+ timeout_seconds: int = 600
+ parallel: bool = True
+ optional: bool = False
+
+ def __post_init__(self) -> None:
+ # Freeze env: a frozen dataclass with a mutable dict still lets callers
+ # scribble onto the shared manifest record (group.env[k] = v). Coerce
+ # to a read-only proxy so the record is genuinely immutable.
+ if not isinstance(self.env, MappingProxyType):
+ object.__setattr__(self, "env", MappingProxyType(dict(self.env)))
+
+ def absolute_workdir(self) -> Path:
+ return (REPO_ROOT / self.workdir).resolve()
+
+ def absolute_paths(self) -> list[Path]:
+ return [(self.absolute_workdir() / p).resolve() for p in self.paths]
+
+
+@dataclass(frozen=True)
+class GroupManifest:
+ groups: dict[str, GroupDefinition]
+
+ def get(self, name: str) -> GroupDefinition:
+ if name not in self.groups:
+ raise KeyError(
+ f"Unknown test group: {name!r}. "
+ "Run `python -m tests.rig list-groups` to see options."
+ )
+ return self.groups[name]
+
+ def names(self) -> list[str]:
+ return sorted(self.groups)
+
+ def names_by_tier(self, tier: Tier) -> list[str]:
+ return sorted(n for n, g in self.groups.items() if g.tier == tier)
+
+ def expand(self, selected: list[str]) -> list[str]:
+ """Return ``selected`` plus the transitive closure of their ``depends_on``,
+ in topological order (dependencies before dependents).
+ """
+ for name in selected:
+ self.get(name) # raises on unknown
+ sorter: graphlib.TopologicalSorter[str] = graphlib.TopologicalSorter()
+ seen: set[str] = set()
+ frontier = list(selected)
+ while frontier:
+ name = frontier.pop()
+ if name in seen:
+ continue
+ seen.add(name)
+ group = self.get(name)
+ sorter.add(name, *group.depends_on)
+ for dep in group.depends_on:
+ if dep not in seen:
+ frontier.append(dep)
+ return list(sorter.static_order())
+
+
+def load_groups(path: Path | None = None) -> GroupManifest:
+ """Parse the YAML manifest and validate it."""
+ manifest_path = path or DEFAULT_MANIFEST
+ raw = yaml.safe_load(manifest_path.read_text())
+ if not isinstance(raw, dict) or "groups" not in raw:
+ raise ValueError(f"{manifest_path}: expected top-level `groups:` mapping")
+
+ defaults = raw.get("defaults") or {}
+ groups: dict[str, GroupDefinition] = {}
+ for name, spec in (raw["groups"] or {}).items():
+ groups[name] = _build_group(name, spec, defaults)
+
+ _validate_no_cycles(groups)
+ _validate_dep_targets_exist(groups)
+ _validate_paths(groups)
+ smoke_gate = defaults.get("platform_gate_group", "e2e-smoke")
+ _validate_platform_groups_depend_on_gate(groups, gate=smoke_gate)
+ return GroupManifest(groups=groups)
+
+
+def _build_group(
+ name: str, spec: dict[str, Any], defaults: dict[str, Any]
+) -> GroupDefinition:
+ tier = spec.get("tier")
+ if tier not in TIERS:
+ raise ValueError(f"group {name!r}: `tier` must be one of {TIERS} (got {tier!r})")
+ runner = spec.get("runner", defaults.get("runner", "pytest"))
+ if runner not in RUNNERS:
+ raise ValueError(
+ f"group {name!r}: `runner` must be one of {RUNNERS} (got {runner!r})"
+ )
+ paths = spec.get("paths") or []
+ if not paths:
+ raise ValueError(f"group {name!r}: at least one `paths` entry is required")
+ try:
+ timeout = int(spec.get("timeout_seconds", defaults.get("timeout_seconds", 600)))
+ except (TypeError, ValueError) as exc:
+ raise ValueError(
+ f"group {name!r}: `timeout_seconds` must be an integer "
+ f"(got {spec.get('timeout_seconds')!r})"
+ ) from exc
+
+ return GroupDefinition(
+ name=name,
+ tier=tier,
+ paths=tuple(paths),
+ runner=runner,
+ workdir=spec.get("workdir", "."),
+ markers=spec.get("markers"),
+ pytest_extra=tuple(spec.get("pytest_extra") or ()),
+ env=dict(spec.get("env") or {}),
+ uv_sync_group=spec.get("uv_sync_group"),
+ pip_install=tuple(spec.get("pip_install") or ()),
+ install_editable=bool(spec.get("install_editable", False)),
+ requires_services=tuple(spec.get("requires_services") or ()),
+ requires_platform=bool(spec.get("requires_platform", False)),
+ depends_on=tuple(spec.get("depends_on") or ()),
+ critical=bool(spec.get("critical", False)),
+ coverage_source=spec.get("coverage_source"),
+ timeout_seconds=timeout,
+ parallel=bool(spec.get("parallel", defaults.get("parallel", True))),
+ optional=bool(spec.get("optional", False)),
+ )
+
+
+def _validate_no_cycles(groups: dict[str, GroupDefinition]) -> None:
+ sorter: graphlib.TopologicalSorter[str] = graphlib.TopologicalSorter()
+ for name, g in groups.items():
+ sorter.add(name, *g.depends_on)
+ try:
+ sorter.prepare()
+ except graphlib.CycleError as exc:
+ raise ValueError(f"dependency cycle in groups.yaml: {exc.args[1]}") from exc
+
+
+def _validate_dep_targets_exist(groups: dict[str, GroupDefinition]) -> None:
+ for name, g in groups.items():
+ for dep in g.depends_on:
+ if dep not in groups:
+ raise ValueError(f"group {name!r} depends_on unknown group {dep!r}")
+
+
+def _validate_paths(groups: dict[str, GroupDefinition]) -> None:
+ for name, g in groups.items():
+ # Optional groups may be placeholders whose paths don't exist yet.
+ # The rig skips them at runtime; don't fail validation here.
+ if g.optional:
+ continue
+ wd = g.absolute_workdir()
+ if not wd.exists():
+ raise ValueError(f"group {name!r}: workdir does not exist: {wd}")
+ for p in g.absolute_paths():
+ if not p.exists():
+ raise ValueError(f"group {name!r}: test path does not exist: {p}")
+
+
+def _validate_platform_groups_depend_on_gate(
+ groups: dict[str, GroupDefinition], *, gate: str
+) -> None:
+ """Every non-gate ``requires_platform`` group must transitively depend on
+ the named gate group. The gate is the smoke test: if it fails, dependent
+ groups skip cleanly rather than running against a half-up stack and
+ reporting misleading failures.
+
+ If the manifest declares ``requires_platform`` groups but doesn't actually
+ define the gate, that's a manifest error — silently disabling the check
+ would defeat the invariant. The gate name is overridable via
+ ``defaults.platform_gate_group`` in ``groups.yaml`` for forks that rename it.
+ """
+ platform_groups = [n for n, g in groups.items() if g.requires_platform and n != gate]
+ if not platform_groups:
+ return # No platform-dependent groups; nothing to enforce.
+ if gate not in groups:
+ raise ValueError(
+ f"`requires_platform` groups present ({sorted(platform_groups)}) "
+ f"but the platform gate {gate!r} is not defined. Either define it, "
+ f"or set `defaults.platform_gate_group` in groups.yaml."
+ )
+ for name in platform_groups:
+ if not _transitively_depends_on(name, gate, groups):
+ raise ValueError(
+ f"group {name!r} requires_platform but does not (transitively) "
+ f"depend on {gate!r}; add it to depends_on so smoke gates this group"
+ )
+
+
+def _transitively_depends_on(
+ name: str, target: str, groups: dict[str, GroupDefinition]
+) -> bool:
+ seen: set[str] = set()
+ frontier = [name]
+ while frontier:
+ current = frontier.pop()
+ if current in seen:
+ continue
+ seen.add(current)
+ for dep in groups[current].depends_on:
+ if dep == target:
+ return True
+ frontier.append(dep)
+ return False
diff --git a/tests/rig/reporting.py b/tests/rig/reporting.py
new file mode 100644
index 0000000000..8bff0a00cb
--- /dev/null
+++ b/tests/rig/reporting.py
@@ -0,0 +1,281 @@
+"""Aggregate per-group results into a single human + machine summary.
+
+Per group, the rig writes:
+ reports//junit.xml — pytest --junitxml
+ reports//report.md — pytest-md-report
+ reports//exit.txt — single integer pytest exit code
+
+This module aggregates those into:
+ reports/summary.json — machine-readable
+ reports/summary.md — human-readable, PR-comment friendly
+ reports/combined-test-report.md — backward-compatible alias for the
+ existing sticky-comment workflow.
+"""
+
+from __future__ import annotations
+
+import logging
+import xml.etree.ElementTree as ET
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Literal
+
+from tests.rig.critical_paths import CriticalPathStatus
+
+log = logging.getLogger(__name__)
+
+ResultStatus = Literal["pass", "empty", "fail"]
+
+_STATUS_ICONS: dict[ResultStatus, str] = {
+ "pass": "✅",
+ "empty": "⚪",
+ "fail": "❌",
+}
+
+
+@dataclass(frozen=True)
+class GroupResult:
+ name: str
+ tier: str
+ exit_code: int
+ passed: int
+ failed: int
+ errors: int
+ skipped: int
+ duration_seconds: float
+
+ @property
+ def status(self) -> ResultStatus:
+ if self.exit_code == 5: # pytest "no tests collected"
+ return "empty"
+ if self.exit_code == 0 and self.failed == 0 and self.errors == 0:
+ return "pass"
+ return "fail"
+
+ @property
+ def status_icon(self) -> str:
+ return _STATUS_ICONS[self.status]
+
+
+def parse_junit(group_name: str, tier: str, reports_dir: Path) -> GroupResult | None:
+ """Parse a group's junit.xml + exit.txt. Returns ``None`` if junit.xml is
+ missing or unparseable, ``GroupResult`` with errors=1 if the XML lacks the
+ expected counter attributes (which would otherwise look spuriously green).
+ """
+ junit_path = reports_dir / group_name / "junit.xml"
+ exit_path = reports_dir / group_name / "exit.txt"
+ if not junit_path.exists():
+ return None
+
+ exit_code = _read_exit_code(exit_path)
+
+ try:
+ tree = ET.parse(junit_path)
+ except ET.ParseError as exc:
+ log.warning("malformed junit.xml for group %r: %s", group_name, exc)
+ return GroupResult(
+ name=group_name,
+ tier=tier,
+ exit_code=exit_code or -1,
+ passed=0,
+ failed=0,
+ errors=1,
+ skipped=0,
+ duration_seconds=0.0,
+ )
+
+ root = tree.getroot()
+ suites = root.findall("testsuite") if root.tag == "testsuites" else [root]
+
+ passed = failed = errors = skipped = 0
+ duration = 0.0
+ saw_attributes = False
+ for s in suites:
+ if "tests" in s.attrib:
+ saw_attributes = True
+ try:
+ total = int(s.attrib.get("tests", "0"))
+ f = int(s.attrib.get("failures", "0"))
+ e = int(s.attrib.get("errors", "0"))
+ sk = int(s.attrib.get("skipped", "0"))
+ duration += float(s.attrib.get("time") or 0)
+ except (TypeError, ValueError) as exc:
+ log.warning(
+ "junit.xml for group %r has non-numeric counters: %s", group_name, exc
+ )
+ return GroupResult(
+ name=group_name,
+ tier=tier,
+ exit_code=exit_code or -1,
+ passed=0,
+ failed=0,
+ errors=1,
+ skipped=0,
+ duration_seconds=duration,
+ )
+ failed += f
+ errors += e
+ skipped += sk
+ passed += max(total - f - e - sk, 0)
+
+ if not saw_attributes:
+ # Junit that parses but has no counters anywhere is almost certainly a
+ # truncated write. Don't count it as green.
+ log.warning(
+ "junit.xml for group %r has no counter attributes; treating as error",
+ group_name,
+ )
+ errors = max(errors, 1)
+
+ return GroupResult(
+ name=group_name,
+ tier=tier,
+ exit_code=exit_code,
+ passed=passed,
+ failed=failed,
+ errors=errors,
+ skipped=skipped,
+ duration_seconds=duration,
+ )
+
+
+def _read_exit_code(exit_path: Path) -> int:
+ if not exit_path.exists():
+ return -1
+ try:
+ return int(exit_path.read_text().strip())
+ except (OSError, ValueError) as exc:
+ log.warning("could not read exit code from %s: %s", exit_path, exc)
+ return -1
+
+
+def write_summary(
+ *,
+ reports_dir: Path,
+ group_results: list[GroupResult],
+ critical_statuses: list[CriticalPathStatus],
+ baseline_corrupt: bool = False,
+) -> None:
+ """Write the per-build summary in JSON and Markdown.
+
+ ``baseline_corrupt=True`` flags a build where the cached baseline could
+ not be parsed. The Markdown summary surfaces a banner so reviewers
+ reading the durable artifact (sticky PR comment, CI step summary) know
+ that regression detection was disabled and any "gap" entries here might
+ actually be regressions.
+ """
+ summary_json = reports_dir / "summary.json"
+ summary_md = reports_dir / "summary.md"
+ combined_md = reports_dir / "combined-test-report.md"
+
+ import json
+
+ summary_json.write_text(
+ json.dumps(
+ {
+ "groups": [asdict(r) for r in group_results],
+ "critical_paths": [
+ {
+ "id": s.path.id,
+ "state": s.state,
+ "covering_groups_run": list(s.covering_groups_run),
+ }
+ for s in critical_statuses
+ ],
+ "baseline_corrupt": baseline_corrupt,
+ },
+ indent=2,
+ )
+ )
+
+ md = _render_markdown(group_results, critical_statuses, baseline_corrupt)
+ summary_md.write_text(md)
+ # Backward-compat alias for the existing sticky-comment workflow.
+ combined_md.write_text(md)
+
+
+def _render_markdown(
+ group_results: list[GroupResult],
+ critical_statuses: list[CriticalPathStatus],
+ baseline_corrupt: bool = False,
+) -> str:
+ lines: list[str] = ["# Unstract test results", ""]
+ if baseline_corrupt:
+ lines.extend(
+ [
+ "> ⚠️ **Baseline cache was corrupt; regression detection "
+ "disabled this run.** Paths classified below as `gap` may "
+ "actually be regressions. Clear the baseline cache and "
+ "re-run to re-validate.",
+ "",
+ ]
+ )
+
+ if group_results:
+ lines.append("## Per-group results")
+ lines.append("")
+ lines.append(
+ "| Status | Group | Tier | Passed | Failed | Errors | Skipped | Duration (s) |"
+ )
+ lines.append("|---|---|---|---:|---:|---:|---:|---:|")
+ for r in group_results:
+ lines.append(
+ f"| {r.status_icon} | `{r.name}` | {r.tier} | {r.passed} | {r.failed} "
+ f"| {r.errors} | {r.skipped} | {r.duration_seconds:.1f} |"
+ )
+ totals = _totals(group_results)
+ lines.append(
+ f"| | **TOTAL** | | **{totals['passed']}** | **{totals['failed']}** "
+ f"| **{totals['errors']}** | **{totals['skipped']}** "
+ f"| **{totals['duration']:.1f}** |"
+ )
+ lines.append("")
+ else:
+ lines.append("_No groups ran in this build._\n")
+
+ if critical_statuses:
+ regressions = [s for s in critical_statuses if s.state == "regression"]
+ gaps = [s for s in critical_statuses if s.state == "gap"]
+ covered = [s for s in critical_statuses if s.state == "covered"]
+
+ lines.append("## Critical paths")
+ lines.append("")
+ if regressions:
+ lines.append("### ❌ Regressions (must be zero)")
+ lines.append("")
+ for s in regressions:
+ lines.append(
+ f"- **{s.path.id}** — {s.path.description} (entry: `{s.path.entry}`)"
+ )
+ lines.append("")
+ if gaps:
+ lines.append("### ⚠️ Critical paths not yet covered")
+ lines.append("")
+ for s in gaps:
+ covers = ", ".join(s.path.covered_by) or "_no groups declared_"
+ lines.append(
+ f"- **{s.path.id}** — {s.path.description} "
+ f"(entry: `{s.path.entry}`; declared coverage: {covers})"
+ )
+ lines.append("")
+ if covered:
+ lines.append("✅ Covered critical paths
")
+ lines.append("")
+ for s in covered:
+ groups_str = ", ".join(s.covering_groups_run)
+ lines.append(f"- **{s.path.id}** — covered by {groups_str}")
+ lines.append("")
+ lines.append("