feat(ci): container-based integration tests + migration gate (closes #3, #5)

.github/workflows/test.yml

@@ -17,8 +17,10 @@ jobs:
     name: Detect Changes
     runs-on: ubuntu-latest
     outputs:
-      backend-matrix: ${{ steps.build-matrix.outputs.matrix }}
+      backend-matrix: ${{ steps.build-matrix.outputs.backend-matrix }}
+      integration-matrix: ${{ steps.build-matrix.outputs.integration-matrix }}
       has-backend: ${{ steps.build-matrix.outputs.has-backend }}
+      has-integration: ${{ steps.build-matrix.outputs.has-integration }}
       frontend: ${{ steps.filter.outputs.frontend }}
     steps:
       - uses: actions/checkout@v4
@@ -87,9 +89,6 @@ jobs:
               - 'libs/**'
               - 'pyproject.toml'
               - 'uv.lock'
-              - 'libs/**'
-              - 'pyproject.toml'
-              - 'uv.lock'
             worker-search:
               - 'services/worker-search/**'
               - 'libs/**'
@@ -100,30 +99,72 @@ jobs:
               - 'libs/**'
               - 'pyproject.toml'
               - 'uv.lock'
+            plugin-search-providers:
+              - 'plugins/backend-engine-search-providers/**'
+              - 'libs/kt-providers/**'
+              - 'libs/kt-core-engine-api/**'
+              - 'libs/kt-db/**'
+              - 'libs/kt-config/**'
+              - 'pyproject.toml'
+              - 'uv.lock'
             frontend:
               - 'frontend/**'
-      - name: Build test matrix from changed packages
+      - name: Build test matrices from changed packages
         id: build-matrix
         run: |
-          MATRIX="[]"
-          add() { MATRIX=$(echo "$MATRIX" | jq --arg n "$1" --arg p "$2" '. + [{"name": $n, "path": $p}]'); }
+          BACKEND="[]"
+          INTEGRATION="[]"
 
-          [ "${{ steps.filter.outputs.kt-config }}" = "true" ]    && add "kt-config"   "libs/kt-config"
-          [ "${{ steps.filter.outputs.kt-db }}" = "true" ]        && add "kt-db"        "libs/kt-db"
-          [ "${{ steps.filter.outputs.kt-models }}" = "true" ]    && add "kt-models"    "libs/kt-models"
-          [ "${{ steps.filter.outputs.kt-providers }}" = "true" ] && add "kt-providers" "libs/kt-providers"
-          [ "${{ steps.filter.outputs.kt-graph }}" = "true" ]     && add "kt-graph"     "libs/kt-graph"
-          [ "${{ steps.filter.outputs.kt-facts }}" = "true" ]     && add "kt-facts"     "libs/kt-facts"
-          [ "${{ steps.filter.outputs.api }}" = "true" ]                  && add "api"                  "services/api"
-          [ "${{ steps.filter.outputs.mcp }}" = "true" ]                  && add "mcp"                  "services/mcp"
-          [ "${{ steps.filter.outputs.worker-bottomup }}" = "true" ]  && add "worker-bottomup"  "services/worker-bottomup"
-          [ "${{ steps.filter.outputs.worker-nodes }}" = "true" ]         && add "worker-nodes"         "services/worker-nodes"
-          [ "${{ steps.filter.outputs.worker-search }}" = "true" ]        && add "worker-search"        "services/worker-search"
-          [ "${{ steps.filter.outputs.worker-sync }}" = "true" ]          && add "worker-sync"          "services/worker-sync"
+          add_backend() {
+            BACKEND=$(echo "$BACKEND" | jq --arg n "$1" --arg p "$2" '. + [{"name": $n, "path": $p}]')
+          }
+          add_integration() {
+            # $3 = pytest -n workers (optional, default "auto"). Serialise
+            # suites that hit external rate-limited APIs (e.g. kt-models
+            # → OpenRouter) by passing workers="1".
+            local workers="${3:-auto}"
+            INTEGRATION=$(echo "$INTEGRATION" \
+              | jq --arg n "$1" --arg p "$2" --arg w "$workers" \
+                  '. + [{"name": $n, "path": $p, "workers": $w}]')
+          }
+          # Detect real integration tests (not just a conftest/__init__) so
+          # empty integration/ dirs don't produce "no tests ran" failures.
+          has_integration_tests() {
+            local path="$1"
+            [ -d "$path/tests/integration" ] || return 1
+            local count
+            count=$(find "$path/tests/integration" -maxdepth 1 -name 'test_*.py' -type f | wc -l)
+            [ "$count" -gt 0 ]
+          }
+          maybe_add() {
+            local name="$1" path="$2" changed="$3" workers="${4:-auto}"
+            [ "$changed" = "true" ] || return 0
+            add_backend "$name" "$path"
+            if has_integration_tests "$path"; then
+              add_integration "$name" "$path" "$workers"
+            fi
+          }
 
-          echo "matrix={\"suite\":$(echo "$MATRIX" | jq -c '.')}" >> "$GITHUB_OUTPUT"
-          echo "has-backend=$([ "$(echo "$MATRIX" | jq 'length')" -gt 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
-          echo "Changed suites: $(echo "$MATRIX" | jq -r '[.[].name] | join(", ")')"
+          maybe_add kt-config               libs/kt-config                              "${{ steps.filter.outputs.kt-config }}"
+          maybe_add kt-db                   libs/kt-db                                  "${{ steps.filter.outputs.kt-db }}"
+          maybe_add kt-models               libs/kt-models                              "${{ steps.filter.outputs.kt-models }}"                      1
+          maybe_add kt-providers            libs/kt-providers                           "${{ steps.filter.outputs.kt-providers }}"
+          maybe_add kt-graph                libs/kt-graph                               "${{ steps.filter.outputs.kt-graph }}"
+          maybe_add kt-facts                libs/kt-facts                               "${{ steps.filter.outputs.kt-facts }}"
+          maybe_add api                     services/api                                "${{ steps.filter.outputs.api }}"
+          maybe_add mcp                     services/mcp                                "${{ steps.filter.outputs.mcp }}"
+          maybe_add worker-bottomup         services/worker-bottomup                    "${{ steps.filter.outputs.worker-bottomup }}"
+          maybe_add worker-nodes            services/worker-nodes                       "${{ steps.filter.outputs.worker-nodes }}"
+          maybe_add worker-search           services/worker-search                      "${{ steps.filter.outputs.worker-search }}"
+          maybe_add worker-sync             services/worker-sync                        "${{ steps.filter.outputs.worker-sync }}"
+          maybe_add plugin-search-providers plugins/backend-engine-search-providers     "${{ steps.filter.outputs.plugin-search-providers }}"
+
+          echo "backend-matrix={\"suite\":$(echo "$BACKEND" | jq -c '.')}" >> "$GITHUB_OUTPUT"
+          echo "integration-matrix={\"suite\":$(echo "$INTEGRATION" | jq -c '.')}" >> "$GITHUB_OUTPUT"
+          echo "has-backend=$([ "$(echo "$BACKEND" | jq 'length')" -gt 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "has-integration=$([ "$(echo "$INTEGRATION" | jq 'length')" -gt 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          echo "Changed unit suites: $(echo "$BACKEND" | jq -r '[.[].name] | join(", ")')"
+          echo "Changed integration suites: $(echo "$INTEGRATION" | jq -r '[.[].name] | join(", ")')"
 
   backend-lint:
     name: Backend Lint
@@ -140,14 +181,113 @@ jobs:
       - run: uv run --frozen ruff check .
       - run: uv run --frozen ruff format --check .
 
+  migrations-check:
+    name: Migrations
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_DB: knowledge_tree
+          POSTGRES_USER: kt
+          POSTGRES_PASSWORD: localdev
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U kt -d knowledge_tree"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+      postgres-write:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: knowledge_tree_write
+          POSTGRES_USER: kt
+          POSTGRES_PASSWORD: localdev
+        ports:
+          - 5435:5432
+        options: >-
+          --health-cmd "pg_isready -U kt -d knowledge_tree_write"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+    env:
+      DATABASE_URL: postgresql+asyncpg://kt:localdev@localhost:5432/knowledge_tree
+      WRITE_DATABASE_URL: postgresql+asyncpg://kt:localdev@localhost:5435/knowledge_tree_write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: uv sync --all-packages --frozen
+      - name: Verify single graph-db head
+        run: |
+          cd libs/kt-db
+          heads=$(uv run --frozen alembic heads 2>/dev/null | grep -c '(head)' || true)
+          if [ "$heads" != "1" ]; then
+            echo "::error::Expected exactly 1 graph-db alembic head, found $heads"
+            uv run --frozen alembic heads
+            exit 1
+          fi
+      - name: Verify single write-db head
+        run: |
+          cd libs/kt-db
+          heads=$(uv run --frozen alembic -c alembic_write.ini heads 2>/dev/null | grep -c '(head)' || true)
+          if [ "$heads" != "1" ]; then
+            echo "::error::Expected exactly 1 write-db alembic head, found $heads"
+            uv run --frozen alembic -c alembic_write.ini heads
+            exit 1
+          fi
+      - name: Apply graph-db migrations
+        run: cd libs/kt-db && uv run --frozen alembic upgrade head
+      - name: Apply write-db migrations
+        run: cd libs/kt-db && uv run --frozen alembic -c alembic_write.ini upgrade head
+      - name: Downgrade and re-upgrade graph-db (sanity)
+        run: |
+          cd libs/kt-db
+          uv run --frozen alembic downgrade -1
+          uv run --frozen alembic upgrade head
+      - name: Downgrade and re-upgrade write-db (sanity)
+        run: |
+          cd libs/kt-db
+          uv run --frozen alembic -c alembic_write.ini downgrade -1
+          uv run --frozen alembic -c alembic_write.ini upgrade head
+
   backend-test:
-    name: Test ${{ matrix.suite.name }}
+    name: Unit ${{ matrix.suite.name }}
     needs: changes
     if: needs.changes.outputs.has-backend == 'true'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(needs.changes.outputs.backend-matrix) }}
+    env:
+      SKIP_AUTH: "true"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: uv sync --all-packages --frozen
+      - name: Run unit tests
+        run: >-
+          uv run --frozen pytest ${{ matrix.suite.path }}/tests/ -x -n auto
+          --ignore=${{ matrix.suite.path }}/tests/integration
+
+  backend-integration-test:
+    name: Integration ${{ matrix.suite.name }}
+    needs: changes
+    if: needs.changes.outputs.has-integration == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.changes.outputs.integration-matrix) }}
     services:
       postgres:
         image: pgvector/pgvector:pg16
@@ -184,12 +324,26 @@ jobs:
           --health-interval 5s
           --health-timeout 5s
           --health-retries 5
+      qdrant:
+        image: qdrant/qdrant:v1.17.1
+        ports:
+          - 6333:6333
+          - 6334:6334
     env:
       DATABASE_URL: postgresql+asyncpg://kt:localdev@localhost:5432/knowledge_tree
       WRITE_DATABASE_URL: postgresql+asyncpg://kt:localdev@localhost:5435/knowledge_tree_write
       REDIS_URL: redis://localhost:6379
+      QDRANT_URL: http://localhost:6333
       SKIP_AUTH: "true"
+      # Fake JWT accepted by Hatchet-lite's ClientConfig validator.
+      # Payload: {server_url, grpc_broadcast_address, sub, iat}.
       HATCHET_CLIENT_TOKEN: "eyJhbGciOiAiSFMyNTYiLCAidHlwIjogIkpXVCJ9.eyJzZXJ2ZXJfdXJsIjogImh0dHA6Ly9sb2NhbGhvc3Q6ODA4MCIsICJncnBjX2Jyb2FkY2FzdF9hZGRyZXNzIjogImxvY2FsaG9zdDo3MDcwIiwgInN1YiI6ICJjaSIsICJpYXQiOiAxNTE2MjM5MDIyfQ.ZmFrZXNpZw"
+      USE_HATCHET: "false"
+      # External-API secrets — BRAVE_KEY intentionally omitted (Serper is
+      # the primary provider; Brave tests skip cleanly via pytest.skip).
+      # OpenAI embeddings go through OpenRouter, so no OPENAI_API_KEY needed.
+      SERPER_KEY: ${{ secrets.SERPER_KEY }}
+      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v4
@@ -199,10 +353,17 @@ jobs:
         with:
           python-version: "3.12"
       - run: uv sync --all-packages --frozen
-      - name: Run tests
-        run: >-
-          uv run --frozen pytest ${{ matrix.suite.path }}/tests/ -x -n auto
-          --ignore=${{ matrix.suite.path }}/tests/integration
+      # Note: migrations applied to public schema for connectivity sanity.
+      # Integration fixtures still use Base.metadata.create_all against
+      # per-worker schemas for isolation under xdist. Model/migration
+      # drift is therefore caught by the `migrations-check` job above,
+      # not here.
+      - name: Apply graph-db migrations
+        run: cd libs/kt-db && uv run --frozen alembic upgrade head
+      - name: Apply write-db migrations
+        run: cd libs/kt-db && uv run --frozen alembic -c alembic_write.ini upgrade head
+      - name: Run integration tests
+        run: uv run --frozen pytest ${{ matrix.suite.path }}/tests/integration -x -n ${{ matrix.suite.workers }}
 
   frontend-test:
     name: Frontend Tests

justfile

@@ -103,34 +103,46 @@ migrate-write:
     cd libs/kt-db && uv run alembic -c alembic_write.ini upgrade head
 
 # ── Testing ───────────────────────────────────────────────────────
+# Unit tests are infra-free and key-free. Integration tests need infra
+# (`just up`) and run only against `tests/integration/` directories.
 
-# Run all lib tests
-test-libs:
-    cd libs/kt-config && uv run pytest -x
-    cd libs/kt-db && uv run pytest -x
-    cd libs/kt-flags && uv run pytest -x
-    cd libs/kt-models && uv run pytest -x
-    cd libs/kt-providers && uv run pytest -x
-    cd libs/kt-graph && uv run pytest -x
-    cd libs/kt-facts && uv run pytest -x
-
-# Run API tests
-test-api:
-    cd services/api && uv run pytest -x
-
-# Run MCP tests
-test-mcp:
-    cd services/mcp && uv run pytest -x
-
-# Run worker tests
-test-workers:
-    cd services/worker-bottomup && uv run pytest -x
-    cd services/worker-search && uv run pytest -x
-    cd services/worker-nodes && uv run pytest -x
-    cd services/worker-ingest && uv run pytest -x
-
-# Run all tests
-test-all: test-libs test-api test-workers
+# Run unit tests for every backend package (no infra required)
+test-unit:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    for pkg in \
+        libs/kt-config libs/kt-db libs/kt-models libs/kt-providers \
+        libs/kt-graph libs/kt-facts libs/kt-hatchet libs/kt-qdrant \
+        libs/kt-agents-core libs/kt-auth libs/kt-core-engine-api \
+        libs/kt-flags libs/kt-rbac \
+        services/api services/mcp \
+        services/worker-bottomup services/worker-nodes services/worker-sync \
+        services/worker-synthesis services/worker-ingest \
+        plugins/backend-engine-search-providers \
+        plugins/backend-engine-concept-extractor; do
+        if [ -d "$pkg/tests" ]; then
+            echo "── $pkg ──"
+            uv run --frozen pytest "$pkg/tests/" --ignore="$pkg/tests/integration" -x -q
+        fi
+    done
+
+# Run integration tests for every package that has an integration/ dir
+# Requires `just up` (postgres, postgres-write, redis, qdrant, hatchet)
+test-integration:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    for pkg in \
+        libs/kt-db libs/kt-facts libs/kt-graph libs/kt-models libs/kt-providers \
+        services/api services/worker-bottomup services/worker-nodes services/worker-sync \
+        plugins/backend-engine-search-providers; do
+        if [ -d "$pkg/tests/integration" ]; then
+            echo "── $pkg (integration) ──"
+            uv run --frozen pytest "$pkg/tests/integration" -x -q
+        fi
+    done
+
+# Run all backend tests (unit + integration). Needs infra up.
+test-all: test-unit test-integration
 
 # Frontend tests
 test-frontend:

lefthook.yml

@@ -28,13 +28,6 @@ pre-commit:
 pre-push:
   parallel: true
   jobs:
-    - name: python-tests
+    - name: python-unit-tests
       glob: "**/*.py"
-      run: |
-        just test-all || {
-          echo ""
-          echo "Tests failed. Make sure infrastructure is running:"
-          echo "  just up   # starts postgres, postgres-write, redis, qdrant, hatchet"
-          echo ""
-          exit 1
-        }
+      run: just test-unit

libs/kt-db/alembic/versions/212efc51d897_drop_convergence_reports_and_divergent_.py

@@ -1,7 +1,7 @@
 """drop convergence_reports and divergent_claims
 
 Revision ID: 212efc51d897
-Revises: 489643109ccd
+Revises: e3eb15f51d7c
 Create Date: 2026-04-17 17:29:09.860226
 
 """
@@ -13,7 +13,7 @@
 
 # revision identifiers, used by Alembic.
 revision: str = "212efc51d897"
-down_revision: Union[str, Sequence[str], None] = "489643109ccd"
+down_revision: Union[str, Sequence[str], None] = "e3eb15f51d7c"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None