diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2609173c..401fe2d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,21 +23,6 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] - services: - postgres: - image: pgvector/pgvector:pg17 - env: - POSTGRES_USER: cortex - POSTGRES_PASSWORD: cortex - POSTGRES_DB: cortex - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U cortex" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: DATABASE_URL: postgresql://cortex:cortex@localhost:5432/cortex @@ -49,8 +34,37 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Enable pg_trgm extension - run: PGPASSWORD=cortex psql -h localhost -U cortex -d cortex -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" + # Provision PostgreSQL + pgvector on the runner itself, instead of a + # Docker-Hub service container. Anonymous `pgvector/pgvector:pg17` pulls + # from registry-1.docker.io are rate-limited and outage-prone (observed: + # "context deadline exceeded" failing container init before any test ran, + # CI run 27190427877). The runner ships PostgreSQL preinstalled; pgvector + # comes from the PGDG apt repo (apt.postgresql.org) — no registry, no pull + # rate limit. source: runner image (actions/runner-images) + PGDG. + - name: Set up PostgreSQL + pgvector (runner-local, no registry pull) + run: | + set -euxo pipefail + sudo systemctl start postgresql + PG_VER="$(pg_lsclusters -h | awk 'NR==1 {print $1}')" + # Ensure the PGDG apt repo (canonical pgvector source); idempotent. + sudo install -d /usr/share/postgresql-common/pgdg + sudo curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ + | sudo tee /etc/apt/sources.list.d/pgdg.list + for i in 1 2 3; do sudo apt-get update && break || sleep 5; done + for i in 1 2 3; do sudo apt-get install -y "postgresql-${PG_VER}-pgvector" && break || sleep 5; done + # Role + DB expected by DATABASE_URL. + sudo -u postgres psql -v ON_ERROR_STOP=1 \ + -c "CREATE ROLE cortex LOGIN SUPERUSER PASSWORD 'cortex';" + sudo -u postgres createdb -O cortex cortex + sudo -u postgres psql -v ON_ERROR_STOP=1 -d cortex \ + -c "CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS pg_trgm;" + # Wait until reachable over TCP with password auth. + for i in $(seq 1 30); do + PGPASSWORD=cortex pg_isready -h localhost -U cortex -d cortex && break || sleep 1 + done + PGPASSWORD=cortex psql -h localhost -U cortex -d cortex -c "SELECT version();" - name: Cache pip uses: actions/cache@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5da865bb..911aab13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,21 +30,6 @@ jobs: name: Test before release runs-on: ubuntu-latest - services: - postgres: - image: pgvector/pgvector:pg17 - env: - POSTGRES_USER: cortex - POSTGRES_PASSWORD: cortex - POSTGRES_DB: cortex - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U cortex" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: DATABASE_URL: postgresql://cortex:cortex@localhost:5432/cortex @@ -56,8 +41,30 @@ jobs: with: python-version: "3.12" - - name: Enable pg_trgm extension - run: PGPASSWORD=cortex psql -h localhost -U cortex -d cortex -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" + # Runner-local PostgreSQL + pgvector (no Docker Hub pull). See ci.yml for + # the rationale: anonymous registry-1.docker.io pulls are rate-limited and + # outage-prone; PGDG apt is the canonical, unmetered pgvector source. + - name: Set up PostgreSQL + pgvector (runner-local, no registry pull) + run: | + set -euxo pipefail + sudo systemctl start postgresql + PG_VER="$(pg_lsclusters -h | awk 'NR==1 {print $1}')" + sudo install -d /usr/share/postgresql-common/pgdg + sudo curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ + | sudo tee /etc/apt/sources.list.d/pgdg.list + for i in 1 2 3; do sudo apt-get update && break || sleep 5; done + for i in 1 2 3; do sudo apt-get install -y "postgresql-${PG_VER}-pgvector" && break || sleep 5; done + sudo -u postgres psql -v ON_ERROR_STOP=1 \ + -c "CREATE ROLE cortex LOGIN SUPERUSER PASSWORD 'cortex';" + sudo -u postgres createdb -O cortex cortex + sudo -u postgres psql -v ON_ERROR_STOP=1 -d cortex \ + -c "CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS pg_trgm;" + for i in $(seq 1 30); do + PGPASSWORD=cortex pg_isready -h localhost -U cortex -d cortex && break || sleep 1 + done + PGPASSWORD=cortex psql -h localhost -U cortex -d cortex -c "SELECT version();" - name: Cache HuggingFace models uses: actions/cache@v4 diff --git a/.gitignore b/.gitignore index 484a76ea..71765e28 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ benchmarks/snapshots/ .memsearch/ app-tauri/src-tauri/target/ + +# Vendored site-packages tree (pip --target install; not part of the repo) +/deps/ diff --git a/mcp_server/core/wiki_coverage.py b/mcp_server/core/wiki_coverage.py index c3419e7a..158adc14 100644 --- a/mcp_server/core/wiki_coverage.py +++ b/mcp_server/core/wiki_coverage.py @@ -1205,6 +1205,8 @@ def audit_all_domains( ".venv", "venv", "env", + "deps", + "site-packages", "__pycache__", ".mypy_cache", ".pytest_cache", diff --git a/mcp_server/core/wiki_drift.py b/mcp_server/core/wiki_drift.py index 499a8367..3366e77b 100644 --- a/mcp_server/core/wiki_drift.py +++ b/mcp_server/core/wiki_drift.py @@ -186,8 +186,17 @@ def _file_exists_under(source_root: str, cited: str) -> bool: if os.path.isfile(full): return True bn = os.path.basename(cited) + # Prune the same vendored / build dirs as list_source_files. Without this, + # a repo carrying a venv/, node_modules/, deps/, or site-packages/ at its + # root makes this per-cited-path fallback walk tens of thousands of files, + # turning one consolidate cycle into a multi-minute stall. The skip set is + # the single source of truth for "not a source tree". + from mcp_server.core.wiki_coverage import _SKIP_DIRECTORIES + for dirpath, dirnames, filenames in os.walk(source_root): - dirnames[:] = [d for d in dirnames if not d.startswith(".")] + dirnames[:] = [ + d for d in dirnames if d not in _SKIP_DIRECTORIES and not d.startswith(".") + ] if bn in filenames: return True return False diff --git a/mcp_server/handlers/add_rule.py b/mcp_server/handlers/add_rule.py index 9afabd02..e21c6edb 100644 --- a/mcp_server/handlers/add_rule.py +++ b/mcp_server/handlers/add_rule.py @@ -19,7 +19,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import IDEMPOTENT_WRITE # ── Schema ──────────────────────────────────────────────────────────────────── @@ -111,7 +111,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/anchor.py b/mcp_server/handlers/anchor.py index e9b0e1e1..b01b9a18 100644 --- a/mcp_server/handlers/anchor.py +++ b/mcp_server/handlers/anchor.py @@ -15,7 +15,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import IDEMPOTENT_WRITE # ── Schema ──────────────────────────────────────────────────────────────────── @@ -76,7 +76,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/assess_coverage.py b/mcp_server/handlers/assess_coverage.py index f337f2f6..07fdcab9 100644 --- a/mcp_server/handlers/assess_coverage.py +++ b/mcp_server/handlers/assess_coverage.py @@ -17,7 +17,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY # ── Schema ──────────────────────────────────────────────────────────────────── @@ -76,7 +76,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/backfill_memories.py b/mcp_server/handlers/backfill_memories.py index 663e3501..78ed0335 100644 --- a/mcp_server/handlers/backfill_memories.py +++ b/mcp_server/handlers/backfill_memories.py @@ -24,7 +24,7 @@ slug_to_domain, ) from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.scanner import read_head_tail from mcp_server.handlers._tool_meta import NON_IDEMPOTENT_WRITE @@ -324,7 +324,7 @@ async def handler(args: dict[str, Any] | None = None) -> dict[str, Any]: """Backfill prior conversations into the memory store.""" parsed = _parse_args(args) settings = get_memory_settings() - store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) ensure_backfill_log(store) candidates = discover_files(parsed["project_filter"], parsed["max_files"]) diff --git a/mcp_server/handlers/checkpoint.py b/mcp_server/handlers/checkpoint.py index da7baf4b..0003aefa 100644 --- a/mcp_server/handlers/checkpoint.py +++ b/mcp_server/handlers/checkpoint.py @@ -16,7 +16,7 @@ from mcp_server.core.replay import format_restoration from mcp_server.handlers._tool_meta import IDEMPOTENT_WRITE from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store logger = logging.getLogger(__name__) @@ -150,7 +150,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/codebase_analyze.py b/mcp_server/handlers/codebase_analyze.py index 2566f52a..256fadc9 100644 --- a/mcp_server/handlers/codebase_analyze.py +++ b/mcp_server/handlers/codebase_analyze.py @@ -31,7 +31,7 @@ ) from mcp_server.handlers.remember import handler as remember_handler from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY # ── Schema ──────────────────────────────────────────────────────────────── @@ -143,7 +143,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: s = get_memory_settings() - _store = MemoryStore(s.DB_PATH, s.EMBEDDING_DIM) + _store = get_shared_store(s.DB_PATH, s.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/consolidate.py b/mcp_server/handlers/consolidate.py index 8d2a600a..49f5794d 100644 --- a/mcp_server/handlers/consolidate.py +++ b/mcp_server/handlers/consolidate.py @@ -27,7 +27,7 @@ get_embedding_engine, ) from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import IDEMPOTENT_WRITE logger = logging.getLogger(__name__) @@ -161,7 +161,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store @@ -183,6 +183,20 @@ def _timed(fn, *args, **kwargs) -> dict[str, Any]: return result +async def _atimed(fn, *args, **kwargs) -> dict[str, Any]: + """Async counterpart of :func:`_timed` for awaitable cycle functions.""" + t0 = time.monotonic() + try: + result = await fn(*args, **kwargs) or {} + except Exception as exc: + ms = int((time.monotonic() - t0) * 1000) + return {"error": f"{type(exc).__name__}: {exc}", "duration_ms": ms} + ms = int((time.monotonic() - t0) * 1000) + if isinstance(result, dict): + result["duration_ms"] = ms + return result + + async def handler(args: dict[str, Any] | None = None) -> dict[str, Any]: """Run maintenance cycles on the memory system.""" args = args or {} @@ -212,7 +226,7 @@ async def handler(args: dict[str, Any] | None = None) -> dict[str, Any]: if args.get("wiki", True): cap_raw = args.get("wiki_max_purges_per_axis", 500) cap = int(cap_raw) if cap_raw is not None and int(cap_raw) > 0 else None - wiki_stats = _timed( + wiki_stats = await _atimed( run_wiki_maintenance, store, apply_stubs=bool(args.get("wiki_apply_stubs", True)), diff --git a/mcp_server/handlers/consolidation/wiki_maintenance.py b/mcp_server/handlers/consolidation/wiki_maintenance.py index b7facada..f68de02f 100644 --- a/mcp_server/handlers/consolidation/wiki_maintenance.py +++ b/mcp_server/handlers/consolidation/wiki_maintenance.py @@ -28,11 +28,30 @@ from __future__ import annotations import logging +import os from typing import Any logger = logging.getLogger(__name__) +def _headless_authoring_enabled() -> bool: + """Opt-in gate for the ``claude -p`` headless authoring drain. + + Default OFF. The drain can spawn up to ~38 ``claude -p`` subprocesses + per cycle (30 anchor + 8 file-doc), each up to 180s, **synchronously + on the consolidate event loop**. Unthrottled, that storm wedges the + machine — and in tests/CI (where ``claude`` may be on PATH on dev + boxes) it also blocks the suite. It stays off until per-cycle load + balancing lands; set ``CORTEX_HEADLESS_AUTHORING=1`` to opt in. + """ + return os.getenv("CORTEX_HEADLESS_AUTHORING", "0").strip().lower() in { + "1", + "true", + "yes", + "on", + } + + # Autonomous mode applies the stub + classifier purge axes — these # remove content that is either placeholder-only or doesn't pass # admission. **Shallow pages are NEVER auto-deleted** (user direction @@ -56,28 +75,25 @@ async def _invoke_wiki_purge(args: dict[str, Any]) -> dict[str, Any]: - """Call the wiki_purge handler in whichever event-loop context we land in.""" - import asyncio - + """Await the wiki_purge handler on the caller's event loop.""" from mcp_server.handlers.wiki_purge import handler as wiki_purge_handler - try: - running_loop = asyncio.get_running_loop() - except RuntimeError: - running_loop = None - if running_loop is None: - return await wiki_purge_handler(args) - # We're already inside an event loop — just await directly. The - # ``async`` def at the top makes that legal. return await wiki_purge_handler(args) -def _run_purge_axis( +async def _run_purge_axis( *, axis: str, apply: bool, max_purges: int | None = None ) -> dict[str, Any]: - """Run wiki_purge with exactly one axis enabled, returning a flat dict.""" - import asyncio - + """Run wiki_purge with exactly one axis enabled, returning a flat dict. + + Awaited directly on the consolidate handler's event loop. An earlier + revision bridged via ``asyncio.run_coroutine_threadsafe(...).result()``, + which self-deadlocked: it scheduled the coroutine on the *same* loop + whose thread the synchronous caller had already blocked, so the + coroutine could never run and every axis stalled to the 120s timeout + (CI Test job hung ~1h). Awaiting keeps the psycopg async pool on its + owning loop and removes the deadlock entirely. + """ purge_args: dict[str, Any] = { "apply": apply, "purge_stubs": axis == "stub", @@ -86,19 +102,10 @@ def _run_purge_axis( } if max_purges is not None: purge_args["max_purges"] = max_purges - try: - running_loop = asyncio.get_running_loop() - except RuntimeError: - running_loop = None - if running_loop is None: - return asyncio.run(_invoke_wiki_purge(purge_args)) - future = asyncio.run_coroutine_threadsafe( - _invoke_wiki_purge(purge_args), running_loop - ) - return future.result(timeout=120) - - -def run_wiki_maintenance( + return await _invoke_wiki_purge(purge_args) + + +async def run_wiki_maintenance( store: Any, *, apply_stubs: bool = _AUTONOMOUS_STUB_APPLY_DEFAULT, @@ -143,7 +150,7 @@ def run_wiki_maintenance( # Stub axis. try: - r = _run_purge_axis( + r = await _run_purge_axis( axis="stub", apply=apply_stubs, max_purges=max_purges_per_axis ) out["stub"]["purged"] = r.get("purged", 0) @@ -155,7 +162,7 @@ def run_wiki_maintenance( # Classifier axis. try: - r = _run_purge_axis( + r = await _run_purge_axis( axis="classifier", apply=apply_classifier_rejects, max_purges=max_purges_per_axis, @@ -172,27 +179,36 @@ def run_wiki_maintenance( # session. The worker here calls `claude -p` directly so the # loop closes without human intervention. See # ``consolidation/headless_authoring.py``. - try: - from mcp_server.handlers.consolidation.headless_authoring import ( - run_headless_authoring_cycle, - ) - - cycle = run_headless_authoring_cycle() - out["headless_authoring"] = { - "pages_with_gaps": cycle.pages_with_gaps, - "drains_attempted": cycle.drains_attempted, - "drains_filled": cycle.drains_filled, - "drains_failed": cycle.drains_failed, - "duration_ms": cycle.duration_ms, - } - except Exception as exc: - logger.debug( - "wiki_maintenance: headless authoring drain failed (non-fatal): %s", - exc, - ) - out["headless_authoring"] = { - "status": f"error: {type(exc).__name__}: {exc}", - } + # + # Opt-in only (default OFF): the drain spawns up to ~38 ``claude -p`` + # subprocesses synchronously on the event loop. Until per-cycle load + # balancing lands it stays gated behind ``CORTEX_HEADLESS_AUTHORING`` + # so consolidate (and the test suite) never blocks on a subprocess + # storm. + if not _headless_authoring_enabled(): + out["headless_authoring"] = {"status": "disabled"} + else: + try: + from mcp_server.handlers.consolidation.headless_authoring import ( + run_headless_authoring_cycle, + ) + + cycle = run_headless_authoring_cycle() + out["headless_authoring"] = { + "pages_with_gaps": cycle.pages_with_gaps, + "drains_attempted": cycle.drains_attempted, + "drains_filled": cycle.drains_filled, + "drains_failed": cycle.drains_failed, + "duration_ms": cycle.duration_ms, + } + except Exception as exc: + logger.debug( + "wiki_maintenance: headless authoring drain failed (non-fatal): %s", + exc, + ) + out["headless_authoring"] = { + "status": f"error: {type(exc).__name__}: {exc}", + } # Per-project coverage dashboards (Meadows L6 information surface). try: diff --git a/mcp_server/handlers/create_trigger.py b/mcp_server/handlers/create_trigger.py index bbe5a72c..451c0d78 100644 --- a/mcp_server/handlers/create_trigger.py +++ b/mcp_server/handlers/create_trigger.py @@ -16,7 +16,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import IDEMPOTENT_WRITE # ── Schema ──────────────────────────────────────────────────────────────────── @@ -91,7 +91,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/curate_wiki.py b/mcp_server/handlers/curate_wiki.py index bbfc334a..58855dc9 100644 --- a/mcp_server/handlers/curate_wiki.py +++ b/mcp_server/handlers/curate_wiki.py @@ -53,7 +53,7 @@ from mcp_server.core.wiki_drift import audit_wiki_drift from mcp_server.handlers._tool_meta import READ_ONLY from mcp_server.infrastructure.config import WIKI_ROOT -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import get_shared_store schema = { @@ -247,7 +247,7 @@ async def handler(args: dict[str, Any] | None = None) -> dict[str, Any]: include_reauthor = bool(args.get("include_reauthor", True)) reauthor_jobs_max = int(args.get("reauthor_jobs_max") or 3) - store = MemoryStore() + store = get_shared_store() # Draw a memory pool. Recently-accessed memories are higher-signal # candidates because they reflect what the user actively works on. if recent_only: diff --git a/mcp_server/handlers/detect_gaps.py b/mcp_server/handlers/detect_gaps.py index 1d41ac16..21a4a175 100644 --- a/mcp_server/handlers/detect_gaps.py +++ b/mcp_server/handlers/detect_gaps.py @@ -17,7 +17,7 @@ from mcp_server.core.blindspot_detector import detect_blind_spots from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.profile_store import load_profiles from mcp_server.handlers._tool_meta import READ_ONLY @@ -101,7 +101,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/drill_down.py b/mcp_server/handlers/drill_down.py index 2c2d7575..83517c10 100644 --- a/mcp_server/handlers/drill_down.py +++ b/mcp_server/handlers/drill_down.py @@ -14,7 +14,7 @@ from mcp_server.core import fractal from mcp_server.infrastructure.embedding_engine import get_embedding_engine from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY from mcp_server.handlers._telemetry_wrap import instrument @@ -77,7 +77,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/forget.py b/mcp_server/handlers/forget.py index b2e2a7f7..fba08b9a 100644 --- a/mcp_server/handlers/forget.py +++ b/mcp_server/handlers/forget.py @@ -9,7 +9,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import DESTRUCTIVE from mcp_server.handlers._telemetry_wrap import instrument @@ -69,7 +69,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/get_causal_chain.py b/mcp_server/handlers/get_causal_chain.py index 9a95fa0d..bb1221d4 100644 --- a/mcp_server/handlers/get_causal_chain.py +++ b/mcp_server/handlers/get_causal_chain.py @@ -13,7 +13,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY from mcp_server.handlers._telemetry_wrap import instrument @@ -104,7 +104,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/get_project_story.py b/mcp_server/handlers/get_project_story.py index 8c93c1ee..c0688fa0 100644 --- a/mcp_server/handlers/get_project_story.py +++ b/mcp_server/handlers/get_project_story.py @@ -14,7 +14,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY # ── Schema ──────────────────────────────────────────────────────────────────── @@ -77,7 +77,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/get_rules.py b/mcp_server/handlers/get_rules.py index e4c48f95..cb589afe 100644 --- a/mcp_server/handlers/get_rules.py +++ b/mcp_server/handlers/get_rules.py @@ -9,7 +9,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY # ── Schema ──────────────────────────────────────────────────────────────────── @@ -70,7 +70,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/ingest_codebase.py b/mcp_server/handlers/ingest_codebase.py index 21b39218..4fb3c5ac 100644 --- a/mcp_server/handlers/ingest_codebase.py +++ b/mcp_server/handlers/ingest_codebase.py @@ -37,7 +37,7 @@ from mcp_server.handlers.ingest_codebase_schema import schema # re-exported from mcp_server.handlers.ingest_helpers import call_upstream, normalise_mcp_payload from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.core.streaming.adaptive_writer import ( AdaptiveBatchWriter, adaptive_drain, @@ -94,7 +94,7 @@ def _get_store() -> MemoryStore: with _store_lock: if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/ingest_prd.py b/mcp_server/handlers/ingest_prd.py index 534f1055..9a6c931a 100644 --- a/mcp_server/handlers/ingest_prd.py +++ b/mcp_server/handlers/ingest_prd.py @@ -29,7 +29,7 @@ from mcp_server.handlers.ingest_helpers import call_upstream, normalise_mcp_payload from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.wiki_store import write_page logger = logging.getLogger(__name__) @@ -107,7 +107,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/memory_stats.py b/mcp_server/handlers/memory_stats.py index 608b3777..27fdaa8f 100644 --- a/mcp_server/handlers/memory_stats.py +++ b/mcp_server/handlers/memory_stats.py @@ -9,7 +9,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY # ── Schema ──────────────────────────────────────────────────────────────── @@ -49,7 +49,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/narrative.py b/mcp_server/handlers/narrative.py index 3ccb3ad8..d30b6d22 100644 --- a/mcp_server/handlers/narrative.py +++ b/mcp_server/handlers/narrative.py @@ -10,7 +10,7 @@ from mcp_server.core.narrative import generate_brief_summary, generate_narrative from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY schema = { @@ -67,7 +67,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/navigate_memory.py b/mcp_server/handlers/navigate_memory.py index 84f3ea94..0d14c11a 100644 --- a/mcp_server/handlers/navigate_memory.py +++ b/mcp_server/handlers/navigate_memory.py @@ -19,7 +19,7 @@ project_to_2d, ) from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY from mcp_server.handlers._telemetry_wrap import instrument @@ -90,7 +90,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/query_methodology.py b/mcp_server/handlers/query_methodology.py index fe2dd900..dbcef43f 100644 --- a/mcp_server/handlers/query_methodology.py +++ b/mcp_server/handlers/query_methodology.py @@ -86,10 +86,10 @@ def _try_get_memory_store(): return _memory_store try: from mcp_server.infrastructure.memory_config import get_memory_settings - from mcp_server.infrastructure.memory_store import MemoryStore + from mcp_server.infrastructure.memory_store import get_shared_store settings = get_memory_settings() - _memory_store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _memory_store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) _memory_available = True return _memory_store except Exception as e: diff --git a/mcp_server/handlers/query_workflow_graph.py b/mcp_server/handlers/query_workflow_graph.py index 1d34a9ed..4a50e6a0 100644 --- a/mcp_server/handlers/query_workflow_graph.py +++ b/mcp_server/handlers/query_workflow_graph.py @@ -28,7 +28,7 @@ from mcp_server.handlers._tool_meta import READ_ONLY from mcp_server.handlers.workflow_graph import build_workflow_graph from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store _store: MemoryStore | None = None @@ -38,7 +38,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: s = get_memory_settings() - _store = MemoryStore(s.DB_PATH, s.EMBEDDING_DIM) + _store = get_shared_store(s.DB_PATH, s.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/rate_memory.py b/mcp_server/handlers/rate_memory.py index bab16692..063feba9 100644 --- a/mcp_server/handlers/rate_memory.py +++ b/mcp_server/handlers/rate_memory.py @@ -16,7 +16,7 @@ from mcp_server.core import reranker, reranker_calibration, thermodynamics from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import IDEMPOTENT_WRITE from mcp_server.handlers._telemetry_wrap import instrument @@ -82,7 +82,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/recall.py b/mcp_server/handlers/recall.py index 34573b5d..314d5227 100644 --- a/mcp_server/handlers/recall.py +++ b/mcp_server/handlers/recall.py @@ -24,7 +24,7 @@ ) from mcp_server.infrastructure.embedding_engine import get_embedding_engine from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store schema = { "title": "Recall (retrieve memories)", @@ -185,7 +185,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: s = get_memory_settings() - _store = MemoryStore(s.DB_PATH, s.EMBEDDING_DIM) + _store = get_shared_store(s.DB_PATH, s.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/recall_hierarchical.py b/mcp_server/handlers/recall_hierarchical.py index 1d07faf6..1caf4418 100644 --- a/mcp_server/handlers/recall_hierarchical.py +++ b/mcp_server/handlers/recall_hierarchical.py @@ -18,7 +18,7 @@ get_embedding_engine, ) from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY from mcp_server.handlers._telemetry_wrap import instrument @@ -117,7 +117,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/record_session_end.py b/mcp_server/handlers/record_session_end.py index 46951dc7..d6fd6be7 100644 --- a/mcp_server/handlers/record_session_end.py +++ b/mcp_server/handlers/record_session_end.py @@ -392,7 +392,7 @@ async def handler(args: dict) -> dict: from mcp_server.handlers.auto_task_record_writer import ( maybe_write_task_record, ) - from mcp_server.infrastructure.memory_store import MemoryStore + from mcp_server.infrastructure.memory_store import get_shared_store task_record_status = maybe_write_task_record( session_id=session_id, @@ -401,7 +401,7 @@ async def handler(args: dict) -> dict: duration_seconds=duration, turn_count=turn_count, tools_used=tools_used or [], - store=MemoryStore(), + store=get_shared_store(), ) except Exception as exc: task_record_status = { diff --git a/mcp_server/handlers/remember.py b/mcp_server/handlers/remember.py index 3451ab49..f2170313 100644 --- a/mcp_server/handlers/remember.py +++ b/mcp_server/handlers/remember.py @@ -24,7 +24,7 @@ from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.embedding_engine import get_embedding_engine from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.profile_store import load_profiles schema = { @@ -188,7 +188,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: s = get_memory_settings() - _store = MemoryStore(s.DB_PATH, s.EMBEDDING_DIM) + _store = get_shared_store(s.DB_PATH, s.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/seed_project.py b/mcp_server/handlers/seed_project.py index 4921527b..6b4e086a 100644 --- a/mcp_server/handlers/seed_project.py +++ b/mcp_server/handlers/seed_project.py @@ -26,7 +26,7 @@ heat_for_tags, ) from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import NON_IDEMPOTENT_WRITE _store: MemoryStore | None = None @@ -36,7 +36,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/sync_instructions.py b/mcp_server/handlers/sync_instructions.py index 8b216055..9b549875 100644 --- a/mcp_server/handlers/sync_instructions.py +++ b/mcp_server/handlers/sync_instructions.py @@ -16,7 +16,7 @@ from typing import Any from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import IDEMPOTENT_WRITE # ── Schema ──────────────────────────────────────────────────────────────────── @@ -82,7 +82,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/validate_memory.py b/mcp_server/handlers/validate_memory.py index f9d70ecb..18cf3155 100644 --- a/mcp_server/handlers/validate_memory.py +++ b/mcp_server/handlers/validate_memory.py @@ -13,7 +13,7 @@ from mcp_server.core.staleness import assess_staleness, collect_all_refs from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.handlers._tool_meta import READ_ONLY from mcp_server.handlers._telemetry_wrap import instrument @@ -95,7 +95,7 @@ def _get_store() -> MemoryStore: global _store if _store is None: settings = get_memory_settings() - _store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + _store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return _store diff --git a/mcp_server/handlers/wiki_api.py b/mcp_server/handlers/wiki_api.py index c7b4c27c..1ac55420 100644 --- a/mcp_server/handlers/wiki_api.py +++ b/mcp_server/handlers/wiki_api.py @@ -77,10 +77,10 @@ def _get_store(): """Lazy store accessor — never raises; returns None if DB missing.""" try: from mcp_server.infrastructure.memory_config import get_memory_settings - from mcp_server.infrastructure.memory_store import MemoryStore + from mcp_server.infrastructure.memory_store import get_shared_store settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) except Exception: return None diff --git a/mcp_server/handlers/wiki_compile.py b/mcp_server/handlers/wiki_compile.py index fca1cf8e..85867fbd 100644 --- a/mcp_server/handlers/wiki_compile.py +++ b/mcp_server/handlers/wiki_compile.py @@ -25,7 +25,7 @@ from mcp_server.core.wiki_layout import slugify from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.pg_store_wiki import ( body_hash, get_draft, @@ -93,7 +93,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) def _domain_for_memory(conn, memory_id: int | None) -> str: diff --git a/mcp_server/handlers/wiki_consolidate.py b/mcp_server/handlers/wiki_consolidate.py index 20c1fd54..54943cb8 100644 --- a/mcp_server/handlers/wiki_consolidate.py +++ b/mcp_server/handlers/wiki_consolidate.py @@ -33,7 +33,7 @@ from mcp_server.core.wiki_thermodynamics import evaluate_page, summarise from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.pg_store_wiki import ( apply_staleness_decisions, apply_thermo_decisions, @@ -117,7 +117,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) def _check_existence(refs: set[str], repo_root: Path) -> dict[str, bool]: diff --git a/mcp_server/handlers/wiki_curate.py b/mcp_server/handlers/wiki_curate.py index 0696c816..07f60ef4 100644 --- a/mcp_server/handlers/wiki_curate.py +++ b/mcp_server/handlers/wiki_curate.py @@ -23,7 +23,7 @@ from mcp_server.core.wiki_schema_loader import load_registry from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.pg_store_wiki import ( insert_memo, list_drafts, @@ -101,7 +101,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) async def handler(args: dict[str, Any] | None = None) -> dict[str, Any]: diff --git a/mcp_server/handlers/wiki_emerge.py b/mcp_server/handlers/wiki_emerge.py index 9b63d4a1..3018bf6e 100644 --- a/mcp_server/handlers/wiki_emerge.py +++ b/mcp_server/handlers/wiki_emerge.py @@ -23,7 +23,7 @@ emerge, ) from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.pg_store_wiki import ( get_concepts_by_entity_overlap, insert_concept, @@ -79,7 +79,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) def _fetch_resolved_claims(conn, limit: int) -> list[dict]: diff --git a/mcp_server/handlers/wiki_extract.py b/mcp_server/handlers/wiki_extract.py index 1a77857a..eb2b9fe4 100644 --- a/mcp_server/handlers/wiki_extract.py +++ b/mcp_server/handlers/wiki_extract.py @@ -20,7 +20,7 @@ from mcp_server.core.claim_extractor import extract_claims from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.pg_store_wiki import ( delete_claims_for_memory, insert_claim_events, @@ -82,7 +82,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) def _memory_rows(conn, memory_id: int | None, limit: int, force: bool) -> list[dict]: diff --git a/mcp_server/handlers/wiki_migrate.py b/mcp_server/handlers/wiki_migrate.py index 404a2bcb..415142d2 100644 --- a/mcp_server/handlers/wiki_migrate.py +++ b/mcp_server/handlers/wiki_migrate.py @@ -224,10 +224,10 @@ async def handler(args: dict) -> dict: """MCP tool entry: run migration, return summary.""" from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.memory_config import get_memory_settings - from mcp_server.infrastructure.memory_store import MemoryStore + from mcp_server.infrastructure.memory_store import get_shared_store settings = get_memory_settings() - store = MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + store = get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) return migrate_wiki(WIKI_ROOT, store._conn) diff --git a/mcp_server/handlers/wiki_refine.py b/mcp_server/handlers/wiki_refine.py index ea9f44d9..2d704e1a 100644 --- a/mcp_server/handlers/wiki_refine.py +++ b/mcp_server/handlers/wiki_refine.py @@ -33,7 +33,7 @@ from mcp_server.core.wiki_schema_loader import load_registry from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.pg_store_wiki import ( get_draft, insert_memo, @@ -71,7 +71,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) def _claims_for_memory(conn, memory_id: int) -> list[dict]: diff --git a/mcp_server/handlers/wiki_resolve.py b/mcp_server/handlers/wiki_resolve.py index 2094befe..22669020 100644 --- a/mcp_server/handlers/wiki_resolve.py +++ b/mcp_server/handlers/wiki_resolve.py @@ -19,7 +19,7 @@ from mcp_server.core.claim_resolver import resolve from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.pg_store_wiki import ( get_claims_by_entity, get_entities_by_memory, @@ -88,7 +88,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) def _fetch_claims(conn, memory_id: int | None, limit: int) -> list[dict]: diff --git a/mcp_server/handlers/wiki_synthesize.py b/mcp_server/handlers/wiki_synthesize.py index a80738e8..9647f745 100644 --- a/mcp_server/handlers/wiki_synthesize.py +++ b/mcp_server/handlers/wiki_synthesize.py @@ -26,7 +26,7 @@ from mcp_server.core.wiki_schema_loader import load_registry from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store from mcp_server.infrastructure.pg_store_wiki import ( find_draft_for_source, insert_draft, @@ -107,7 +107,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) # ── Kind inference ──────────────────────────────────────────────────── diff --git a/mcp_server/handlers/wiki_view.py b/mcp_server/handlers/wiki_view.py index 6ecb743d..9d333cc0 100644 --- a/mcp_server/handlers/wiki_view.py +++ b/mcp_server/handlers/wiki_view.py @@ -25,7 +25,7 @@ from mcp_server.core.wiki_view_executor import compile_view from mcp_server.infrastructure.config import WIKI_ROOT from mcp_server.infrastructure.memory_config import get_memory_settings -from mcp_server.infrastructure.memory_store import MemoryStore +from mcp_server.infrastructure.memory_store import MemoryStore, get_shared_store schema = { @@ -86,7 +86,7 @@ def _get_store() -> MemoryStore: settings = get_memory_settings() - return MemoryStore(settings.DB_PATH, settings.EMBEDDING_DIM) + return get_shared_store(settings.DB_PATH, settings.EMBEDDING_DIM) async def handler(args: dict[str, Any] | None = None) -> dict[str, Any]: diff --git a/mcp_server/infrastructure/memory_store.py b/mcp_server/infrastructure/memory_store.py index d39b8caa..cb09848f 100644 --- a/mcp_server/infrastructure/memory_store.py +++ b/mcp_server/infrastructure/memory_store.py @@ -8,9 +8,20 @@ import logging import os +import threading logger = logging.getLogger(__name__) +# Process-wide store cache. 37 MCP handlers each used to construct their own +# store via MemoryStore(...), and each store eagerly opens psycopg pools +# (min2/max8 interactive + min1/max2 batch). conftest only reset 5 of them, so +# connections leaked past 60 and the 1800s batch-pool acquire timeout produced +# the 30-minute CI hangs. Caching one store per (backend, url, dim) caps live +# connections at a single store's two pools regardless of handler count, and +# fixes the same connection-quota leak in production. +_shared_lock = threading.Lock() +_shared_stores: dict[tuple[str, str, int], object] = {} + def _try_pg(database_url: str): """Try connecting to PostgreSQL. Returns PgMemoryStore or None.""" @@ -47,67 +58,139 @@ def __new__( *, database_url: str | None = None, ): - from mcp_server.infrastructure.memory_config import get_memory_settings + return _construct_store(db_path, embedding_dim, database_url=database_url) - settings = get_memory_settings() - runtime = settings.RUNTIME - backend = settings.STORE_BACKEND - url = ( - database_url or os.environ.get("DATABASE_URL", "") or settings.DATABASE_URL - ) - # In CLI mode, "auto" means PostgreSQL is required - if runtime == "cli" and backend == "auto": - backend = "postgresql" +def get_shared_store( + db_path: str = "", + embedding_dim: int = 384, + *, + database_url: str | None = None, +): + """Return a process-wide cached store, one per (backend, url, dim) key. - if backend == "sqlite": - return _make_sqlite(db_path or settings.SQLITE_FALLBACK_PATH, embedding_dim) + Handlers MUST use this instead of constructing MemoryStore(...) directly: + each store owns two psycopg pools, so one cached store caps live + connections regardless of how many of the 37 handlers ask for it. See the + module-level note on _shared_stores for the CI-hang / quota-leak history. + """ + key = _resolve_key(db_path, embedding_dim, database_url) + with _shared_lock: + store = _shared_stores.get(key) + if store is None: + store = _construct_store(db_path, embedding_dim, database_url=database_url) + _shared_stores[key] = store + return store - if backend == "postgresql": - if url: - store, err = _try_pg_verbose(url) - else: - store, err = None, "DATABASE_URL not set" - if store is not None: - return store - # Inspection-mode fallback — Glama's sandbox, CI smoke - # tests, and first-glance experimenters launch Cortex with - # no DATABASE_URL. Rather than hard-fail and leave them - # unable to even see the tool surface, drop to SQLite with - # a loud warning. Real production users who have - # configured Postgres will see the PG connect succeed; - # only unset/unreachable installs trip this path. - allow_fallback = not url or os.environ.get( - "CORTEX_ALLOW_SQLITE_FALLBACK", "" - ).lower() in ("1", "true", "yes") - if allow_fallback: - logger.warning( - "PostgreSQL unavailable (%s); falling back to SQLite. " - "This is expected for inspection/sandbox launches; " - "production installs should set DATABASE_URL.", - err, - ) - return _make_sqlite( - db_path or settings.SQLITE_FALLBACK_PATH, embedding_dim - ) - raise RuntimeError( - f"PostgreSQL connection failed (url={url or ''}): {err}\n" - "Cortex requires PostgreSQL in CLI mode.\n" - "Run: bash setup.sh to configure PostgreSQL.\n" - "If DATABASE_URL is set, verify it points to a reachable Postgres instance " - "(host/port/credentials/database exists).\n" - "Or set CORTEX_RUNTIME=cowork (or CORTEX_ALLOW_SQLITE_FALLBACK=1) " - "to allow SQLite fallback." - ) - # "auto" in cowork mode: try PG, fall back to SQLite - if url: - store = _try_pg(url) - if store is not None: - return store +def reset_shared_store() -> None: + """Close and evict all cached shared stores (test teardown / shutdown). + + Releases every store's psycopg pools so connections do not leak across + test modules. Subsequent get_shared_store() calls reconstruct lazily. + """ + with _shared_lock: + for store in _shared_stores.values(): + close = getattr(store, "close", None) + if callable(close): + try: + close() + except Exception: # pragma: no cover - defensive teardown + logger.warning("error closing shared store", exc_info=True) + _shared_stores.clear() + + +def _resolve_backend_url( + db_path: str, embedding_dim: int, database_url: str | None +) -> tuple[str, str]: + """Resolve the (backend, url) a construction would target — the cache key + discriminators. Mirrors the branch selection in _construct_store.""" + from mcp_server.infrastructure.memory_config import get_memory_settings + + settings = get_memory_settings() + backend = settings.STORE_BACKEND + url = database_url or os.environ.get("DATABASE_URL", "") or settings.DATABASE_URL + if settings.RUNTIME == "cli" and backend == "auto": + backend = "postgresql" + return backend, url + + +def _resolve_key( + db_path: str, embedding_dim: int, database_url: str | None +) -> tuple[str, str, int]: + backend, url = _resolve_backend_url(db_path, embedding_dim, database_url) + return (backend, url, embedding_dim) + + +def _construct_store( + db_path: str = "", + embedding_dim: int = 384, + *, + database_url: str | None = None, +): + """Build a fresh store using runtime-aware backend selection. + + CLI mode: PostgreSQL required (auto → postgresql). Raises on failure. + Cowork mode: tries PostgreSQL, falls back to SQLite. + Explicit sqlite backend always works (for testing). + """ + from mcp_server.infrastructure.memory_config import get_memory_settings + + settings = get_memory_settings() + runtime = settings.RUNTIME + backend = settings.STORE_BACKEND + url = database_url or os.environ.get("DATABASE_URL", "") or settings.DATABASE_URL + # In CLI mode, "auto" means PostgreSQL is required + if runtime == "cli" and backend == "auto": + backend = "postgresql" + + if backend == "sqlite": return _make_sqlite(db_path or settings.SQLITE_FALLBACK_PATH, embedding_dim) + if backend == "postgresql": + if url: + store, err = _try_pg_verbose(url) + else: + store, err = None, "DATABASE_URL not set" + if store is not None: + return store + # Inspection-mode fallback — Glama's sandbox, CI smoke + # tests, and first-glance experimenters launch Cortex with + # no DATABASE_URL. Rather than hard-fail and leave them + # unable to even see the tool surface, drop to SQLite with + # a loud warning. Real production users who have + # configured Postgres will see the PG connect succeed; + # only unset/unreachable installs trip this path. + allow_fallback = not url or os.environ.get( + "CORTEX_ALLOW_SQLITE_FALLBACK", "" + ).lower() in ("1", "true", "yes") + if allow_fallback: + logger.warning( + "PostgreSQL unavailable (%s); falling back to SQLite. " + "This is expected for inspection/sandbox launches; " + "production installs should set DATABASE_URL.", + err, + ) + return _make_sqlite(db_path or settings.SQLITE_FALLBACK_PATH, embedding_dim) + raise RuntimeError( + f"PostgreSQL connection failed (url={url or ''}): {err}\n" + "Cortex requires PostgreSQL in CLI mode.\n" + "Run: bash setup.sh to configure PostgreSQL.\n" + "If DATABASE_URL is set, verify it points to a reachable Postgres instance " + "(host/port/credentials/database exists).\n" + "Or set CORTEX_RUNTIME=cowork (or CORTEX_ALLOW_SQLITE_FALLBACK=1) " + "to allow SQLite fallback." + ) + + # "auto" in cowork mode: try PG, fall back to SQLite + if url: + store = _try_pg(url) + if store is not None: + return store + + return _make_sqlite(db_path or settings.SQLITE_FALLBACK_PATH, embedding_dim) + def _make_sqlite(path: str, embedding_dim: int): """Create SQLite fallback store.""" diff --git a/mcp_server/infrastructure/pg_store_relationships.py b/mcp_server/infrastructure/pg_store_relationships.py index 5380949e..eb95a3a8 100644 --- a/mcp_server/infrastructure/pg_store_relationships.py +++ b/mcp_server/infrastructure/pg_store_relationships.py @@ -50,11 +50,23 @@ def delete_relationships_batch(self, rel_ids: list[int]) -> int: return len(rel_ids) def insert_relationship(self, data: dict[str, Any]) -> int: + # Idempotent on the directed tuple (source, target, type). Re-ingest + # (e.g. incremental codebase re-analysis) replays the same structural + # edges; without ON CONFLICT this raises UniqueViolation on + # uq_relationships_directed. On conflict we refresh the edge instead + # of duplicating: keep the strongest weight, mark it re-reinforced. + # Source: uq_relationships_directed (pg_schema.py §A3); issue #13. row = self._execute( "INSERT INTO relationships " "(source_entity_id, target_entity_id, relationship_type, weight, " "is_causal, confidence, created_at, last_reinforced) " - "VALUES (%s, %s, %s, %s, %s, %s, COALESCE(%s, NOW()), NOW()) RETURNING id", + "VALUES (%s, %s, %s, %s, %s, %s, COALESCE(%s, NOW()), NOW()) " + "ON CONFLICT (source_entity_id, target_entity_id, relationship_type) " + "DO UPDATE SET " + " weight = GREATEST(relationships.weight, EXCLUDED.weight), " + " confidence = GREATEST(relationships.confidence, EXCLUDED.confidence), " + " last_reinforced = NOW() " + "RETURNING id", ( data["source_entity_id"], data["target_entity_id"], diff --git a/mcp_server/server/http_file_diff.py b/mcp_server/server/http_file_diff.py index 7324827a..66d20382 100644 --- a/mcp_server/server/http_file_diff.py +++ b/mcp_server/server/http_file_diff.py @@ -91,28 +91,36 @@ def _allowed_probe_roots() -> "list[str]": return roots -def _under_allowed_root(p: "Path") -> bool: # noqa: F821 - """True iff ``p`` resolves inside any allowed probe root. - - Uses ``Path.resolve().is_relative_to()`` — the canonical, CodeQL- - recognised path-traversal sanitiser (CWE-22) — so the ``?name=`` / - ``?path=`` query data can never reach a filesystem op that escapes - ``$HOME`` / cwd / temp. +def _contained_resolved(p: "str | Path") -> "Path | None": # noqa: F821 + """Resolve ``p`` and return it ONLY if it lands inside an allowed probe + root; otherwise ``None``. + + Sanitise-and-return: the returned Path is the *validated* value, and + callers must use it (never the raw input) for any subsequent filesystem + op. This places the ``is_relative_to`` barrier (the canonical CWE-22 + path-traversal sanitiser) directly on the tainted→sink dataflow, which + CodeQL recognises — so ``?name=`` / ``?path=`` query data can never + reach a filesystem op that escapes ``$HOME`` / cwd / temp. """ from pathlib import Path try: target = Path(p).resolve(strict=False) except (OSError, ValueError): - return False + return None for root in _allowed_probe_roots(): try: base = Path(root).resolve(strict=False) except (OSError, ValueError): continue if target == base or target.is_relative_to(base): - return True - return False + return target + return None + + +def _under_allowed_root(p: "Path") -> bool: # noqa: F821 + """Back-compat boolean wrapper around :func:`_contained_resolved`.""" + return _contained_resolved(p) is not None def _git_root_for_name(name: str, find_git_root) -> "Path | None": # noqa: F821 @@ -158,11 +166,11 @@ def _git_root_for_name(name: str, find_git_root) -> "Path | None": # noqa: F821 # ``_under_allowed_root`` (HOME / cwd / temp) so a crafted ``?name=`` # still can't probe ``/etc`` / ``/root`` (CWE-22). if clean.startswith(("/", "\\")): - try: - target = Path(clean).resolve(strict=False) - except (OSError, ValueError): - return find_git_root() - if not _under_allowed_root(target): + # Sanitise-and-return: ``target`` is the validated resolved path + # (gated by is_relative_to), so the value reaching ``is_dir()`` / + # ``git rev-parse`` below carries the CWE-22 barrier inline. + target = _contained_resolved(clean) + if target is None: return find_git_root() # Walk up to the first directory that exists (handles a file, or # an intermediate dir, deleted after capture). Capped at 64. diff --git a/mcp_server/server/http_standalone_trace.py b/mcp_server/server/http_standalone_trace.py index b6e530bd..be7afd42 100644 --- a/mcp_server/server/http_standalone_trace.py +++ b/mcp_server/server/http_standalone_trace.py @@ -659,27 +659,16 @@ def _to_repo_relative(path: str) -> str: from pathlib import Path from mcp_server.infrastructure.git_diff import find_git_root - from mcp_server.server.http_file_diff import _allowed_probe_roots - - try: - ap = Path(p).resolve(strict=False) - except (OSError, ValueError): - return p.lstrip("/") - # CWE-22 containment: only derive a repo for paths that resolve inside - # an allowed root (HOME / cwd / temp). ``is_relative_to`` is the - # canonical path-traversal sanitiser — a crafted ``?path=`` cannot - # reach ``/etc`` / ``/root``. ``?path=`` is loopback-only and normally - # carries the user's own file node path; this just bounds the surface. - contained = False - for r in _allowed_probe_roots(): - try: - base = Path(r).resolve(strict=False) - except (OSError, ValueError): - continue - if ap == base or ap.is_relative_to(base): - contained = True - break - if not contained: + from mcp_server.server.http_file_diff import _contained_resolved + + # Sanitise-and-return (CWE-22): ``ap`` is None unless ``p`` resolves + # inside an allowed root (HOME / cwd / temp). ``is_relative_to`` is the + # canonical path-traversal barrier, applied inline before ``ap`` reaches + # any filesystem op — a crafted ``?path=`` cannot reach ``/etc`` / + # ``/root``. ``?path=`` is loopback-only and normally carries the user's + # own file node path; this just bounds the surface. + ap = _contained_resolved(p) + if ap is None: return p.lstrip("/") try: root = find_git_root(ap.parent) diff --git a/tests_py/conftest.py b/tests_py/conftest.py index 11155a28..8a53d074 100644 --- a/tests_py/conftest.py +++ b/tests_py/conftest.py @@ -155,45 +155,45 @@ def _clean_sqlite_store() -> None: pass -def _reset_all_singletons() -> None: - """Reset handler module-level singletons so they reconnect fresh. +# Handler module-level caches that hold (a reference to) the shared store or +# its derivatives. Nulling them forces re-fetch from get_shared_store() after +# the shared store is closed; otherwise a handler would hand back a store whose +# psycopg pools are already closed. +_HANDLER_CACHE_ATTRS = ("_store", "_memory_store", "_embeddings", "_memory_available") + - Closes any SQLite store connections before nulling to prevent leaked - file handles and 'database is locked' errors in subsequent tests. +def _reset_all_singletons() -> None: + """Reset the shared store and handler-level caches so the next test + reconnects fresh. + + The 37 handlers no longer each own a store — they fetch one process-wide + instance via get_shared_store(), whose two psycopg pools are the only + connections held. reset_shared_store() closes those pools (fixing the CI + connection leak that drove live connections past 60 and triggered the + 30-minute batch-pool acquire hangs). We then null every handler cache by + iterating the handlers package, so the list cannot drift out of date. """ - modules_and_attrs = [ - ("mcp_server.handlers.recall", ["_store", "_embeddings"]), - ("mcp_server.handlers.remember", ["_store", "_embeddings"]), - ("mcp_server.handlers.consolidate", ["_store", "_embeddings"]), - ("mcp_server.handlers.checkpoint", ["_store"]), - ("mcp_server.handlers.memory_stats", ["_store"]), - ] + try: + from mcp_server.infrastructure.memory_store import reset_shared_store - # Close any open SQLite store connections before nulling - closed_ids: set[int] = set() - for mod_name, attrs in modules_and_attrs: - try: - mod = importlib.import_module(mod_name) - store = getattr(mod, "_store", None) - if store is not None and id(store) not in closed_ids: - if hasattr(store, "close"): - try: - store.close() - except Exception: - pass - closed_ids.add(id(store)) - except ImportError: - pass + reset_shared_store() + except ImportError: + pass + + import pkgutil - # Now null all singletons - for mod_name, attrs in modules_and_attrs: + import mcp_server.handlers as handlers_pkg + + for _finder, mod_name, _ispkg in pkgutil.iter_modules(handlers_pkg.__path__): try: - mod = importlib.import_module(mod_name) - for attr in attrs: - if hasattr(mod, attr): - setattr(mod, attr, None) - except ImportError: - pass + mod = importlib.import_module(f"mcp_server.handlers.{mod_name}") + except Exception: + continue + for attr in _HANDLER_CACHE_ATTRS: + if hasattr(mod, attr): + # All these caches use None as their "recompute me" sentinel + # (_memory_available starts None = "not yet checked"). + setattr(mod, attr, None) try: from mcp_server.infrastructure.memory_config import get_memory_settings diff --git a/tests_py/infrastructure/test_mcp_client.py b/tests_py/infrastructure/test_mcp_client.py index ea7a0368..4a19bdaa 100644 --- a/tests_py/infrastructure/test_mcp_client.py +++ b/tests_py/infrastructure/test_mcp_client.py @@ -258,8 +258,7 @@ async def _drain(): new_callable=AsyncMock, return_value=proc, ): - with patch("asyncio.sleep", new_callable=AsyncMock): - await client.connect() + await client.connect() assert client.connected is True assert client.protocol_version == "2025-11-25" @@ -322,8 +321,7 @@ async def _drain(): new_callable=AsyncMock, return_value=proc, ): - with patch("asyncio.sleep", new_callable=AsyncMock): - await client.connect() + await client.connect() assert client.connected is True assert client.protocol_version == PROTOCOL_VERSION @@ -375,9 +373,8 @@ async def _drain(): new_callable=AsyncMock, return_value=proc, ): - with patch("asyncio.sleep", new_callable=AsyncMock): - with pytest.raises(McpConnectionError, match="Handshake failed"): - await client.connect() + with pytest.raises(McpConnectionError, match="Handshake failed"): + await client.connect() assert client.connected is False @@ -423,8 +420,7 @@ async def _drain(): new_callable=AsyncMock, return_value=proc, ) as mock_exec: - with patch("asyncio.sleep", new_callable=AsyncMock): - await client.connect() + await client.connect() call_kwargs = mock_exec.call_args assert call_kwargs[1]["cwd"] == "/tmp" diff --git a/tests_py/invariants/test_I2_canonical_writer.py b/tests_py/invariants/test_I2_canonical_writer.py index 195fdf58..e0001545 100644 --- a/tests_py/invariants/test_I2_canonical_writer.py +++ b/tests_py/invariants/test_I2_canonical_writer.py @@ -33,12 +33,12 @@ # OR be added here with a source-commented ADR justification. _ALLOWED_WRITERS: set[tuple[str, int]] = { # Canonical single-row writer (all callers route through this). - # bump_heat_raw — shifted 448→477 by the viz server-streaming merge - # (25eb33a) which added code above it; still the one canonical site. - ("infrastructure/pg_store.py", 477), + # bump_heat_raw (def at line 526) — shifted 477→543 by code added above + # it in pg_store.py; still the one canonical single-row site. + ("infrastructure/pg_store.py", 543), # A3 batched writer (homeostatic cohort branch + any other batch consumer). - # update_memories_heat_batch — shifted 508→537 by the same merge. - ("infrastructure/pg_store.py", 537), + # update_memories_heat_batch (def at line 587) — shifted 537→603 likewise. + ("infrastructure/pg_store.py", 603), # SQLite parity. ("infrastructure/sqlite_store.py", 284), ("infrastructure/sqlite_store.py", 328),