diff --git a/mcp_server/core/wiki_axis_registry.py b/mcp_server/core/wiki_axis_registry.py index c990a02f..d37d4d99 100644 --- a/mcp_server/core/wiki_axis_registry.py +++ b/mcp_server/core/wiki_axis_registry.py @@ -142,6 +142,11 @@ def _re(pattern: str) -> re.Pattern[str]: return re.compile(pattern, re.IGNORECASE) +def _re_ml(pattern: str) -> re.Pattern[str]: + """Compile a regex with IGNORECASE + MULTILINE for ``^``-anchored line patterns.""" + return re.compile(pattern, re.IGNORECASE | re.MULTILINE) + + # Each tuple = (name, display_name, patterns, tag_aliases, kwargs). _DEFAULT_KINDS: tuple[AxisValue, ...] = ( AxisValue( @@ -149,10 +154,20 @@ def _re(pattern: str) -> re.Pattern[str]: axis=AXIS_KIND, display_name="ADR (Architecture Decision Record)", patterns=( + # Inline decision markers (Nygard prose). _re(r"\b(decided to|decision:|the decision is|chose .+ because)\b"), _re(r"\b(rejected .+ (due to|because)|we will use|selected .+ over)\b"), + # Nygard heading skeleton — the canonical ADR section structure. + # Pilot 2026-05-13 found that 3 of 8 sampled ADRs had ``## Decision`` + # heading without a colon, so the prose-only patterns missed them. + _re_ml(r"^##+\s*Decision\s*$"), + _re_ml(r"^##+\s*Consequences\s*$"), ), - tag_aliases=("decision", "adr", "architecture"), + # ``architecture`` removed (pilot 2026-05-13): it is too broad to be an + # ADR-only tag — 8 of 8 sampled rfc/ pages were misrouted to adr/ + # because they carried the ``architecture`` tag. Architecture-tagged + # content is more often spec/rfc/explanation than a single decision. + tag_aliases=("decision", "adr"), description="Nygard/MADR-style record of a single architectural decision.", ), AxisValue( @@ -354,12 +369,22 @@ def _re(pattern: str) -> re.Pattern[str]: name="security", axis=AXIS_AUDIENCE, display_name="Security", + # Pilot 2026-05-13 found bare ``crypto`` (Node built-in module) false- + # positiving as a security signal. The patterns now require the full + # suffix (``cryptograph(y|ic)``) or the longer security-domain words. + # Same for ``auth`` — must be ``authentication``/``authorization`` to + # count, not the abbreviation. patterns=( _re( - r"\b(auth(entication|orization)?|crypto(graphy)?|vulnerab(le|ility)|cve|threat model)\b" + r"\b(authentication|authorization|cryptograph(y|ic)|vulnerab(le|ility)|cve|threat model)\b" + ), + _re( + r"\b(credential|oauth|sso|encryption|decrypt(ed|ion)?|" + r"key rotation|HSM|secret manager)\b" ), - _re(r"\b(secret|token|credential|session|sso|oauth|tls|encryption)\b"), ), + # Tags remain the strongest signal — a page explicitly tagged ``security`` + # is unambiguous. tag_aliases=("security", "auth", "crypto", "vulnerability", "secops"), description="Security engineers and reviewers.", ), diff --git a/mcp_server/shared/wiki_classification.py b/mcp_server/shared/wiki_classification.py index 2964a038..6b20f88e 100644 --- a/mcp_server/shared/wiki_classification.py +++ b/mcp_server/shared/wiki_classification.py @@ -30,7 +30,7 @@ # writes. The registry does not list these; ``normalize_legacy_kind`` # remaps them on read. LEGACY_KINDS: Final[frozenset[str]] = frozenset( - {"notes", "specs", "conventions", "lessons", "guides", "files"} + {"notes", "specs", "conventions", "lessons", "guides", "files", "adrs"} ) LEGACY_KIND_TO_MODERN: Final[dict[str, str]] = { @@ -40,6 +40,9 @@ "lessons": "explanation", "guides": "how-to", "files": "reference", + # The wiki has a few pages under ``adrs/`` (plural) — observed during the + # 2026-05-13 Phase 2 pilot. Treated as the same legacy kind as ``adr``. + "adrs": "adr", } diff --git a/scripts/wiki-pilot-report.md b/scripts/wiki-pilot-report.md new file mode 100644 index 00000000..10a039a6 --- /dev/null +++ b/scripts/wiki-pilot-report.md @@ -0,0 +1,192 @@ +# ADR-2244 Phase 2 — Pilot migration report + +Wiki root: `/Users/cdeust/.claude/methodology/wiki` + +## Summary + +- **Sample size:** 100 +- **Admitted by new classifier:** 88 (88.0%) +- **Rejected (admission gate):** 12 (12.0%) +- **Kind kept (legacy → modern direct map):** 77 (87.5% of admitted) +- **Kind changed:** 11 (12.5% of admitted) + +## Distribution — legacy kinds in the sample + +| Legacy kind | Pages | +|---|---:| +| `notes` | 23 | +| `reference` | 10 | +| `specs` | 9 | +| `adr` | 8 | +| `explanation` | 8 | +| `rfc` | 8 | +| `guides` | 8 | +| `conventions` | 8 | +| `lessons` | 8 | +| `adrs` | 8 | +| `README.md` | 1 | +| `architecture` | 1 | + +## Distribution — proposed modern kinds + +| Proposed kind | Pages | +|---|---:| +| `explanation` | 54 | +| `adr` | 16 | +| `` | 12 | +| `reference` | 10 | +| `rfc` | 8 | + +## Transition matrix (legacy → proposed) + +| From | To | Count | +|---|---|---:| +| `notes` | `explanation` | 23 | +| `reference` | `reference` | 10 | +| `adr` | `adr` | 8 | +| `explanation` | `explanation` | 8 | +| `specs` | `` | 8 | +| `rfc` | `rfc` | 8 | +| `guides` | `explanation` | 8 | +| `conventions` | `explanation` | 8 | +| `adrs` | `adr` | 8 | +| `lessons` | `explanation` | 4 | +| `lessons` | `` | 4 | +| `README.md` | `explanation` | 1 | +| `specs` | `explanation` | 1 | +| `architecture` | `explanation` | 1 | + +## Proposed facet distributions (admitted pages only) + +### Lifecycle + +| Value | Pages | +|---|---:| +| `seedling` | 72 | +| `proposed` | 16 | + +### Audience (multi-valued — counted per occurrence) + +| Value | Pages | +|---|---:| +| `developer` | 86 | +| `ops` | 15 | +| `security` | 10 | + +### Provenance + +| Value | Pages | +|---|---:| +| `auto-generated` | 46 | +| `human` | 42 | + +## Rejection reasons + +| Reason | Pages | +|---|---:| +| admission gate rejected (audit-tag, noise, or low score) | 12 | + +## Per-page proposals + +| Path | Legacy → Proposed | Lifecycle | Audience | Provenance | Status | +|---|---|---|---|---|---| +| `README.md` | `README.md` → `explanation` | `seedling` | `developer` | `human` | 🔁 changed | +| `adr/_general/2236-decision-003-felder-silverman-model.md.md` | `adr` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adr/agentic-ai/86789-decision-0004-validation-tool-optional-triple.md.md` | `adr` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adr/agentic-ai/96925-decision-0005-prd-spec-subtree-approach.md.md` | `adr` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adr/_general/2238-decision-005-agglomerative-over-kmeans-clustering.md.md` | `adr` → `adr` | `proposed` | `ops` | `human` | ✅ kept | +| `adr/_general/2234-decision-001-zero-dependencies.md.md` | `adr` → `adr` | `proposed` | `developer`, `ops` | `human` | ✅ kept | +| `adr/agentic-ai/96928-decision-0008-claude-plugin-path-placement.md.md` | `adr` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adr/agentic-ai/86792-decision-0007-better-sqlite3-native-build.md.md` | `adr` → `adr` | `proposed` | `developer`, `ops` | `human` | ✅ kept | +| `adr/agentic-ai/86787-decision-0002-analyze-codebase-serial-vs-parallel.md.md` | `adr` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `explanation/codebase-alteration-bench/23744-file-storage-repository.py.md` | `explanation` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `explanation/codebase-alteration-bench/23738-file-auth-crypto.py.md` | `explanation` → `explanation` | `seedling` | `developer`, `security` | `auto-generated` | ✅ kept | +| `explanation/codebase-alteration-bench/23739-file-auth-middleware.py.md` | `explanation` → `explanation` | `seedling` | `developer`, `security` | `auto-generated` | ✅ kept | +| `explanation/codebase-alteration-bench/23768-file-api-health.py.md` | `explanation` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `explanation/codebase-alteration-bench/23779-file-api-routes.py.md` | `explanation` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `explanation/codebase-alteration-bench/23753-file-models-user.py.md` | `explanation` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `explanation/codebase-alteration-bench/23771-file-auth-middleware.py.md` | `explanation` → `explanation` | `seedling` | `developer`, `security` | `auto-generated` | ✅ kept | +| `explanation/codebase-alteration-bench/23778-file-api-health.py.md` | `explanation` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `specs/_general/1173-reviewed-by-alexander-patterns-liskov-substitutability-hamilton.md` | `specs` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `specs/_general/2133-tool-bash.md` | `specs` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `specs/_general/2107-tool-bash.md` | `specs` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `specs/2026/2026-04-17-multiple-use-cases-and-a-pagingsource-in.md` | `specs` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `specs/_general/2087-tool-bash.md` | `specs` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `specs/_general/2207-tool-bash.md` | `specs` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `specs/2026/2026-04-17-jdk-21-temurin.md` | `specs` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `specs/_general/1820-spec-module-catalog-dcp-wealth-android.md` | `specs` → `explanation` | `seedling` | `developer` | `human` | 🔁 changed | +| `architecture/2026/2026-04-21-overview.md` | `architecture` → `explanation` | `seedling` | `developer`, `ops` | `human` | 🔁 changed | +| `rfc/repo-a/24695-project-structure-repo-a.md` | `rfc` → `rfc` | `seedling` | `developer` | `human` | ✅ kept | +| `rfc/repo-b/24071-project-structure-repo-b.md` | `rfc` → `rfc` | `seedling` | `developer` | `human` | ✅ kept | +| `rfc/repo-b/24655-project-structure-repo-b.md` | `rfc` → `rfc` | `seedling` | `developer` | `human` | ✅ kept | +| `rfc/repo-a/24068-project-structure-repo-a.md` | `rfc` → `rfc` | `seedling` | `developer` | `human` | ✅ kept | +| `rfc/repo-b/24698-project-structure-repo-b.md` | `rfc` → `rfc` | `seedling` | `developer` | `human` | ✅ kept | +| `rfc/repo-a/24664-project-structure-repo-a.md` | `rfc` → `rfc` | `seedling` | `developer` | `human` | ✅ kept | +| `rfc/repo-b/24077-project-structure-repo-b.md` | `rfc` → `rfc` | `seedling` | `developer` | `human` | ✅ kept | +| `rfc/repo-a/24658-project-structure-repo-a.md` | `rfc` → `rfc` | `seedling` | `developer` | `human` | ✅ kept | +| `notes/codebase-alteration-bench/12160-file-storage-user_repo.py.md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/codebase-alteration-bench/9312-file-storage-user_repo.py.md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-architect-prd-builder/101422-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/113856-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/109874-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/agentic-ai/97741-file-packages-mcp-servers-reasoning-src-backend.ts.md` | `notes` → `explanation` | `seedling` | `developer`, `ops` | `auto-generated` | ✅ kept | +| `notes/agentic-ai/98169-file-packages-memory-src-shared-hash.ts.md` | `notes` → `explanation` | `seedling` | `developer`, `security` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/113272-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `guides/2026/2026-04-21-patterns.md` | `guides` → `explanation` | `seedling` | `developer` | `human` | 🔁 changed | +| `guides/2026/2026-04-21-user-flow.md` | `guides` → `explanation` | `seedling` | `security` | `human` | 🔁 changed | +| `guides/2026/2026-04-21-journeys.md` | `guides` → `explanation` | `seedling` | `developer`, `ops`, `security` | `human` | 🔁 changed | +| `guides/2026/2026-04-21-glossary.md` | `guides` → `explanation` | `seedling` | `developer`, `ops` | `human` | 🔁 changed | +| `guides/2026/2026-04-21-security.md` | `guides` → `explanation` | `seedling` | `developer`, `ops`, `security` | `human` | 🔁 changed | +| `guides/2026/2026-04-21-testing.md` | `guides` → `explanation` | `seedling` | `developer`, `security` | `human` | 🔁 changed | +| `guides/2026/2026-04-21-00-day-one.md` | `guides` → `explanation` | `seedling` | `developer`, `ops` | `human` | 🔁 changed | +| `guides/2026/2026-04-21-design-system.md` | `guides` → `explanation` | `seedling` | `developer` | `human` | 🔁 changed | +| `conventions/agentic-ai/86774-convention-p-align-center.md` | `conventions` → `explanation` | `seedling` | `developer`, `ops` | `human` | ✅ kept | +| `conventions/ai-architect-mcp/99075-convention-file-mcp-ai_architect_mcp-_adapters-git_adapter.py.md` | `conventions` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `conventions/agentic-ai/98438-convention-file-....md` | `conventions` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `conventions/ai-architect-mcp/99125-convention-file-mcp-ai_architect_mcp-_interview-scorers-outline_flow.py.md` | `conventions` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `conventions/agentic-ai/96936-convention-ci-cd-.github-workflows-ci.yml.md` | `conventions` → `explanation` | `seedling` | `developer`, `ops` | `human` | ✅ kept | +| `conventions/agentic-ai/96915-convention-phase_3_plan.md.md` | `conventions` → `explanation` | `seedling` | `developer`, `ops` | `human` | ✅ kept | +| `conventions/agentic-ai/96914-convention-patterns.md.md` | `conventions` → `explanation` | `seedling` | `developer` | `human` | ✅ kept | +| `conventions/ai-architect-mcp/99082-convention-file-mcp-ai_architect_mcp-_adapters-github_adapter.py.md` | `conventions` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `lessons/cortex/97502-lesson-file-tests_py-hooks-test_auto_recall.py.md` | `lessons` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `lessons/2026/2026-04-17-1-49-0.md` | `lessons` → `explanation` | `seedling` | `developer` | `human` | ✅ kept | +| `lessons/2026/2026-04-17-tool-bash.md` | `lessons` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `lessons/2026/2026-04-21-route-simulation-findings.md` | `lessons` → `explanation` | `seedling` | `developer` | `human` | ✅ kept | +| `lessons/agentic-ai/96917-lesson-phase_plan.md.md` | `lessons` → `explanation` | `seedling` | `developer`, `ops` | `human` | ✅ kept | +| `lessons/2026/2026-04-17-found-a-crash-when-coming-from-publication-push-notification.md` | `lessons` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `lessons/_general/1855-known-issues-and-technical-debt-dcp-wealth-android.md` | `lessons` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `lessons/2026/2026-04-17-c1-plaintext-backbase-artifactory-credentials-in-git.md` | `lessons` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `adrs/2026/2026-04-08-author-the-wiki-as-a-first-class-layer.md` | `adrs` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adrs/2026/2026-04-15-rotate-backbase-artifactory-credentials-committed-in-setting.md` | `adrs` → `adr` | `proposed` | `developer`, `ops` | `human` | ✅ kept | +| `adrs/2026/2026-04-15-network-security-config-allows-cleartext-and-lacks-certifica.md` | `adrs` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adrs/2026/2026-04-15-sgn-credentials-shipped-in-apk-via-ci-sed-injection.md` | `adrs` → `adr` | `proposed` | `developer`, `security` | `human` | ✅ kept | +| `adrs/2026/2026-04-15-maskingaction-only-masks-checkable-views-textview-pii-leaks.md` | `adrs` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adrs/2026/2026-04-08-manual-jacoco-agent-configuration-instead-of-the-jacoco-grad.md` | `adrs` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adrs/2026/2026-04-21-portfolio-details-declares-hilt-plugin-kapt-but-uses-ko.md` | `adrs` → `adr` | `proposed` | `developer` | `human` | ✅ kept | +| `adrs/2026/2026-04-15-myapplicationdependencies-is-a-1180-line-god-composition-roo.md` | `adrs` → `adr` | `proposed` | `developer`, `security` | `human` | ✅ kept | +| `reference/codebase/plugins-codebase-src-rust-src-clustering-rs-cluster-graph.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/plugins-codebase-src-rust-src-graph-store-rs-tests-test-create-and-query.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/packages-codebase-rust-tests-stage9-integration-rs-test-semantic-diff-missing-pa.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/packages-codebase-rust-tests-stage3d-integration-rs-test-search-partial-name.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/packages-reasoning-memory-pii-scanner-py-main.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/packages-codebase-rust-src-graph-store-rs-tests-test-cypher-str-escape-rules.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/plugins-codebase-src-rust-src-prd-validator-rs-tests-test-regex-extract-backtick.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/plugins-codebase-src-rust-src-search-rrf-rs-tests-test-rrf-two-lists-fusion.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/packages-mcp-servers-reasoning-src-index-ts-main.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/112457-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/111282-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-architect-prd-builder/104250-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/112572-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd/104329-file-library-.build-checkouts-nimble-sources-nimble-dsl-require.swift.md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd/104454-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `specs/_general/2096-tool-bash.md` | `specs` → — | — | — | — | ❌ admission gate rejected (audit-tag, noise, or low score) | +| `notes/ai-architect-mcp/98901-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `reference/codebase/packages-codebase-rust-src-lsp-client-rs-tests-test-parse-definition-array.md` | `reference` → `reference` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-architect-prd-builder/101325-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/113609-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/113989-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-architect-prd-builder/102607-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/codebase-alteration-bench/7578-file-storage-user_repo.py.md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-architect-prd-builder/103842-file-....md` | `notes` → `explanation` | `seedling` | `developer`, `ops` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/113348-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | +| `notes/ai-prd-generator/112784-file-....md` | `notes` → `explanation` | `seedling` | `developer` | `auto-generated` | ✅ kept | diff --git a/scripts/wiki_pilot_migration.py b/scripts/wiki_pilot_migration.py new file mode 100644 index 00000000..e77fa656 --- /dev/null +++ b/scripts/wiki_pilot_migration.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 +"""Pilot migration analyzer — Phase 2 of ADR-2244. + +Walks the methodology wiki, runs each page's body through the new +data-driven classifier (``mcp_server.core.wiki_classifier.classify_memory``, +post-#27/#28), and produces a Markdown report showing the proposed +modern 4-tuple (kind, lifecycle, audience, provenance) for each page +alongside its current legacy ``kind``. + +Goal: human-reviewable accuracy check before any bulk re-bucketing +(Phase 4). The ADR-2244 acceptance criterion is ≥ 90% kind agreement +with human judgment on a ~100-page representative sample. + +Usage +----- + + uv run scripts/wiki_pilot_migration.py \\ + --wiki ~/.claude/methodology/wiki \\ + --sample-size 100 \\ + --out scripts/wiki-pilot-report.md + +By default samples are stratified across the current ``kind`` directories +so the report exercises ADRs, specs, lessons, notes, references, etc. +without being swamped by the 7,820 file-doc notes. + +Read-only. The script never writes to the wiki itself. +""" + +from __future__ import annotations + +import argparse +import random +import re +import sys +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +# Ensure mcp_server is importable when run from the Cortex repo root. +_REPO_ROOT = Path(__file__).resolve().parent.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from mcp_server.core.wiki_classifier import classify_memory # noqa: E402 + + +_FRONTMATTER_RE = re.compile(r"\A---\s*\n(?P.*?)\n---\s*\n?", re.DOTALL) + + +@dataclass(frozen=True) +class PageRecord: + """One row of the migration report.""" + + path: str + legacy_kind: str + title: str + content_length: int + proposed_kind: str | None + proposed_lifecycle: str | None + proposed_audience: tuple[str, ...] + proposed_provenance: str | None + rejected: bool + rejection_reason: str + kind_agreement: str # "kept" | "changed" | "n/a" + + +def _parse_frontmatter(text: str) -> tuple[dict[str, object], str]: + """Cheap YAML-ish parser; returns (frontmatter_dict, body). + + Handles three patterns observed in the wiki: + 1. ``key: value`` scalars. + 2. ``key: [a, b, c]`` inline lists. + 3. ``key:`` followed by indented `` - item`` block lists. + + Lists land as Python ``list[str]``; scalars as ``str``. Indentation + inside multi-line scalars (folded blocks etc.) is not preserved — + sufficient for the pilot since we only care about ``tags``, ``kind``, + and ``title``. + """ + m = _FRONTMATTER_RE.match(text) + if not m: + return {}, text + + fm: dict[str, object] = {} + current_list_key: str | None = None + current_list: list[str] = [] + + def _close_list() -> None: + nonlocal current_list_key, current_list + if current_list_key is not None: + fm[current_list_key] = current_list + current_list_key = None + current_list = [] + + for raw in m.group("fm").splitlines(): + stripped = raw.strip() + if not stripped: + _close_list() + continue + # Block-list item: indented, starts with `-`. + if ( + current_list_key is not None + and raw.startswith(" ") + and stripped.startswith("- ") + ): + current_list.append(stripped[2:].strip().strip("'\"")) + continue + # Otherwise this line ends any open list. + _close_list() + if ":" not in stripped: + continue + key, _, value = stripped.partition(":") + key = key.strip() + value = value.strip() + if value == "": + # Empty value → either a block list follows, or scalar with no value. + current_list_key = key + current_list = [] + continue + if value.startswith("[") and value.endswith("]"): + fm[key] = [ + t.strip().strip("'\"") for t in value[1:-1].split(",") if t.strip() + ] + continue + fm[key] = value.strip("'\"") + + _close_list() + return fm, text[m.end() :] + + +def _extract_tags(fm: dict[str, object]) -> list[str]: + """Pull tags from frontmatter. + + Accepts the value as ``list[str]`` (block list, inline list) or + ``str`` (comma-separated scalar). Empty/missing → ``[]``. + """ + raw = fm.get("tags", "") + if isinstance(raw, list): + return [str(t).strip() for t in raw if str(t).strip()] + if isinstance(raw, str) and raw: + return [t.strip() for t in raw.split(",") if t.strip()] + return [] + + +def _legacy_kind_from_path(rel_path: str) -> str: + """First path component is the legacy kind directory.""" + return rel_path.split("/", 1)[0] + + +def _strip_h1(body: str) -> str: + """Drop a leading H1 heading if present (it's usually the title).""" + lines = body.lstrip().splitlines() + if lines and lines[0].startswith("# "): + return "\n".join(lines[1:]).strip() + return body.strip() + + +def _evaluate( + path: str, + text: str, +) -> PageRecord: + fm, body = _parse_frontmatter(text) + tags = _extract_tags(fm) + title = str(fm.get("title", "")).strip() + content = _strip_h1(body) + if not content: + content = title # use the title for nearly-empty pages + legacy_kind = _legacy_kind_from_path(path) + + try: + verdict = classify_memory(content, tags) + except Exception as exc: # noqa: BLE001 + return PageRecord( + path=path, + legacy_kind=legacy_kind, + title=title, + content_length=len(content), + proposed_kind=None, + proposed_lifecycle=None, + proposed_audience=(), + proposed_provenance=None, + rejected=True, + rejection_reason=f"classifier raised: {type(exc).__name__}: {exc}", + kind_agreement="n/a", + ) + + if verdict is None: + return PageRecord( + path=path, + legacy_kind=legacy_kind, + title=title, + content_length=len(content), + proposed_kind=None, + proposed_lifecycle=None, + proposed_audience=(), + proposed_provenance=None, + rejected=True, + rejection_reason="admission gate rejected (audit-tag, noise, or low score)", + kind_agreement="n/a", + ) + + # Map legacy directory name → modern kind for the "kept vs changed" view. + from mcp_server.shared.wiki_classification import normalize_legacy_kind + + normalized_legacy = normalize_legacy_kind(legacy_kind) + agreement = "kept" if verdict.kind == normalized_legacy else "changed" + + return PageRecord( + path=path, + legacy_kind=legacy_kind, + title=title, + content_length=len(content), + proposed_kind=verdict.kind, + proposed_lifecycle=verdict.lifecycle, + proposed_audience=tuple(verdict.audience), + proposed_provenance=verdict.provenance, + rejected=False, + rejection_reason="", + kind_agreement=agreement, + ) + + +def _collect_pages(wiki_root: Path) -> list[Path]: + """All .md pages under wiki_root excluding ``.generated/``.""" + out: list[Path] = [] + for md in wiki_root.rglob("*.md"): + rel = md.relative_to(wiki_root) + if rel.parts and rel.parts[0].startswith("."): + continue + out.append(md) + return out + + +def _stratified_sample( + pages: list[Path], + wiki_root: Path, + sample_size: int, + rng: random.Random, +) -> list[Path]: + """Sample evenly across the legacy ``kind`` directories. + + Each kind contributes up to ``sample_size / N_kinds`` pages. Kinds + with fewer pages contribute everything they have; the remainder is + redistributed so the total approaches ``sample_size``. + """ + by_kind: dict[str, list[Path]] = defaultdict(list) + for p in pages: + rel = p.relative_to(wiki_root) + by_kind[rel.parts[0]].append(p) + + n_kinds = len(by_kind) + if n_kinds == 0: + return [] + + per_kind = max(sample_size // n_kinds, 1) + sampled: list[Path] = [] + leftover_quota = sample_size + for kind, files in by_kind.items(): + take = min(per_kind, len(files), leftover_quota) + sampled.extend(rng.sample(files, take)) + leftover_quota -= take + + if leftover_quota > 0: + remaining = [p for p in pages if p not in set(sampled)] + rng.shuffle(remaining) + sampled.extend(remaining[:leftover_quota]) + + return sampled + + +def _format_report(records: list[PageRecord], wiki_root: Path) -> str: + """Render the migration report as Markdown.""" + n = len(records) + n_rejected = sum(1 for r in records if r.rejected) + n_admitted = n - n_rejected + n_kept = sum(1 for r in records if r.kind_agreement == "kept") + n_changed = sum(1 for r in records if r.kind_agreement == "changed") + + by_legacy: dict[str, int] = defaultdict(int) + by_proposed: dict[str, int] = defaultdict(int) + by_transition: dict[tuple[str, str | None], int] = defaultdict(int) + by_lifecycle: dict[str | None, int] = defaultdict(int) + by_audience: dict[str, int] = defaultdict(int) + by_provenance: dict[str | None, int] = defaultdict(int) + rejection_reasons: dict[str, int] = defaultdict(int) + + for r in records: + by_legacy[r.legacy_kind] += 1 + by_proposed[r.proposed_kind or ""] += 1 + by_transition[(r.legacy_kind, r.proposed_kind)] += 1 + if not r.rejected: + by_lifecycle[r.proposed_lifecycle] += 1 + for a in r.proposed_audience: + by_audience[a] += 1 + by_provenance[r.proposed_provenance] += 1 + else: + rejection_reasons[r.rejection_reason] += 1 + + lines: list[str] = [] + lines.append("# ADR-2244 Phase 2 — Pilot migration report") + lines.append("") + lines.append(f"Wiki root: `{wiki_root}`") + lines.append("") + lines.append("## Summary") + lines.append("") + lines.append(f"- **Sample size:** {n}") + lines.append( + f"- **Admitted by new classifier:** {n_admitted} ({_pct(n_admitted, n)})" + ) + lines.append( + f"- **Rejected (admission gate):** {n_rejected} ({_pct(n_rejected, n)})" + ) + lines.append( + f"- **Kind kept (legacy → modern direct map):** {n_kept} ({_pct(n_kept, n_admitted)} of admitted)" + ) + lines.append( + f"- **Kind changed:** {n_changed} ({_pct(n_changed, n_admitted)} of admitted)" + ) + lines.append("") + lines.append("## Distribution — legacy kinds in the sample") + lines.append("") + lines.append("| Legacy kind | Pages |") + lines.append("|---|---:|") + for k, c in sorted(by_legacy.items(), key=lambda kv: -kv[1]): + lines.append(f"| `{k}` | {c} |") + lines.append("") + lines.append("## Distribution — proposed modern kinds") + lines.append("") + lines.append("| Proposed kind | Pages |") + lines.append("|---|---:|") + for k, c in sorted(by_proposed.items(), key=lambda kv: -kv[1]): + lines.append(f"| `{k}` | {c} |") + lines.append("") + lines.append("## Transition matrix (legacy → proposed)") + lines.append("") + lines.append("| From | To | Count |") + lines.append("|---|---|---:|") + for (frm, to), c in sorted(by_transition.items(), key=lambda kv: -kv[1]): + to_str = to if to is not None else "" + lines.append(f"| `{frm}` | `{to_str}` | {c} |") + lines.append("") + lines.append("## Proposed facet distributions (admitted pages only)") + lines.append("") + lines.append("### Lifecycle") + lines.append("") + lines.append("| Value | Pages |") + lines.append("|---|---:|") + for k, c in sorted(by_lifecycle.items(), key=lambda kv: -kv[1]): + lines.append(f"| `{k}` | {c} |") + lines.append("") + lines.append("### Audience (multi-valued — counted per occurrence)") + lines.append("") + lines.append("| Value | Pages |") + lines.append("|---|---:|") + for k, c in sorted(by_audience.items(), key=lambda kv: -kv[1]): + lines.append(f"| `{k}` | {c} |") + lines.append("") + lines.append("### Provenance") + lines.append("") + lines.append("| Value | Pages |") + lines.append("|---|---:|") + for k, c in sorted(by_provenance.items(), key=lambda kv: -kv[1]): + lines.append(f"| `{k}` | {c} |") + lines.append("") + if rejection_reasons: + lines.append("## Rejection reasons") + lines.append("") + lines.append("| Reason | Pages |") + lines.append("|---|---:|") + for reason, c in sorted(rejection_reasons.items(), key=lambda kv: -kv[1]): + lines.append(f"| {reason} | {c} |") + lines.append("") + + lines.append("## Per-page proposals") + lines.append("") + lines.append( + "| Path | Legacy → Proposed | Lifecycle | Audience | Provenance | Status |" + ) + lines.append("|---|---|---|---|---|---|") + for r in records: + if r.rejected: + status = f"❌ {r.rejection_reason}" + transition = f"`{r.legacy_kind}` → —" + lifecycle = "—" + audience = "—" + provenance = "—" + else: + status = "✅ kept" if r.kind_agreement == "kept" else "🔁 changed" + transition = f"`{r.legacy_kind}` → `{r.proposed_kind}`" + lifecycle = f"`{r.proposed_lifecycle}`" + audience = ", ".join(f"`{a}`" for a in r.proposed_audience) + provenance = f"`{r.proposed_provenance}`" + lines.append( + f"| `{r.path}` | {transition} | {lifecycle} | {audience} | {provenance} | {status} |" + ) + lines.append("") + return "\n".join(lines) + + +def _pct(num: int, denom: int) -> str: + if denom == 0: + return "0.0%" + return f"{num / denom * 100:.1f}%" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--wiki", + type=Path, + default=Path.home() / ".claude" / "methodology" / "wiki", + help="Wiki root directory.", + ) + parser.add_argument( + "--sample-size", + type=int, + default=100, + help="Pages to sample. Stratified by legacy kind directory.", + ) + parser.add_argument( + "--out", + type=Path, + default=Path(__file__).parent / "wiki-pilot-report.md", + help="Path to write the Markdown report.", + ) + parser.add_argument( + "--seed", + type=int, + default=20260512, + help="RNG seed for the sample (deterministic by default).", + ) + parser.add_argument( + "--all", + action="store_true", + help="Evaluate every page (ignores --sample-size).", + ) + args = parser.parse_args() + + wiki_root: Path = args.wiki.expanduser().resolve() + if not wiki_root.is_dir(): + print(f"error: wiki root not found: {wiki_root}", file=sys.stderr) + return 2 + + print(f"scanning {wiki_root}", file=sys.stderr) + pages = _collect_pages(wiki_root) + print(f"found {len(pages)} pages", file=sys.stderr) + + if args.all: + sampled = pages + else: + rng = random.Random(args.seed) + sampled = _stratified_sample(pages, wiki_root, args.sample_size, rng) + print(f"evaluating {len(sampled)} pages", file=sys.stderr) + + records: list[PageRecord] = [] + for p in sampled: + rel = p.relative_to(wiki_root) + try: + text = p.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + print(f" skip {rel}: {exc}", file=sys.stderr) + continue + records.append(_evaluate(str(rel), text)) + + report = _format_report(records, wiki_root) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(report, encoding="utf-8") + print(f"wrote report: {args.out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests_py/core/test_wiki_classifier.py b/tests_py/core/test_wiki_classifier.py index 3f84cbf8..d23ca65a 100644 --- a/tests_py/core/test_wiki_classifier.py +++ b/tests_py/core/test_wiki_classifier.py @@ -281,3 +281,60 @@ def test_security_tag_routes_to_security_audience() -> None: result = classify_memory(content, tags=["security", "decision"]) assert result is not None assert "security" in result.audience + + +# ── Calibration regressions from the Phase 2 pilot (2026-05-13) ─────── + + +def test_adr_detected_from_nygard_heading_skeleton() -> None: + """Pilot 2026-05-13 found 3 of 8 ADRs misclassified because their body + used the canonical ``## Decision`` heading without a ``Decision:`` colon. + The ADR pattern now matches the heading skeleton. + """ + content = ( + "## Status\n\nAccepted\n\n" + "## Context\n\n" + "MCP plugins run inside Claude Code's process. Any external " + "dependency introduces supply chain risk.\n\n" + "## Decision\n\n" + "Use zero external npm dependencies. Rely on Node.js built-ins.\n\n" + "## Consequences\n\n" + "Gain: no supply chain attack surface. Lose: more hand-written code." + ) + result = classify_memory(content, tags=["adr"]) + assert result is not None + assert result.kind == "adr" + + +def test_architecture_tag_alone_does_not_route_to_adr() -> None: + """Pilot 2026-05-13 found 8 of 8 RFC pages misrouted to ADR because + they carried the ``architecture`` tag, which used to be in adr.tag_aliases. + ``architecture`` was removed from adr aliases — those pages now stay RFC + (or fall through to explanation if no other signal hits). + """ + content = ( + "## Top-level layout\n\n- README.md\n- pyproject.toml\n\n" + "Project structure: repo-a. Primary languages: unknown." + ) + result = classify_memory(content, tags=["architecture", "project-structure"]) + # Must NOT be adr (the regression we just fixed). + if result is not None: + assert result.kind != "adr" + + +def test_crypto_module_name_does_not_flag_security_audience() -> None: + """Pilot 2026-05-13 found ADR-001 (zero dependencies) tagged ``security`` + audience because its body listed ``crypto`` among Node built-in modules. + The security pattern now requires ``cryptograph(y|ic)`` — the full word — + so a bare module name no longer fires the audience. + """ + content = ( + "Decision: use zero external dependencies. Rely on Node.js built-in " + "modules: fs, path, os, http, crypto, and node:test. No external " + "supply chain. Hand-write any utility a library would provide." + ) + result = classify_memory(content, tags=["decision", "adr"]) + assert result is not None + assert result.kind == "adr" + # The bare word ``crypto`` (module name) should NOT trigger security. + assert "security" not in result.audience