diff --git a/packages/codev/src/__tests__/prompt-behavior-metrics.test.ts b/packages/codev/src/__tests__/prompt-behavior-metrics.test.ts index d422764ee..b066838e5 100644 --- a/packages/codev/src/__tests__/prompt-behavior-metrics.test.ts +++ b/packages/codev/src/__tests__/prompt-behavior-metrics.test.ts @@ -176,9 +176,40 @@ describe('behavioural metrics (M12a / T14)', () => { } }); + // The exact project set the committed baseline (n=160) was computed over. + // FROZEN by definition: the baseline is a point-in-time artifact, so its + // reproduction must pin its sample. Do NOT add new projects here to make a + // failure go away — a change in these numbers over THIS set means the + // measurement code's semantics changed, which is what this test guards. + const BASELINE_SAMPLE = [ + '0092-terminal-file-links', + '0120-codex-sdk-integration', + '0124-test-suite-consolidation', + '403-af-send-typing-awareness', + '456-dashboard-statistics-tab-in-ri', + '462-add-spike-protocol-for-technic', + '467-add-open-files-shells-section-', + '468-af-rename-command-to-rename-cu', + '589-support-non-github-repositorie', + '723-improve-arch-md-lessons-learne', + '746-spir-architect-s-baked-archite', + '755-multi-architect-support-per-ar', + '761-surface-multiple-architects-in', + '778-gemini-cli-antigravity-cli-jun', + '786-multi-architect-feature-is-und', + '823-multi-architect-coordination-b', + '927-needs-attention-surface-prs-vi', + '987-engineering-wisdom-is-write-on', + ]; + it('reproduces the committed baseline numbers on this repo', () => { // Guards against a refactor silently changing what the baseline means. - const m = measureBehavior(REPO_ROOT); + // Pinned to the baseline's own sample so projects that post-date the + // baseline (each adding review verdicts to codev/projects/) cannot + // perturb the reproduction — the un-pinned form broke the moment any + // later project ran its first consultation (160 → 163 via project 1286). + const m = measureBehavior(REPO_ROOT, { includeProjects: BASELINE_SAMPLE }); + expect(m.sampleProjects).toEqual(BASELINE_SAMPLE); expect(m.b1_totalVerdicts).toBe(160); expect(m.b1_requestChangesRate).toBeCloseTo(51.88, 1); expect(m.b2_roundsPerPhaseMean).toBeCloseTo(1.12, 2); diff --git a/packages/codev/src/lib/prompt-behavior-metrics.ts b/packages/codev/src/lib/prompt-behavior-metrics.ts index e325c440f..11507adfc 100644 --- a/packages/codev/src/lib/prompt-behavior-metrics.ts +++ b/packages/codev/src/lib/prompt-behavior-metrics.ts @@ -158,11 +158,12 @@ function walk(dir: string, pred: (p: string) => boolean): string[] { * review history for protocols that loop that way, so pir/bugfix/air contribute * nothing. Sorted for deterministic output. */ -function collectReviewMetrics(root: string, excludeProjects: string[]) { +function collectReviewMetrics(root: string, excludeProjects: string[], includeProjects?: string[]) { const projDir = path.join(root, 'codev', 'projects'); const files = fs.existsSync(projDir) ? fs .readdirSync(projDir) + .filter((d) => (includeProjects ? includeProjects.includes(d) : true)) .filter((d) => !excludeProjects.includes(d)) .map((d) => path.join(projDir, d, 'status.yaml')) .filter((f) => fs.existsSync(f)) @@ -264,6 +265,18 @@ export interface MeasureOptions { * test failed the moment this project's own iter-1 review landed). */ excludeProjects?: string[]; + /** + * Project directory names to measure EXCLUSIVELY (an allowlist), applied + * before `excludeProjects`. Unset means "every project on disk". + * + * A frozen baseline needs a frozen sample: the committed baseline artifact + * was computed over the projects that existed at baseline time, so a + * reproduction against live history must pin that set — otherwise every + * subsequent project that runs consultations perturbs the numbers and the + * reproduction fails for reasons that have nothing to do with the metrics + * (discovered when project 1286's first review iteration moved B1 160 → 163). + */ + includeProjects?: string[]; /** * Basename prefixes excluded from B3's prose scan, for the same reason: this * project's own thread/review discuss scar rules at length and grow with @@ -278,7 +291,7 @@ export const SELF_FILE_PREFIXES = ['spir-1252_', '1252-']; /** Collect all behavioural metrics for a repo root. Deterministic over B1–B4. */ export function measureBehavior(root: string, opts: MeasureOptions = {}): BehaviorMetrics { - const rv = collectReviewMetrics(root, opts.excludeProjects ?? [SELF_PROJECT_DIR]); + const rv = collectReviewMetrics(root, opts.excludeProjects ?? [SELF_PROJECT_DIR], opts.includeProjects); const scar = collectScarHits(root, opts.excludeFilePrefixes ?? SELF_FILE_PREFIXES); return { b1_requestChangesRate: round2(rv.rcRate * 100),