From 227bff70f08b2b5d7740cc03bb162c75625c09e9 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Tue, 25 Aug 2026 11:17:39 -0700 Subject: [PATCH 1/2] Overhaul Testsmith prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teach Testsmith to design permanent cases from success_criteria with risk prioritization, a setup/action/expect template, and Corbits report shape. Keep blinders on design-only — refuse Tester/Builder drift despite mounted writes; no gaas twin. Closes CL-7033 --- src/agent/directors/testsmith/package.test.ts | 76 +++++++++++++++++-- src/agent/directors/testsmith/package.ts | 74 ++++++++++++++---- 2 files changed, 129 insertions(+), 21 deletions(-) diff --git a/src/agent/directors/testsmith/package.test.ts b/src/agent/directors/testsmith/package.test.ts index ab37e84c9..e47abccd8 100644 --- a/src/agent/directors/testsmith/package.test.ts +++ b/src/agent/directors/testsmith/package.test.ts @@ -11,10 +11,67 @@ describe("testsmithPackage", () => { expect(testsmithPackage.systemPrompt.startsWith("Placeholder")).toBe(false); }); - test("systemPrompt states PRIMARY INTENT for test design", () => { - expect(testsmithPackage.systemPrompt).toContain("PRIMARY INTENT"); - expect(testsmithPackage.systemPrompt).toMatch(/test strategy|test cases|design/i); - expect(testsmithPackage.systemPrompt).toMatch(/do not implement|not implement/i); + test("systemPrompt identity is Testsmith / TestsmithDirector", () => { + const p = testsmithPackage.systemPrompt; + expect(p).toMatch(/TestsmithDirector \(Testsmith\)/); + expect(p).toContain("PRIMARY INTENT"); + expect(p).toMatch(/permanent test/i); + }); + + test("systemPrompt teaches design-in-report workflow and case template", () => { + const p = testsmithPackage.systemPrompt; + expect(p).toMatch(/Design-in-report workflow/i); + expect(p).toMatch(/Blinders on|BLINDERS ON/i); + expect(p).toContain("success_criteria"); + expect(p).toMatch(/Case template/i); + expect(p).toMatch(/\*\*Setup\*\*/); + expect(p).toMatch(/\*\*Action\*\*/); + expect(p).toMatch(/\*\*Expect\*\*/); + expect(p).toMatch(/what not to test/i); + expect(p).toMatch(/unit \| integration \| e2e/); + }); + + test("systemPrompt teaches risk prioritization", () => { + const p = testsmithPackage.systemPrompt; + expect(p).toMatch(/Risk prioritization/i); + expect(p).toMatch(/Cover first/i); + expect(p).toMatch(/Defer or omit/i); + }); + + test("systemPrompt is design lane only (not Tester / Builder / orchestrator)", () => { + const p = testsmithPackage.systemPrompt; + expect(p).toMatch(/Do not become Builder/i); + expect(p).toMatch(/that is Tester/i); + expect(p).toMatch(/do not use them/i); + expect(p).toMatch(/fleet orchestration/i); + expect(p).toMatch(/DONE GATE/i); + expect(p).toMatch(/Hand off/i); + }); + + test("systemPrompt states Corbits report shape", () => { + const p = testsmithPackage.systemPrompt; + expect(p).toContain("## Summary"); + expect(p).toContain("## Findings"); + expect(p).toContain("## Blockers"); + expect(p).toContain("## Paths"); + expect(p).toMatch(/Corbits report shape/i); + }); + + test("systemPrompt has no tool-schema restatement or fake caps", () => { + const p = testsmithPackage.systemPrompt; + expect(p).not.toMatch(/parameters?:/i); + expect(p).not.toMatch(/fan-out/i); + expect(p).not.toMatch(/at most \d+/i); + expect(p).not.toMatch(/turn budget/i); + expect(p).not.toMatch(/scheduler/i); + }); + + test("systemPrompt is not a gaasbot twin", () => { + const p = testsmithPackage.systemPrompt; + expect(p).not.toMatch(/Gaasbot/i); + expect(p).not.toMatch(/risk counsel/i); + expect(p).not.toMatch(/ship-with-note/i); + expect(p).not.toMatch(/filed-for-later/i); }); test("spawn.maySpawn is false (leaf)", () => { @@ -33,14 +90,17 @@ describe("testsmithPackage", () => { expect(testsmithPackage.modelRole).toBe("test"); }); - test("primaryIntent is design-only and not primary verifier", () => { - expect(testsmithPackage.primaryIntent).toMatch(/design/i); - expect(testsmithPackage.primaryIntent).toMatch(/not.*verifier|do not run as primary verifier/i); + test("primaryIntent is permanent-design and not primary verifier", () => { + expect(testsmithPackage.primaryIntent).toMatch(/permanent test cases/i); + expect(testsmithPackage.primaryIntent).toMatch( + /not.*verifier|do not run as primary verifier/i, + ); }); - test("outOfLane refuses product implement and runtime verify role", () => { + test("outOfLane refuses product implement, verifier role, and landing tests", () => { const joined = testsmithPackage.outOfLane.join(" "); expect(joined).toMatch(/implement/i); expect(joined).toMatch(/verifier|tester/i); + expect(joined).toMatch(/landing test/i); }); }); diff --git a/src/agent/directors/testsmith/package.ts b/src/agent/directors/testsmith/package.ts index 6783f7554..6996fc8ec 100644 --- a/src/agent/directors/testsmith/package.ts +++ b/src/agent/directors/testsmith/package.ts @@ -2,34 +2,82 @@ import type { DirectorPackage } from "../types.js"; import { REVIEW_TOOLS } from "../tool-sets.js"; /** - * Testsmith: test design specialist — strategy and cases only; never implements product - * and is not the runtime verifier (that is tester). + * Testsmith leaf (CL-7033). + * Design permanent test strategy and cases in the report — never implement product, + * never replace Tester as the runtime verifier. */ export const testsmithPackage: DirectorPackage = { id: "testsmith", primaryIntent: - "Design test strategy and cases; do not implement product; do not run as primary verifier", + "Design permanent test cases; do not implement product; do not run as primary verifier", outOfLane: [ "implementing product code", "shipping features", "acting as primary runtime verifier (tester)", "fixing failing product code", + "landing test files as the implementer", "orchestration", ], - description: "Test design specialist — strategy and cases in the report only", - systemPrompt: `You are TestsmithDirector, a specialist in Corbits Code. + description: "Test design specialist — permanent cases in the report only", + systemPrompt: `You are TestsmithDirector (Testsmith), a specialist in Corbits Code. -PRIMARY INTENT: design test strategy and test cases for the brief. Produce clear, agent-ready coverage plans. Do not implement product code. Do not act as the primary runtime verifier (that is Tester). +PRIMARY INTENT: design permanent test strategy and cases for the brief. Produce agent-ready coverage the suite should keep. Do not implement product code. Do not act as the primary runtime verifier (that is Tester). Do not become Builder. -Design in the report. Prefer: -- risk-based coverage and acceptance criteria from the brief -- unit / integration / e2e boundaries when relevant -- concrete cases: setup, action, expected result, edge/failure modes -- what not to test and why +You are the test-design lane only — not Tester, not Builder, not Counsel, not an orchestrator. Do not spawn specialists. Write tools are mounted with no path lock — do not use them. Leave product and test-file edits to Builder; leave suite/repro execution to Tester. -OUT OF LANE: fixing production code, becoming the implementer, running the full verify-and-fix loop, fleet orchestration. +BLINDERS ON: Design from the brief's success_criteria / acceptance criteria and stated risks — not from "whatever the code does today." Read/search only to ground paths, public APIs, and existing suite shape. Do not soften cases to match current buggy behavior. Stay on this brief; do not wander into peer work or fleet orchestration. -Read and search the codebase to ground the design; do not mutate product code.`, +# Design-in-report workflow + +1. Map every success_criteria item to concrete permanent cases (or Blockers if you cannot). +2. Rank by risk: correctness/data integrity and user-visible breaks first; then API contract and regression of known failure modes; defer style theater and impossible paths. +3. Name the boundary for each case: unit | integration | e2e — pick the cheapest layer that can prove the claim. +4. Write each case with the template below. Prefer a few sharp permanent cases over a fog of speculative ones. +5. Explicitly list what not to test and why (impossible paths, over-engineering theater, pure typechecker/library happy paths the project already trusts). +6. Hand off: Builder lands the tests; Tester runs them. You design only. + +# Case template + +For every permanent case include: +- **Name** — short, stable identifier a Builder can paste into a test title +- **Boundary** — unit | integration | e2e +- **Risk** — why this case earns a permanent seat (what breaks if it is missing) +- **Setup** — fixtures, state, mocks/fakes (prefer inject clocks/I/O over sleeping/network) +- **Action** — the single behavior under test +- **Expect** — observable result (return, state, error shape, side effect) +- **Edge / failure** — invalid input, missing branch, or failure mode that must stay covered + +# Risk prioritization + +Cover first: +- Invariants that protect customers/data and stated success_criteria +- Public API sync/async and signature contracts when the brief specifies them +- Regression of defects the brief or Findings already named + +Defer or omit: +- Speculative abstractions and defensive cases for impossible states +- Style nits and "while we're here" coverage +- Re-testing a well-maintained library's happy path + +# Corbits report shape + +When done, stop tooling and reply with ONLY this envelope: + +## Summary +One or two sentences: strategy and coverage scope designed. + +## Findings +Permanent cases (name + boundary + setup/action/expect + risk), coverage map of each success_criteria item → cases (or blocked), and what not to test with why. + +## Blockers +Open questions, missing acceptance criteria, or assumptions. Write "None." if clear. + +## Paths +Files/suites you read to ground the design (one per line). Write "None." if none. + +DONE GATE: Stop when every success_criteria item has permanent cases (or Blockers). Do not invent architecture or expand the brief after criteria are covered. If the brief is ambiguous, report Blockers — do not become Counsel or Greybeard. + +OUT OF LANE: implementing product or tests, becoming Tester/Builder, running the full verify-and-fix loop, fleet orchestration, architecture essays, exploration maps as primary.`, tools: { allow: REVIEW_TOOLS }, spawn: { maySpawn: false }, tier: "leaf", From 97190ef4a1d6c4b25ba91aed97b5906fcebebc59 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Tue, 25 Aug 2026 11:19:37 -0700 Subject: [PATCH 2/2] Format Testsmith director package with Prettier --- src/agent/directors/testsmith/package.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/agent/directors/testsmith/package.test.ts b/src/agent/directors/testsmith/package.test.ts index e47abccd8..505889193 100644 --- a/src/agent/directors/testsmith/package.test.ts +++ b/src/agent/directors/testsmith/package.test.ts @@ -92,9 +92,7 @@ describe("testsmithPackage", () => { test("primaryIntent is permanent-design and not primary verifier", () => { expect(testsmithPackage.primaryIntent).toMatch(/permanent test cases/i); - expect(testsmithPackage.primaryIntent).toMatch( - /not.*verifier|do not run as primary verifier/i, - ); + expect(testsmithPackage.primaryIntent).toMatch(/not.*verifier|do not run as primary verifier/i); }); test("outOfLane refuses product implement, verifier role, and landing tests", () => {