From 318199dc44b67cd2e58134b936db5b7f04e7c3d2 Mon Sep 17 00:00:00 2001 From: Sam Tukra Date: Sat, 30 May 2026 12:45:50 +0100 Subject: [PATCH] =?UTF-8?q?feat(memory):=20wire=20self-evolution=20?= =?UTF-8?q?=E2=80=94=20seed/load/learn=20+=20zo=20learnings=20promote?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Batch A of the process-hardening work — make ZO actually learn per project. The EvolutionEngine + seed_priors were fully built and unit-tested but never wired into a run (PR-009: built != wired): - Seed: _maybe_seed_priors writes plan domain_priors into the project PRIORS.md on first run (idempotent). - Load: _prompt_memory injects project priors ("accumulated learnings") into every lead prompt, not just decision summaries. - Learn: EvolutionEngine wired in; _record_learning appends a durable auto-learning prior on the loop's DEAD_END/PLATEAU verdicts. - Promote: new src/zo/promote.py + `zo learnings promote` — fail-closed sanitizer (generic-category + blocklist-clear only; block-not-strip; no-blocklist -> nothing). Adversarially tested. Audit-driven cleanup (repo-cleanup-audit swarm, confidentiality cleared): fix 4 latent log_error(message=) bugs in orchestrator failure branches (+ regression test); make the experiment-checklist refresh best-effort; docs-site drift (21 agents + training-checker accordion + self-evolution/zo-learnings docs); gitignore .DS_Store/.agents/.codex/ AGENTS.md + untrack .claude/.DS_Store; fix draft.py docstring. +32 tests (780 -> 812 on Python 3.11 & 3.12). ruff src/ clean, validate-docs 0 failures. PR-041 captures the build-not-wired lesson. Co-Authored-By: Claude Opus 4.8 --- .claude/.DS_Store | Bin 8196 -> 0 bytes .gitignore | 7 + README.md | 8 +- docs/COMMANDS.md | 10 + docs/concepts/memory-and-continuity.mdx | 15 +- docs/concepts/overview.mdx | 2 +- docs/concepts/the-team.mdx | 9 +- docs/demo.html | 4 +- docs/installation.mdx | 2 +- docs/introduction.mdx | 2 +- memory/zo-platform/DECISION_LOG.md | 24 +++ memory/zo-platform/PRIORS.md | 24 +++ memory/zo-platform/STATE.md | 7 +- .../sessions/session-035-2026-05-30.md | 28 +++ src/zo/cli.py | 59 ++++++ src/zo/draft.py | 2 +- src/zo/experiments.py | 23 +- src/zo/orchestrator.py | 101 ++++++++- src/zo/promote.py | 197 ++++++++++++++++++ tests/unit/test_cli.py | 4 +- tests/unit/test_experiments.py | 46 ++++ tests/unit/test_orchestrator.py | 118 +++++++++++ tests/unit/test_promote.py | 192 +++++++++++++++++ tests/unit/test_training_metrics.py | 19 ++ 24 files changed, 876 insertions(+), 27 deletions(-) delete mode 100644 .claude/.DS_Store create mode 100644 memory/zo-platform/sessions/session-035-2026-05-30.md create mode 100644 src/zo/promote.py create mode 100644 tests/unit/test_promote.py diff --git a/.claude/.DS_Store b/.claude/.DS_Store deleted file mode 100644 index 540af9803b54219f0114e4e3f73a3830afef8058..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMYitx%6u#fI&>1_xv=%7HWVe=Au-NTGuz^w$4+PJ&g_;) zYD$a`jETlq;xFR!XQM`g#-Qjg6JOCZB$7yonuve=#UD&S&z(C+)g z0{D%dsSfaUq5+KubX-thm28UY0f8!l5(7e=3Xr)8AwINbLN(mC?%yO z2ahG)DWhie|_5Ql@QWeFZaP(30Lx)6O`f2`6V~ zTwgaS1d1XnecIxqN9!AEq78Miv6|@74fU~_Xk){=u`xwnSY5kmZ{LtLY&*w=F9^;9 zHgB45PtWX_<>s4(V@1h=V`l@$WaXApb)MeY)!n1^cA5Q-oivXaecA$-yxU3ao~)54 zuR8l3D>rPreOg(6%IeQqDbHxKEWViL7P$Rc$FlY~uIZVM-QDFGhrEKJDWa|(*U5U# zzOg<{%els0gKU|3qlGMzsdB#C?h$Ur-DhN7qSf)3a8#L?zc{jd_1gNo?}=~S+4jKX z0D7w%zTBIkmuzz~++xO;=vDeES>Dg*o3jRuVp?6J z*C<*_Zg(j_!bGiFr)aLyK14VWs*UP8S<5ILRw`{+VrqlBN!E5Loq6H8n5rJ!05}S4!wRWVy#3Ol6JTqlChOMyo9QA1uvtm8}`m zbFzLk?yu7IZf>e6lQA;n^wP>D<$6-@khSkfM>FlbNw}xaLcm|HE+DTuzD7J&pH7|k zpZpjr6@>)><#(OVRkbeq)!|@9Z+Wf+DCW z$1*I(TGS$jI&8!yY{nMsLOVLJ4`~d+f(-{A#_%Y{aSTu3Sv-g5@d94KYj_=R;w?rRCBJX{A&v#iV+vLE0v5m-a~AQd%02 z289PwA?XfG^-MZU{0yfFC0%fuNE0(BU1a-?o%gA&H+<54KcA=BtrZIwEsm~OQ`@+H zbJJBY3hWl(dTko`$e)rgKJtBZOh>KAM>WzxsbL(NEiJVYp2G4ZUtM*(8qwxZ@(6b2 zDq`nS;t2MRnn*;GC|?B|tBa^w5oM#|>)usEECOc3Mpe~{DKCX=i%u*Hm_##~;aMjC z(|G@uon`0Q1$L2L!d#R>Lj?n5uYla7E%ZD|!9Db@t!?Zw_a|w+tf?M&K_+0L3lImISHJc9`*6 zJ4*c_>b&uF_AUj2t5Jx2psJfP!(l7!m7{zJgu`qCZV|Ka`5-2cM; F|0h9AfU*Dp diff --git a/.gitignore b/.gitignore index 1b8418b..15ec79e 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,10 @@ docs/source-design/ # Local client-identifier blocklist for validate-docs.sh (NEVER commit). # See CONTRIBUTING.md → Confidentiality for the rationale. scripts/.client-blocklist + +# macOS Finder metadata + local agent-tooling artifacts (not part of ZO) +.DS_Store +**/.DS_Store +.agents/ +.codex/ +AGENTS.md diff --git a/README.md b/README.md index 2e58646..084b000 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@
[![Status](https://img.shields.io/badge/status-validated-D87A57?style=flat-square&labelColor=12110F)](#status) -[![Tests](https://img.shields.io/badge/tests-780_passing-D87A57?style=flat-square&labelColor=12110F)](#status) +[![Tests](https://img.shields.io/badge/tests-812_passing-D87A57?style=flat-square&labelColor=12110F)](#status) [![Agents](https://img.shields.io/badge/agents-21_defined-D87A57?style=flat-square&labelColor=12110F)](#agent-teams) [![Docs](https://img.shields.io/badge/docs-zerooperators.com-D87A57?style=flat-square&labelColor=12110F)](https://docs.zerooperators.com) @@ -356,7 +356,7 @@ Adds **Phase 0: Literature Review** (prior art survey, baseline definition). Pha │ ├── Agent(name="oracle-qa", team_name="project") │ │ └── Agents communicate peer-to-peer via SendMessage │ │ │ -│ The Lead knows all 20 agents and creates new ones on the │ +│ The Lead knows all 21 agents and creates new ones on the │ │ fly if the project needs expertise not in the roster. │ │ │ ├─────────────────────────────────────────────────────────────┤ @@ -444,7 +444,7 @@ zero-operators/ │ ├── semantic.py # fastembed + SQLite semantic search │ ├── comms.py # JSONL event logger (5 event types) │ └── evolution.py # Self-evolving post-mortem protocol -├── .claude/agents/ # 20 agent definitions +├── .claude/agents/ # 21 agent definitions ├── specs/ # 8 specification documents ├── plans/ # Project plan files ├── memory/ # Per-project state (STATE.md, DECISION_LOG, PRIORS) @@ -495,7 +495,7 @@ delivery-repo/ | Phase | What | Status | |-------|------|--------| -| 0 | Agent definitions (20) + Claude Code setup | Done | +| 0 | Agent definitions (21) + Claude Code setup | Done | | 1 | Plan parser, target parser, comms logger, setup | Done | | 2 | Memory layer, semantic index | Done | | 3 | Orchestration engine + lifecycle wrapper | Done | diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index c7cb3d2..df3d3e0 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -182,6 +182,16 @@ zo experiments diff EXP_A EXP_B --project NAME [--repo PATH] - **show**: full details for one experiment including the content of every authored markdown artefact. - **diff**: side-by-side comparison of two experiments' metrics and shortfalls. Useful for sibling comparisons (two parallel variants) and parent-child comparisons (did the iteration actually improve?). +### zo learnings promote + +Promote generic, client-sanitised learnings from a project's `.zo/memory/PRIORS.md` to the platform `memory/zo-platform/PRIORS.md`. + +``` +zo learnings promote --project NAME [--repo PATH] [--dry-run] +``` + +**Fail-closed by design** (the platform repo is public): only priors in generic categories (`auto-learning`, `evolution`) that clear the client blocklist (`scripts/.client-blocklist`) are promoted. A prior that is plan-seeded / `domain`, or that matches a client identifier, is **blocked** — reported for manual review, never auto-rewritten. With no blocklist file configured, **nothing** is promoted. `--dry-run` screens and reports without writing. Every run prints an auditable promoted / blocked / duplicate report. + --- ## Slash Commands diff --git a/docs/concepts/memory-and-continuity.mdx b/docs/concepts/memory-and-continuity.mdx index cfffb76..3a9521f 100644 --- a/docs/concepts/memory-and-continuity.mdx +++ b/docs/concepts/memory-and-continuity.mdx @@ -17,7 +17,7 @@ Every project gets a `memory/` directory with four canonical files (or `.zo/memo Append-only audit trail. Every architectural decision, gate passage, scope change. Each entry has a timestamp, type, title, decision, rationale, alternatives considered, outcome. - Domain knowledge accumulated through running ZO. Each prior references the failure that triggered it. After 23 sessions, ZO has 34 documented priors. + Domain knowledge accumulated through running ZO. Each prior references the failure that triggered it. ZO has 40+ documented priors, each tracing to a real failure. Per-session summary files (`session-NNN-YYYY-MM-DD.md`). Written at session end. Captures what was attempted, what shipped, what's next. @@ -122,7 +122,7 @@ This prevents accumulation of irrelevant reasoning and keeps token costs predict ## Self-evolution in practice -The 34 priors in [`memory/zo-platform/PRIORS.md`](https://github.com/SamPlvs/zero-operators/blob/main/memory/zo-platform/PRIORS.md) are the cumulative output of this protocol. A few examples: +The 40+ priors in [`memory/zo-platform/PRIORS.md`](https://github.com/SamPlvs/zero-operators/blob/main/memory/zo-platform/PRIORS.md) are the cumulative output of this protocol. A few examples: - **PR-001**: `claude --print --dangerously-skip-permissions` exits immediately. Captured after a tmux pane stayed blank during MNIST testing. - **PR-005**: Aspirational rules without enforcement are dead letter. Captured after a documentation cascade was repeatedly ignored despite being written in CLAUDE.md. @@ -131,6 +131,17 @@ The 34 priors in [`memory/zo-platform/PRIORS.md`](https://github.com/SamPlvs/zer Each prior was earned by a real failure. The same mistake never happens twice. +## Per-project priors: seed → load → learn → promote + +Priors aren't only hand-written. ZO maintains a project's `PRIORS.md` automatically across a run: + +- **Seed** — at first session the plan's `## Domain Context and Priors` are written into the project `PRIORS.md`, so the team starts with the human's domain knowledge instead of a blank slate. +- **Load** — every Lead prompt injects the current project priors ("accumulated learnings — honor these before repeating past mistakes"), so agents see them before acting. +- **Learn** — when the autonomous Phase-4 loop hits a dead-end or plateau, the orchestrator records the failure and appends a durable `auto-learning` prior, so the next iteration (or a later session) doesn't repeat it. +- **Promote** — generic learnings can graduate to the platform with `zo learnings promote --project NAME --repo PATH`. It is **fail-closed**: only generic-category priors that clear the client blocklist are promoted; anything project-specific or matching a client identifier is blocked and reported, never auto-rewritten. With no blocklist configured, nothing is promoted. + +This is how one project's experience compounds — within the project, and (sanitised) across the platform. + ## Next diff --git a/docs/concepts/overview.mdx b/docs/concepts/overview.mdx index 399c8a5..b9d4b19 100644 --- a/docs/concepts/overview.mdx +++ b/docs/concepts/overview.mdx @@ -13,7 +13,7 @@ ZO has five primitives. Master these and the rest of the system follows. Every project must define a hard, verifiable success metric. Without a measurable criterion, autonomous agents become hallucinating cost centers. - 20 specialised personas, orchestrator, data engineer, model builder, oracle, XAI, code reviewer, and more, communicate peer-to-peer through Claude Code's native team APIs. + 21 specialised personas, orchestrator, data engineer, model builder, oracle, XAI, code reviewer, and more, communicate peer-to-peer through Claude Code's native team APIs. Six sequential phases, each separated by a gate. Automated gates run validation; blocking gates pause for a human. diff --git a/docs/concepts/the-team.mdx b/docs/concepts/the-team.mdx index 12051f0..731dcdc 100644 --- a/docs/concepts/the-team.mdx +++ b/docs/concepts/the-team.mdx @@ -1,9 +1,9 @@ --- title: "The team" -description: "20 specialised AI personas, each with a defined role, tier, and contract. The team coordinates peer-to-peer through Claude Code's native APIs." +description: "21 specialised AI personas, each with a defined role, tier, and contract. The team coordinates peer-to-peer through Claude Code's native APIs." --- -ZO ships with **20 agent definitions** in `.claude/agents/`. Each is a Markdown file with YAML frontmatter (name, model tier, role, team) plus a structured prompt body covering ownership, off-limits, contracts, coordination protocol, and a self-validation checklist. +ZO ships with **21 agent definitions** in `.claude/agents/`. Each is a Markdown file with YAML frontmatter (name, model tier, role, team) plus a structured prompt body covering ownership, off-limits, contracts, coordination protocol, and a self-validation checklist. ## The two teams @@ -11,7 +11,7 @@ ZO uses two distinct team configurations: - Executes the projects defined in `plan.md`. **11 launch + phase-in agents** covering data, model, oracle, code review, testing, XAI, domain evaluation, ML engineering, and infrastructure. + Executes the projects defined in `plan.md`. **12 launch + phase-in agents** covering data, model, oracle, code review, testing, XAI, domain evaluation, ML engineering, infrastructure, and live training monitoring. Used to build and maintain ZO itself. **6 specialised agents** including software architect, backend engineer, frontend engineer, platform code reviewer, platform test engineer, and documentation agent. @@ -75,6 +75,9 @@ Activated for specific phases or by plan opt-in: Compute resource allocation, experiment tracking setup, artifact storage, logging. Format-following work, appropriate for the smaller model. + + Phase 4 live training monitor — the Lead spawns one per model run as `training-{modelname}-checker`. Tails the active experiment's `metrics.jsonl` / `training_status.json`, alerts on NaN/divergence/gradient-blowup/overfit/stall so a broken run dies early, and writes a mechanistic `diagnosis.md` plus next-round suggestions (pairs with Research Scout's general-AI literature track). + ## Platform Build Team diff --git a/docs/demo.html b/docs/demo.html index c8f44d3..6d05031 100644 --- a/docs/demo.html +++ b/docs/demo.html @@ -997,7 +997,7 @@

ZERO OPERATORS

-
20agents
+
21agents
Defined
@@ -1255,7 +1255,7 @@

USER WORKFLOW

AGENT TEAMS

-
20 agents across project delivery, platform build, draft scouts, and init
+
21 agents across project delivery, platform build, draft scouts, and init
Project Delivery Team -- Launch Agents
diff --git a/docs/installation.mdx b/docs/installation.mdx index bad7be1..d5c58d8 100644 --- a/docs/installation.mdx +++ b/docs/installation.mdx @@ -41,7 +41,7 @@ cd zero-operators Resolves and installs `pyproject.toml` dependencies, pydantic, fastembed, click, rich, pyyaml. - Confirms 20 agent `.md` files in `.claude/agents/`. + Confirms 21 agent `.md` files in `.claude/agents/`. Confirms 24 commands in `.claude/commands/`. diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 25e4bf1..be74816 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -17,7 +17,7 @@ The human is the research director. The plan is the only communication medium. A Every project defines a hard, verifiable success metric. No deliverable is complete until the oracle confirms it. - A team of 20 specialised agents, orchestrator, data, model, oracle, XAI, and more, coordinates over a contract-first protocol. + A team of 21 specialised agents, orchestrator, data, model, oracle, XAI, and more, coordinates over a contract-first protocol. `STATE.md`, `DECISION_LOG.md`, `PRIORS.md`, and a semantic index give every session continuity with the last. diff --git a/memory/zo-platform/DECISION_LOG.md b/memory/zo-platform/DECISION_LOG.md index 2da9b9a..82fdc47 100644 --- a/memory/zo-platform/DECISION_LOG.md +++ b/memory/zo-platform/DECISION_LOG.md @@ -1149,3 +1149,27 @@ The `--no-headlines` flag is preserved (not removed) for backwards compatibility - Opus for the checker — deferred: it both tails and reasons, but Sonnet matches oracle-qa's tier and is cost-appropriate for a long-running monitor; revisit if diagnosis depth is insufficient. **Outcome:** Agent roster 20 → 21; full doc cascade (setup.sh EXPECTED_AGENTS + count + pass-msg, README badge + roster + status line, lead-orchestrator count + roster, specs/agents.md counts + entry, plans, PRD §9 + tree). +20 tests (760 → 780 + 7 skipped), pytest green on Python 3.11 AND 3.12 (PR-039 matrix), ruff `src/` clean; test-file additions ruff-clean (2 pre-existing warnings in test_training_metrics.py left out of scope per PR-039). validate-docs 9 pass / 0 fail / 2 warn (client-blocklist skip + known grep-768-vs-pytest-780 test-badge parameterization gap; README badge updated 743 → 780 to reflect the true pytest count). PR-040 added to PRIORS. **Next:** Batch A (per-project self-evolution: seed plan priors → load full project priors into prompts → wire EvolutionEngine into failure paths → automated sanitized promotion to platform PRIORS per the user's choice), then D (optimization audit: broaden ml-engineer + new software-engineer agent), then E (swarm reinforcement + idle-agent shutdown). Branch `claude/training-intelligence`. + +--- + +## Decision: 2026-05-30T16:00:00Z +**Type:** FEATURE + EVOLUTION +**Title:** Batch A — per-project self-evolution (seed/load/write/promote): wiring the dead EvolutionEngine + seed_priors; plus audit-driven cleanup (4 latent log_error bugs, docs-site drift, hygiene) + +**Decision:** Second process-hardening batch from the user's "make ZO learn so I stop re-telling it." Investigation + the `repo-cleanup-audit` swarm confirmed the self-evolution machinery was fully implemented but DEAD — `EvolutionEngine` never imported/called, `seed_priors` never invoked, `_prompt_memory` injected only decision summaries. Wired end-to-end: + +1. **Seed** — `Orchestrator._maybe_seed_priors()` (from `start_session`) writes the plan's `domain_priors` into the project `.zo/memory/PRIORS.md` when PRIORS is empty (idempotent; never clobbers accumulated learnings). +2. **Load** — `_prompt_memory` injects up to 8 non-superseded project priors (statement + evidence) under "Project priors (accumulated learnings — honor these…)", distinct from semantic decision matches. +3. **Write** — `EvolutionEngine(memory, comms, zo_root)` instantiated in `__init__`; new `_record_learning()` (uses `evolution.record_failure` + `memory.append_prior`) fires on the autonomous loop's DEAD_END/PLATEAU verdicts in `_auto_iterate_if_needed`, persisting a durable `auto-learning` prior. +4. **Promote** — new `src/zo/promote.py` (`screen_prior`, `promote_learnings`, `load_blocklist`, `PromotionReport`) + `zo learnings promote --project --repo [--dry-run]`. **Fail-closed** automated sanitized promotion: only `auto-learning`/`evolution` categories that clear `scripts/.client-blocklist` are promoted; `domain`/plan-seeded/blocklist-hit priors are BLOCKED + reported (never auto-rewritten); a missing blocklist promotes nothing. Returns an auditable promoted/blocked/duplicate report. + +**Rationale:** The user chose "automated sanitized promotion" (auto-strip + promote, no per-item approval). On the legal-critical public-repo path I implemented **block-not-strip**: stripping a client term from a sentence leaves garbled/misleading text and can miss adjacent project-specific words, so refusing (and reporting for manual rewrite) is safer than guessing — while still automated and approval-free for clean priors. The seed/load/write wiring is the PR-009 lesson ("built ≠ wired") applied to the evolution engine: all machinery existed and was unit-tested, it was simply never connected to the run. + +**Audit-driven (repo-cleanup-audit swarm, 6 agents, 27 findings):** confidentiality CLEARED. Fixed **4 latent `log_error(message=)` bugs** in `orchestrator.py` failure branches (`CommsLogger.log_error` has no `message` param and requires `description` → `TypeError` if the branch fired; never caught because rarely executed) — surfaced by wiring the dead engine; added a regression test forcing the `_generate_test_report` failure branch. Checklist auto-refresh made **best-effort** (`_safe_refresh_checklist` + `contextlib.suppress(OSError)`). Fixed **docs-site drift** (docs/*.mdx + demo.html + README → 21 agents; training-checker accordion in `the-team.mdx`; self-evolution section + `zo learnings` in `memory-and-continuity.mdx`/`COMMANDS.md`). Hygiene: untracked + gitignored `.claude/.DS_Store`, gitignored `.agents/`/`.codex/`/`AGENTS.md` (per user), fixed deprecated `source_dir` example in `draft.py`. + +**Alternatives considered:** +- Strip-and-promote (user's literal wording) — rejected on the legal path; block-not-strip is safer and the sanitizer design was delegated to me. Trivial to switch to strip later. +- Full `run_postmortem` on every dead-end — deferred; `record_failure` + clean `append_prior` avoids the engine's auto-phrasing ("Add new prior: …") and keeps prior statements readable. +- Category-allowlist-only (skip blocklist) — rejected; defence-in-depth wants generic-category AND blocklist-clear AND no-blocklist-refusal. + +**Outcome:** modified `src/zo/{orchestrator,experiments,cli,draft}.py`; new `src/zo/promote.py`; docs (`the-team`, `memory-and-continuity`, `overview`, `introduction`, `installation`, `demo.html`, `COMMANDS.md`, `README.md`); `.gitignore`; new `tests/unit/test_promote.py` (15) + additions to test_orchestrator/test_experiments/test_training_metrics/test_cli. **+32 tests (780 → 812 + 7 skipped), green on Python 3.11 AND 3.12, ruff `src/` clean, validate-docs 0 failures.** PR-041 added. **Deferred (audit #13/#15/#16/#17):** semantic reindex at session-end, agent failure-reporting protocol, `end_session` DECISION_LOG/PRIORS integration, `zo retrospective` CLI. Branch `claude/self-evolution`, stacked on #95. diff --git a/memory/zo-platform/PRIORS.md b/memory/zo-platform/PRIORS.md index 8eff354..e765ddc 100644 --- a/memory/zo-platform/PRIORS.md +++ b/memory/zo-platform/PRIORS.md @@ -1274,3 +1274,27 @@ For PR #92 the corrective sequence was: identify the actual CI gates by reading ### Verified Solution `training-checker` agent (always-fires Phase-4 spawn instruction + `AGENT_PHASE_MAP` entry + role/ownership/off-limits maps); `should_checkpoint` helper + "Checkpointing and Disaster Recovery (REQUIRED)" section in `model-builder.md` (DL every-10-epochs + best + last resumable; ML per-fold + best + HPO-state); `render_checklist` / `write_checklist` baked into `mint_experiment` / `update_result` / `update_status` / `update_next_ideas` so `.zo/experiments/CHECKLIST.md` always tracks the registry; research-scout general-AI track for Phase-4 iteration. +20 tests (760 → 780, both Python 3.11 & 3.12). Ships in the Batch B+C PR (branch `claude/training-intelligence`). **Cross-reference:** PR-005, PR-009, PR-035 — same enforcement-over-aspiration family. + +--- + +## PR-041: Built-But-Unwired Infrastructure Is a Latent Liability — Wiring Dead Code Surfaces Latent Bugs + +**Source:** Session 035 (2026-05-30), Batch A — self-evolution wiring +**Root cause category:** missing_rule (built-not-wired; PR-009 family) + +**Failure:** ZO did not learn per project despite shipping a complete, unit-tested self-evolution stack. `EvolutionEngine` (full 5-step post-mortem) and `MemoryManager.seed_priors` were implemented and covered by `test_evolution.py` / `test_memory.py`, but **neither was ever invoked in production** — `EvolutionEngine` had zero imports outside its own module, `seed_priors` had zero callers, and `_prompt_memory` surfaced only decision summaries, not priors. The user kept re-supplying domain knowledge and lessons because the loop meant to seed/load/record them was never connected. Wiring it then surfaced **4 latent `log_error(message=)` bugs** in `orchestrator.py` failure branches: `CommsLogger.log_error` takes `description` (no `message` param), so those calls would `TypeError` the moment their `except` branch fired — never caught because the branches almost never execute and no test forced them. + +### Rules + +1. **"Implemented + unit-tested" is not "wired."** A module with zero production callers is a liability, not a feature — it rots, drifts from its callers' contracts, and gives false confidence ("we have a self-evolution engine") while delivering nothing. PR-009 ("built modules must be wired"), restated for the evolution engine and the priors loop. + - **How to apply:** in the PR that builds a module, grep for its callers; if there are none outside tests, wire it or don't merge it. A green unit test on an uncalled module proves it compiles, not that the platform uses it. + +2. **Wiring previously-dead code surfaces latent bugs in the newly-reachable paths — budget for it and regression-test those branches.** The 4 `log_error` bugs sat dormant because the failure branches never executed; making the engine live made them reachable. + - **How to apply:** when wiring dead code, treat its now-reachable error/except branches as new surface area — read them, fix what no longer matches current signatures, and add at least one test that forces each newly-reachable failure branch (here: monkeypatch the failing call, assert the branch swallows + logs without raising). + +3. **On a public-repo promotion path, fail-closed (block) beats strip-and-hope.** The user chose "auto-strip + promote"; the implementation blocks instead — stripping a client term out of a sentence leaves garbled, misleading text and can miss adjacent project-specific words (a reactor/tag name not in the blocklist). Promoting only already-clean priors (generic category AND blocklist-clear AND a blocklist exists), and reporting the rest for manual rewrite, never emits a half-sanitised statement to the public repo. + - **How to apply:** for any automated flow that writes to the public/platform repo, default to refuse-and-report over auto-redact; redaction is lossy and unverifiable against a legal constraint. + +### Verified Solution + +Seed (`_maybe_seed_priors`), load (`_prompt_memory` injects project priors), write (`_record_learning` on loop DEAD_END/PLATEAU via `EvolutionEngine.record_failure` + `append_prior`), promote (`src/zo/promote.py` fail-closed sanitizer + `zo learnings promote`). 4 `log_error(message=)` → `description=` fixes + a forced-failure-branch regression test. New `tests/unit/test_promote.py` (15 adversarial cases) + seed/load/write/bug-fix tests. +32 tests (780 → 812, both Python 3.11 & 3.12). **Cross-reference:** PR-009 (built ≠ wired), PR-005/PR-035/PR-040 (enforcement over aspiration), PR-024/PR-030 (confidentiality — the promotion sanitizer reuses the validate-docs client blocklist). diff --git a/memory/zo-platform/STATE.md b/memory/zo-platform/STATE.md index 51d1856..3631aae 100644 --- a/memory/zo-platform/STATE.md +++ b/memory/zo-platform/STATE.md @@ -8,7 +8,9 @@ status: complete ## Current Position -**Session 034 hand-off — pick up here.** Shipped **Batch B+C (training intelligence)** — the first of the process-hardening batches from the user's "harden the rules I keep repeating so they take effect" request (PR for B+C; batches A/D/E queued). Framing: PR-005 applied to the autonomous *run* — turn aspirational agent prose into enforced platform behaviour. Four hardenings: **(1) new `training-checker` agent** (Sonnet, phase-in, project) — a per-model-run live monitor the Lead spawns as `training-{modelname}-checker`; tails the active experiment's `metrics.jsonl`/`training_status.json`, alerts Model Builder+Lead on NaN/divergence/gradient-blowup/overfit/stall (kill broken runs early), writes mechanistic `diagnosis.md` + feeds `next.md`. Enforcement = always-fires Phase-4 instruction in `Orchestrator._prompt_experiment_context` (NOT reliant on the plan's active-agent list, which `_agents_for_phase` filters), backed by the agent file + `AGENT_PHASE_MAP["training-checker"]=["phase_4"]` + role/ownership/off-limits maps. **(2) checkpoint cadence + disaster recovery** — new importable `should_checkpoint(epoch, total_epochs, every=10, is_best=)` in `training_metrics.py` (replaces the contract's undefined pseudocode); `model-builder.md` gains a "Checkpointing and Disaster Recovery (REQUIRED)" section: DL every-10-epochs + best + last with fully-resumable state (optimizer/scheduler/AMP-scaler/epoch/RNG) + resume-from-`last.pt`; ML per-fold + best + persist HPO study state. **(3) research-scout general-AI track** — Phase-4-iteration section surveying general ML literature (time-series/sequence modelling, optimization, regularization) method-first, pairing with the checker per failure mode (additive to its domain problem-class survey). **(4) auto-maintained experiment checklist** — `experiments.render_checklist()/write_checklist()`; `.zo/experiments/CHECKLIST.md` regenerated deterministically on EVERY registry mutation (mint/update_result/update_status/update_next_ideas) → exp→hypothesis→metric→Δ→tier→top-shortfall + "Next planned", no agent action needed. Agent roster **20→21** + full cascade (setup.sh/README/lead-orchestrator/specs/plans/PRD). **+20 tests (760→780)**, green on Python 3.11 AND 3.12 (PR-039 matrix), ruff `src/` clean. validate-docs 9 pass/0 fail/2 warn (client-blocklist skip + known grep-768-vs-pytest-780 badge gap; README badge updated 743→780). PR-040 captures the lesson. **Next action when picking up:** Batch **A — self-evolution** (seed plan priors → load full project priors into prompts → wire `EvolutionEngine` into failure paths → **automated sanitized promotion** to platform PRIORS per the user's choice), then **D** (optimization audit: broaden ml-engineer + new `software-engineer` agent), then **E** (swarm reinforcement + idle-agent shutdown >30-40min). Standing Tier-1 roadmap (caveman ablation, onboarding hardening) still queued. +**Session 035 hand-off — pick up here.** Shipped **Batch A (self-evolution)** — the root fix for "ZO doesn't learn per project" (branch `claude/self-evolution`, stacked on #95; PR opened). The machinery existed but was DEAD: `EvolutionEngine` was never imported/called, `seed_priors` was never invoked, `_prompt_memory` injected only decision summaries (not priors). Now wired end-to-end: **(1) Seed** — `Orchestrator._maybe_seed_priors()` (from `start_session`) writes plan `domain_priors` into the project `.zo/memory/PRIORS.md` on first run (idempotent — only when PRIORS empty). **(2) Load** — `_prompt_memory` injects up to 8 non-superseded project priors (statement + evidence) under "Project priors (accumulated learnings — honor these…)", separate from semantic decision matches. **(3) Write** — `EvolutionEngine` instantiated in `__init__`; new `_record_learning()` (uses `evolution.record_failure` + `append_prior`) fires on the autonomous loop's DEAD_END/PLATEAU verdicts in `_auto_iterate_if_needed`, persisting a durable `auto-learning` prior so the next iteration/session doesn't repeat the dead-end. **(4) Promote** — new `src/zo/promote.py` + `zo learnings promote` CLI: **fail-closed** automated sanitized promotion (user chose "automated sanitized"; I went block-not-strip on the legal-critical path) — only `auto-learning`/`evolution` categories that clear `scripts/.client-blocklist` are promoted; `domain`/plan-seeded/blocklist-hit → BLOCKED + reported (never auto-rewritten); no blocklist → nothing promoted. Adversarially tested (a prior with a client term and a `domain` prior are both blocked). **Audit-driven bonus** (the `repo-cleanup-audit` swarm, 6 agents, confirmed confidentiality CLEARED): fixed **4 latent `log_error(message=)` bugs** (no `message` param → `TypeError` in failure branches; found by wiring the dead engine) + regression test; checklist auto-refresh made **best-effort**; **docs-site drift** fixed (docs/ + README → 21 agents, training-checker accordion in the-team.mdx, self-evolution section + `zo learnings` in memory-and-continuity.mdx/COMMANDS.md); hygiene (`.DS_Store` untracked + gitignored, `.agents/`/`.codex/`/`AGENTS.md` gitignored, `draft.py` docstring). **+32 tests (780→812), green on Python 3.11 AND 3.12, ruff `src/` clean, validate-docs 0 failures.** PR-041 captures the lesson. **Deferred (audit #13/#15/#16/#17, NOT done):** semantic reindex at session-end, agent failure-reporting protocol, end_session DECISION_LOG/PRIORS integration, `zo retrospective` CLI — queued. **Next:** Batch **D** (optimization audit + `software-engineer` agent), Batch **E** (swarm reinforcement + idle-agent shutdown); standing Tier-1 (caveman, onboarding). Two PRs stacked: #95 (training intelligence) → self-evolution. + +**Session 034 hand-off (prior).** Shipped **Batch B+C (training intelligence)** — the first of the process-hardening batches from the user's "harden the rules I keep repeating so they take effect" request (PR for B+C; batches A/D/E queued). Framing: PR-005 applied to the autonomous *run* — turn aspirational agent prose into enforced platform behaviour. Four hardenings: **(1) new `training-checker` agent** (Sonnet, phase-in, project) — a per-model-run live monitor the Lead spawns as `training-{modelname}-checker`; tails the active experiment's `metrics.jsonl`/`training_status.json`, alerts Model Builder+Lead on NaN/divergence/gradient-blowup/overfit/stall (kill broken runs early), writes mechanistic `diagnosis.md` + feeds `next.md`. Enforcement = always-fires Phase-4 instruction in `Orchestrator._prompt_experiment_context` (NOT reliant on the plan's active-agent list, which `_agents_for_phase` filters), backed by the agent file + `AGENT_PHASE_MAP["training-checker"]=["phase_4"]` + role/ownership/off-limits maps. **(2) checkpoint cadence + disaster recovery** — new importable `should_checkpoint(epoch, total_epochs, every=10, is_best=)` in `training_metrics.py` (replaces the contract's undefined pseudocode); `model-builder.md` gains a "Checkpointing and Disaster Recovery (REQUIRED)" section: DL every-10-epochs + best + last with fully-resumable state (optimizer/scheduler/AMP-scaler/epoch/RNG) + resume-from-`last.pt`; ML per-fold + best + persist HPO study state. **(3) research-scout general-AI track** — Phase-4-iteration section surveying general ML literature (time-series/sequence modelling, optimization, regularization) method-first, pairing with the checker per failure mode (additive to its domain problem-class survey). **(4) auto-maintained experiment checklist** — `experiments.render_checklist()/write_checklist()`; `.zo/experiments/CHECKLIST.md` regenerated deterministically on EVERY registry mutation (mint/update_result/update_status/update_next_ideas) → exp→hypothesis→metric→Δ→tier→top-shortfall + "Next planned", no agent action needed. Agent roster **20→21** + full cascade (setup.sh/README/lead-orchestrator/specs/plans/PRD). **+20 tests (760→780)**, green on Python 3.11 AND 3.12 (PR-039 matrix), ruff `src/` clean. validate-docs 9 pass/0 fail/2 warn (client-blocklist skip + known grep-768-vs-pytest-780 badge gap; README badge updated 743→780). PR-040 captures the lesson. **Next action when picking up:** Batch **A — self-evolution** (seed plan priors → load full project priors into prompts → wire `EvolutionEngine` into failure paths → **automated sanitized promotion** to platform PRIORS per the user's choice), then **D** (optimization audit: broaden ml-engineer + new `software-engineer` agent), then **E** (swarm reinforcement + idle-agent shutdown >30-40min). Standing Tier-1 roadmap (caveman ablation, onboarding hardening) still queued. **Session 033 hand-off (prior).** Documentation-only cascade completing a gap from PR #92. `--bypass-permissions` (added to `zo build` / `zo continue` in PR #92, session 031) was documented in `docs/cli/build.mdx` (full "Permission prompts" section + truth table) and `docs/cli/overview.mdx` (shared options table) but NOT in `docs/COMMANDS.md`, the terminal-command reference that documents the sibling flags (`--no-tmux`, `--low-token`, `--gate-mode`, `--no-headlines`). Added `--bypass-permissions` in 4 places: `zo build` usage block, a new "Permission prompts:" subsection under build, `zo continue` usage block, and the continue "same semantics as `zo build`" bullet. Framing matches build.mdx — auto-approves every Claude Code tool-call prompt, independent of `--gate-mode` (which gates ZO phases, not individual tool calls), implied by `--gate-mode full-auto`, off by default, tmux settings-overlay vs headless `--dangerously-skip-permissions`. validate-docs 10 passed / 0 failed / 1 pre-existing warning (client-blocklist skip). No code/tests touched (docs + memory only), so the ruff/pytest CI matrix is unaffected. **Minor pre-existing drift noted, NOT fixed (out of scope):** README test badge reads 743 while STATE reports 760 post-#92/#93 (validate-docs "within tolerance", passes). **Next action when picking up:** unchanged from session 032 — monitor Discussions for early external-user signal on Tier 2 sequencing, then resume Tier 1 (caveman ablation → onboarding hardening). @@ -135,6 +137,7 @@ ZO **v1.0.2** + **`--low-token` mode** (session-024) — cost-saving preset for - [x] v1.0.2-post: `get_current_phase` returns ACTIVE phases on resume — second bug surfaced during the same prod-001 investigation. `Orchestrator.get_current_phase` previously only returned GATED or PENDING phases; ACTIVE phases (set by `apply_human_decision(ITERATE)` at line 707, the autonomous loop's CONTINUE verdict at line 1046, or persisted from an interrupted session) were silently skipped. Result: any `zo continue` after a Ctrl-C / network disconnect / OS reboot mid-phase would say "All phases complete. Nothing to do." and drop all in-flight work. Fix adds an ACTIVE branch between the GATED and PENDING resolution loops. Resolution order is now GATED > ACTIVE > PENDING (with deps met). BLOCKED is intentionally still skipped (requires human escalation). New `TestGetCurrentPhase` tests: `test_returns_active_phase_on_resume`, `test_gated_takes_priority_over_active`, `test_active_takes_priority_over_pending`, `test_blocked_phase_not_returned`, `test_real_resume_via_state_md_round_trip` (end-to-end through SessionState write → start_session → decompose_plan → get_current_phase). +5 tests (738 → 743 + 7 skipped). ruff `src/zo/orchestrator.py tests/unit/test_orchestrator.py` clean. Captured as PR-037. - [x] B+C: Training intelligence — `training-checker` agent (per-run live monitor `training-{modelname}-checker`: NaN/divergence/overfit alerts + mechanistic `diagnosis.md` + `next.md`, enforced via always-fires Phase-4 prompt instruction in `_prompt_experiment_context`); `should_checkpoint` helper + "Checkpointing and Disaster Recovery (REQUIRED)" section in model-builder.md (DL every-10-epochs+best+last resumable; ML per-fold+best+HPO-state); research-scout general-AI-research track (Phase-4, time-series/sequence-modelling, method-first); auto-maintained `.zo/experiments/CHECKLIST.md` (`render_checklist`/`write_checklist`, regenerated on every registry mutation). Agent roster 20→21 + full cascade. +20 tests (760→780), Python 3.11 & 3.12. PR-040. +- [x] A: Self-evolution — wired the dead `EvolutionEngine` + `seed_priors`: `_maybe_seed_priors` (plan priors → project PRIORS on first run), `_prompt_memory` injects project priors, `_record_learning` on loop DEAD_END/PLATEAU appends an `auto-learning` prior; new `src/zo/promote.py` + `zo learnings promote` (fail-closed sanitizer: generic-category + blocklist-clear only, block-not-strip, no-blocklist→nothing). Fixed 4 latent `log_error(message=)` bugs; checklist refresh best-effort; docs-site 20→21 + training-checker accordion + self-evolution/`zo learnings` docs; `.DS_Store`/`.agents`/`.codex`/`AGENTS.md` gitignore; `draft.py` docstring. +32 tests (780→812), Python 3.11 & 3.12. PR-041. ## Known Issues @@ -174,7 +177,7 @@ last_session: session-028 (Q3/H2 roadmap finalised — three workstreams from ex branch: claude/beautiful-bhabha-69ae22 (worktree, roadmap commit pending) v1_status: COMPLETE — all 8 PRD §9 acceptance criteria met, all Known Issues closed docs_site: deployed at docs.zerooperators.com via Mintlify (mint.json + 16 pages: 3 get-started + 7 concepts + 4 cli + 2 reference; redirect rule /concepts/agents → /concepts/the-team for stale-cache hardening). Build verified live 2026-04-27 — all sibling concept pages return 200 except `/concepts/the-team` which 404'd due to MDX 2 prose `<500 lines, functions <50 lines` parser bug on line 51 (`<` followed by digit treated as JSX opening tag). Fixed by replacing `<500` / `<50` with `under 500` / `under 50` natural-prose. (Original planning URL `docs.zero-operators.dev` documented in DECISION_LOG was changed to the no-hyphen `docs.zerooperators.com` during actual setup; README badge + docs/README updated to match.) -test_count: 780 passed, 7 skipped (ZO platform); 16 passed (mnist demo); 19 passed (cifar10 demo); 297 passed (prod-001) +test_count: 812 passed, 7 skipped (ZO platform); 16 passed (mnist demo); 19 passed (cifar10 demo); 297 passed (prod-001) benchmark: scripts/benchmark_low_token.sh harness in place; first completed measured bench 2026-04-27 — **\$7.75 end-to-end** (\$4.48 lead Sonnet + \$3.27 sub-agents Sonnet, captured via `npx ccusage --instances`) vs. ~\$11 default-mode reference = **~30% reduction**. Earlier 70-80% projection refuted: structural ceiling is ~25-30% because sub-agents already run on Sonnet in default mode (`.md` frontmatter), so `--low-token` only affects the lead's ~30-40% cost share. Sub-agent model override (`_prompt_low_token_overrides()`) confirmed working end-to-end via `ps aux` — Claude Code 2.1.107 honours `model="claude-sonnet-4-6"` on `Agent()` spawns; the earlier 2.1.92-ignores-param hypothesis no longer load-bearing. Path to higher savings (50-60% target): Haiku for code-reviewer/test-engineer/oracle-qa, Phase-1 trim (data-engineer only), skip Phase-5 deep analysis. Path to 70-80% target: SDK refactor for prompt caching + Batch API + Files API. Both deferred behind PR B (cost-benchmark honesty cascade). demo_results: mnist-digit-classifier: 99.66% test accuracy (Tier 3 could_pass), 64s on MPS, 8 epochs, 468K params diff --git a/memory/zo-platform/sessions/session-035-2026-05-30.md b/memory/zo-platform/sessions/session-035-2026-05-30.md new file mode 100644 index 0000000..222579a --- /dev/null +++ b/memory/zo-platform/sessions/session-035-2026-05-30.md @@ -0,0 +1,28 @@ +# Session 035 — 2026-05-30 + +## Focus +Batch A (self-evolution) — the root fix for the user's "ZO doesn't learn per project" complaint. Wire the dead `EvolutionEngine` + `seed_priors` and add a fail-closed promotion path. Plus the on-theme cleanup the `repo-cleanup-audit` swarm surfaced. + +## Shipped +- **Seed** — `Orchestrator._maybe_seed_priors()` (from `start_session`): plan `domain_priors` → project `.zo/memory/PRIORS.md` on first run (idempotent). +- **Load** — `_prompt_memory` injects up to 8 non-superseded project priors (statement + evidence). +- **Write** — `EvolutionEngine` instantiated; `_record_learning()` fires on loop DEAD_END/PLATEAU → durable `auto-learning` prior. +- **Promote** — new `src/zo/promote.py` + `zo learnings promote`: fail-closed sanitizer (generic-category + blocklist-clear only; block-not-strip; no-blocklist→nothing), adversarially tested. +- **Audit-driven:** 4 latent `log_error(message=)` bugs fixed (+ regression test); checklist refresh best-effort; docs-site 20→21 + training-checker accordion + self-evolution/`zo learnings` docs; `.DS_Store` untracked + gitignored, `.agents`/`.codex`/`AGENTS.md` gitignored, `draft.py` docstring. + +## Verification +- pytest 812 passed / 7 skipped on Python 3.11 AND 3.12. +- ruff `src/` clean; new test files clean; validate-docs 0 failures. + +## Memory +- DECISION_LOG: 2026-05-30T16:00Z FEATURE + EVOLUTION entry. +- PRIORS: PR-041 — built-but-unwired infrastructure is a latent liability; wiring dead code surfaces latent bugs; fail-closed > strip on public-repo paths. + +## Deferred (audit #13/#15/#16/#17) +Semantic reindex at session-end, agent failure-reporting protocol, `end_session` DECISION_LOG/PRIORS integration, `zo retrospective` CLI. + +## Next +Batch D (optimization audit + `software-engineer` agent), Batch E (swarm reinforcement + idle-agent shutdown). Standing Tier-1: caveman, onboarding. + +## PR +Branch `claude/self-evolution` (stacked on #95) → PR against `main`. diff --git a/src/zo/cli.py b/src/zo/cli.py index c96eb19..f6b7dac 100644 --- a/src/zo/cli.py +++ b/src/zo/cli.py @@ -2629,6 +2629,65 @@ def experiments_diff( console.print(f" [{_AMBER}]+ {b.id} only:[/] {s}") +@cli.group() +def learnings() -> None: + """Promote generic, client-sanitized learnings to the ZO platform.""" + + +@learnings.command("promote") +@click.option("--project", "-p", required=True, help="Project name") +@click.option( + "--repo", type=click.Path(exists=True, file_okay=False), default=None, + help="Path to delivery repo with .zo/ directory.", +) +@click.option( + "--dry-run", is_flag=True, + help="Screen and report without writing to platform PRIORS.", +) +def learnings_promote(project: str, repo: str | None, dry_run: bool) -> None: + """Promote clean generic priors from a project to platform PRIORS.md. + + Automated and fail-closed: only generic-category priors that clear the + client blocklist are promoted. Anything that trips the blocklist, is + plan-seeded/domain, or (when no blocklist is configured) gets blocked + and reported for manual review — priors are never auto-rewritten. + """ + from zo.promote import promote_learnings + + delivery = Path(repo).resolve() if repo else None + pctx = _load_project_context(project, delivery_repo=delivery) + target = pctx.make_target() + delivery_repo = Path(target.target_repo) + if not delivery_repo.is_dir(): + console.print( + f"[red bold]Delivery repo not found:[/] {delivery_repo}\n" + f"Pass [bold]--repo PATH[/] to override.", + ) + raise SystemExit(1) + + report = promote_learnings(delivery_repo, _zo_root(), dry_run=dry_run) + + if not report.blocklist_loaded: + console.print( + f"[{_AMBER}]⚠ No client blocklist at scripts/.client-blocklist[/] — " + "nothing promoted (fail-closed). Configure it to enable promotion.", + ) + suffix = " [dim](dry-run)[/]" if dry_run else "" + console.print(f"\n[bold]Promotion:[/] {report.summary}{suffix}\n") + for p in report.promoted: + console.print(f" [green]✓ promoted[/] ({p.category}) {p.statement}") + for p, reason in report.blocked: + console.print(f" [{_AMBER}]✗ blocked[/] ({p.category}) {p.statement}") + console.print(f" [{_DIM}]{reason}[/]") + for p in report.skipped_duplicate: + console.print(f" [{_DIM}]= duplicate[/] {p.statement}") + if report.written: + console.print( + f"\n[green]Wrote {len(report.promoted)} prior(s) to " + "memory/zo-platform/PRIORS.md[/]", + ) + + @cli.command("watch-training") @click.option("--project", "-p", required=True, help="Project name") @click.option("--interval", "-i", default=2.0, help="Refresh interval in seconds") diff --git a/src/zo/draft.py b/src/zo/draft.py index 9df5d1b..29ab811 100644 --- a/src/zo/draft.py +++ b/src/zo/draft.py @@ -6,7 +6,7 @@ Typical usage:: from zo.draft import PlanDrafter - drafter = PlanDrafter(source_dir=Path("docs"), project_name="alpha", zo_root=Path(".")) + drafter = PlanDrafter(source_paths=[Path("docs")], project_name="alpha", zo_root=Path(".")) count = drafter.index_documents() plan_path = drafter.generate_plan() valid = drafter.validate_draft(plan_path) diff --git a/src/zo/experiments.py b/src/zo/experiments.py index 2a16c1f..1d4d723 100644 --- a/src/zo/experiments.py +++ b/src/zo/experiments.py @@ -49,6 +49,7 @@ from __future__ import annotations +import contextlib import json import re from datetime import UTC, datetime @@ -339,6 +340,20 @@ def write_checklist( return path +def _safe_refresh_checklist( + registry_dir: Path, registry: ExperimentRegistry, +) -> None: + """Best-effort ``CHECKLIST.md`` refresh after a registry mutation. + + The checklist is a derived view; a render/write failure (disk full, + permission denied) must never break the registry write that already + succeeded. Strict ``write_checklist`` stays available for callers + that want the error surfaced (and for tests). + """ + with contextlib.suppress(OSError): + write_checklist(registry_dir, registry) + + def _checklist_cell(text: str, limit: int = 70) -> str: """Collapse whitespace, escape pipes, and truncate for a table cell.""" text = " ".join((text or "").split()).replace("|", "\\|") @@ -477,7 +492,7 @@ def mint_experiment( ) registry.experiments.append(exp) save_registry(registry_dir, registry) - write_checklist(registry_dir, registry) + _safe_refresh_checklist(registry_dir, registry) return exp @@ -511,7 +526,7 @@ def update_result( exp.result = result exp.status = ExperimentStatus.COMPLETE save_registry(registry_dir, registry) - write_checklist(registry_dir, registry) + _safe_refresh_checklist(registry_dir, registry) return exp @@ -527,7 +542,7 @@ def update_status( ) exp.status = status save_registry(registry_dir, registry) - write_checklist(registry_dir, registry) + _safe_refresh_checklist(registry_dir, registry) return exp @@ -543,7 +558,7 @@ def update_next_ideas( ) exp.next_ideas = list(next_ideas) save_registry(registry_dir, registry) - write_checklist(registry_dir, registry) + _safe_refresh_checklist(registry_dir, registry) return exp diff --git a/src/zo/orchestrator.py b/src/zo/orchestrator.py index 01ce7f7..0155ac0 100644 --- a/src/zo/orchestrator.py +++ b/src/zo/orchestrator.py @@ -23,7 +23,14 @@ from textwrap import dedent from typing import TYPE_CHECKING -from zo._memory_models import DecisionEntry, OperatingMode, SessionState +from zo._evolution_models import FailureRecord, FailureSeverity +from zo._memory_models import ( + Confidence, + DecisionEntry, + OperatingMode, + PriorEntry, + SessionState, +) from zo._orchestrator_models import ( AgentContract, GateDecision, @@ -40,6 +47,7 @@ LOW_TOKEN_PHASE_DROPS, MODE_PHASE_FACTORY, ) +from zo.evolution import EvolutionEngine from zo.plan import Plan, WorkflowMode if TYPE_CHECKING: @@ -191,6 +199,7 @@ def __init__( self._workflow: WorkflowDecomposition | None = None self._session_state: SessionState | None = None self._plan_hash: str = self._compute_plan_hash() + self._evolution = EvolutionEngine(memory, comms, self._zo_root) @property def workflow(self) -> WorkflowDecomposition | None: @@ -232,6 +241,7 @@ def start_session(self) -> SessionState: self._session_state = state self._memory.write_state(state) + self._maybe_seed_priors() self._comms.log_decision( agent="orchestrator", title=f"Session started in {state.mode} mode", @@ -241,6 +251,28 @@ def start_session(self) -> SessionState: ) return state + def _maybe_seed_priors(self) -> None: + """Seed project PRIORS.md from the plan's domain priors on first run. + + Idempotent: only seeds when PRIORS.md has no entries yet, so it + runs once at project start and never clobbers accumulated + learnings on later sessions. + """ + if not self._plan.domain_priors.strip(): + return + if self._memory.read_priors(): + return + self._memory.seed_priors(self._plan.domain_priors) + self._comms.log_decision( + agent="orchestrator", + title="Seeded project priors from plan domain priors", + rationale=( + f"{len(self._plan.domain_priors.splitlines())} line(s) seeded " + "into PRIORS.md (was empty)" + ), + outcome="seeded", + ) + def end_session(self, summary: SessionSummary | None = None) -> None: """End the current session, persisting phase states.""" if self._session_state is not None: @@ -849,8 +881,8 @@ def _generate_test_report(self, phase: PhaseDefinition) -> None: self._comms.log_error( agent="orchestrator", error_type="test_report_generation_failed", - message=f"Failed to generate test report for {phase.phase_id}: {exc}", severity="warning", + description=f"Failed to generate test report for {phase.phase_id}: {exc}", ) def _generate_notebook(self, phase: PhaseDefinition) -> None: @@ -877,8 +909,8 @@ def _generate_notebook(self, phase: PhaseDefinition) -> None: self._comms.log_error( agent="orchestrator", error_type="notebook_generation_failed", - message=f"Failed to generate notebook for {phase.phase_id}: {exc}", severity="warning", + description=f"Failed to generate notebook for {phase.phase_id}: {exc}", ) # -- Experiment capture layer (Phase 4) --------------------------------- @@ -1003,8 +1035,8 @@ def _finalize_experiments( self._comms.log_error( agent="orchestrator", error_type="experiment_result_parse_failed", - message=f"Failed to parse {exp.id}/result.md: {exc}", severity="warning", + description=f"Failed to parse {exp.id}/result.md: {exc}", ) missing.append( f".zo/experiments/{exp.id}/result.md (parse failed)", @@ -1063,6 +1095,21 @@ def _auto_iterate_if_needed( )) if decision.verdict != LoopVerdict.CONTINUE: + # Capture a durable learning for the failure-ish verdicts so the + # next iteration/session/project doesn't repeat the dead-end. + if decision.verdict in (LoopVerdict.DEAD_END, LoopVerdict.PLATEAU): + self._record_learning( + title=( + f"{phase.phase_id} hit {decision.verdict} after " + f"{decision.completed_count} experiments" + ), + root_cause=decision.reason, + rule_gap=( + f"Phase 4 stopped at {decision.verdict} (last exp: " + f"{decision.last_exp_id or 'none'}). Diversify the " + "approach earlier instead of iterating near-duplicates." + ), + ) # Stop — caller proceeds with phase completion. return None @@ -1076,6 +1123,39 @@ def _auto_iterate_if_needed( rationale=f"Autonomous iteration: {decision.reason}", ) + def _record_learning( + self, *, title: str, root_cause: str, rule_gap: str, + ) -> None: + """Persist a failure-driven learning (self-evolution). + + Documents the failure to DECISION_LOG + comms via the + EvolutionEngine and appends a durable prior to the project's + PRIORS.md so future iterations/sessions don't repeat it. + Best-effort: never raises into the gate path. + """ + try: + self._evolution.record_failure(FailureRecord( + title=title, + detected_by="orchestrator", + severity=FailureSeverity.MINOR, + phase=self._EXPERIMENT_PHASE, + description=root_cause, + immediate_impact=rule_gap, + )) + self._memory.append_prior(PriorEntry( + category="auto-learning", + statement=rule_gap, + evidence=f"{title}: {root_cause}", + confidence=Confidence.MEDIUM, + )) + except Exception as exc: # noqa: BLE001 + self._comms.log_error( + agent="orchestrator", + error_type="record_learning_failed", + severity="warning", + description=f"Failed to record learning '{title}': {exc}", + ) + def _abort_running_experiments(self, phase_id: str) -> None: """Mark all running experiments in a phase as ABORTED. @@ -1291,8 +1371,8 @@ def _generate_snapshot( self._comms.log_error( agent="orchestrator", error_type="snapshot_generation_failed", - message=f"Failed to write snapshot for {phase.phase_id}: {exc}", severity="warning", + description=f"Failed to write snapshot for {phase.phase_id}: {exc}", ) def _recent_decisions_for_phase( @@ -1506,6 +1586,17 @@ def _prompt_memory(self) -> str: f"Mode: {state.mode}", f"Phase: {state.phase}", f"Blockers: {state.active_blockers or 'none'}", ] + priors = [ + p for p in self._memory.read_priors() if p.superseded_by is None + ] + if priors: + lines.append( + "\n**Project priors (accumulated learnings — honor these " + "before repeating past mistakes):**" + ) + for p in priors[:8]: + evidence = f" — _{p.evidence}_" if p.evidence else "" + lines.append(f"- ({p.category}) {p.statement}{evidence}") relevant = self._semantic.query(self._plan.objective, top_k=3) if relevant: lines.append("\n**Relevant past decisions:**") diff --git a/src/zo/promote.py b/src/zo/promote.py new file mode 100644 index 0000000..b42a615 --- /dev/null +++ b/src/zo/promote.py @@ -0,0 +1,197 @@ +"""Promote generic, client-sanitized learnings from a project to platform PRIORS. + +Self-evolution closes the loop only if learnings discovered *during a project* +can lift the platform for the next one. This module does that promotion — +**automated** (no per-item approval) but **fail-closed**, because the platform +repo is public and client confidentiality is a legal non-negotiable. + +A project's ``.zo/memory/PRIORS.md`` is read; each prior is screened and only +the ones that are unambiguously generic reach ``memory/zo-platform/PRIORS.md``: + +* Only priors in generic categories (``auto-learning`` / ``evolution``) are + candidates. Plan-seeded ``domain`` priors are project-specific by + construction and are never promoted. +* Any prior whose text matches the client blocklist (the same + ``scripts/.client-blocklist`` ``validate-docs`` uses) is **blocked** — not + auto-stripped. Stripping a client term out of a sentence leaves garbled, + misleading text and can miss adjacent project-specific words; refusing is + safer. Blocked priors are reported so a human can rewrite them generically. +* If no blocklist file is present there is no client-term protection, so + **nothing is promoted** and the report says why. + +Every run returns a :class:`PromotionReport` (promoted / blocked+reason / +skipped-duplicate) so the automated promotion is fully auditable after the fact. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from zo._memory_formats import parse_priors, render_prior +from zo._memory_models import PriorEntry + +if TYPE_CHECKING: + from pathlib import Path + +__all__ = [ + "PromotionReport", + "load_blocklist", + "screen_prior", + "promote_learnings", + "PROMOTABLE_CATEGORIES", +] + +# Prior categories eligible for promotion. Everything else (notably the +# plan-seeded ``domain`` category) is treated as project-specific and skipped. +PROMOTABLE_CATEGORIES: frozenset[str] = frozenset({"auto-learning", "evolution"}) + +# Evidence markers that flag a prior as plan-seeded / project-local. +_PROJECT_LOCAL_EVIDENCE = ("seeded from plan",) + +_BLOCKLIST_RELPATH = ("scripts", ".client-blocklist") + + +@dataclass +class PromotionReport: + """Outcome of a promotion run — auditable record of every decision.""" + + promoted: list[PriorEntry] = field(default_factory=list) + blocked: list[tuple[PriorEntry, str]] = field(default_factory=list) + skipped_duplicate: list[PriorEntry] = field(default_factory=list) + blocklist_loaded: bool = False + written: bool = False + + @property + def summary(self) -> str: + """One-line human summary.""" + return ( + f"{len(self.promoted)} promoted, {len(self.blocked)} blocked, " + f"{len(self.skipped_duplicate)} duplicate " + f"(blocklist {'loaded' if self.blocklist_loaded else 'MISSING'})" + ) + + +def load_blocklist(zo_root: Path) -> list[str]: + """Load lowercased client-blocklist patterns from ``scripts/.client-blocklist``. + + Returns an empty list when the file is absent (gitignored / not yet + configured). Comment (``#``) and blank lines are skipped. + """ + path = zo_root.joinpath(*_BLOCKLIST_RELPATH) + if not path.is_file(): + return [] + patterns: list[str] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if line and not line.startswith("#"): + patterns.append(line.lower()) + return patterns + + +def _prior_text(prior: PriorEntry) -> str: + """All free text of a prior, lowercased, for blocklist screening.""" + return f"{prior.category} {prior.statement} {prior.evidence}".lower() + + +def _matches_blocklist(prior: PriorEntry, blocklist: list[str]) -> str | None: + """Return the offending pattern if the prior trips the blocklist, else None.""" + text = _prior_text(prior) + for pattern in blocklist: + try: + hit = pattern in text or re.search(pattern, text) is not None + except re.error: + hit = pattern in text + if hit: + return pattern + return None + + +def screen_prior( + prior: PriorEntry, blocklist: list[str], *, blocklist_loaded: bool, +) -> tuple[bool, str]: + """Decide whether a single prior may be promoted. Fail-closed. + + Returns ``(ok, reason)``. ``ok`` is True only for an unambiguously + generic prior that clears the blocklist. + """ + if not blocklist_loaded: + return False, "no client blocklist configured — promotion refused" + if prior.category not in PROMOTABLE_CATEGORIES: + return False, f"category '{prior.category}' is not promotable (project-specific)" + if any(m in prior.evidence.lower() for m in _PROJECT_LOCAL_EVIDENCE): + return False, "plan-seeded prior (project-specific)" + offending = _matches_blocklist(prior, blocklist) + if offending is not None: + return False, f"matches client blocklist pattern {offending!r}" + return True, "clean" + + +def _genericized(prior: PriorEntry) -> PriorEntry: + """Re-tag a promoted prior so no project-specific provenance leaks.""" + return PriorEntry( + category=f"promoted/{prior.category}", + statement=prior.statement, + evidence="promoted from a project (client-sanitized)", + confidence=prior.confidence, + ) + + +def promote_learnings( + delivery_repo: Path, zo_root: Path, *, dry_run: bool = False, +) -> PromotionReport: + """Promote clean generic priors from a project to platform PRIORS. + + Args: + delivery_repo: Project/delivery repo root (reads + ``{delivery_repo}/.zo/memory/PRIORS.md``). + zo_root: ZO platform repo root (writes + ``{zo_root}/memory/zo-platform/PRIORS.md``, reads the blocklist). + dry_run: When True, screen and report but write nothing. + + Returns: + A :class:`PromotionReport` recording every promote/block/dedup decision. + """ + report = PromotionReport() + blocklist = load_blocklist(zo_root) + report.blocklist_loaded = bool(blocklist) + + src = delivery_repo / ".zo" / "memory" / "PRIORS.md" + if not src.is_file(): + return report + project_priors = parse_priors(src.read_text(encoding="utf-8")) + + platform_path = zo_root / "memory" / "zo-platform" / "PRIORS.md" + existing = ( + parse_priors(platform_path.read_text(encoding="utf-8")) + if platform_path.is_file() else [] + ) + seen = {p.statement.strip().lower() for p in existing} + + to_write: list[PriorEntry] = [] + for prior in project_priors: + ok, reason = screen_prior( + prior, blocklist, blocklist_loaded=report.blocklist_loaded, + ) + if not ok: + report.blocked.append((prior, reason)) + continue + key = prior.statement.strip().lower() + if key in seen: + report.skipped_duplicate.append(prior) + continue + seen.add(key) + report.promoted.append(prior) + to_write.append(_genericized(prior)) + + if to_write and not dry_run: + platform_path.parent.mkdir(parents=True, exist_ok=True) + with open(platform_path, "a", encoding="utf-8") as fh: + for entry in to_write: + if platform_path.stat().st_size > 0: + fh.write("\n") + fh.write(render_prior(entry)) + report.written = True + + return report diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 98a982e..5270097 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -27,7 +27,7 @@ class TestCliGroup: def test_cli_group_has_all_commands(self) -> None: expected = { "build", "continue", "init", "status", "draft", "preflight", - "gates", "watch-training", "migrate", "experiments", + "gates", "watch-training", "migrate", "experiments", "learnings", } actual = set(cli.commands.keys()) assert expected == actual @@ -1057,7 +1057,9 @@ def test_banner_renders_low_token_badge( ) -> None: """``_show_banner(low_token=True)`` includes the badge text.""" from io import StringIO + from rich.console import Console + import zo.cli as cli_module # Redirect zo.cli.console to a buffer so we can inspect output. diff --git a/tests/unit/test_experiments.py b/tests/unit/test_experiments.py index 6b5d742..85ec55c 100644 --- a/tests/unit/test_experiments.py +++ b/tests/unit/test_experiments.py @@ -662,3 +662,49 @@ def test_update_result_refreshes_checklist(self, tmp_path: Path) -> None: text = (tmp_path / "CHECKLIST.md").read_text(encoding="utf-8") assert "acc=0.99" in text assert "must_pass" in text + + +class TestChecklistHardening: + def test_checklist_cell_none_and_empty(self) -> None: + from zo.experiments import _checklist_cell + assert _checklist_cell(None) == "—" # type: ignore[arg-type] + assert _checklist_cell("") == "—" + + def test_checklist_cell_truncates_and_escapes_pipes(self) -> None: + from zo.experiments import _checklist_cell + assert _checklist_cell("a | b") == "a \\| b" + out = _checklist_cell("x" * 100, limit=10) + assert out.endswith("…") + assert len(out) <= 10 + + def test_checklist_cell_collapses_whitespace(self) -> None: + from zo.experiments import _checklist_cell + assert _checklist_cell("a\n b\t c") == "a b c" + + def test_render_no_next_planned_when_all_empty(self) -> None: + reg = ExperimentRegistry(project="demo") + reg.experiments.append(Experiment( + id="exp-001", phase="phase_4", hypothesis="h", next_ideas=[], + )) + assert "## Next planned" not in render_checklist(reg) + + def test_write_checklist_raises_on_bad_path(self, tmp_path: Path) -> None: + bad = tmp_path / "afile" + bad.write_text("x", encoding="utf-8") # a file, not a directory + with pytest.raises(OSError): # noqa: PT011 -- any OSError is fine here + write_checklist(bad, ExperimentRegistry(project="demo")) + + def test_refresh_is_best_effort( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # A checklist-write failure must NOT break the registry mutation. + def _boom(*_a: object, **_k: object) -> object: + raise OSError("disk full") + + monkeypatch.setattr("zo.experiments.write_checklist", _boom) + exp = mint_experiment(tmp_path, project="demo", phase="phase_4") + result = update_result(tmp_path, exp.id, ExperimentResult( + oracle_tier="must_pass", + primary_metric=PrimaryMetric(name="acc", value=0.9), + )) + assert result.status == ExperimentStatus.COMPLETE diff --git a/tests/unit/test_orchestrator.py b/tests/unit/test_orchestrator.py index 247f0e8..cfb2b90 100644 --- a/tests/unit/test_orchestrator.py +++ b/tests/unit/test_orchestrator.py @@ -10,6 +10,7 @@ import pytest +from zo._memory_models import PriorEntry from zo._orchestrator_models import ( AgentContract, GateDecision, @@ -1470,3 +1471,120 @@ def test_non_training_phase_omits_monitor( phase_1 = next(p for p in decomp.phases if p.phase_id == "phase_1") prompt = orch.build_lead_prompt(phase_1) assert "training-{modelname}-checker" not in prompt + + +# --------------------------------------------------------------------------- +# Self-evolution (Batch A): seed / load / write +# --------------------------------------------------------------------------- + + +def _orch_with_memory( + plan: Plan, tmp_path: Path, *, repo: Path | None = None, +) -> tuple[Orchestrator, MemoryManager]: + """Build an Orchestrator and return it with its MemoryManager handle.""" + memory = MemoryManager(project_dir=tmp_path, project_name="test-project") + memory.initialize_project() + comms = CommsLogger( + log_dir=tmp_path / "logs" / "comms", + project="test-project", session_id="s1", + ) + semantic = SemanticIndex(db_path=tmp_path / "index.db") + target = _make_target() + if repo is not None: + target = TargetConfig( + project="test-project", target_repo=str(repo), + target_branch="main", worktree_base=str(tmp_path / "wt"), + git_author_name="ZO Test", git_author_email="zo@test.dev", + agent_working_dirs={}, zo_only_paths=[".zo/"], + enforce_isolation=False, + ) + orch = Orchestrator( + plan=plan, target=target, memory=memory, comms=comms, + semantic=semantic, zo_root=REPO_ROOT, gate_mode=GateMode.AUTO, + ) + return orch, memory + + +class TestSelfEvolution: + """Batch A — seed plan priors, load priors into prompt, learn on failure.""" + + def test_seed_priors_on_first_session( + self, plan: Plan, tmp_path: Path, + ) -> None: + plan.domain_priors = "- Bearing vibration is the key signal\n- Watch drift" + orch, memory = _orch_with_memory(plan, tmp_path) + assert memory.read_priors() == [] + orch.start_session() + statements = [p.statement for p in memory.read_priors()] + assert any("Bearing vibration" in s for s in statements) + + def test_seed_is_noop_when_priors_exist( + self, plan: Plan, tmp_path: Path, + ) -> None: + plan.domain_priors = "- brand new prior text" + orch, memory = _orch_with_memory(plan, tmp_path) + memory.append_prior(PriorEntry( + category="domain", statement="pre-existing", evidence="x", + )) + orch.start_session() + statements = [p.statement for p in memory.read_priors()] + assert "brand new prior text" not in statements + assert "pre-existing" in statements + + def test_seed_is_noop_without_domain_priors( + self, plan: Plan, tmp_path: Path, + ) -> None: + plan.domain_priors = "" + orch, memory = _orch_with_memory(plan, tmp_path) + orch.start_session() + assert memory.read_priors() == [] + + def test_prompt_memory_includes_project_priors( + self, plan: Plan, tmp_path: Path, + ) -> None: + orch, memory = _orch_with_memory(plan, tmp_path) + memory.append_prior(PriorEntry( + category="auto-learning", + statement="Diversify the model family earlier", + evidence="phase_4 plateau", + )) + orch.start_session() + orch.decompose_plan() + phase_1 = next( + p for p in orch.workflow.phases if p.phase_id == "phase_1" + ) + prompt = orch.build_lead_prompt(phase_1) + assert "accumulated learnings" in prompt + assert "Diversify the model family earlier" in prompt + + def test_record_learning_appends_prior( + self, plan: Plan, tmp_path: Path, + ) -> None: + orch, memory = _orch_with_memory(plan, tmp_path) + orch.start_session() + orch._record_learning( # noqa: SLF001 + title="phase_4 hit dead_end after 3 experiments", + root_cause="hypotheses near-duplicate", + rule_gap="Diversify earlier", + ) + priors = memory.read_priors() + assert any(p.category == "auto-learning" for p in priors) + assert any("Diversify earlier" in p.statement for p in priors) + + def test_generate_test_report_failure_is_swallowed( + self, plan: Plan, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Regression guard for the log_error(message=) bug: the failure + # branch must use the correct signature and NOT raise. + repo = tmp_path / "delivery" + (repo / "tests").mkdir(parents=True) + orch, _ = _orch_with_memory(plan, tmp_path, repo=repo) + + def _boom(*_a: object, **_k: object) -> object: + raise RuntimeError("boom") + + monkeypatch.setattr("zo.test_report.generate_test_report", _boom) + phase = next( + p for p in orch.decompose_plan().phases if p.phase_id == "phase_1" + ) + orch._generate_test_report(phase) # noqa: SLF001 -- must not raise diff --git a/tests/unit/test_promote.py b/tests/unit/test_promote.py new file mode 100644 index 0000000..9fafad6 --- /dev/null +++ b/tests/unit/test_promote.py @@ -0,0 +1,192 @@ +"""Unit tests for the learnings promotion sanitizer (zo.promote). + +The promoter writes to the PUBLIC platform repo, so these tests are +adversarial: a client identifier in a prior must be BLOCKED, project-local +priors must never promote, and with no blocklist configured nothing +promotes (fail-closed). +""" + +from __future__ import annotations + +from pathlib import Path # noqa: TC003 -- used at runtime in fixtures + +from zo._memory_formats import render_prior +from zo._memory_models import Confidence, PriorEntry +from zo.promote import ( + PROMOTABLE_CATEGORIES, + load_blocklist, + promote_learnings, + screen_prior, +) + + +def _prior( + category: str, statement: str, evidence: str = "", +) -> PriorEntry: + return PriorEntry( + category=category, statement=statement, evidence=evidence, + confidence=Confidence.MEDIUM, + ) + + +def _setup( + tmp_path: Path, + priors: list[PriorEntry], + *, + blocklist: tuple[str, ...] | None = ("acme", "widgetco"), + platform_existing: tuple[PriorEntry, ...] = (), +) -> tuple[Path, Path]: + """Build a delivery repo + zo_root; return (delivery_repo, zo_root).""" + zo_root = tmp_path / "zo" + (zo_root / "scripts").mkdir(parents=True) + if blocklist is not None: + (zo_root / "scripts" / ".client-blocklist").write_text( + "# client blocklist\n" + "\n".join(blocklist) + "\n", + encoding="utf-8", + ) + platform = zo_root / "memory" / "zo-platform" + platform.mkdir(parents=True) + (platform / "PRIORS.md").write_text( + "\n".join(render_prior(p) for p in platform_existing), encoding="utf-8", + ) + delivery = tmp_path / "proj" + mem = delivery / ".zo" / "memory" + mem.mkdir(parents=True) + (mem / "PRIORS.md").write_text( + "\n".join(render_prior(p) for p in priors), encoding="utf-8", + ) + return delivery, zo_root + + +class TestLoadBlocklist: + def test_reads_patterns_skips_comments_and_blanks(self, tmp_path: Path) -> None: + (tmp_path / "scripts").mkdir() + (tmp_path / "scripts" / ".client-blocklist").write_text( + "# header\nAcme\n\nWidgetCo\n", encoding="utf-8", + ) + assert load_blocklist(tmp_path) == ["acme", "widgetco"] + + def test_absent_file_returns_empty(self, tmp_path: Path) -> None: + assert load_blocklist(tmp_path) == [] + + +class TestScreenPrior: + def test_clean_generic_passes(self) -> None: + ok, _ = screen_prior( + _prior("auto-learning", "Diversify the approach earlier"), + ["acme"], blocklist_loaded=True, + ) + assert ok is True + + def test_domain_category_blocked(self) -> None: + ok, reason = screen_prior( + _prior("domain", "anything"), ["acme"], blocklist_loaded=True, + ) + assert ok is False + assert "not promotable" in reason + + def test_plan_seeded_blocked(self) -> None: + ok, reason = screen_prior( + _prior("auto-learning", "x", evidence="seeded from plan.md"), + ["acme"], blocklist_loaded=True, + ) + assert ok is False + assert "plan-seeded" in reason + + def test_blocklist_hit_in_statement_blocked(self) -> None: + ok, reason = screen_prior( + _prior("auto-learning", "On Acme the model overfit"), + ["acme"], blocklist_loaded=True, + ) + assert ok is False + assert "blocklist" in reason + + def test_blocklist_hit_case_insensitive(self) -> None: + ok, _ = screen_prior( + _prior("auto-learning", "ACME had an issue"), + ["acme"], blocklist_loaded=True, + ) + assert ok is False + + def test_no_blocklist_blocks_everything(self) -> None: + ok, reason = screen_prior( + _prior("auto-learning", "totally generic learning"), + [], blocklist_loaded=False, + ) + assert ok is False + assert "no client blocklist" in reason + + def test_promotable_categories_exclude_domain(self) -> None: + assert "auto-learning" in PROMOTABLE_CATEGORIES + assert "domain" not in PROMOTABLE_CATEGORIES + + +class TestPromoteLearnings: + def test_clean_prior_promoted_and_genericized(self, tmp_path: Path) -> None: + delivery, zo_root = _setup( + tmp_path, + [_prior("auto-learning", "Diversify approach earlier", "phase_4 dead_end")], + ) + report = promote_learnings(delivery, zo_root) + assert len(report.promoted) == 1 + assert report.written is True + platform = (zo_root / "memory" / "zo-platform" / "PRIORS.md").read_text( + encoding="utf-8", + ) + assert "Diversify approach earlier" in platform + # Provenance genericized — the project-specific evidence is gone. + assert "promoted from a project" in platform + assert "phase_4 dead_end" not in platform + + def test_domain_and_blocklisted_blocked(self, tmp_path: Path) -> None: + delivery, zo_root = _setup(tmp_path, [ + _prior("auto-learning", "Clean generic learning"), + _prior("domain", "Bearing BPFO is diagnostic", "seeded from plan.md"), + _prior("auto-learning", "Acme reactor overfit fast"), + ]) + report = promote_learnings(delivery, zo_root) + assert len(report.promoted) == 1 + assert len(report.blocked) == 2 + + def test_no_blocklist_promotes_nothing(self, tmp_path: Path) -> None: + delivery, zo_root = _setup( + tmp_path, [_prior("auto-learning", "generic")], blocklist=None, + ) + report = promote_learnings(delivery, zo_root) + assert report.promoted == [] + assert report.blocklist_loaded is False + assert len(report.blocked) == 1 + + def test_dry_run_writes_nothing(self, tmp_path: Path) -> None: + delivery, zo_root = _setup( + tmp_path, [_prior("auto-learning", "generic learning")], + ) + platform_path = zo_root / "memory" / "zo-platform" / "PRIORS.md" + before = platform_path.read_text(encoding="utf-8") + report = promote_learnings(delivery, zo_root, dry_run=True) + assert len(report.promoted) == 1 + assert report.written is False + assert platform_path.read_text(encoding="utf-8") == before + + def test_dedup_against_existing_platform(self, tmp_path: Path) -> None: + existing = (_prior("promoted/auto-learning", "Diversify earlier", "x"),) + delivery, zo_root = _setup( + tmp_path, [_prior("auto-learning", "Diversify earlier")], + platform_existing=existing, + ) + report = promote_learnings(delivery, zo_root) + assert report.promoted == [] + assert len(report.skipped_duplicate) == 1 + + def test_missing_source_priors_empty_report(self, tmp_path: Path) -> None: + zo_root = tmp_path / "zo" + (zo_root / "scripts").mkdir(parents=True) + (zo_root / "scripts" / ".client-blocklist").write_text( + "acme\n", encoding="utf-8", + ) + (zo_root / "memory" / "zo-platform").mkdir(parents=True) + delivery = tmp_path / "proj" + delivery.mkdir() + report = promote_learnings(delivery, zo_root) + assert report.promoted == [] + assert report.blocked == [] diff --git a/tests/unit/test_training_metrics.py b/tests/unit/test_training_metrics.py index c9ab11c..0eb8099 100644 --- a/tests/unit/test_training_metrics.py +++ b/tests/unit/test_training_metrics.py @@ -341,3 +341,22 @@ def test_zero_total_epochs(self) -> None: def test_negative_epoch(self) -> None: assert should_checkpoint(-1, 100, every=10) is False + + def test_single_epoch_run(self) -> None: + # The only epoch is also the final epoch. + assert should_checkpoint(0, 1, every=10) is True + + def test_every_zero_clamps_to_one(self) -> None: + assert should_checkpoint(0, 5, every=0) is True + assert should_checkpoint(3, 5, every=0) is True + + def test_every_negative_clamps_to_one(self) -> None: + assert should_checkpoint(2, 5, every=-3) is True + + def test_every_greater_than_total(self) -> None: + assert should_checkpoint(5, 10, every=20) is False + assert should_checkpoint(9, 10, every=20) is True # final epoch + + def test_epoch_at_or_beyond_total(self) -> None: + assert should_checkpoint(10, 10, every=10) is False + assert should_checkpoint(10, 10, every=10, is_best=True) is True