From 72a687415c11253223dfa80d347b7dcc5006aeb2 Mon Sep 17 00:00:00 2001 From: Alexander Nikolskiy Date: Fri, 24 Jul 2026 21:50:45 +0300 Subject: [PATCH 1/6] =?UTF-8?q?feat(research):=20R13=20=E2=80=94=20=D0=BA?= =?UTF-8?q?=D0=B0=D1=82=D0=B0=D0=BB=D0=BE=D0=B3=20best-practice=20=D0=BE?= =?UTF-8?q?=D1=81=D0=B5=D0=B9=20sweep=20+=20denylist=20=D0=BE=D1=81=D0=B8?= =?UTF-8?q?=20=D0=BF=D0=BB=D0=B5=D1=87=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/research/sweep-axis-catalog.test.ts | 99 +++++++++++++++++ src/research/sweep-axis-catalog.ts | 135 ++++++++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 src/research/sweep-axis-catalog.test.ts create mode 100644 src/research/sweep-axis-catalog.ts diff --git a/src/research/sweep-axis-catalog.test.ts b/src/research/sweep-axis-catalog.test.ts new file mode 100644 index 0000000..a6d47f7 --- /dev/null +++ b/src/research/sweep-axis-catalog.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from 'vitest'; + +import { + SWEEP_AXIS_CATALOG, + SWEEP_AXIS_CATALOG_PROMPT, + isDenylistedParam, +} from './sweep-axis-catalog.ts'; + +describe('SWEEP_AXIS_CATALOG_PROMPT', () => { + it('states the plateau-not-peak and degradation-point rules', () => { + expect(SWEEP_AXIS_CATALOG_PROMPT).toMatch(/wide plateau, not (a )?peak/i); + expect(SWEEP_AXIS_CATALOG_PROMPT).toMatch(/degradation point/i); + }); + + it('names every non-denylisted axis from report-13 G13', () => { + for (const marker of ['hold time', 'entry threshold', 'stop', 'take', 'cooldown', 'sizing', 'regime']) { + expect(SWEEP_AXIS_CATALOG_PROMPT.toLowerCase()).toContain(marker); + } + }); + + it('denylists the leverage axis until a liquidation model exists', () => { + expect(SWEEP_AXIS_CATALOG_PROMPT).toMatch(/leverage/i); + expect(SWEEP_AXIS_CATALOG_PROMPT).toMatch(/liquidation/i); + expect(SWEEP_AXIS_CATALOG_PROMPT.toLowerCase()).toMatch(/never (sweep|include|propose).{0,40}leverage|leverage.{0,40}(denylist|forbidden|never)/); + }); +}); + +describe('SWEEP_AXIS_CATALOG', () => { + it('marks exactly one axis denylisted — leverage', () => { + const denylisted = SWEEP_AXIS_CATALOG.filter((a) => a.denylisted === true); + expect(denylisted).toHaveLength(1); + expect(denylisted[0]!.axis).toBe('leverage'); + }); + + it('every axis has a non-empty promptGuidance string', () => { + for (const a of SWEEP_AXIS_CATALOG) { + expect(a.promptGuidance.length).toBeGreaterThan(0); + } + }); + + it('covers the six onboarding axes plus the denylisted leverage axis', () => { + expect(SWEEP_AXIS_CATALOG.map((a) => a.axis).sort()).toEqual([ + 'cooldown', + 'entry_thresholds', + 'hold_time', + 'leverage', + 'regime_as_axis', + 'sizing', + 'stops_takes', + ]); + }); + + it('matchesParam classifies real profile param names deterministically', () => { + const byAxis = Object.fromEntries(SWEEP_AXIS_CATALOG.map((a) => [a.axis, a])); + + expect(byAxis.hold_time!.matchesParam('maxHoldMin')).toBe(true); + expect(byAxis.hold_time!.matchesParam('entry.fastBouncePct')).toBe(false); + + expect(byAxis.entry_thresholds!.matchesParam('dump.minDropPct')).toBe(true); + expect(byAxis.entry_thresholds!.matchesParam('entry.fastBouncePct')).toBe(true); + expect(byAxis.entry_thresholds!.matchesParam('oiFilter.minOi')).toBe(true); + expect(byAxis.entry_thresholds!.matchesParam('liqFilter.minNotional')).toBe(true); + expect(byAxis.entry_thresholds!.matchesParam('tpLadder.tp1Pct')).toBe(false); + + expect(byAxis.stops_takes!.matchesParam('hardStopPct')).toBe(true); + expect(byAxis.stops_takes!.matchesParam('tpLadder.tp1Pct')).toBe(true); + expect(byAxis.stops_takes!.matchesParam('dump.minDropPct')).toBe(false); + + expect(byAxis.cooldown!.matchesParam('watch.cooldownMin')).toBe(true); + expect(byAxis.cooldown!.matchesParam('warmup.maxSignalAgeMin')).toBe(true); + expect(byAxis.cooldown!.matchesParam('entry.fastBouncePct')).toBe(false); + + expect(byAxis.sizing!.matchesParam('dca.stepPct')).toBe(true); + expect(byAxis.sizing!.matchesParam('positionSizePct')).toBe(true); + expect(byAxis.sizing!.matchesParam('hardStopPct')).toBe(false); + + expect(byAxis.regime_as_axis!.matchesParam('regimeFilter.enabled')).toBe(true); + expect(byAxis.regime_as_axis!.matchesParam('dump.minDropPct')).toBe(false); + + expect(byAxis.leverage!.matchesParam('leverage.multiplier')).toBe(true); + expect(byAxis.leverage!.matchesParam('marginMode')).toBe(true); + expect(byAxis.leverage!.matchesParam('hardStopPct')).toBe(false); + }); +}); + +describe('isDenylistedParam', () => { + it('flags leverage/margin-named params', () => { + expect(isDenylistedParam('leverage.multiplier')).toBe(true); + expect(isDenylistedParam('marginMode')).toBe(true); + expect(isDenylistedParam('positionLeveragePct')).toBe(true); + }); + + it('does not flag unrelated tunable params', () => { + expect(isDenylistedParam('entry.fastBouncePct')).toBe(false); + expect(isDenylistedParam('tpLadder.tp1Pct')).toBe(false); + expect(isDenylistedParam('dump.minDropPct')).toBe(false); + expect(isDenylistedParam('maxHoldMin')).toBe(false); + }); +}); diff --git a/src/research/sweep-axis-catalog.ts b/src/research/sweep-axis-catalog.ts new file mode 100644 index 0000000..5aaa734 --- /dev/null +++ b/src/research/sweep-axis-catalog.ts @@ -0,0 +1,135 @@ +/** + * R13 (research-validation-hardening item 6, report-13 gap G13): a deterministically-checkable + * catalog of sweep-grid axes for onboarding a new strategy. The `sweep-designer` prompt is + * intentionally minimal for WFO rounds ("cheap to run, not exhaustive, bracket the baseline") — + * correct there, but it leaves onboarding without a best-practice sweep over the axes that + * actually matter (hold time, entry thresholds, stops/takes, cooldown, sizing, regime-as-axis), + * without a requirement to look for a plateau (not a peak) and the parameter's degradation + * point, and without a denylist for the leverage axis. + * + * `leverage` is DENYLISTED: the engine has no margin/liquidation model (report-13 G7, §3 point + * 5) — sweeping leverage without one is a systematic look-ahead bias, since a leveraged run + * "survives" drawdowns that would have triggered a real liquidation. Denylisted until a + * liquidation model lands in the engine (P2); today the axis is simply forbidden. + * + * Numeric thresholds are deliberately absent here: the only pinned battery thresholds are + * `battery-policy@1` (DSR 0.95 / WFE 0.5 / plateau 0.5× peak — control-center + * `docs/architecture/battery-policy.md`), which govern the post-hoc break-battery (R11), not + * grid design. This catalog only prescribes WHICH axes to sweep and the plateau/degradation + * shape to look for — not a magnitude. + */ + +export interface SweepAxis { + /** Stable machine-readable axis id (snake_case). */ + axis: string; + /** Prompt-ready guidance for this axis — folded into SWEEP_AXIS_CATALOG_PROMPT verbatim. */ + promptGuidance: string; + /** Does a given profile param name belong to this axis? Deterministic, no LLM involved. */ + matchesParam(name: string): boolean; + /** true only for axes that must never be swept today (see module doc). */ + denylisted?: boolean; +} + +/** Case-insensitive "prefix OR substring" name matcher — mirrors the style of + * `ENTRY_AFFECTING_NAME_PREFIXES` in `src/domain/wfo.ts`, extended with substrings for axes + * that aren't tied to a fixed namespace prefix (e.g. `hardStopPct`, `positionSizePct`). */ +function nameMatcher(prefixes: readonly string[], substrings: readonly string[]) { + return (name: string): boolean => { + if (prefixes.some((prefix) => name.startsWith(prefix))) return true; + const lower = name.toLowerCase(); + return substrings.some((s) => lower.includes(s)); + }; +} + +export const SWEEP_AXIS_CATALOG: readonly SweepAxis[] = [ + { + axis: 'hold_time', + promptGuidance: + 'Hold time — sweep the max/target holding duration (e.g. maxHoldMin) around the baseline. ' + + 'A strategy whose edge only survives at one exact hold time is fragile; look for a wide ' + + 'plateau, not a peak, and include a point past the point where performance starts to degrade.', + matchesParam: nameMatcher(['hold.'], ['hold']), + }, + { + axis: 'entry_thresholds', + promptGuidance: + 'Entry thresholds — sweep entry-signal filter thresholds (dump/entry/OI/liquidation filters, ' + + 'e.g. dump.minDropPct, entry.fastBouncePct, oiFilter.minOi, liqFilter.minNotional). Bracket the ' + + 'baseline on both sides and include the expected degradation point (too loose → noise trades, ' + + 'too tight → starves the strategy of trades) rather than stopping at the current value.', + matchesParam: nameMatcher(['dump.', 'entry.', 'oiFilter.', 'liqFilter.'], []), + }, + { + axis: 'stops_takes', + promptGuidance: + 'Stops/takes — sweep stop-loss and take-profit levels (e.g. hardStopPct, tpLadder.tp1Pct). ' + + 'A single sharp optimum here is a red flag for overfitting; prefer a grid wide enough to reveal ' + + 'a plateau of acceptable values and the degradation point where the exit starts hurting returns.', + matchesParam: nameMatcher(['tpLadder.'], ['stop', 'take', 'tp1', 'tp2', 'tpladder']), + }, + { + axis: 'cooldown', + promptGuidance: + 'Cooldown — sweep post-signal cooldown / warmup windows (e.g. watch.cooldownMin, ' + + 'warmup.maxSignalAgeMin). Too short re-enters on the same move; too long misses the next one — ' + + 'sweep enough points to see both failure modes, not just the current setting.', + matchesParam: nameMatcher(['watch.cooldown', 'warmup.maxSignalAge'], ['cooldown']), + }, + { + axis: 'sizing', + promptGuidance: + 'Sizing — sweep position-sizing / DCA step params (e.g. dca.stepPct, positionSizePct). Sizing ' + + 'interacts with drawdown and the risk/reward tradeoff is the researcher\'s job to surface, not ' + + 'to pick a winner by Sharpe alone; report the plateau, not just the top point.', + matchesParam: nameMatcher(['dca.'], ['size', 'sizing']), + }, + { + axis: 'regime_as_axis', + promptGuidance: + 'Regime-as-axis — when the profile has a regime/market-condition filter, sweep it as a first-class ' + + 'axis (e.g. regimeFilter.enabled, regime thresholds) instead of leaving it fixed. A strategy that ' + + 'only works in one regime needs that stated explicitly, not discovered later in paper trading.', + matchesParam: nameMatcher(['regime.', 'regimeFilter.'], ['regime']), + }, + { + axis: 'leverage', + denylisted: true, + promptGuidance: + 'Leverage — DENYLISTED. Never sweep, propose, or include a leverage/margin axis in a grid: the ' + + 'engine has no margin/liquidation model, so a leveraged backtest systematically survives ' + + 'drawdowns that would have been liquidated in reality (look-ahead bias). This is forbidden until ' + + 'a liquidation model exists in the engine — do not propose leverage params even as a stretch axis.', + matchesParam: nameMatcher(['leverage.', 'margin.'], ['leverage', 'margin']), + }, +]; + +/** Non-axis-specific rules the sweep-designer prompt must always carry, regardless of which + * axes apply to a given profile. */ +const CATALOG_RULES = [ + 'AXIS CATALOG — when designing an onboarding sweep, draw candidate axes from this catalog ' + + '(hold time, entry thresholds, stops/takes, cooldown, sizing, regime-as-axis) rather than only ' + + 'the params that happen to be tunable in isolation.', + 'For every swept axis, look for a wide plateau, not a peak: a single isolated best value that is ' + + 'much better than its neighbors is evidence of overfitting, not of a real edge.', + 'Always include the expected degradation point of each swept parameter — a value past where ' + + 'performance is known or expected to fall off — not only values near the current baseline.', +].join(' '); + +/** Full prompt-ready text: per-axis guidance + the plateau/degradation rules + the leverage + * denylist, in one string so sweep-designer.agent.ts and researcher-capabilities.ts can embed + * it verbatim (pattern: RESEARCHER_CAPABILITIES / RESEARCHER_INSTRUCTIONS). */ +export const SWEEP_AXIS_CATALOG_PROMPT = [ + CATALOG_RULES, + ...SWEEP_AXIS_CATALOG.map((a) => a.promptGuidance), +].join('\n'); + +/** Denylisted axes only (today: leverage). Exported so callers that need the catalog restricted + * to enforcement — not just prompt text — don't have to re-derive the filter. */ +export const DENYLISTED_SWEEP_AXES: readonly SweepAxis[] = SWEEP_AXIS_CATALOG.filter((a) => a.denylisted === true); + +/** True when a profile param name belongs to any denylisted axis (today: leverage/margin + * naming). Used by `validateSweepGrid` (src/domain/wfo.ts) to reject a grid key deterministically, + * independent of whether the param happens to be marked `tunable` in the profile. */ +export function isDenylistedParam(name: string): boolean { + return DENYLISTED_SWEEP_AXES.some((axis) => axis.matchesParam(name)); +} From 4dfb6ee7aadd1c17ffa34f37e5480f07cf125951 Mon Sep 17 00:00:00 2001 From: Alexander Nikolskiy Date: Fri, 24 Jul 2026 21:57:19 +0300 Subject: [PATCH 2/6] =?UTF-8?q?feat(research):=20R13=20=E2=80=94=20=D0=BA?= =?UTF-8?q?=D0=B0=D1=82=D0=B0=D0=BB=D0=BE=D0=B3=20=D0=BE=D1=81=D0=B5=D0=B9?= =?UTF-8?q?=20=D0=B2=20=D0=BF=D1=80=D0=BE=D0=BC=D0=BF=D1=82=D1=8B=20sweep-?= =?UTF-8?q?designer/researcher?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/adapters/wfo/mastra-agents.prompt.test.ts | 13 +++++++++ src/mastra/agents/agents.test.ts | 27 ++++++++++++++++++- .../agents/researcher-capabilities.test.ts | 22 +++++++++++++++ src/mastra/agents/researcher-capabilities.ts | 5 ++++ src/mastra/agents/sweep-designer.agent.ts | 12 +++++++-- 5 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/adapters/wfo/mastra-agents.prompt.test.ts b/src/adapters/wfo/mastra-agents.prompt.test.ts index 7c84218..60640ca 100644 --- a/src/adapters/wfo/mastra-agents.prompt.test.ts +++ b/src/adapters/wfo/mastra-agents.prompt.test.ts @@ -65,6 +65,19 @@ describe('WFO Mastra prompt builders — outcome embargo', () => { expect(prompts[0]!).toContain('"sharpe":1.1'); }); + // R13 (research-validation-hardening item 6): the axis catalog + plateau/degradation rules + // live in the agent's static instructions (sweep-designer.agent.ts), not the per-call prompt + // built here — asserts the two stay separate so the catalog isn't duplicated into every call. + it('sweep-designer per-call prompt does not duplicate the static axis-catalog instructions', async () => { + const { agent, prompts } = capturingAgent({ grid: {}, rationale: 'r' }); + const input: SweepInput = { + profile, baselineTrainSummary: dirtyMetrics, + tunableParams: [], restrictToEntryParams: false, maxPoints: 4, + }; + await new MastraSweepDesigner(agent, 'test').design(input); + expect(prompts[0]!).not.toMatch(/wide plateau, not (a )?peak/i); + }); + it('result-interpreter prompt scrubs embargo keys nested inside topN', async () => { const { agent, prompts } = capturingAgent({ decision: 'stop' }); const topN = [{ diff --git a/src/mastra/agents/agents.test.ts b/src/mastra/agents/agents.test.ts index 15762d5..dd431fa 100644 --- a/src/mastra/agents/agents.test.ts +++ b/src/mastra/agents/agents.test.ts @@ -6,6 +6,10 @@ import { createResearcherAgent, RESEARCHER_AGENT_ID } from './researcher.agent.t import { createCriticAgent, CRITIC_AGENT_ID } from './critic.agent.ts'; import { createBuilderAgent, BUILDER_AGENT_ID } from './builder.agent.ts'; import { createTurnInterpreterAgent, TURN_INTERPRETER_AGENT_ID } from './turn-interpreter.agent.ts'; +import { + createSweepDesignerAgent, SWEEP_DESIGNER_AGENT_ID, SWEEP_DESIGNER_INSTRUCTIONS, +} from './sweep-designer.agent.ts'; +import { SWEEP_AXIS_CATALOG_PROMPT } from '../../research/sweep-axis-catalog.ts'; const model = createAnthropic({ apiKey: 'dummy' })('claude-sonnet-4-6'); @@ -17,11 +21,32 @@ describe('mastra agent factories', () => { [createCriticAgent(model), CRITIC_AGENT_ID, 'Critic'], [createBuilderAgent(model), BUILDER_AGENT_ID, 'Builder'], [createTurnInterpreterAgent(model), TURN_INTERPRETER_AGENT_ID, 'Turn Interpreter'], + [createSweepDesignerAgent(model), SWEEP_DESIGNER_AGENT_ID, 'SweepDesigner'], ] as const; - expect(cases).toHaveLength(5); + expect(cases).toHaveLength(6); for (const [agent, id, name] of cases) { expect(agent.id).toBe(id); expect(agent.name).toBe(name); } }); }); + +// R13 (research-validation-hardening item 6, report-13 G13): sweep-designer must carry the +// deterministic axis catalog (hold time / entry thresholds / stops-takes / cooldown / sizing / +// regime-as-axis), the "wide plateau, not a peak" + "degradation point" rules, and the leverage +// denylist — pinned the same way RESEARCHER_INSTRUCTIONS pins RESEARCHER_CAPABILITIES. +describe('SWEEP_DESIGNER_INSTRUCTIONS', () => { + it('embeds the axis catalog verbatim', () => { + expect(SWEEP_DESIGNER_INSTRUCTIONS).toContain(SWEEP_AXIS_CATALOG_PROMPT); + }); + + it('carries the plateau-not-peak and degradation-point rules', () => { + expect(SWEEP_DESIGNER_INSTRUCTIONS).toMatch(/wide plateau, not (a )?peak/i); + expect(SWEEP_DESIGNER_INSTRUCTIONS).toMatch(/degradation point/i); + }); + + it('carries the leverage denylist', () => { + expect(SWEEP_DESIGNER_INSTRUCTIONS).toMatch(/leverage/i); + expect(SWEEP_DESIGNER_INSTRUCTIONS).toMatch(/liquidation/i); + }); +}); diff --git a/src/mastra/agents/researcher-capabilities.test.ts b/src/mastra/agents/researcher-capabilities.test.ts index 82c8b4b..ac1416f 100644 --- a/src/mastra/agents/researcher-capabilities.test.ts +++ b/src/mastra/agents/researcher-capabilities.test.ts @@ -60,3 +60,25 @@ describe('researcher capability framings', () => { expect(RESEARCHER_CAPABILITIES).toMatch(/runner-owned/); }); }); + +// R13 (research-validation-hardening item 6, report-13 G13): the researcher's proposed +// thresholds feed sweep grids downstream — it must be told to reason in terms of a wide +// plateau (not a fragile peak), to expect a degradation point, and that leverage/margin are +// denylisted from any sweep, not just runner-owned. +describe('RESEARCHER_CAPABILITIES axis/plateau awareness', () => { + it('names the sweep axes hypotheses feed into', () => { + for (const marker of ['hold time', 'entry threshold', 'stop', 'take', 'cooldown', 'sizing', 'regime']) { + expect(RESEARCHER_CAPABILITIES.toLowerCase()).toContain(marker); + } + }); + + it('carries the plateau-not-peak and degradation-point rules', () => { + expect(RESEARCHER_CAPABILITIES).toMatch(/wide plateau, not (a )?peak/i); + expect(RESEARCHER_CAPABILITIES).toMatch(/degradation point/i); + }); + + it('denylists a leverage axis in hypothesis proposals', () => { + expect(RESEARCHER_CAPABILITIES).toMatch(/leverage/i); + expect(RESEARCHER_CAPABILITIES).toMatch(/denylist/i); + }); +}); diff --git a/src/mastra/agents/researcher-capabilities.ts b/src/mastra/agents/researcher-capabilities.ts index e386896..d3c5523 100644 --- a/src/mastra/agents/researcher-capabilities.ts +++ b/src/mastra/agents/researcher-capabilities.ts @@ -8,6 +8,11 @@ export const RESEARCHER_CAPABILITIES = [ 'Per-trade context gives indicator snapshots at the entry bar (@entry), the exit bar (@exit), and a post-exit bar (@post, ~60m after exit) of each surfaced trade (losing trades in the loss-reduction pass, winning trades in the profit-improvement pass), plus a micro table spanning the exit. Use them to reason about both entry quality (what conditions preceded the loss → entry filters) and exit quality (was the stop too tight or the exit premature — did price reverse or keep moving favourably after exit → tighten_stop / widen_stop / exit-timing / trailing).', 'GENERALIZE — every rule must be symbol-agnostic: express the observed pattern as a market regime keyed on the indicators above, never on a specific symbol or its absolute price levels. The observed trades are examples of a regime, not the target. Cite specific trades only in `rationale` as evidence; keep `params` clean (numeric thresholds / enums) — no trade names or prices in params.', 'Execution, fills, leverage and risk sizing stay runner-owned — never prescribe them.', + // R13 (research-validation-hardening item 6, report-13 G13): the thresholds a hypothesis + // proposes feed a sweep grid downstream (hold time, entry thresholds, stops/takes, cooldown, + // sizing, regime-as-axis) — frame each proposed threshold as robust across a range, not a + // single fragile number. + 'SWEEP-GRID AWARENESS — your proposed thresholds get swept across hold time, entry thresholds, stops/takes, cooldown, sizing, and regime-as-axis: state each threshold so it can hold across a wide plateau, not a peak, and name the degradation point where you expect it to stop working, rather than proposing one exact fragile value. Never propose a leverage or margin axis — it is denylisted from any sweep (no liquidation model in the engine), on top of already being runner-owned.', ].join('\n'); // Profit-improvement pass framing — used when focus === 'profit_improvement'. The @post tail shows diff --git a/src/mastra/agents/sweep-designer.agent.ts b/src/mastra/agents/sweep-designer.agent.ts index b258961..ce9453d 100644 --- a/src/mastra/agents/sweep-designer.agent.ts +++ b/src/mastra/agents/sweep-designer.agent.ts @@ -1,18 +1,26 @@ import { Agent } from '@mastra/core/agent'; import type { ProviderModel } from '../../adapters/llm/model-provider.ts'; +import { SWEEP_AXIS_CATALOG_PROMPT } from '../../research/sweep-axis-catalog.ts'; export const SWEEP_DESIGNER_AGENT_ID = 'sweep-designer'; -const INSTRUCTIONS = [ +const BASE_INSTRUCTIONS = [ 'You design a small parameter sweep grid for a walk-forward-optimization (WFO) round over a trading strategy.', 'Given the tunable params and the baseline train-period metrics, propose a COMBINED grid (a map of param name to a short', 'array of candidate values) spanning at most a few points per param and a modest total cartesian size — this is meant to be', 'cheap to run, not exhaustive. Prefer values bracketing the current baseline value (e.g. below/above it) over arbitrary ones.', 'When restrictToEntryParams is true, ONLY include entry-affecting params (params that can change whether trades fire at all,', 'e.g. entry filters, dump/OI/liquidation filters, cooldowns) — exclude exit/risk-only params from the grid in that case.', + 'NEVER propose a leverage or margin axis — it is denylisted until the engine has a liquidation model (see below).', 'Always give a short "rationale" string explaining why these params and ranges were chosen.', ].join(' '); +// R13 (research-validation-hardening item 6, report-13 G13): appends the deterministic axis +// catalog — best-practice axes to sweep at onboarding, the "wide plateau, not a peak" + +// "degradation point" rules, and the leverage denylist — same pattern as +// RESEARCHER_INSTRUCTIONS embedding RESEARCHER_CAPABILITIES. +export const SWEEP_DESIGNER_INSTRUCTIONS = `${BASE_INSTRUCTIONS}\n\n${SWEEP_AXIS_CATALOG_PROMPT}`; + export function createSweepDesignerAgent(model: ProviderModel): Agent { - return new Agent({ id: SWEEP_DESIGNER_AGENT_ID, name: 'SweepDesigner', instructions: INSTRUCTIONS, model }); + return new Agent({ id: SWEEP_DESIGNER_AGENT_ID, name: 'SweepDesigner', instructions: SWEEP_DESIGNER_INSTRUCTIONS, model }); } From 8b9e79865967d50b677cb7e3683363167cc62c70 Mon Sep 17 00:00:00 2001 From: Alexander Nikolskiy Date: Fri, 24 Jul 2026 22:02:02 +0300 Subject: [PATCH 3/6] =?UTF-8?q?feat(research):=20R13=20=E2=80=94=20=D0=B4?= =?UTF-8?q?=D0=B5=D1=82=D0=B5=D1=80=D0=BC=D0=B8=D0=BD=D0=B8=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D0=BD=D0=BD=D1=8B=D0=B9=20=D0=B3=D0=B5=D0=B9=D1=82?= =?UTF-8?q?=20denylisted=5Faxis=20=D0=B2=20validateSweepGrid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/domain/wfo.test.ts | 48 ++++++++++++++++++++++++++++++++++++++++++ src/domain/wfo.ts | 9 ++++++++ 2 files changed, 57 insertions(+) diff --git a/src/domain/wfo.test.ts b/src/domain/wfo.test.ts index 4dc8f71..3b5c11f 100644 --- a/src/domain/wfo.test.ts +++ b/src/domain/wfo.test.ts @@ -97,4 +97,52 @@ describe('validateSweepGrid', () => { const r = validateSweepGrid({}, { tunableParamNames, restrictToEntryParams: false, entryAffecting }); expect(r).toEqual({ ok: false, reason: 'empty_grid' }); }); + + // R13 (research-validation-hardening item 6, report-13 G13/§3.5): leverage is denylisted from + // any sweep grid until the engine has a liquidation model — a systematic gate, independent of + // whether the profile happens to mark the param tunable. + it('a leverage-named key → denylisted_axis, even when it is a declared tunable param', () => { + const r = validateSweepGrid( + { 'leverage.multiplier': [1, 2] }, + { tunableParamNames: ['leverage.multiplier'], restrictToEntryParams: false, entryAffecting: [] }, + ); + expect(r).toEqual({ ok: false, reason: 'denylisted_axis:leverage.multiplier' }); + }); + + it('a margin-named key → denylisted_axis', () => { + const r = validateSweepGrid( + { marginMode: [1, 2] }, + { tunableParamNames: ['marginMode'], restrictToEntryParams: false, entryAffecting: [] }, + ); + expect(r).toEqual({ ok: false, reason: 'denylisted_axis:marginMode' }); + }); + + it('a leverage-named key that is NOT a declared tunable param → denylisted_axis takes priority over non_tunable_param', () => { + const r = validateSweepGrid( + { 'leverage.multiplier': [1, 2] }, + { tunableParamNames, restrictToEntryParams: false, entryAffecting }, + ); + expect(r).toEqual({ ok: false, reason: 'denylisted_axis:leverage.multiplier' }); + }); + + it('existing reasons are unaffected by the denylist check', () => { + expect(validateSweepGrid( + { 'entry.fastBouncePct': [1, 2], 'tpLadder.tp1Pct': [3, 4] }, + { tunableParamNames, restrictToEntryParams: false, entryAffecting }, + )).toEqual({ ok: true }); + expect(validateSweepGrid( + { 'entry.fastBouncePct': [1, 2], 'unknown.param': [1] }, + { tunableParamNames, restrictToEntryParams: false, entryAffecting }, + )).toEqual({ ok: false, reason: 'non_tunable_param:unknown.param' }); + expect(validateSweepGrid( + { 'tpLadder.tp1Pct': [1, 2] }, + { tunableParamNames, restrictToEntryParams: true, entryAffecting }, + )).toEqual({ ok: false, reason: 'non_entry_param_in_exploratory:tpLadder.tp1Pct' }); + expect(validateSweepGrid( + { 'entry.fastBouncePct': [] }, + { tunableParamNames, restrictToEntryParams: false, entryAffecting }, + )).toEqual({ ok: false, reason: 'empty_values:entry.fastBouncePct' }); + expect(validateSweepGrid({}, { tunableParamNames, restrictToEntryParams: false, entryAffecting })) + .toEqual({ ok: false, reason: 'empty_grid' }); + }); }); diff --git a/src/domain/wfo.ts b/src/domain/wfo.ts index 85523d4..b4ed25e 100644 --- a/src/domain/wfo.ts +++ b/src/domain/wfo.ts @@ -1,6 +1,7 @@ import { z } from 'zod'; import type { StrategyParameter } from './strategy-profile.ts'; +import { isDenylistedParam } from '../research/sweep-axis-catalog.ts'; /** Alias for the `StrategyProfile.profile.parameters[]` element type. */ export type ProfileParam = StrategyParameter; @@ -72,6 +73,11 @@ export type SweepGridValidation = { ok: true } | { ok: false; reason: string }; * upstream confirms its keys are actual tunable params of the profile, nor that a 0-trade * exploratory sweep stayed restricted to entry-affecting params. The Fake designer used in * tests happens to respect both constraints; a real LLM is not guaranteed to. + * + * R13 (research-validation-hardening item 6, report-13 G13/§3.5): also rejects any key on a + * denylisted axis (today: leverage/margin — no liquidation model in the engine). Checked BEFORE + * the tunable-param check: the denylist is an absolute gate, independent of whether the profile + * happens to mark the param `tunable`. */ export function validateSweepGrid( grid: Record, @@ -80,6 +86,9 @@ export function validateSweepGrid( const keys = Object.keys(grid); if (keys.length === 0) return { ok: false, reason: 'empty_grid' }; for (const key of keys) { + if (isDenylistedParam(key)) { + return { ok: false, reason: `denylisted_axis:${key}` }; + } if (!opts.tunableParamNames.includes(key)) { return { ok: false, reason: `non_tunable_param:${key}` }; } From 3f8f3fa2a746cb895789fc8d1fc5cb063671103c Mon Sep 17 00:00:00 2001 From: Alexander Nikolskiy Date: Fri, 24 Jul 2026 22:14:37 +0300 Subject: [PATCH 4/6] =?UTF-8?q?fix(research):=20R13=20=E2=80=94=20=D1=81?= =?UTF-8?q?=D1=83=D0=B7=D0=B8=D1=82=D1=8C=20=D0=BC=D0=B0=D1=82=D1=87=D0=B5?= =?UTF-8?q?=D1=80=D1=8B=20=D0=BE=D1=81=D0=B5=D0=B9=20hold=5Ftime/sizing=20?= =?UTF-8?q?(=D1=83=D1=81=D1=82=D1=80=D0=B0=D0=BD=D0=B8=D1=82=D1=8C=20?= =?UTF-8?q?=D0=BA=D0=BE=D0=BB=D0=BB=D0=B8=D0=B7=D0=B8=D0=B8=20'threshold'?= =?UTF-8?q?=E2=86=92hold,=20'resize'=E2=86=92size)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit matchesParam использовал bare-подстроку includes('hold') / includes('size'), из-за чего entry.thresholdPct/dump.dropThreshold ложно матчились на hold_time (…thresHOLD), а resize* — на sizing. До R12 это трогало лишь текст промпта, но onboarding-сетка (Task 4) строится на matchesParam — поэтому сужаем до компаундных токенов + namespace-префиксов и пиним red-тестами. Co-Authored-By: Claude Fable 5 --- src/research/sweep-axis-catalog.test.ts | 25 +++++++++++++++++++++++++ src/research/sweep-axis-catalog.ts | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/research/sweep-axis-catalog.test.ts b/src/research/sweep-axis-catalog.test.ts index a6d47f7..b6a2f82 100644 --- a/src/research/sweep-axis-catalog.test.ts +++ b/src/research/sweep-axis-catalog.test.ts @@ -54,6 +54,8 @@ describe('SWEEP_AXIS_CATALOG', () => { const byAxis = Object.fromEntries(SWEEP_AXIS_CATALOG.map((a) => [a.axis, a])); expect(byAxis.hold_time!.matchesParam('maxHoldMin')).toBe(true); + expect(byAxis.hold_time!.matchesParam('holdTimeMin')).toBe(true); + expect(byAxis.hold_time!.matchesParam('hold.maxBars')).toBe(true); expect(byAxis.hold_time!.matchesParam('entry.fastBouncePct')).toBe(false); expect(byAxis.entry_thresholds!.matchesParam('dump.minDropPct')).toBe(true); @@ -83,6 +85,29 @@ describe('SWEEP_AXIS_CATALOG', () => { }); }); +describe('SWEEP_AXIS_CATALOG — substring-collision regressions (R13 Task 4)', () => { + const byAxis = Object.fromEntries(SWEEP_AXIS_CATALOG.map((a) => [a.axis, a])); + + it('threshold-named entry params never match hold_time (the "…thresHOLD" trap)', () => { + for (const name of ['entry.thresholdPct', 'dump.dropThreshold', 'oiFilter.oiThreshold']) { + expect(byAxis.hold_time!.matchesParam(name)).toBe(false); + } + // …and they DO land on entry_thresholds where they belong. + expect(byAxis.entry_thresholds!.matchesParam('entry.thresholdPct')).toBe(true); + expect(byAxis.entry_thresholds!.matchesParam('dump.dropThreshold')).toBe(true); + expect(byAxis.entry_thresholds!.matchesParam('oiFilter.oiThreshold')).toBe(true); + }); + + it('resize-named params never match sizing (the "reSIZE" trap)', () => { + for (const name of ['resizeWindow', 'gridResizeEnabled']) { + expect(byAxis.sizing!.matchesParam(name)).toBe(false); + } + // real sizing params still match. + expect(byAxis.sizing!.matchesParam('positionSizePct')).toBe(true); + expect(byAxis.sizing!.matchesParam('dca.stepPct')).toBe(true); + }); +}); + describe('isDenylistedParam', () => { it('flags leverage/margin-named params', () => { expect(isDenylistedParam('leverage.multiplier')).toBe(true); diff --git a/src/research/sweep-axis-catalog.ts b/src/research/sweep-axis-catalog.ts index 5aaa734..1544397 100644 --- a/src/research/sweep-axis-catalog.ts +++ b/src/research/sweep-axis-catalog.ts @@ -48,7 +48,7 @@ export const SWEEP_AXIS_CATALOG: readonly SweepAxis[] = [ 'Hold time — sweep the max/target holding duration (e.g. maxHoldMin) around the baseline. ' + 'A strategy whose edge only survives at one exact hold time is fragile; look for a wide ' + 'plateau, not a peak, and include a point past the point where performance starts to degrade.', - matchesParam: nameMatcher(['hold.'], ['hold']), + matchesParam: nameMatcher(['hold.'], ['maxhold', 'holdtime', 'holdmin', 'holdbar']), }, { axis: 'entry_thresholds', @@ -81,7 +81,7 @@ export const SWEEP_AXIS_CATALOG: readonly SweepAxis[] = [ 'Sizing — sweep position-sizing / DCA step params (e.g. dca.stepPct, positionSizePct). Sizing ' + 'interacts with drawdown and the risk/reward tradeoff is the researcher\'s job to surface, not ' + 'to pick a winner by Sharpe alone; report the plateau, not just the top point.', - matchesParam: nameMatcher(['dca.'], ['size', 'sizing']), + matchesParam: nameMatcher(['dca.'], ['positionsize', 'sizepct']), }, { axis: 'regime_as_axis', From 0044bdc6a6cb02b525fcb476e56e4a45419bbde1 Mon Sep 17 00:00:00 2001 From: Alexander Nikolskiy Date: Fri, 24 Jul 2026 22:47:19 +0300 Subject: [PATCH 5/6] =?UTF-8?q?feat(research):=20R13=20=E2=80=94=20onboard?= =?UTF-8?q?ing-=D0=B1=D0=B0=D1=82=D0=B0=D1=80=D0=B5=D1=8F=20=D1=81=D0=B5?= =?UTF-8?q?=D1=82=D0=BE=D0=BA=20=D0=BF=D1=80=D0=B8=20strategy.onboard=20(l?= =?UTF-8?q?og-only,=20=D0=B2=D1=81=D0=B5=20=D1=82=D0=BE=D1=87=D0=BA=D0=B8?= =?UTF-8?q?=20=D0=B2=20trial=20ledger)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Детерминированная маленькая сетка вокруг baseline-значений tunable-параметров бандла по каталогу осей (без LLM, denylisted оси исключены), прогон через ParamGridRunner (role 'train', тот же datasetScope/бандл) между успешным baseline-прогоном и enqueue strategy.wfo. Все точки попадают в trial ledger backtester сервер-сайд; событие strategy.onboard_battery.completed несёт counts + lonePeak/plateau (без магнитуд), сводка — artifact (без миграции). Fail-soft: любой сбой → strategy.onboard_battery.skipped, цепочка baseline→wfo не прерывается; off-режим байт-идентичен. Флаг LAB_ONBOARD_BATTERY_MODE (off|log), fail-closed резолвер (enforce → throw, стадия log-only), проведён env-schema/env/composition/app-services/make-services; ENV.md перегенерирован. trialFamilyHint в strategy-lane отсутствует — задокументировано как хвост (wire-контракт не тронут). Co-Authored-By: Claude Fable 5 --- ENV.md | 3 +- src/composition.ts | 2 + src/config/env-schema.test.ts | 16 ++ src/config/env-schema.ts | 10 + src/config/env.test.ts | 17 ++ src/config/env.ts | 6 + src/orchestrator/app-services.ts | 9 + .../strategy-baseline.handler.test.ts | 103 ++++++++++- .../handlers/strategy-baseline.handler.ts | 79 +++++++- src/research/onboard-battery.test.ts | 125 +++++++++++++ src/research/onboard-battery.ts | 172 ++++++++++++++++++ test/support/make-services.ts | 5 +- 12 files changed, 543 insertions(+), 4 deletions(-) create mode 100644 src/research/onboard-battery.test.ts create mode 100644 src/research/onboard-battery.ts diff --git a/ENV.md b/ENV.md index 74ff69c..b94a717 100644 --- a/ENV.md +++ b/ENV.md @@ -8,7 +8,7 @@ Значения секретов здесь не появляются никогда — только имя и форма; живые значения — в `.env.vps` на хостах и SOPS/age-контуре. -Переменных: 122. Точка чтения — `src/config/env.ts` (loadEnv) и явно перечисленные consumers. +Переменных: 123. Точка чтения — `src/config/env.ts` (loadEnv) и явно перечисленные consumers. | Имя | Тип | Обяз. | Default | Secret | Flag | Описание | | --- | --- | --- | --- | --- | --- | --- | @@ -47,6 +47,7 @@ | `LAB_CONSOLIDATION_TOL_REL` | float | — | `0.001` | — | — | Относительный допуск parity-гейта консолидации | | `LAB_HYPOTHESIS_HOLDOUT` | enum(off, log) | — | `off` | — | off/log → off | Флаг E4b-паттерна (R12a, research-validation-hardening item 5): режим раскатки лёгкого holdout-подтверждения проксистатуса PAPER_CANDIDATE (task hypothesis.holdout), запускающего break_battery@1 (R11) на уровне гипотезы. off — holdout не enqueue-ится; log — enqueue-ится, персистит и логирует, вердикты не меняет. Состояние enforce намеренно отклоняется резолвером до калибровки порогов battery-policy@1 | | `LAB_MARKET_HISTORY_URL` | url | — | — | — | — | URL market-history-поверхности; при отсутствии — LAB_OPS_READ_URL, затем http://mock-platform:8839 | +| `LAB_ONBOARD_BATTERY_MODE` | enum(off, log) | — | `off` | — | off/log → off | Флаг E4b-паттерна (R13, research-validation-hardening item 6): режим детерминированной onboarding-батареи сеток вокруг baseline-значений перед первым WFO. off — батарея не запускается; log — прогоняет маленькую сетку через ParamGridRunner (все точки в trial ledger сервер-сайд), эмитит strategy.onboard_battery.completed/skipped со счётчиками и lonePeak-эвиденс, вердикты и цепочку baseline→wfo не меняет. Состояние enforce намеренно отклоняется резолвером: стадия log-only, калибровки порогов у неё нет (battery-policy@1) | | `LAB_OPS_READ_FIXTURE_DIR` | string | — | — | — | — | Каталог фикстур для fixture-режима bot-results/trade-evidence; дефолт — встроенный каталог фикстур | | `LAB_OPS_READ_TOKEN` | string | — | — | да | — | Bearer-токен ops-read-поверхности | | `LAB_OPS_READ_URL` | url | — | — | — | — | Базовый URL ops-read-поверхности (bot-results/trade-evidence/market-history); фолбэк http://127.0.0.1:8839 в селекторах | diff --git a/src/composition.ts b/src/composition.ts index ee1c0ca..ec773c3 100644 --- a/src/composition.ts +++ b/src/composition.ts @@ -479,6 +479,8 @@ export function composeRuntime() { paperEvidenceRequired: env.LAB_PAPER_EVIDENCE_REQUIRED, cycleScorecards, hypothesisHoldoutMode: env.LAB_HYPOTHESIS_HOLDOUT, + paramGridRunner, + onboardBatteryMode: env.LAB_ONBOARD_BATTERY_MODE, }; const router = new WorkflowRouter(); diff --git a/src/config/env-schema.test.ts b/src/config/env-schema.test.ts index 67b0896..dca4a7b 100644 --- a/src/config/env-schema.test.ts +++ b/src/config/env-schema.test.ts @@ -195,6 +195,19 @@ describe('env-schema: доменные инварианты lab', () => { expect(v!.description).toMatch(/калибр/); }); + it('LAB_ONBOARD_BATTERY_MODE — флаг off|log, default off, enforce отклонён (стадия log-only)', () => { + const v = byName(doc, 'LAB_ONBOARD_BATTERY_MODE'); + expect(v).toBeDefined(); + expect(v!.flag).toBe(true); + expect(v!.type).toBe('enum'); + expect(v!.enum_values).toEqual(['off', 'log']); + expect(v!.flag_states).toEqual(['off', 'log']); + expect(v!.default_state).toBe('off'); + expect(v!.default).toBe('off'); + expect(v!.description).toMatch(/enforce/); + expect(v!.description).toMatch(/log-only/); + }); + it('все токены/ключи — secret с default null', () => { const secretByPattern = doc.variables.filter((v) => /_(TOKEN|KEY)$/.test(v.name)); expect(secretByPattern.length).toBeGreaterThanOrEqual(10); @@ -309,6 +322,9 @@ describe('env-schema: fail-fast loadEnv сохранён (негативы)', () it('enforce для LAB_BREAK_BATTERY_MODE отклоняется', () => { expect(() => loadEnv({ LAB_BREAK_BATTERY_MODE: 'enforce' } as NodeJS.ProcessEnv)).toThrow(/enforce/); }); + it('enforce для LAB_ONBOARD_BATTERY_MODE отклоняется', () => { + expect(() => loadEnv({ LAB_ONBOARD_BATTERY_MODE: 'enforce' } as NodeJS.ProcessEnv)).toThrow(/enforce/); + }); it('неизвестные значения fail-closed осей бросают', () => { expect(() => loadEnv({ TRADING_PLATFORM_INTEGRATION: 'backtestr' } as NodeJS.ProcessEnv)).toThrow(); expect(() => loadEnv({ LAB_AGENTS_ADAPTER: 'Mastra' } as NodeJS.ProcessEnv)).toThrow(); diff --git a/src/config/env-schema.ts b/src/config/env-schema.ts index 6dae530..aec8903 100644 --- a/src/config/env-schema.ts +++ b/src/config/env-schema.ts @@ -237,6 +237,16 @@ const VARIABLES: EnvVariableSpec[] = [ default: 'off', description: 'Флаг E4b-паттерна (R12a, research-validation-hardening item 5): режим раскатки лёгкого holdout-подтверждения проксистатуса PAPER_CANDIDATE (task hypothesis.holdout), запускающего break_battery@1 (R11) на уровне гипотезы. off — holdout не enqueue-ится; log — enqueue-ится, персистит и логирует, вердикты не меняет. Состояние enforce намеренно отклоняется резолвером до калибровки порогов battery-policy@1', }), + v({ + name: 'LAB_ONBOARD_BATTERY_MODE', + type: 'enum', + enum_values: ['off', 'log'], + flag: true, + flag_states: ['off', 'log'], + default_state: 'off', + default: 'off', + description: 'Флаг E4b-паттерна (R13, research-validation-hardening item 6): режим детерминированной onboarding-батареи сеток вокруг baseline-значений перед первым WFO. off — батарея не запускается; log — прогоняет маленькую сетку через ParamGridRunner (все точки в trial ledger сервер-сайд), эмитит strategy.onboard_battery.completed/skipped со счётчиками и lonePeak-эвиденс, вердикты и цепочку baseline→wfo не меняет. Состояние enforce намеренно отклоняется резолвером: стадия log-only, калибровки порогов у неё нет (battery-policy@1)', + }), // --- селекторные оси (boot-safe селекторы читают свой env из composition.ts) --- v({ name: 'LAB_SIGNED_EVIDENCE_SOURCE', type: 'enum', enum_values: ['none', 'fixture', 'http'], default: 'none', description: 'Источник подписанной backtest-evidence; fixture вне NODE_ENV=test требует LAB_ALLOW_FIXTURE_EVIDENCE=true (fail-closed)', consumers: ['src/adapters/platform/select-signed-evidence.ts'] }), v({ name: 'LAB_ALLOW_FIXTURE_EVIDENCE', type: 'bool', default: 'false', description: 'Явное разрешение fixture-evidence вне NODE_ENV=test (self-signed, никогда для прод-гейтов)', consumers: ['src/adapters/platform/select-signed-evidence.ts'] }), diff --git a/src/config/env.test.ts b/src/config/env.test.ts index 3c4817e..cb250b1 100644 --- a/src/config/env.test.ts +++ b/src/config/env.test.ts @@ -669,3 +669,20 @@ describe('loadEnv — LAB_HYPOTHESIS_HOLDOUT (R12a hypothesis holdout)', () => { expect(() => loadEnv({ LAB_HYPOTHESIS_HOLDOUT: 'bogus' } as NodeJS.ProcessEnv)).toThrow(/off\|log/); }); }); + +describe('loadEnv — LAB_ONBOARD_BATTERY_MODE (R13 onboarding battery)', () => { + it("defaults to 'off' (battery never runs)", () => { + expect(loadEnv({} as NodeJS.ProcessEnv).LAB_ONBOARD_BATTERY_MODE).toBe('off'); + expect(loadEnv({ LAB_ONBOARD_BATTERY_MODE: '' } as NodeJS.ProcessEnv).LAB_ONBOARD_BATTERY_MODE).toBe('off'); + }); + + it("accepts 'off' and 'log'", () => { + expect(loadEnv({ LAB_ONBOARD_BATTERY_MODE: 'off' } as NodeJS.ProcessEnv).LAB_ONBOARD_BATTERY_MODE).toBe('off'); + expect(loadEnv({ LAB_ONBOARD_BATTERY_MODE: 'log' } as NodeJS.ProcessEnv).LAB_ONBOARD_BATTERY_MODE).toBe('log'); + }); + + it("fail-closed: 'enforce' (log-only stage) and unknown values throw", () => { + expect(() => loadEnv({ LAB_ONBOARD_BATTERY_MODE: 'enforce' } as NodeJS.ProcessEnv)).toThrow(/enforce/); + expect(() => loadEnv({ LAB_ONBOARD_BATTERY_MODE: 'bogus' } as NodeJS.ProcessEnv)).toThrow(/off\|log/); + }); +}); diff --git a/src/config/env.ts b/src/config/env.ts index 5208910..5d49d62 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -3,6 +3,7 @@ import { MODEL_PROVIDERS, type ModelProvider } from '../adapters/llm/model-provi import { DEFAULT_PRESERVATION_THRESHOLDS, type PreservationThresholds } from '../validation/trade-preservation.ts'; import { resolveBreakBatteryMode, type BreakBatteryMode } from '../research/break-battery.ts'; import { resolveHypothesisHoldoutMode, type HypothesisHoldoutMode } from '../research/hypothesis-holdout.ts'; +import { resolveOnboardBatteryMode, type OnboardBatteryMode } from '../research/onboard-battery.ts'; export interface Env { DATABASE_URL?: string; @@ -161,6 +162,10 @@ export interface Env { * 'log' (enqueues the lightweight holdout + break battery, log-only, NEVER changes verdict). * 'enforce' rejected until battery calibration closes (battery-policy@1). */ LAB_HYPOTHESIS_HOLDOUT: HypothesisHoldoutMode; + /** R13 onboarding-battery rollout mode: 'off' (default — battery never runs) | 'log' (runs the + * deterministic onboarding grid before the first WFO, seeds the trial ledger + logs lone-peak + * evidence, NEVER changes any verdict/chain). 'enforce' rejected — the stage is log-only. */ + LAB_ONBOARD_BATTERY_MODE: OnboardBatteryMode; } function parseModelProvider(value: string | undefined): ModelProvider { @@ -337,6 +342,7 @@ export function loadEnv(source: NodeJS.ProcessEnv = process.env): Env { LAB_CONSOLIDATION_TOL_ABS: parseFloatOr(source.LAB_CONSOLIDATION_TOL_ABS, 0.01), LAB_BREAK_BATTERY_MODE: resolveBreakBatteryMode(source.LAB_BREAK_BATTERY_MODE), LAB_HYPOTHESIS_HOLDOUT: resolveHypothesisHoldoutMode(source.LAB_HYPOTHESIS_HOLDOUT), + LAB_ONBOARD_BATTERY_MODE: resolveOnboardBatteryMode(source.LAB_ONBOARD_BATTERY_MODE), ...loadRagEnv(source), }; } diff --git a/src/orchestrator/app-services.ts b/src/orchestrator/app-services.ts index 7c4aa90..e0bed9e 100644 --- a/src/orchestrator/app-services.ts +++ b/src/orchestrator/app-services.ts @@ -43,6 +43,8 @@ import type { StrategyConsolidatorPort } from '../ports/strategy-consolidator.po import type { ConsolidationTolerances } from '../validation/consolidation-evaluator.ts'; import type { CycleScorecardRepository } from '../ports/cycle-scorecard.repository.ts'; import type { HypothesisHoldoutMode } from '../research/hypothesis-holdout.ts'; +import type { OnboardBatteryMode } from '../research/onboard-battery.ts'; +import type { ParamGridRunner } from '../research/param-grid-runner.ts'; /** * Fail-soft retrieval indexer seam. The concrete StrategyRetrievalIndexer satisfies it; @@ -139,4 +141,11 @@ export interface AppServices { * LAB_HYPOTHESIS_HOLDOUT); consumed by backtestCompletedHandler's PAPER_CANDIDATE branch to * decide whether to enqueue 'hypothesis.holdout' (Task 3). 'off' (default) enqueues nothing. */ hypothesisHoldoutMode: HypothesisHoldoutMode; + /** Deterministic onboarding grid runner (R13); the strategy-baseline handler drives it directly + * for the log-only onboarding battery. Same instance the ExperimentService uses for WFO. */ + paramGridRunner: ParamGridRunner; + /** R13 rollout mode for the log-only onboarding battery (env LAB_ONBOARD_BATTERY_MODE); + * consumed by strategyBaselineHandler between a passing baseline and the strategy.wfo enqueue. + * 'off' (default) runs nothing — byte-identical to pre-R13 behavior. */ + onboardBatteryMode: OnboardBatteryMode; } diff --git a/src/orchestrator/handlers/strategy-baseline.handler.test.ts b/src/orchestrator/handlers/strategy-baseline.handler.test.ts index 00e8b9b..d7631bc 100644 --- a/src/orchestrator/handlers/strategy-baseline.handler.test.ts +++ b/src/orchestrator/handlers/strategy-baseline.handler.test.ts @@ -11,6 +11,8 @@ import type { ResearchTask } from '../../domain/types.ts'; import type { StrategyProfile } from '../../domain/strategy-profile.ts'; import type { StrategyRevision } from '../../domain/strategy-revision.ts'; import type { RunStrategyBaselineValidationInput } from '../../research/experiment-service.ts'; +import type { GridRunOutput } from '../../research/param-grid-runner.ts'; +import type { StrategyParameter } from '../../domain/strategy-profile.ts'; function profile(): StrategyProfile { const now = '2026-01-01T00:00:00Z'; @@ -21,6 +23,19 @@ function profile(): StrategyProfile { }; } +/** Profile carrying real tunable params so the R13 onboarding battery has axes to sweep. The base + * `profile()` uses `profile: {} as never`, which yields zero eligible axes (skip path). */ +function param(over: Partial & { name: string }): StrategyParameter { + return { value: 10, unit: null, description: '', tunable: true, ...over }; +} +function profileWithParams(params: StrategyParameter[]): StrategyProfile { + return { ...profile(), profile: { parameters: params } as never }; +} + +function fakeGridOutput(over: Partial = {}): GridRunOutput { + return { allResults: [], ranked: [], submitted: 6, rejected: 0, ...over }; +} + function taskOf(payload: Record): ResearchTask { const now = '2026-01-01T00:00:00Z'; return { id: 't1', taskType: 'strategy.baseline', source: 'operator', correlationId: 'c1', status: 'running', payload, createdAt: now, updatedAt: now }; @@ -31,6 +46,8 @@ async function makeFakeServices(opts: { strategyBuilder?: AppServices['strategyBuilder']; revisions?: AppServices['revisions']; verdict?: 'PASS' | 'FAIL' | 'MODIFY' | 'INCONCLUSIVE' | 'PAPER_CANDIDATE'; + onboardBatteryMode?: 'off' | 'log'; + profileParams?: StrategyParameter[]; } = {}): Promise<{ services: AppServices; queued: AppServices['taskQueue'] extends { queued: infer Q } ? Q : never; @@ -41,6 +58,7 @@ async function makeFakeServices(opts: { const services = makeServices({ ...(opts.strategyBuilder ? { strategyBuilder: opts.strategyBuilder } : {}), ...(opts.revisions ? { revisions: opts.revisions } : {}), + ...(opts.onboardBatteryMode ? { onboardBatteryMode: opts.onboardBatteryMode } : {}), }); const originalPut = services.artifacts.put.bind(services.artifacts); services.artifacts.put = async (content, meta) => { @@ -48,7 +66,7 @@ async function makeFakeServices(opts: { return originalPut(content, meta); }; - await services.strategyProfiles.create(profile()); + await services.strategyProfiles.create(opts.profileParams ? profileWithParams(opts.profileParams) : profile()); const experimentCalls: (RunStrategyBaselineValidationInput & { returnedExperimentId?: string })[] = []; let counter = 0; @@ -228,3 +246,86 @@ describe('strategyBaselineHandler', () => { expect((queued as unknown[]).filter((t) => (t as { taskType: string }).taskType === 'strategy.wfo')).toHaveLength(1); }); }); + +describe('strategyBaselineHandler — R13 onboarding battery', () => { + it('off (default): never touches ParamGridRunner and emits no onboard_battery event', async () => { + const { services } = await makeFakeServices({ profileParams: [param({ name: 'maxHoldMin', value: 60 })] }); + const runGridSpy = vi.spyOn(services.paramGridRunner, 'runGrid'); + const appendSpy = vi.spyOn(services.events, 'append'); + + await strategyBaselineHandler(taskOf({ strategyProfileId: 'prof-1' }), services); + + expect(runGridSpy).not.toHaveBeenCalled(); + expect(appendSpy.mock.calls.map((c) => (c[0] as { type: string }).type)) + .not.toContain('strategy.onboard_battery.completed'); + }); + + it('log: sweeps a deterministic grid (denylisted params excluded), emits completed with counts, still enqueues wfo', async () => { + const { services, queued } = await makeFakeServices({ + onboardBatteryMode: 'log', + profileParams: [ + param({ name: 'maxHoldMin', value: 60 }), + param({ name: 'dump.minDropPct', value: 4 }), + param({ name: 'leverage.multiplier', value: 3 }), + ], + }); + const runGridSpy = vi.spyOn(services.paramGridRunner, 'runGrid').mockResolvedValue( + fakeGridOutput({ + submitted: 9, rejected: 1, + ranked: [ + { point: {}, paramsHash: 'a', status: 'completed', strategyBacktestRunId: 'a', metrics: {} as never, lowConfidence: false, lonePeak: true, neighborCount: 2 }, + { point: {}, paramsHash: 'b', status: 'completed', strategyBacktestRunId: 'b', metrics: {} as never, lowConfidence: false, lonePeak: false, neighborCount: 2 }, + ], + allResults: [{ point: {}, paramsHash: 'a', status: 'completed', strategyBacktestRunId: 'a' }], + }), + ); + const appendSpy = vi.spyOn(services.events, 'append'); + + await strategyBaselineHandler(taskOf({ strategyProfileId: 'prof-1' }), services); + + // grid built off matchesParam, leverage EXCLUDED + expect(runGridSpy).toHaveBeenCalledTimes(1); + const grid = runGridSpy.mock.calls[0]![0].grid; + expect(Object.keys(grid).sort()).toEqual(['dump.minDropPct', 'maxHoldMin']); + expect(Object.keys(grid)).not.toContain('leverage.multiplier'); + + const completed = appendSpy.mock.calls.map((c) => c[0] as { type: string; payload: Record }) + .find((e) => e.type === 'strategy.onboard_battery.completed'); + expect(completed).toBeDefined(); + expect(completed!.payload).toMatchObject({ points: 9, rejected: 1, ranked: 2, lonePeak: 1, plateau: 1 }); + + // chain intact: strategy.wfo still enqueued + expect((queued as unknown[]).filter((t) => (t as { taskType: string }).taskType === 'strategy.wfo')).toHaveLength(1); + }); + + it('log with no eligible axes: emits skipped(no_eligible_axes), never calls runGrid, still enqueues wfo', async () => { + // base profile() carries profile: {} as never -> zero params -> zero axes + const { services, queued } = await makeFakeServices({ onboardBatteryMode: 'log' }); + const runGridSpy = vi.spyOn(services.paramGridRunner, 'runGrid'); + const appendSpy = vi.spyOn(services.events, 'append'); + + await strategyBaselineHandler(taskOf({ strategyProfileId: 'prof-1' }), services); + + expect(runGridSpy).not.toHaveBeenCalled(); + const skipped = appendSpy.mock.calls.map((c) => c[0] as { type: string; payload: Record }) + .find((e) => e.type === 'strategy.onboard_battery.skipped'); + expect(skipped?.payload.reason).toBe('no_eligible_axes'); + expect((queued as unknown[]).filter((t) => (t as { taskType: string }).taskType === 'strategy.wfo')).toHaveLength(1); + }); + + it('log: a runner throw degrades to skipped and NEVER breaks the baseline->wfo chain', async () => { + const { services, queued } = await makeFakeServices({ + onboardBatteryMode: 'log', + profileParams: [param({ name: 'maxHoldMin', value: 60 })], + }); + vi.spyOn(services.paramGridRunner, 'runGrid').mockRejectedValue(new Error('runner boom')); + const appendSpy = vi.spyOn(services.events, 'append'); + + await strategyBaselineHandler(taskOf({ strategyProfileId: 'prof-1' }), services); + + const skipped = appendSpy.mock.calls.map((c) => c[0] as { type: string; payload: Record }) + .find((e) => e.type === 'strategy.onboard_battery.skipped'); + expect(skipped?.payload.reason).toMatch(/runner boom/); + expect((queued as unknown[]).filter((t) => (t as { taskType: string }).taskType === 'strategy.wfo')).toHaveLength(1); + }); +}); diff --git a/src/orchestrator/handlers/strategy-baseline.handler.ts b/src/orchestrator/handlers/strategy-baseline.handler.ts index 8e03469..403de68 100644 --- a/src/orchestrator/handlers/strategy-baseline.handler.ts +++ b/src/orchestrator/handlers/strategy-baseline.handler.ts @@ -8,7 +8,15 @@ import { RESEARCH_RUN_METRICS } from '../../domain/platform-comparison.ts'; import { getAuthoringDoc } from '@trdlabs/backtester-sdk/builder'; import { createAndEnqueueTask } from '../task-intake.ts'; import { event } from './backtest-support.ts'; -import type { ArtifactRef } from '../../domain/types.ts'; +import { + buildOnboardBatteryGrid, + summarizeOnboardBatteryRun, + ONBOARD_BATTERY_MAX_POINTS, +} from '../../research/onboard-battery.ts'; +import type { AppServices } from '../app-services.ts'; +import type { ArtifactRef, ResearchTask } from '../../domain/types.ts'; +import type { StrategyProfile } from '../../domain/strategy-profile.ts'; +import type { PlatformRunConfig } from '../../ports/research-platform.port.ts'; export const StrategyBaselinePayloadSchema = z.object({ strategyProfileId: z.string().min(1), @@ -22,6 +30,72 @@ export const StrategyBaselinePayloadSchema = z.object({ consolidatedRevisionId: z.string().optional(), }); +/** + * R13 log-only onboarding battery: after a fresh strategy's baseline passes and BEFORE the first + * full WFO, sweep a small DETERMINISTIC grid around the profile's baseline param values (built + * mechanically from SWEEP_AXIS_CATALOG — no LLM), submitting each point through ParamGridRunner so + * every run lands in the backtester's server-side trial ledger. Emits + * `strategy.onboard_battery.completed` with counts + lone-peak evidence (NO magnitudes), persists + * a summary artifact, and NEVER changes a verdict/status/the baseline→wfo chain. + * + * FAIL-SOFT + FAIL-CLOSED: only runs when `onboardBatteryMode === 'log'` ('off' default is + * byte-identical to pre-R13). Any error — no eligible axes, runner throw — degrades to a + * `strategy.onboard_battery.skipped` event; this function NEVER throws, so the caller's WFO + * enqueue is unaffected. + */ +async function runOnboardBattery( + task: ResearchTask, + services: AppServices, + ctx: { profile: StrategyProfile; strategyBundle: AssembledStrategyBundle; experimentId: string; run: AppServices['defaultPlatformRun'] }, +): Promise { + if (services.onboardBatteryMode !== 'log') return; + const { profile, strategyBundle, experimentId, run } = ctx; + try { + const built = buildOnboardBatteryGrid(profile.profile?.parameters); + if (built.axes.length === 0) { + await services.events.append(event(task.id, 'strategy.onboard_battery.skipped', { + strategyProfileId: profile.id, experimentId, reason: 'no_eligible_axes', + })); + return; + } + + const trainRun: PlatformRunConfig = { + datasetId: run.datasetId, symbols: run.symbols, timeframe: run.timeframe, + period: { from: run.period.from, to: run.period.to }, seed: run.seed, + }; + const output = await services.paramGridRunner.runGrid({ + experimentId, + strategyBundle, + strategyProfileId: profile.id, + trainRun, + grid: built.grid, + metrics: RESEARCH_RUN_METRICS, + maxPoints: ONBOARD_BATTERY_MAX_POINTS, + topN: built.pointCount, + minTradesTrain: 1, + foldId: 0, + }); + + const summary = summarizeOnboardBatteryRun(output); + const summaryRef = await services.artifacts.put( + JSON.stringify({ strategyProfileId: profile.id, experimentId, axes: built.axes, ...summary }), + { kind: 'onboard_battery_summary', mime_type: 'application/json', producer: 'strategy-baseline-handler' }, + ); + await services.events.append(event(task.id, 'strategy.onboard_battery.completed', { + strategyProfileId: profile.id, experimentId, axes: built.axes, summaryRef, ...summary, + })); + } catch (err) { + try { + await services.events.append(event(task.id, 'strategy.onboard_battery.skipped', { + strategyProfileId: profile.id, experimentId, + reason: err instanceof Error ? err.message : String(err), + })); + } catch { + /* swallow — the onboarding battery must never break the baseline→wfo chain */ + } + } +} + export const strategyBaselineHandler: WorkflowHandler = async (task, services) => { const parsed = validateWithSchema(StrategyBaselinePayloadSchema, task.payload); if (parsed.status === 'invalid') throw new Error(`invalid strategy.baseline payload: ${JSON.stringify(parsed.issues)}`); @@ -88,6 +162,9 @@ export const strategyBaselineHandler: WorkflowHandler = async (task, services) = // that generate enough trades. Revision re-baselines (revisionId present) stay strict. const allowWfoOnInconclusiveForFreshProfile = !revisionId && baselineValidationStatus === 'inconclusive'; if (baselineValidationStatus === 'passed' || allowWfoOnInconclusiveForFreshProfile) { + // R13: log-only onboarding battery between the passing baseline and the WFO enqueue. Fail-soft + // and never throws — the strategy.wfo enqueue below runs regardless of the battery's outcome. + await runOnboardBattery(task, services, { profile, strategyBundle: bundle, experimentId, run }); await createAndEnqueueTask( { taskType: 'strategy.wfo', diff --git a/src/research/onboard-battery.test.ts b/src/research/onboard-battery.test.ts new file mode 100644 index 0000000..b0ce3f0 --- /dev/null +++ b/src/research/onboard-battery.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from 'vitest'; + +import { + ONBOARD_BATTERY_MAX_POINTS, + ONBOARD_BATTERY_STEP, + buildOnboardBatteryGrid, + resolveOnboardBatteryMode, + summarizeOnboardBatteryRun, +} from './onboard-battery.ts'; +import type { StrategyParameter } from '../domain/strategy-profile.ts'; +import type { GridRunOutput } from './param-grid-runner.ts'; +import type { GridResult, RankedPoint } from './top-n-prefilter.ts'; + +function param(over: Partial & { name: string }): StrategyParameter { + return { value: 10, unit: null, description: '', tunable: true, ...over }; +} + +describe('resolveOnboardBatteryMode', () => { + it("defaults to 'off' on undefined / empty / 'off'", () => { + expect(resolveOnboardBatteryMode(undefined)).toBe('off'); + expect(resolveOnboardBatteryMode('')).toBe('off'); + expect(resolveOnboardBatteryMode('off')).toBe('off'); + }); + + it("accepts 'log'", () => { + expect(resolveOnboardBatteryMode('log')).toBe('log'); + }); + + it("fail-closed: 'enforce' throws (log-only stage) and unknown values throw off|log", () => { + expect(() => resolveOnboardBatteryMode('enforce')).toThrow(/enforce/); + expect(() => resolveOnboardBatteryMode('enforce')).toThrow(/battery-policy/); + expect(() => resolveOnboardBatteryMode('bogus')).toThrow(/off\|log/); + }); +}); + +describe('buildOnboardBatteryGrid', () => { + it('returns an empty grid (no axes) for undefined / empty / no-tunable params', () => { + expect(buildOnboardBatteryGrid(undefined)).toMatchObject({ axes: [], pointCount: 0 }); + expect(buildOnboardBatteryGrid([])).toMatchObject({ axes: [], pointCount: 0 }); + expect(buildOnboardBatteryGrid([param({ name: 'maxHoldMin', tunable: false })])).toMatchObject({ axes: [], pointCount: 0 }); + }); + + it('brackets a tunable numeric axis with [down, base, up] (integer-preserving)', () => { + const built = buildOnboardBatteryGrid([param({ name: 'maxHoldMin', value: 60 })]); + expect(built.axes).toEqual(['maxHoldMin']); + expect(built.grid.maxHoldMin).toEqual([30, 60, 90]); + expect(built.pointCount).toBe(3); + }); + + it('keeps float baselines as floats (non-integer baseline skips the int-rounding path)', () => { + const built = buildOnboardBatteryGrid([param({ name: 'dump.minDropPct', value: 2.4 })], { step: 0.5 }); + expect(built.grid['dump.minDropPct']).toEqual([1.2, 2.4, 3.6]); + }); + + it('EXCLUDES denylisted (leverage/margin) params even when tunable', () => { + const built = buildOnboardBatteryGrid([ + param({ name: 'maxHoldMin', value: 60 }), + param({ name: 'leverage.multiplier', value: 3 }), + param({ name: 'marginMode', value: 2 }), + ]); + expect(built.axes).toEqual(['maxHoldMin']); + expect(Object.keys(built.grid)).not.toContain('leverage.multiplier'); + expect(Object.keys(built.grid)).not.toContain('marginMode'); + }); + + it('excludes params that match no catalog axis', () => { + const built = buildOnboardBatteryGrid([param({ name: 'someRandomKnob', value: 5 })]); + expect(built.axes).toEqual([]); + }); + + it('skips axes whose bracket collapses to a single value (e.g. baseline 0)', () => { + const built = buildOnboardBatteryGrid([param({ name: 'maxHoldMin', value: 0 })]); + expect(built.axes).toEqual([]); + }); + + it('caps the cartesian product at maxPoints — greedy in NAME order (3rd axis dropped)', () => { + // Three eligible axes, 3 values each -> 3*3*3 = 27 > 12; only the first two (by name) survive. + const built = buildOnboardBatteryGrid([ + param({ name: 'watch.cooldownMin', value: 10 }), + param({ name: 'maxHoldMin', value: 60 }), + param({ name: 'dump.minDropPct', value: 4 }), + ]); + expect(built.pointCount).toBeLessThanOrEqual(ONBOARD_BATTERY_MAX_POINTS); + // name order: dump.minDropPct < maxHoldMin < watch.cooldownMin + expect(built.axes).toEqual(['dump.minDropPct', 'maxHoldMin']); + expect(built.pointCount).toBe(9); + }); + + it('is deterministic — identical input yields identical grid', () => { + const inp = [param({ name: 'maxHoldMin', value: 60 }), param({ name: 'dump.minDropPct', value: 4 })]; + expect(buildOnboardBatteryGrid(inp)).toEqual(buildOnboardBatteryGrid(inp)); + }); + + it('uses the default step/maxPoints constants', () => { + expect(ONBOARD_BATTERY_STEP).toBe(0.5); + expect(ONBOARD_BATTERY_MAX_POINTS).toBe(12); + }); +}); + +describe('summarizeOnboardBatteryRun', () => { + function completed(id: string, lonePeak: boolean): RankedPoint { + return { + point: { x: id }, paramsHash: id, status: 'completed', strategyBacktestRunId: id, + metrics: { totalTrades: 5, sharpe: 1, profitFactor: 1, maxDrawdownPct: 1, netPnlPct: 1 } as RankedPoint['metrics'], + lowConfidence: false, lonePeak, neighborCount: 2, + }; + } + function rejected(id: string): GridResult { + return { point: { x: id }, paramsHash: id, status: 'rejected', strategyBacktestRunId: id }; + } + + it('reports counts and lone-peak/plateau split — no magnitudes', () => { + const ranked = [completed('a', false), completed('b', true), completed('c', false)]; + const output: GridRunOutput = { + allResults: [...ranked, rejected('d')], + ranked, + submitted: 4, + rejected: 1, + }; + const summary = summarizeOnboardBatteryRun(output); + expect(summary).toEqual({ points: 4, completed: 3, rejected: 1, ranked: 3, lonePeak: 1, plateau: 2 }); + // guard: only counts/booleans leak — no sharpe/pnl keys + expect(JSON.stringify(summary)).not.toMatch(/sharpe|pnl|drawdown/i); + }); +}); diff --git a/src/research/onboard-battery.ts b/src/research/onboard-battery.ts new file mode 100644 index 0000000..22c5b2d --- /dev/null +++ b/src/research/onboard-battery.ts @@ -0,0 +1,172 @@ +/** + * R13 (research-validation-hardening item 6): the log-only "onboarding battery" — a small, + * DETERMINISTIC grid swept around a fresh strategy's baseline parameter values right after its + * baseline validation passes and BEFORE the first full WFO round. Its only jobs are to (a) seed + * the backtester's server-side trial ledger with a handful of neighbor points so the very first + * DSR/plateau evidence isn't a single lone run, and (b) surface lone-peak-vs-plateau evidence + * around the baseline as a log signal. It NEVER changes a verdict, status, or the + * strategy.onboard → baseline → wfo chain. + * + * NO LLM: the grid is built mechanically from the profile's tunable params and the + * `SWEEP_AXIS_CATALOG` (R13 Task 1) axis matchers — the sweep-designer agent is deliberately not + * called here (YAGNI; onboarding wants a cheap deterministic bracket, not a designed sweep). + * + * Denylisted axes (today: leverage/margin — no liquidation model in the engine) are excluded + * exactly like `validateSweepGrid`'s `denylisted_axis:*` gate, so a leverage param never enters + * the battery grid even if the profile marks it tunable. + * + * Size is intentionally tiny — see `ONBOARD_BATTERY_MAX_POINTS`. Each grid point is a real + * backtest submission; this stage pays for itself only if it stays cheap. + */ +import { expandGrid } from './param-grid.ts'; +import { isDenylistedParam, SWEEP_AXIS_CATALOG } from './sweep-axis-catalog.ts'; +import type { GridRunOutput } from './param-grid-runner.ts'; +import type { ParameterGrid } from '../domain/research-experiment.ts'; +import type { StrategyParameter } from '../domain/strategy-profile.ts'; + +/** Rollout mode. `enforce` intentionally absent — the onboarding battery is log-only by + * construction (it seeds the ledger and logs evidence; it never gates anything), mirroring + * `resolveBreakBatteryMode` / `resolveHypothesisHoldoutMode`. */ +export type OnboardBatteryMode = 'off' | 'log'; + +/** + * Fail-closed parser for LAB_ONBOARD_BATTERY_MODE (repo convention: a present-but-unrecognized + * value is a deploy typo, not a request for the default). Mirrors `resolveHypothesisHoldoutMode`. + * `enforce` is rejected EXPLICITLY: this stage has no enforce semantics — silently mapping it to + * `log` (or `off`) would misstate what the flag does. + */ +export function resolveOnboardBatteryMode(raw: string | undefined): OnboardBatteryMode { + if (raw === undefined || raw === '' || raw === 'off') return 'off'; + if (raw === 'log') return 'log'; + if (raw === 'enforce') { + throw new Error( + 'LAB_ONBOARD_BATTERY_MODE=enforce is not available — the onboarding battery is log-only by ' + + 'design (seeds the trial ledger, never changes a verdict); calibration/enforce is out of ' + + 'scope (battery-policy@1, control-center docs/architecture/battery-policy.md). Use off|log.', + ); + } + throw new Error(`LAB_ONBOARD_BATTERY_MODE must be one of off|log, got '${raw}'`); +} + +/** + * Conservative ceiling on the onboarding grid's cartesian size. 12 is deliberately small: each + * point is a real backtest run, this stage runs on EVERY fresh onboarding, and its purpose is a + * cheap bracket around the baseline — not an exhaustive sweep (that's the WFO's job). With + * ±STEP giving 3 values per axis, 12 admits at most 2 swept axes (3×3=9 ≤ 12; a 3rd axis would + * be 27 > 12), which the greedy name-sorted selection in `buildOnboardBatteryGrid` enforces. + */ +export const ONBOARD_BATTERY_MAX_POINTS = 12; + +/** Fractional perturbation applied on each side of a baseline value (±50%). Wide enough that the + * neighbor points can actually reveal a lone peak vs. a plateau, deterministic, unitless. */ +export const ONBOARD_BATTERY_STEP = 0.5; + +export interface OnboardBatteryGrid { + /** The grid handed to `ParamGridRunner.runGrid` (a `Record`). Empty `{}` + * when no eligible axis exists. */ + grid: ParameterGrid; + /** Param names actually swept (grid keys), sorted. Empty ⇒ nothing to run. */ + axes: string[]; + /** `expandGrid(grid).length` — 0 when `axes` is empty (an empty grid expands to one no-op + * point, which we must NOT run). */ + pointCount: number; +} + +/** True when a param name belongs to at least one NON-denylisted catalog axis. */ +function matchesNonDenylistedAxis(name: string): boolean { + return SWEEP_AXIS_CATALOG.some((axis) => axis.denylisted !== true && axis.matchesParam(name)); +} + +/** `[down, base, up]` around `v`, integer-preserving and deduped. Fewer than 2 distinct values + * (e.g. baseline 0, where both sides collapse to 0) ⇒ no sweep signal on this axis. */ +function perturbValues(v: number, step: number): number[] { + const preserveInt = Number.isInteger(v); + // Integer baselines stay integers; float baselines are rounded to 6 dp so the grid carries clean, + // deterministic values (2.4 * 1.5 === 3.5999999999999996 → 3.6) instead of FP noise into the + // paramsHash / trial ledger. + const norm = [v * (1 - step), v, v * (1 + step)].map((x) => + preserveInt ? Math.round(x) : Math.round(x * 1e6) / 1e6, + ); + const seen = new Set(); + const out: number[] = []; + for (const x of norm) { + if (!seen.has(x)) { + seen.add(x); + out.push(x); + } + } + return out; +} + +/** + * Builds the deterministic onboarding grid from a profile's tunable params. Selection rules: + * - param is `tunable`; + * - its baseline `value` is a finite number (string/bool/null params can't be bracketed + * numerically — out of scope, YAGNI); + * - it belongs to a non-denylisted catalog axis, AND is not denylisted (double guard — the + * leverage/margin axis never enters the grid); + * - the ±STEP bracket yields ≥ 2 distinct values. + * Eligible params are taken in NAME order, greedily, while the running cartesian product stays + * ≤ maxPoints (so a 3rd axis that would blow the ceiling is simply left out) — fully + * deterministic, no LLM, no RNG. + */ +export function buildOnboardBatteryGrid( + params: readonly StrategyParameter[] | undefined, + opts: { maxPoints?: number; step?: number } = {}, +): OnboardBatteryGrid { + const maxPoints = opts.maxPoints ?? ONBOARD_BATTERY_MAX_POINTS; + const step = opts.step ?? ONBOARD_BATTERY_STEP; + + const eligible = (params ?? []) + .filter((p) => p.tunable) + .filter((p): p is StrategyParameter & { value: number } => typeof p.value === 'number' && Number.isFinite(p.value)) + .filter((p) => matchesNonDenylistedAxis(p.name) && !isDenylistedParam(p.name)) + .slice() + .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + + const grid: ParameterGrid = {}; + const axes: string[] = []; + let product = 1; + for (const p of eligible) { + const values = perturbValues(p.value, step); + if (values.length < 2) continue; + const next = product * values.length; + if (next > maxPoints) break; + grid[p.name] = values; + axes.push(p.name); + product = next; + } + + const pointCount = axes.length === 0 ? 0 : expandGrid(grid, maxPoints).length; + return { grid, axes, pointCount }; +} + +/** Log-only summary of a battery run — COUNTS and boolean evidence only, never metric + * magnitudes (Outcome-Embargo spirit: no Sharpe/PnL leaks into events/artifacts here). */ +export interface OnboardBatterySummary { + /** Grid points submitted (= `expandGrid` length). */ + points: number; + /** Points that completed (a backtest with a result). */ + completed: number; + /** Points the runner rejected / left pending. */ + rejected: number; + /** Points that survived ranking (completed with > 0 trades). */ + ranked: number; + /** Ranked points flagged as a lone peak (overfit signature) — count only. */ + lonePeak: number; + /** Ranked points NOT flagged lone-peak (plateau-ish) — count only. */ + plateau: number; +} + +export function summarizeOnboardBatteryRun(output: GridRunOutput): OnboardBatterySummary { + const completed = output.allResults.filter((r) => r.status === 'completed').length; + const lonePeak = output.ranked.filter((r) => r.lonePeak).length; + return { + points: output.submitted, + completed, + rejected: output.rejected, + ranked: output.ranked.length, + lonePeak, + plateau: output.ranked.length - lonePeak, + }; +} diff --git a/test/support/make-services.ts b/test/support/make-services.ts index fc86e4b..e54d8ff 100644 --- a/test/support/make-services.ts +++ b/test/support/make-services.ts @@ -68,6 +68,7 @@ export function makeServices(overrides: Partial = {}): AppServices platformRunId: 'plat-revision-fake', }), }; + const paramGridRunner = new ParamGridRunner({ strategyRunExecutor }); const experimentService = new ExperimentService({ experiments: overrides.experiments ?? experiments, runTrades: overrides.runTrades ?? runTrades, @@ -87,7 +88,7 @@ export function makeServices(overrides: Partial = {}): AppServices gate1: new FakeGate1(), sweepDesigner: new FakeSweepDesigner(), resultInterpreter: new FakeResultInterpreter(), - paramGridRunner: new ParamGridRunner({ strategyRunExecutor }), + paramGridRunner, strategyBacktests, revisions, }); @@ -150,6 +151,8 @@ export function makeServices(overrides: Partial = {}): AppServices preservationThresholds: overrides.preservationThresholds ?? DEFAULT_PRESERVATION_THRESHOLDS, cycleScorecards: new InMemoryCycleScorecardRepository(), hypothesisHoldoutMode: overrides.hypothesisHoldoutMode ?? 'off', + paramGridRunner, + onboardBatteryMode: overrides.onboardBatteryMode ?? 'off', ...overrides, }; } From 8ee5dc510cec949ce0c3f84727e3e053fdc8e3fc Mon Sep 17 00:00:00 2001 From: Alexander Nikolskiy Date: Fri, 24 Jul 2026 22:54:59 +0300 Subject: [PATCH 6/6] =?UTF-8?q?docs(research):=20R13=20=E2=80=94=20=D0=BE?= =?UTF-8?q?=D1=82=D1=80=D0=B0=D0=B6=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B2=20roa?= =?UTF-8?q?dmap=20(=D1=81=D0=B4=D0=B5=D0=BB=D0=B0=D0=BD=D0=BE/=D1=80=D0=B5?= =?UTF-8?q?=D1=88=D0=B5=D0=BD=D0=B8=D1=8F/=D1=85=D0=B2=D0=BE=D1=81=D1=82)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...026-07-23-research-validation-hardening.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/roadmaps/2026-07-23-research-validation-hardening.md b/docs/roadmaps/2026-07-23-research-validation-hardening.md index e5bb402..d4f911c 100644 --- a/docs/roadmaps/2026-07-23-research-validation-hardening.md +++ b/docs/roadmaps/2026-07-23-research-validation-hardening.md @@ -147,6 +147,36 @@ Principle for every item: the verdict stays with deterministic versioned code (b) register every hypothesis as a family trial in the backtester trial ledger so FAIL/MODIFY retries and many-hypotheses selection get discounted via DSR automatically. +- **R13 (2026-07-24) — done.** Typed axis catalog `SWEEP_AXIS_CATALOG` + (`hold_time`, `entry_thresholds`, `stops_takes`, `cooldown`, `sizing`, + `regime_as_axis`, plus `leverage` denylisted) with a prompt export; the "wide + plateau, not peak" and "include the expected degradation point" rules plus + the leverage denylist are wired into the sweep-designer prompts + (`SWEEP_DESIGNER_INSTRUCTIONS`) and researcher-capabilities. A deterministic + `denylisted_axis:*` gate lands in `validateSweepGrid` (the denylist check + runs first, the four pre-existing reason codes are untouched). Onboarding + battery: flag `LAB_ONBOARD_BATTERY_MODE` (`off` default → `log`, fail-closed) + hooks into `strategy-baseline.handler` right before `strategy.wfo` is + enqueued — a deterministic ±50% grid around the baseline values, built from + the axis catalog (no LLM, capped at 12 points, axes picked greedily by name), + runs through `ParamGridRunner`; every point lands server-side in the trial + ledger, and a `strategy.onboard_battery.completed` event (counts + + `lonePeak`, no magnitudes) plus an artifact summary close the run. Fail-soft: + the baseline→wfo chain never breaks on battery failure. The + `hold_time`/`sizing` matchers were narrowed to remove threshold/resize + collisions. + + Decisions: the grid is built without an LLM (determinism over creativity at + this stage); at most 2 axes combine (3×3 ≤ 12 cap); the result surfaces via + event + artifact rather than a new migration; `enforce` is not implemented — + this is a log-only stage. + + Tail: `trialFamilyHint` is absent from the strategy lane — + `StrategyExperimentRunRequest` carries no such field (the wire contract was + left untouched), so onboarding-battery points land in the trial ledger + without a profile hint and are grouped by `moduleRef.id` + window instead — a + separate task if that grouping turns out to matter. Merging the battery's + plateau points into the first WFO round is deferred as YAGNI. - **R13 — sweep axis catalog + onboarding battery**: extend `src/mastra/agents/sweep-designer.agent.ts` (and researcher-capabilities) with a deterministically-checkable axis catalog (hold time, entry