Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions infrastructure/step_function.json
Original file line number Diff line number Diff line change
Expand Up @@ -393,14 +393,14 @@

"CheckSkipBacktester": {
"Type": "Choice",
"Comment": "Skip-gate. {\"skip_backtester\": true} bypasses the backtester step (which runs backtest + parity + evaluator as consolidated spot stages post-2026-04-24) and jumps directly to the Saturday health check. The legacy {\"skip_evaluator\": true} and {\"freeze_evaluator\": true} inputs are no longer honored; freeze-evaluator is now a spot CLI flag (--freeze-evaluator) invoked manually outside the SF.",
"Comment": "Skip-gate. {\"skip_backtester\": true} bypasses the backtester step (which runs backtest + parity + evaluator as consolidated spot stages post-2026-04-24) and jumps to the eval-judge skip-gate (preserving the eval-pipeline states downstream — the bug fixed 2026-05-03 was that this routed directly to SaturdayHealthCheck, silently bypassing CheckSkipEvalJudge → ComputeEvalCadence → eval-judge → eval-rolling-mean for any operator who passed skip_backtester=true). The legacy {\"skip_evaluator\": true} and {\"freeze_evaluator\": true} inputs are no longer honored; freeze-evaluator is now a spot CLI flag (--freeze-evaluator) invoked manually outside the SF.",
"Choices": [
{
"And": [
{"Variable": "$.skip_backtester", "IsPresent": true},
{"Variable": "$.skip_backtester", "BooleanEquals": true}
],
"Next": "SaturdayHealthCheck"
"Next": "CheckSkipEvalJudge"
}
],
"Default": "Backtester"
Expand Down
22 changes: 22 additions & 0 deletions tests/test_sf_eval_judge_wiring.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,28 @@ def test_success_routes_to_eval_skip_gate(self, states):
# ── Skip gate ─────────────────────────────────────────────────────────────


class TestSkipBacktesterPreservesEvalJudge:
"""Pins the 2026-05-03 fix: when an operator passes
skip_backtester=true (e.g. for SF skip-most validation runs), the
skip path must still flow through the eval-judge skip-gate, not
leap straight to SaturdayHealthCheck.

Caught by SF eval-pipeline-validation-5 when Research succeeded
+ new-format captures landed on S3 but the eval-judge state
silently never fired because skip_backtester=true had been
short-circuiting past it.
"""

def test_skip_backtester_routes_to_eval_skip_gate(self, states):
skip = states["CheckSkipBacktester"]
choice = skip["Choices"][0]
# The skip-true branch must preserve the eval-judge state path.
assert choice["Next"] == "CheckSkipEvalJudge"
# Critically NOT routed to SaturdayHealthCheck — that was the
# silent-bypass bug.
assert choice["Next"] != "SaturdayHealthCheck"


class TestSkipEvalJudge:
def test_skip_flag_bypasses_to_health_check(self, states):
skip = states["CheckSkipEvalJudge"]
Expand Down
Loading