diff --git a/src/tool_system/tools/tasks_v2.py b/src/tool_system/tools/tasks_v2.py index 5cb8197c..7adf07c8 100644 --- a/src/tool_system/tools/tasks_v2.py +++ b/src/tool_system/tools/tasks_v2.py @@ -541,6 +541,148 @@ def _task_update_call(tool_input: dict[str, Any], context: ToolContext) -> ToolR ) +# A stall must survive this many *additional* blocking polls past the first +# (which only records the baseline) before we nudge. So the nudge fires on the +# 4th identical poll — ~2 min at the 30s default — paired with a wall-clock +# floor below so a model using tiny timeouts can't trip it in seconds. +_STUCK_POLL_THRESHOLD = 3 +# The stall must also have persisted at least this long in wall-clock, so +# ``block=True, timeout=1000`` (1s polls) cannot fire the guard on a job that +# has barely started. +_STUCK_MIN_WALL_SECONDS = 60.0 + + +def _bg_output_size(entry: dict[str, Any]) -> int: + """Monotonic byte size of a background task's log, or -1 if unknowable. + + The FILE size is the progress signal, deliberately not the length of the + ``output`` string the poll returned: ``read_background_output`` caps that + at the last 200KB and re-seeks, so a chatty, actively-progressing job's + reported output plateaus near the cap and its length goes constant even as + the job writes. Length would then read as "stuck" and falsely nudge it. + The on-disk size keeps growing past the cap, so it stays a true progress + signal for high-volume jobs. (Critic M1, 2026-07-27.) + """ + path = entry.get("output_path") + if not path: + return -1 + try: + from pathlib import Path as _P + + return _P(str(path)).stat().st_size + except OSError: + return -1 + + +def _guard_repeated_polls( + task_id: str, + result: ToolResult, + context: ToolContext, +) -> ToolResult: + """Nudge the model to stop when it keeps polling a stalled background task. + + A background command that cannot finish on its own — an unbounded search, + a brute-force, a server with no timeout — turns ``TaskOutput`` into an + absorbing state: the model polls, gets "still running, no new output", + polls again, and never reconsiders. On terminal-bench 2.1 crack-7z-hash + (deepseek-v4-pro, 2026-07-27) an agent launched an incremental + john-the-ripper brute-force in the background, then spent its final 20 + minutes and ~15 consecutive polls waiting on it until the harness killed + the trial at 30 minutes. opus-5 finished the same task in 18 seconds. + + Model-agnostic and tool-level. For each background bash task, count + consecutive *blocking* polls (``retrieval_status="timeout"``, task still + running) whose on-disk log size did not grow. Once that count reaches + ``_STUCK_POLL_THRESHOLD`` AND the stall has lasted ``_STUCK_MIN_WALL_SECONDS``, + prepend a hint pointing at TaskStop. The hint repeats on every further + stuck poll, so a model that ignores it once still sees it. Any growth in + the log resets the counter — a slow but progressing job is never nudged, + including one whose returned output has saturated the 200KB read window. + + The hint is PREPENDED (not appended) and also exposed as a top-level + ``stuck_task_hint`` field, so it survives large-result persistence: an + output over ~50KB is replaced with a disk pointer plus a HEAD preview + (``content[:max_bytes]``), which would truncate a tail-appended note away. + (Critic M2, 2026-07-27.) + + Both poll modes count. This is deliberate and load-bearing: on the + crack-7z-hash trajectory, 13 of the 15 polls were ``block=False`` (which + returns ``retrieval_status="success"`` on a still-running task, not + ``"timeout"``). Gating on the timeout status alone — the obvious reading — + would make the guard inert for the exact failure it exists to catch. The + signal that matters is "task still running, log not growing", regardless + of how the model polled. Counting fast ``block=False`` polls is safe only + because ``_STUCK_MIN_WALL_SECONDS`` gates on elapsed time, not poll count, + so a burst of instant non-blocking polls cannot fire the nudge on a job + that just started. + + Agent subtasks (``LocalAgent``) are excluded: a "no new output" poll there + is normal, not stuck. + """ + import time + + out = result.output + if not isinstance(out, dict): + return result + task = out.get("task") + if not isinstance(task, dict): + return result + if task.get("task_type") != "bash_background": + return result + + entry = context.background_bash_tasks.get(task_id) + if not isinstance(entry, dict): + return result + + # A terminal task needs no nudge — the model will stop polling naturally. + # Reset so a later stall on a reused id starts fresh. Any non-running + # status lands here regardless of retrieval_status. + if task.get("status") != "running": + entry["_stuck_polls"] = 0 + entry.pop("_stuck_since", None) + return result + + cur_size = _bg_output_size(entry) + prev_size = entry.get("_stuck_last_size") + if prev_size is not None and cur_size == prev_size: + entry["_stuck_polls"] = int(entry.get("_stuck_polls", 0)) + 1 + entry.setdefault("_stuck_since", time.monotonic()) + else: + entry["_stuck_polls"] = 0 + entry["_stuck_last_size"] = cur_size + entry.pop("_stuck_since", None) + + stalled_for = time.monotonic() - entry.get("_stuck_since", time.monotonic()) + if ( + entry.get("_stuck_polls", 0) < _STUCK_POLL_THRESHOLD + or stalled_for < _STUCK_MIN_WALL_SECONDS + ): + return result + + polls = entry["_stuck_polls"] + 1 # +1: the first poll set the baseline + hint = ( + f"[stuck-task guard] This background task has been polled {polls} " + f"times with no new output for ~{stalled_for / 60:.0f} min and is " + f"still running. If it may not terminate on its own — an unbounded " + f"search, a brute-force, a wait for something that will not arrive — " + f"stop it with TaskStop(task_id={task_id!r}) and take a different " + f"approach instead of polling again. If it is genuinely making slow " + f"progress, keep waiting." + ) + # Copy so the stored snapshot / other readers are untouched. Prepend to the + # output AND surface a top-level field, so the note lands in the head + # preview if this result is persisted for size. + new_task = dict(task) + new_task["output"] = hint + "\n\n" + str(task.get("output") or "") + # ``stuck_task_hint`` FIRST so it heads the serialized JSON: a >50KB result + # is persisted to disk with a head preview (``content[:max_bytes]``), and a + # leading key survives that where a trailing one would not. The prepend + # into ``task.output`` is the second line of defense for the in-context + # (un-persisted) read. + new_out = {"stuck_task_hint": hint, **out, "task": new_task} + return ToolResult(name=result.name, output=new_out) + + async def _task_output_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult: """Return the current state / output of a runtime task or todo. @@ -583,8 +725,12 @@ async def _task_output_call(tool_input: dict[str, Any], context: ToolContext) -> runtime = context.runtime_tasks.get(task_id) if runtime is not None: if not block: - return _runtime_task_to_output(task_id, runtime, context) - return await _poll_runtime_until_terminal(task_id, timeout_seconds, context) + result = _runtime_task_to_output(task_id, runtime, context) + else: + result = await _poll_runtime_until_terminal( + task_id, timeout_seconds, context + ) + return _guard_repeated_polls(task_id, result, context) # Branch 2 — TaskCreate / tasks_v2 todos. Same key space, different # semantics; no polling — output text is set or not at the moment diff --git a/tests/test_task_output_stuck_guard.py b/tests/test_task_output_stuck_guard.py new file mode 100644 index 00000000..9bf57024 --- /dev/null +++ b/tests/test_task_output_stuck_guard.py @@ -0,0 +1,241 @@ +"""The stuck-background-task guard on TaskOutput. + +A background command that cannot finish on its own turns TaskOutput into an +absorbing state: the model polls "still running, no new output" forever. On +terminal-bench 2.1 crack-7z-hash (deepseek-v4-pro, 2026-07-27) an agent +launched a john-the-ripper brute-force in the background and spent its final +~15 polls / 20 minutes waiting on it until the harness killed the trial; +opus-5 finished the same task in 18 seconds. + +The guard nudges toward TaskStop after a run of no-progress polls, and must +not fire on progressing jobs (M1: even ones whose reported output has +saturated the 200KB read window), finished jobs, non-blocking polls, or +agent subtasks. The hint must survive large-result persistence (M2). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from src.tool_system.context import ToolContext +from src.tool_system.protocol import ToolResult +from src.tool_system.tools import tasks_v2 +from src.tool_system.tools.tasks_v2 import ( + _STUCK_MIN_WALL_SECONDS, + _STUCK_POLL_THRESHOLD, + _guard_repeated_polls, +) + + +@pytest.fixture(autouse=True) +def _no_wall_floor(monkeypatch): + """Drop the wall-clock floor to 0 so poll-count logic is testable directly. + + The floor itself is covered by TestWallClockFloor with the real value. + """ + monkeypatch.setattr(tasks_v2, "_STUCK_MIN_WALL_SECONDS", 0.0) + + +def _ctx(tmp_path: Path) -> ToolContext: + ctx = ToolContext(workspace_root=tmp_path) + log = tmp_path / "b1.log" + log.write_bytes(b"") + ctx.background_bash_tasks["b1"] = {"output_path": str(log)} + return ctx + + +def _set_log(ctx: ToolContext, nbytes: int) -> None: + """Set the on-disk log to a given size — the guard's progress signal.""" + Path(ctx.background_bash_tasks["b1"]["output_path"]).write_bytes(b"x" * nbytes) + + +def _running(output: str = "") -> ToolResult: + """A poll that timed out with the task still running (the stuck shape).""" + return ToolResult( + name="TaskOutput", + output={ + "retrieval_status": "timeout", + "task": { + "task_id": "b1", + "task_type": "bash_background", + "status": "running", + "output": output, + }, + }, + ) + + +def _hint_in(result: ToolResult) -> bool: + return "stuck-task guard" in str(result.output.get("task", {}).get("output", "")) + + +class TestStuckGuardFires: + def test_nudges_after_threshold_polls_with_no_growth(self, tmp_path): + ctx = _ctx(tmp_path) + _set_log(ctx, 100) # silent job: size never changes + results = [_guard_repeated_polls("b1", _running(), ctx) + for _ in range(_STUCK_POLL_THRESHOLD + 2)] + assert not any(_hint_in(r) for r in results[:_STUCK_POLL_THRESHOLD]) + assert _hint_in(results[_STUCK_POLL_THRESHOLD]) + + def test_hint_recommends_taskstop_with_the_id(self, tmp_path): + ctx = _ctx(tmp_path) + _set_log(ctx, 50) + for _ in range(_STUCK_POLL_THRESHOLD + 2): + out = _guard_repeated_polls("b1", _running("frozen"), ctx) + text = out.output["task"]["output"] + assert "TaskStop(task_id='b1')" in text + assert "frozen" in text, "the real output must be preserved, not replaced" + + def test_hint_is_prepended_and_also_top_level(self, tmp_path): + """M2: survive head-truncation of a persisted large result.""" + ctx = _ctx(tmp_path) + _set_log(ctx, 50) + for _ in range(_STUCK_POLL_THRESHOLD + 2): + out = _guard_repeated_polls("b1", _running("REAL_OUTPUT"), ctx) + text = out.output["task"]["output"] + assert text.startswith("[stuck-task guard]"), "hint must be at the head" + assert out.output["stuck_task_hint"].startswith("[stuck-task guard]") + + def test_hint_repeats_so_it_cannot_be_missed_once(self, tmp_path): + ctx = _ctx(tmp_path) + _set_log(ctx, 10) + for _ in range(_STUCK_POLL_THRESHOLD + 4): + out = _guard_repeated_polls("b1", _running("x"), ctx) + assert _hint_in(out), "the nudge must persist across further stuck polls" + + +class TestStuckGuardStaysQuiet: + def test_progressing_job_is_never_nudged(self, tmp_path): + ctx = _ctx(tmp_path) + for i in range(_STUCK_POLL_THRESHOLD + 5): + _set_log(ctx, 100 * (i + 1)) # file grows every poll + out = _guard_repeated_polls("b1", _running("line"), ctx) + assert not _hint_in(out) + + def test_saturated_output_but_growing_file_is_not_nudged(self, tmp_path): + """M1: reported output length is constant (200KB cap) but the file + grows — a chatty progressing job. Must NOT nudge.""" + ctx = _ctx(tmp_path) + capped = "y" * 200_000 # what the poll returns, unchanged each time + for i in range(_STUCK_POLL_THRESHOLD + 5): + _set_log(ctx, 200_000 + 5_000 * i) # underlying file keeps growing + out = _guard_repeated_polls("b1", _running(capped), ctx) + assert not _hint_in(out), "file growth means progress, not stuck" + + def test_counter_resets_when_file_advances(self, tmp_path): + ctx = _ctx(tmp_path) + sizes = [100, 100, 200, 200] # two stalls, growth, two stalls — never 3 + for s in sizes: + _set_log(ctx, s) + out = _guard_repeated_polls("b1", _running("o"), ctx) + assert not _hint_in(out) + + def test_finished_task_is_not_nudged(self, tmp_path): + ctx = _ctx(tmp_path) + _set_log(ctx, 100) + done = ToolResult( + name="TaskOutput", + output={ + "retrieval_status": "success", + "task": {"task_id": "b1", "task_type": "bash_background", + "status": "completed", "output": "result"}, + }, + ) + for _ in range(_STUCK_POLL_THRESHOLD + 2): + out = _guard_repeated_polls("b1", done, ctx) + assert not _hint_in(out) + + def test_non_blocking_polls_DO_count(self, tmp_path): + """block=False returns retrieval_status="success" on a running task, + but the guard MUST still count it — 13 of the 15 crack-7z-hash polls + were block=False, so gating on the timeout status would make the guard + inert for the exact failure it exists to catch. + """ + ctx = _ctx(tmp_path) + _set_log(ctx, 100) + nb = ToolResult( + name="TaskOutput", + output={ + "retrieval_status": "success", # what block=False returns + "task": {"task_id": "b1", "task_type": "bash_background", + "status": "running", "output": ""}, + }, + ) + results = [_guard_repeated_polls("b1", nb, ctx) + for _ in range(_STUCK_POLL_THRESHOLD + 2)] + assert _hint_in(results[_STUCK_POLL_THRESHOLD]), ( + "block=False polls on a stuck job must fire the guard" + ) + + def test_agent_subtask_is_not_guarded(self, tmp_path): + """A local-agent poll with no new output is normal, not stuck. + + The entry IS registered so the type gate — not a missing-entry + short-circuit — is what keeps it quiet. + """ + ctx = _ctx(tmp_path) + log = tmp_path / "a1.log" + log.write_bytes(b"data") + ctx.background_bash_tasks["a1"] = {"output_path": str(log)} + agent_poll = ToolResult( + name="TaskOutput", + output={ + "retrieval_status": "not_ready", + "task": {"task_id": "a1", "task_type": "local_agent", + "status": "running", "output": ""}, + }, + ) + for _ in range(_STUCK_POLL_THRESHOLD + 2): + out = _guard_repeated_polls("a1", agent_poll, ctx) + assert "stuck-task guard" not in str(out.output) + + +class TestWallClockFloor: + def test_tight_polls_do_not_fire_before_the_floor(self, tmp_path, monkeypatch): + """m4: block=True with a 1s timeout can hit the poll count in seconds; + the wall-clock floor must hold the nudge back.""" + monkeypatch.setattr(tasks_v2, "_STUCK_MIN_WALL_SECONDS", 60.0) + ctx = _ctx(tmp_path) + _set_log(ctx, 100) + # Many polls, but no monotonic-time advance in the test → 0 wall seconds. + for _ in range(_STUCK_POLL_THRESHOLD + 6): + out = _guard_repeated_polls("b1", _running(), ctx) + assert not _hint_in(out), "poll count reached but wall floor not met" + + +class TestHintSurvivesPersistence: + """M2 end-to-end: a >50KB guarded result is persisted with a head + preview; the hint must land in what the model still sees inline.""" + + def test_hint_survives_large_result_persistence(self, tmp_path, monkeypatch): + import json + from src.services.tool_execution.tool_result_persistence import ( + maybe_persist_large_tool_result, + ) + + monkeypatch.setattr(tasks_v2, "_STUCK_MIN_WALL_SECONDS", 0.0) + ctx = _ctx(tmp_path) + _set_log(ctx, 300) # silent brute-force: constant size + big = "Z" * 200_000 # what the poll returns — far over the 50K threshold + for _ in range(_STUCK_POLL_THRESHOLD + 2): + out = _guard_repeated_polls("b1", _running(big), ctx) + + serialized = json.dumps(out.output) + assert len(serialized) > 50_000, "precondition: result is large enough to persist" + + block = {"type": "tool_result", "content": serialized} + persisted = maybe_persist_large_tool_result( + block, + "TaskOutput", + threshold=50_000, + tool_results_dir=tmp_path / "tr", + ) + rendered = json.dumps(persisted) + assert "stuck-task guard" in rendered, ( + "the nudge was truncated away by head-preview — the model would " + "poll on, defeating the guard on exactly the large-output jobs" + ) + assert "TaskStop(task_id='b1')" in rendered