Fix deferred tool discovery and Harbor deadline handling

eval/harbor/README.md

@@ -78,6 +78,9 @@ PYTHONPATH=$PWD/eval/harbor harbor run \
 ```
 
 Notes:
+- Official Terminal-Bench submissions may not modify task timeouts or
+  resources. Keep Harbor's default timeout policy for leaderboard-comparable
+  runs; use timeout multipliers only for explicitly labeled diagnostics.
 - `effort=high` maps to `clawcodex --effort high` →
   `output_config.effort` on effort-capable models (Opus 4.6/4.8,
   Sonnet 4.6, Fable 5). Requires clawcodex > 1.2.1 in the container —

eval/harbor/clawcodex_agent.py

@@ -86,6 +86,7 @@
     ToolCall,
     Trajectory,
 )
+from time_budget import build_deadline_prompt, resolve_agent_timeout_seconds
 
 # Host env vars forwardable into the container (clawcodex's builtin provider
 # key candidates — see src/providers/__init__.py in the clawcodex repo), keyed
@@ -157,19 +158,21 @@ def _save_host_oauth(credentials: dict[str, Any]) -> None:
     os.replace(tmp, path)
 
 
-# Refresh when under this much runway. Must exceed the longest agent
-# timeout (terminal-bench tasks: 900s) with margin, because containers get
-# an access token WITHOUT a refresh token (see the injection notes) and so
-# cannot refresh mid-trial.
-_MIN_TOKEN_RUNWAY_SEC = 1800
+# Refresh when under this much runway. Containers intentionally receive an
+# access token WITHOUT a refresh token (see the injection notes), so the
+# token must outlive the whole trial. Long problem-solving evaluations can
+# legitimately run beyond the benchmark's original 15-60 minute budgets;
+# keep two hours available so callers can raise Harbor's agent timeout
+# without introducing an unrelated mid-trial authentication failure.
+_MIN_TOKEN_RUNWAY_SEC = 2 * 60 * 60
 
 
 def fresh_subscription_credentials() -> dict[str, Any]:
     """Host-side equivalent of clawcodex's ``get_valid_credentials``.
 
     Returns the credentials dict, refreshing (and persisting back to the
     host file) when the access token has under ``_MIN_TOKEN_RUNWAY_SEC`` of
-    runway, so every trial starts with more runway than its agent timeout.
+    runway, so long-running trials do not inherit a nearly-expired token.
     Raises RuntimeError with a remedial message when no usable credentials
     exist.
     """
@@ -425,19 +428,30 @@ async def _inject_subscription_credentials(
             env={"CLAWCODEX_OAUTH_JSON": json.dumps(credentials)},
         )
 
-    async def _seed_container_settings(self, environment: BaseEnvironment) -> None:
-        """Seed ``settings.effort`` in the container's global config.
+    async def _seed_container_settings(
+        self,
+        environment: BaseEnvironment,
+        *,
+        deadline_prompt: str | None = None,
+    ) -> None:
+        """Seed session-wide effort and deadline guidance in global config.
 
         clawcodex's ``--effort`` flag governs the MAIN loop only; subagents
         (Agent tool) resolve effort from ``settings.effort``. Seeding the
         container's global config (home-anchored ``~/.clawcodex/config.json``
         — the global-config path deliberately does not follow
-        CLAWCODEX_CONFIG_DIR) makes the requested effort session-wide.
+        CLAWCODEX_CONFIG_DIR) makes the requested effort session-wide and
+        appends Harbor's real wall-clock deadline to the model instructions.
         """
+        settings: dict[str, Any] = {}
         effort = self._resolved_flags.get("effort")
-        if not effort:
+        if effort:
+            settings["effort"] = effort
+        if deadline_prompt:
+            settings["append_system_prompt"] = deadline_prompt
+        if not settings:
             return
-        payload = json.dumps({"settings": {"effort": effort}})
+        payload = json.dumps({"settings": settings})
         await self.exec_as_agent(
             environment,
             command=(
@@ -460,7 +474,19 @@ async def run(
         self._captured_instruction = instruction
         if self._subscription:
             await self._inject_subscription_credentials(environment)
-        await self._seed_container_settings(environment)
+        timeout_seconds = resolve_agent_timeout_seconds(
+            environment.environment_dir.parent / "task.toml",
+            environment.trial_paths.lock_path,
+        )
+        deadline_prompt = (
+            build_deadline_prompt(timeout_seconds)
+            if timeout_seconds is not None
+            else None
+        )
+        await self._seed_container_settings(
+            environment,
+            deadline_prompt=deadline_prompt,
+        )
 
         parts: list[str] = [
             "clawcodex",

eval/harbor/time_budget.py

@@ -0,0 +1,61 @@
+"""Deadline helpers shared by Harbor adapters.
+
+Harbor owns the agent-phase timeout but does not pass it to ``Agent.run``.
+The resolved inputs are nevertheless available beside the environment and
+trial, so adapters can make the deadline visible to an otherwise unaware
+agent without encoding dataset or task names.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+import tomllib
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def resolve_agent_timeout_seconds(task_toml: Path, lock_json: Path) -> float | None:
+    """Return Harbor's effective agent timeout from resolved trial inputs."""
+    try:
+        task = tomllib.loads(task_toml.read_text(encoding="utf-8"))
+        lock = json.loads(lock_json.read_text(encoding="utf-8"))
+        base = float(task["agent"]["timeout_sec"])
+        multiplier = lock.get("agent_timeout_multiplier")
+        if multiplier is None:
+            multiplier = lock.get("timeout_multiplier", 1.0)
+        effective = base * float(multiplier)
+    except (KeyError, OSError, TypeError, ValueError, json.JSONDecodeError):
+        return None
+    return effective if effective > 0 else None
+
+
+def build_deadline_prompt(
+    timeout_seconds: float,
+    *,
+    started_at: float | None = None,
+) -> str:
+    """Build model-neutral guidance for a real external execution deadline."""
+    start = time.time() if started_at is None else started_at
+    deadline = start + timeout_seconds
+    # Reserve 15%, bounded so short tasks still get two minutes and very long
+    # tasks do not abandon productive work excessively early.
+    reserve = min(10 * 60, max(2 * 60, timeout_seconds * 0.15))
+    finalize_at = deadline - reserve
+
+    def stamp(value: float) -> str:
+        return datetime.fromtimestamp(value, tz=timezone.utc).isoformat(
+            timespec="seconds"
+        )
+
+    return (
+        "This run has a hard external execution deadline at "
+        f"{stamp(deadline)} ({timeout_seconds / 60:.1f} minutes from start). "
+        f"By {stamp(finalize_at)}, preserve the best valid deliverable, stop "
+        "broad exploration, and switch to the narrowest checks needed for the "
+        "explicit requirements. Do not start optional audits, repeated passing "
+        "checks, or long refinements that cannot finish before the deadline. "
+        "If the core result already works, finish and return control rather "
+        "than consuming the remaining budget. You can use `date -u` to compare "
+        "the current time with these timestamps."
+    )

src/tool_system/tool_search.py

@@ -275,7 +275,16 @@ def extract_discovered_tool_names(messages: list[Any]) -> set[str]:
     discovered: set[str] = set()
 
     for msg in messages:
-        msg_type = getattr(msg, "type", None) or (msg.get("type") if isinstance(msg, dict) else None)
+        msg_type = getattr(msg, "type", None)
+        if isinstance(msg, dict):
+            # Internal Message objects use ``type`` while provider-ready API
+            # messages use ``role``.  _call_model_sync passes the latter into
+            # filter_tools_for_request(), so ignoring ``role`` made every
+            # freshly returned tool_reference invisible on the next request.
+            # The reference remained in history but its schema stayed
+            # filtered out, causing providers to reject the dangling
+            # reference as an invalid request.
+            msg_type = msg_type or msg.get("type") or msg.get("role")
 
         # Compact boundary carries pre-compact discovered set
         if msg_type == "system":

src/tool_system/tools/tool_search.py

@@ -31,11 +31,22 @@ def _map_result_to_api(output: Any, tool_use_id: str) -> dict[str, Any]:
 
 
 def make_tool_search_tool(registry: ToolRegistry) -> Tool:
+    def _is_available_tool(tool: Tool | None) -> bool:
+        """Only advertise tools that can be present on the next request."""
+        if tool is None:
+            return False
+        try:
+            return bool(tool.is_enabled())
+        except Exception:
+            # A broken runtime gate must not produce a reference whose schema
+            # the request builder will subsequently omit.
+            return False
+
     def _deferred_count() -> int:
         return sum(
             1
             for tool in registry.list_tools()
-            if tool.should_defer or tool.is_mcp
+            if (tool.should_defer or tool.is_mcp) and _is_available_tool(tool)
         )
 
     def _tool_search_call(tool_input: dict[str, Any], context: ToolContext) -> ToolResult:
@@ -51,7 +62,7 @@ def _tool_search_call(tool_input: dict[str, Any], context: ToolContext) -> ToolR
         if lowered.startswith("select:"):
             name = q.split(":", 1)[1].strip()
             tool = registry.get(name)
-            matches = [tool.name] if tool else []
+            matches = [tool.name] if _is_available_tool(tool) else []
             return ToolResult(
                 name="ToolSearch",
                 output={
@@ -63,6 +74,8 @@ def _tool_search_call(tool_input: dict[str, Any], context: ToolContext) -> ToolR
 
         scored: list[tuple[int, str]] = []
         for t in registry.list_tools():
+            if not _is_available_tool(t):
+                continue
             hay = f"{t.name}\n{t.prompt()}".lower()
             if lowered in t.name.lower():
                 scored.append((0, t.name))

tests/test_harbor_time_budget.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+
+sys.path.insert(0, str(Path(__file__).parents[1] / "eval" / "harbor"))
+
+from time_budget import build_deadline_prompt, resolve_agent_timeout_seconds
+
+
+def test_resolve_agent_timeout_uses_agent_multiplier(tmp_path: Path) -> None:
+    task = tmp_path / "task.toml"
+    lock = tmp_path / "lock.json"
+    task.write_text("[agent]\ntimeout_sec = 900\n", encoding="utf-8")
+    lock.write_text(
+        json.dumps(
+            {
+                "timeout_multiplier": 3,
+                "agent_timeout_multiplier": 2,
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    assert resolve_agent_timeout_seconds(task, lock) == 1800
+
+
+def test_resolve_agent_timeout_falls_back_to_global_multiplier(
+    tmp_path: Path,
+) -> None:
+    task = tmp_path / "task.toml"
+    lock = tmp_path / "lock.json"
+    task.write_text("[agent]\ntimeout_sec = 1200\n", encoding="utf-8")
+    lock.write_text('{"timeout_multiplier": 1.5}', encoding="utf-8")
+
+    assert resolve_agent_timeout_seconds(task, lock) == 1800
+
+
+def test_deadline_prompt_reserves_finalization_time() -> None:
+    prompt = build_deadline_prompt(1800, started_at=0)
+
+    assert "30.0 minutes from start" in prompt
+    assert "1970-01-01T00:25:30+00:00" in prompt
+    assert "preserve the best valid deliverable" in prompt
+    assert "repeated passing checks" in prompt
+
+
+def test_invalid_timeout_inputs_disable_attachment(tmp_path: Path) -> None:
+    assert (
+        resolve_agent_timeout_seconds(
+            tmp_path / "missing-task.toml",
+            tmp_path / "missing-lock.json",
+        )
+        is None
+    )

tests/test_tool_search.py

@@ -27,16 +27,25 @@
     build_default_registry,
 )
 from src.tool_system.protocol import ToolCall
+from src.tool_system.registry import ToolRegistry
+from src.tool_system.tools.tool_search import make_tool_search_tool
 
 
-def _make_tool(name: str, *, should_defer: bool = False, is_mcp: bool = False):
+def _make_tool(
+    name: str,
+    *,
+    should_defer: bool = False,
+    is_mcp: bool = False,
+    is_enabled=None,
+):
     return build_tool(
         name=name,
         input_schema={"type": "object", "properties": {}},
         call=lambda i, c: None,
         prompt=f"Tool {name}",
         should_defer=should_defer,
         is_mcp=is_mcp,
+        is_enabled=is_enabled,
     )
 
 
@@ -181,6 +190,20 @@ def test_tool_reference_in_user_message(self):
         result = extract_discovered_tool_names(msgs)
         self.assertIn("mcp__tool_x", result)
 
+    def test_tool_reference_in_provider_role_message(self):
+        """Provider-ready history uses role=user, not type=user."""
+        msgs = [{
+            "role": "user",
+            "content": [{
+                "type": "tool_result",
+                "tool_use_id": "123",
+                "content": [
+                    {"type": "tool_reference", "tool_name": "Grep"},
+                ],
+            }],
+        }]
+        self.assertEqual(extract_discovered_tool_names(msgs), {"Grep"})
+
     def test_no_tool_reference(self):
         msgs = [{
             "type": "user",
@@ -244,6 +267,33 @@ def test_discovered_tools_kept(self):
             names = [t.name for t in result]
             self.assertIn("mcp_tool", names)
 
+    def test_provider_role_history_loads_discovered_tool(self):
+        """Regression: real API messages use role and previously lost refs."""
+        with patch.dict(os.environ, {
+            "ENABLE_TOOL_SEARCH": "true",
+            "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS": "",
+        }):
+            tools = [
+                _make_tool("Read"),
+                _make_tool("ToolSearch"),
+                _make_tool("Grep", should_defer=True),
+            ]
+            messages = [{
+                "role": "user",
+                "content": [{
+                    "type": "tool_result",
+                    "tool_use_id": "search-1",
+                    "content": [
+                        {"type": "tool_reference", "tool_name": "Grep"},
+                    ],
+                }],
+            }]
+            result = filter_tools_for_request(
+                tools, "claude-opus-4-8", messages=messages,
+            )
+
+        self.assertIn("Grep", {tool.name for tool in result})
+
     def test_missing_tool_search_keeps_deferred_tools_reachable(self):
         with patch.dict(os.environ, {
             "ENABLE_TOOL_SEARCH": "true",
@@ -306,6 +356,29 @@ def test_tool_search_result_loads_deferred_schema_next_request(self):
 
         self.assertIn("Grep", {tool.name for tool in loaded})
 
+    def test_tool_search_does_not_reference_disabled_tool(self):
+        registry = ToolRegistry([
+            _make_tool(
+                "TodoWrite",
+                should_defer=True,
+                is_enabled=lambda: False,
+            ),
+        ])
+        registry.register(make_tool_search_tool(registry))
+
+        result = registry.dispatch(
+            ToolCall(
+                name="ToolSearch",
+                input={"query": "select:TodoWrite"},
+                tool_use_id="search-1",
+            ),
+            ToolContext(workspace_root=Path.cwd()),
+        )
+        search_tool = registry.get("ToolSearch")
+        assert search_tool is not None
+        block = search_tool.map_result_to_api(result.output, "search-1")
+
+        self.assertEqual(block["content"], "No matching deferred tools found")
 
 if __name__ == "__main__":
     unittest.main()