diff --git a/agent/agent.py b/agent/agent.py index 9f5f8df..07d5ecf 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -1,6 +1,7 @@ """OpenTag's triage-first Deep Agent.""" import os +from collections.abc import Mapping from pathlib import Path from copilotkit import CopilotKitMiddleware @@ -30,6 +31,21 @@ ) VALID_VERBOSITY_LEVELS = frozenset({"low", "medium", "high"}) +# Super-steps the graph may take in one turn -- NOT a tool-call budget. The +# middleware chain makes one tool call cost four super-steps (model -> +# TodoListMiddleware.after_model -> CopilotKitMiddleware.after_model -> tools) +# on top of a six-step baseline, so the usable budget is about (limit - 6) / 4. +# +# Measured, not estimated: see test_one_tool_call_costs_four_super_steps and +# test_the_old_limit_allowed_only_four_tool_calls. The previous value of 25 +# completed at most FOUR tool calls in a turn, which is why research questions +# died mid-answer. 60 completes thirteen. +DEFAULT_RECURSION_LIMIT = 60 + +# Below this the agent cannot complete a single tool call and every research +# turn dies; treat it as a misconfiguration rather than a tuning choice. +MIN_RECURSION_LIMIT = 10 + # Deep Agents adds shell execution and a general-purpose delegation tool by # default. OpenTag has no sandbox for execute, and delegating routine turns to # another agent adds latency without improving triage. @@ -56,6 +72,36 @@ def _validated_openai_setting( return value +# A turn with no tool calls costs six super-steps, and each tool call four. +# Both measured; see test_one_tool_call_costs_four_super_steps. +_TURN_BASELINE_STEPS = 6 +_STEPS_PER_TOOL_CALL = 4 + + +def tool_call_budget(limit: int) -> int: + """Tool calls a super-step limit actually buys, which is what people mean.""" + return max(0, (limit - _TURN_BASELINE_STEPS) // _STEPS_PER_TOOL_CALL) + + +def recursion_limit(env: Mapping[str, str] = os.environ) -> int: + """Read AGENT_RECURSION_LIMIT, or fall back to the tuned default.""" + raw = env.get("AGENT_RECURSION_LIMIT") + if raw is None or not raw.strip(): + return DEFAULT_RECURSION_LIMIT + try: + value = int(raw) + except ValueError as error: + raise RuntimeError( + f'Invalid AGENT_RECURSION_LIMIT: "{raw}" — must be an integer' + ) from error + if value < MIN_RECURSION_LIMIT: + raise RuntimeError( + f"Invalid AGENT_RECURSION_LIMIT: {value} — must be at least " + f"{MIN_RECURSION_LIMIT}, or the agent cannot finish a tool call" + ) + return value + + def build_agent(): """Build the OpenTag triage graph.""" api_key = os.environ.get("OPENAI_API_KEY") @@ -111,4 +157,10 @@ def build_agent(): print(f"[AGENT] internal-source tools: {len(internal_tools)}") print(f"[AGENT] Main tools: {[t.name for t in main_tools]}") - return agent_graph.with_config({"recursion_limit": 25}) + limit = recursion_limit() + print( + f"[AGENT] recursion limit: {limit} " + f"({tool_call_budget(limit)} tool calls per turn)" + ) + + return agent_graph.with_config({"recursion_limit": limit}) diff --git a/agent/tests/test_agent_configuration.py b/agent/tests/test_agent_configuration.py index 94c2eee..2ab49ba 100644 --- a/agent/tests/test_agent_configuration.py +++ b/agent/tests/test_agent_configuration.py @@ -7,7 +7,9 @@ from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool from langchain_openai import ChatOpenAI as RealChatOpenAI +from langgraph.errors import GraphRecursionError from pydantic import Field @@ -97,9 +99,180 @@ def test_build_agent_rejects_invalid_openai_tuning(monkeypatch, name, value): def test_build_agent_bounds_graph_recursion(monkeypatch): + monkeypatch.delenv("AGENT_RECURSION_LIMIT", raising=False) _, captured = build_with_captured_configuration(monkeypatch) - assert captured["config"] == {"recursion_limit": 25} + assert captured["config"] == { + "recursion_limit": agent_mod.DEFAULT_RECURSION_LIMIT + } + + +class ScriptedModel(BaseChatModel): + """Makes exactly `budget` tool calls, one per turn, then answers.""" + + model_name: str = "gpt-5.5" + budget: int = 0 + made: int = Field(default=0) + + @property + def _llm_type(self): + return "scripted" + + def _get_ls_params(self, **_kwargs): + return { + "ls_provider": "openai", + "ls_model_name": self.model_name, + "ls_model_type": "chat", + } + + def bind_tools(self, tools, **_kwargs): + del tools + return self + + def _generate(self, messages, stop=None, run_manager=None, **_kwargs): + del messages, stop, run_manager + if self.made >= self.budget: + return ChatResult( + generations=[ChatGeneration(message=AIMessage(content="done"))] + ) + self.made += 1 + return ChatResult( + generations=[ + ChatGeneration( + message=AIMessage( + content="", + tool_calls=[ + { + "name": "lookup", + "args": {"query": f"q{self.made}"}, + "id": f"call-{self.made}", + } + ], + ) + ) + ] + ) + + +def graph_making_tool_calls(monkeypatch, count, limit=None): + """The real agent graph, driven by a model that makes `count` tool calls.""" + + @tool + def lookup(query: str) -> str: + """Look something up.""" + return f"result for {query}" + + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + for name in ( + "TAVILY_API_KEY", + "GITHUB_PERSONAL_ACCESS_TOKEN", + "POSTHOG_PERSONAL_API_KEY", + "LINEAR_API_KEY", + "NOTION_MCP_AUTH_TOKEN", + ): + monkeypatch.delenv(name, raising=False) + if limit is None: + monkeypatch.delenv("AGENT_RECURSION_LIMIT", raising=False) + else: + monkeypatch.setenv("AGENT_RECURSION_LIMIT", str(limit)) + + monkeypatch.setattr( + agent_mod, "ChatOpenAI", lambda **_kwargs: ScriptedModel(budget=count) + ) + monkeypatch.setattr(agent_mod, "internal_source_tools", lambda: [lookup]) + return agent_mod.build_agent() + + +def super_steps_for(monkeypatch, count): + graph = graph_making_tool_calls(monkeypatch, count, limit=1000) + return sum( + 1 + for _ in graph.stream( + {"messages": [{"role": "user", "content": "go"}]}, + config={"configurable": {"thread_id": f"steps-{count}"}}, + stream_mode="updates", + ) + ) + + +def completes(monkeypatch, count, limit): + graph = graph_making_tool_calls(monkeypatch, count, limit=limit) + try: + graph.invoke( + {"messages": [{"role": "user", "content": "go"}]}, + config={"configurable": {"thread_id": f"survive-{count}-{limit}"}}, + ) + return True + except GraphRecursionError: + return False + + +def test_one_tool_call_costs_four_super_steps(monkeypatch): + """The limit counts super-steps, and the middleware chain multiplies them. + + model -> TodoListMiddleware.after_model -> CopilotKitMiddleware.after_model + -> tools. Measuring it here keeps the budget arithmetic honest: if a + middleware is added or removed, the cost per tool call changes and the + limit has to move with it. + """ + baseline = super_steps_for(monkeypatch, 0) + one = super_steps_for(monkeypatch, 1) + five = super_steps_for(monkeypatch, 5) + + assert one - baseline == 4 + assert five - baseline == 20 + + +def test_the_old_limit_allowed_only_four_tool_calls(monkeypatch): + """Why 25 was failing real turns -- it was never a 25-tool-call budget.""" + assert completes(monkeypatch, 4, limit=25) is True + assert completes(monkeypatch, 5, limit=25) is False + + +@pytest.mark.parametrize("limit", [25, 40, 60, 100]) +def test_the_reported_tool_call_budget_is_the_real_one(monkeypatch, limit): + """What startup prints has to match what the graph actually completes. + + The startup line is the only place an operator learns what a limit buys, + so it must not drift from the graph -- including if a middleware changes + the per-tool-call cost. + """ + budget = agent_mod.tool_call_budget(limit) + + assert completes(monkeypatch, budget, limit=limit) is True + assert completes(monkeypatch, budget + 1, limit=limit) is False + + +def test_the_default_limit_covers_a_research_turn(monkeypatch): + """A GitHub search, pagination, and a chart must fit in one turn.""" + assert completes( + monkeypatch, 12, limit=agent_mod.DEFAULT_RECURSION_LIMIT + ) is True + + +def test_recursion_limit_is_configurable(monkeypatch): + monkeypatch.setenv("AGENT_RECURSION_LIMIT", "120") + _, captured = build_with_captured_configuration(monkeypatch) + + assert captured["config"] == {"recursion_limit": 120} + + +def test_a_blank_recursion_limit_falls_back_to_the_default(): + assert ( + agent_mod.recursion_limit({"AGENT_RECURSION_LIMIT": " "}) + == agent_mod.DEFAULT_RECURSION_LIMIT + ) + assert agent_mod.recursion_limit({}) == agent_mod.DEFAULT_RECURSION_LIMIT + + +def test_a_nonsense_recursion_limit_is_rejected(): + with pytest.raises(RuntimeError, match="AGENT_RECURSION_LIMIT"): + agent_mod.recursion_limit({"AGENT_RECURSION_LIMIT": "lots"}) + + +def test_a_recursion_limit_too_low_to_finish_a_tool_call_is_rejected(): + with pytest.raises(RuntimeError, match="at least"): + agent_mod.recursion_limit({"AGENT_RECURSION_LIMIT": "4"}) def test_build_agent_refreshes_current_date_before_each_model_call(monkeypatch):