From bb2feddee0a006c56e7c9d80bbf6d10ab805f0a4 Mon Sep 17 00:00:00 2001 From: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Fri, 7 Aug 2026 17:14:33 -0400 Subject: [PATCH 1/2] fix(buzz-agent): budget summarizer reasoning separately so it cannot starve the handoff summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reasoning models spend output tokens thinking before any visible text, and that spend counts against the summary call's max_tokens. On deepseek-v4-flash this starved the handoff summarizer completely: in a 89-task benchmark run, 13 consecutive handoff attempts across 5 trials length-stopped inside the reasoning channel, returned empty content, and every one degraded to lossy history truncation (~40 minutes of burned reasoning) before a stochastically-short reasoning run finally fit. Give reasoning its own equal-sized budget on top of the text budget (reasoning.max_tokens) and exclude it from the response (reasoning.exclude) — summarize() only reads content. max_tokens is raised to cover both budgets so the text budget the caller asked for is actually available for text. Non-reasoning endpoints ignore the reasoning object; deliberately not paired with provider.require_parameters (see apply_openrouter_mutations). Verification: - cargo test -p buzz-agent (422 unit + 110 integration, all pass) - cargo fmt / clippy clean Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> --- crates/buzz-agent/src/llm.rs | 47 ++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index a091790425..f1f1370db4 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -2249,10 +2249,26 @@ fn openrouter_summary_body( user_prompt: &str, max_output_tokens: u32, ) -> Value { + // Reasoning models spend output tokens thinking before emitting any + // visible text, and that spend counts against `max_tokens`. Left + // unseparated, a model can burn the entire cap mid-reasoning and return an + // empty `content` — observed with deepseek-v4-flash, where 13 consecutive + // handoff attempts length-stopped inside the reasoning channel and every + // one degraded to lossy history truncation. Give reasoning its own + // equal-sized budget on top of the text budget so `max_output_tokens` + // remains what the caller means: visible summary text. `exclude` keeps the + // reasoning out of the response body; `summarize()` only reads `content`. + // Non-reasoning endpoints ignore the `reasoning` object (see + // `apply_openrouter_mutations` on why it is never paired with + // `provider.require_parameters`). json!({ "model": effective_model, "stream": false, - "max_tokens": max_output_tokens, + "max_tokens": max_output_tokens.saturating_mul(2), + "reasoning": { + "max_tokens": max_output_tokens, + "exclude": true, + }, "messages": [ { "role": "system", "content": system_prompt }, { "role": "user", "content": user_prompt }, @@ -6263,8 +6279,14 @@ mod tests { assert!(body.get("max_completion_tokens").is_none()); } + /// The summary body reserves `max_output_tokens` for visible text by + /// granting reasoning a separate, equal budget on top and excluding it + /// from the response. Without the separation, a reasoning model can spend + /// the entire cap thinking and length-stop with empty `content`, which + /// `summarize()` reports as an empty summary and the handoff degrades to + /// lossy truncation. #[test] - fn openrouter_summary_carries_neither_reasoning_nor_provider() { + fn openrouter_summary_budgets_reasoning_separately_and_carries_no_provider() { let body = openrouter_summary_body( "anthropic/claude-opus-4-7", "summarize", @@ -6274,14 +6296,25 @@ mod tests { assert_eq!(body["model"], "anthropic/claude-opus-4-7"); assert_eq!(body["messages"][0]["role"], "system"); assert_eq!(body["messages"][1]["content"], "text to summarize"); - assert_eq!(body["max_tokens"], 1024); + assert_eq!( + body["max_tokens"], 2048, + "total cap must cover the text budget plus the reasoning budget" + ); + assert_eq!( + body["reasoning"]["max_tokens"], 1024, + "reasoning gets its own budget so it cannot starve the summary text" + ); + assert_eq!( + body["reasoning"]["exclude"], true, + "reasoning must not be included in the response; summarize() reads only content" + ); assert!( - body.get("max_completion_tokens").is_none(), - "summary body must use OpenRouter's token-limit spelling" + body["reasoning"].get("effort").is_none(), + "budget-based cap only; effort stays unset for the summary call" ); assert!( - body.get("reasoning").is_none(), - "summary body must not carry reasoning" + body.get("max_completion_tokens").is_none(), + "summary body must use OpenRouter's token-limit spelling" ); assert!( body.get("provider").is_none(), From 5f53236161a9b411a1ae1a7a9e4635e910b3ac12 Mon Sep 17 00:00:00 2001 From: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Fri, 7 Aug 2026 18:49:33 -0400 Subject: [PATCH 2/2] fix(buzz-agent): reserve the provider's actual summary completion cap in the handoff input budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up (PR #5248, atishpatel): the previous commit doubled OpenRouter's top-level max_tokens to 16384 (text budget + reasoning budget) but build_handoff_prompt still reserved only 8192 output tokens when sizing the input. At the maximum constructed prompt, input plus the actual completion allowance could exceed the configured context window by 8192 — making the summarizer itself context-reject exactly when a handoff is needed. Introduce summary_completion_cap(provider, budget): 2x for OpenRouter (where reasoning gets its own equal budget), unchanged for Anthropic, OpenAI, Databricks, and DatabricksV2 (whose summary bodies request the caller's budget as-is). openrouter_summary_body and build_handoff_prompt both derive from it, so the wire cap and the input reservation cannot drift apart again. A regression test pins the join: budget + fixed prompt + actual completion cap <= window at the 1-byte/token upper bound, and asserts the old single reservation observably violates it. Also per review suggestion: HANDOFF_SYSTEM_PROMPT now says 'Keep the visible plain-text summary under N tokens' with N derived from HANDOFF_MAX_OUTPUT_TOKENS via format!, removing the duplicated literal and the ambiguity about whether hidden reasoning counts against the figure. Verification: - cargo test -p buzz-agent (425 unit + 110 integration, all pass) - cargo fmt / clippy clean Co-authored-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Signed-off-by: Eva <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> --- crates/buzz-agent/src/handoff.rs | 92 +++++++++++++++++++++++++++++--- crates/buzz-agent/src/llm.rs | 28 ++++++++-- 2 files changed, 108 insertions(+), 12 deletions(-) diff --git a/crates/buzz-agent/src/handoff.rs b/crates/buzz-agent/src/handoff.rs index 5fdbc3079d..4748678059 100644 --- a/crates/buzz-agent/src/handoff.rs +++ b/crates/buzz-agent/src/handoff.rs @@ -3,6 +3,7 @@ use crate::config::{ HANDOFF_MAX_OUTPUT_TOKENS, HANDOFF_MAX_TOOL_NAMES, HANDOFF_MIN_PROMPT_BUDGET_BYTES, HANDOFF_ORIGINAL_TASK_MAX_BYTES, MAX_CONTEXT_RECOVERIES_PER_RUN, }; +use crate::llm::summary_completion_cap; use crate::types::HistoryItem; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -35,10 +36,21 @@ pub(crate) enum ContextRecovery { Exhausted, } -const HANDOFF_SYSTEM_PROMPT: &str = "You are generating a context handoff summary for the next \ -turn of an autonomous agent. Be concise but thorough. Cover: what the original task was, what \ -you accomplished, key decisions made, what remains, and one concrete next step. Output plain \ -text only — no tool calls, no JSON. Stay under 8192 tokens."; +/// System prompt for the handoff summarizer. `LazyLock` + `format!` so the +/// token figure is derived from [`HANDOFF_MAX_OUTPUT_TOKENS`] instead of a +/// duplicated literal, and "visible plain-text summary" makes explicit that +/// the limit is on summary text, not on any hidden reasoning the model does +/// first (which is budgeted separately on the wire — see +/// `openrouter_summary_body`). +static HANDOFF_SYSTEM_PROMPT: std::sync::LazyLock = std::sync::LazyLock::new(|| { + format!( + "You are generating a context handoff summary for the next turn of an autonomous agent. \ + Be concise but thorough. Cover: what the original task was, what you accomplished, key \ + decisions made, what remains, and one concrete next step. Output plain text only — no \ + tool calls, no JSON. Keep the visible plain-text summary under \ + {HANDOFF_MAX_OUTPUT_TOKENS} tokens." + ) +}); impl RunCtx<'_> { pub(crate) async fn maybe_handoff(&mut self, handoff_attempts: &mut usize) -> HandoffOutcome { @@ -170,7 +182,7 @@ impl RunCtx<'_> { _ = self.cancel.changed() => return HandoffOutcome::Cancelled, r = self.llm.summarize( self.cfg, - HANDOFF_SYSTEM_PROMPT, + &HANDOFF_SYSTEM_PROMPT, &prompt, HANDOFF_MAX_OUTPUT_TOKENS, self.effective_model, @@ -331,7 +343,7 @@ impl RunCtx<'_> { Some(explicit) => explicit.saturating_sub(fixed_bytes), None => handoff_prompt_budget_bytes( self.cfg.max_context_tokens, - HANDOFF_MAX_OUTPUT_TOKENS, + summary_completion_cap(self.cfg.provider, HANDOFF_MAX_OUTPUT_TOKENS), fixed_bytes, ), }; @@ -513,8 +525,9 @@ fn byte_fallback_threshold( mod tests { use super::{ byte_fallback_threshold, estimate_tokens_from_bytes, handoff_prompt_budget_bytes, - token_threshold, + summary_completion_cap, token_threshold, HANDOFF_SYSTEM_PROMPT, }; + use crate::config::{Provider, HANDOFF_MAX_OUTPUT_TOKENS}; #[test] fn handoff_prompt_budget_reserves_summary_output_and_fixed_prompt() { @@ -526,6 +539,71 @@ mod tests { assert_eq!(handoff_prompt_budget_bytes(1_000, 2_000, 10_000), 0); } + /// OpenRouter's summary request grants reasoning an equal budget on top of + /// the visible-text budget, so its completion cap is 2× the handoff text + /// budget; the input budget must reserve that doubled cap. At the + /// 1-byte/token upper bound, prompt bytes bound prompt tokens, so the join + /// to pin is: (budget + fixed prompt) + actual completion cap ≤ window. + /// Reserving only `HANDOFF_MAX_OUTPUT_TOKENS` would break this by exactly + /// one extra reasoning budget at the maximum constructed prompt. + #[test] + fn openrouter_prompt_budget_reserves_doubled_completion_cap() { + let cap = summary_completion_cap(Provider::OpenRouter, HANDOFF_MAX_OUTPUT_TOKENS); + assert_eq!( + cap, + 2 * HANDOFF_MAX_OUTPUT_TOKENS, + "OpenRouter doubles: text + reasoning" + ); + let window = 200_000u64; + let fixed = 1_000usize; + let budget = handoff_prompt_budget_bytes(window, cap, fixed); + assert_eq!(budget, 182_616); // 200_000 - 16_384 - 1_000 + let max_prompt_tokens = estimate_tokens_from_bytes(budget + fixed); + assert!( + max_prompt_tokens + u64::from(cap) <= window, + "input + completion allowance must fit the configured window" + ); + // The old single reservation violates the same join — the regression + // this guards against. + let stale_budget = handoff_prompt_budget_bytes(window, HANDOFF_MAX_OUTPUT_TOKENS, fixed); + assert!( + estimate_tokens_from_bytes(stale_budget + fixed) + u64::from(cap) > window, + "reserving only the text budget must be observable as an overflow here" + ); + } + + /// Anthropic/OpenAI/Databricks summary bodies request exactly the caller's + /// budget, so their input reservation is unchanged. + #[test] + fn non_openrouter_completion_cap_is_the_callers_budget() { + for provider in [ + Provider::Anthropic, + Provider::OpenAi, + Provider::Databricks, + Provider::DatabricksV2, + ] { + assert_eq!( + summary_completion_cap(provider, HANDOFF_MAX_OUTPUT_TOKENS), + HANDOFF_MAX_OUTPUT_TOKENS + ); + } + } + + /// The prompt's token figure is derived from `HANDOFF_MAX_OUTPUT_TOKENS` + /// and names the *visible plain-text summary* as its target, so hidden + /// reasoning (budgeted separately on the wire) is not the referent. + #[test] + fn handoff_system_prompt_derives_limit_and_targets_visible_text() { + let expected = format!( + "Keep the visible plain-text summary under {HANDOFF_MAX_OUTPUT_TOKENS} tokens." + ); + assert!( + HANDOFF_SYSTEM_PROMPT.contains(&expected), + "prompt must derive its token figure from HANDOFF_MAX_OUTPUT_TOKENS: {}", + *HANDOFF_SYSTEM_PROMPT + ); + } + #[test] fn token_threshold_uses_fraction_when_output_is_small() { // 200k window, 1k output. fractional = 0.9*200000 = 180000; diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index f1f1370db4..289adbd1ad 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -2237,12 +2237,30 @@ pub(crate) fn build_token_source(cfg: &Config) -> Result, A } } +/// Completion-token cap that [`Llm::summarize`] actually requests from +/// `provider`, given the caller's visible-text budget. OpenRouter grants +/// reasoning a separate, equal budget on top of the text budget (see +/// [`openrouter_summary_body`]), so its top-level cap is double the caller's +/// budget; every other provider requests the caller's budget unchanged. +/// Callers that reserve output headroom in an input budget +/// (`handoff_prompt_budget_bytes`) must reserve THIS value, not the text +/// budget — otherwise input plus the actual completion allowance can exceed +/// the configured context window. +pub(crate) fn summary_completion_cap(provider: Provider, max_output_tokens: u32) -> u32 { + match provider { + Provider::OpenRouter => max_output_tokens.saturating_mul(2), + Provider::Anthropic | Provider::OpenAi | Provider::Databricks | Provider::DatabricksV2 => { + max_output_tokens + } + } +} + /// Build the request body for `Llm::summarize` on `Provider::OpenRouter`. /// Extracted so tests can assert on the actual wire shape instead of a -/// hand-rolled literal — summaries never carry `reasoning` (see -/// `apply_openrouter_mutations`, which the summary path never calls). -/// It spells the token limit `max_tokens` directly for the same reason: the -/// mutation that renames it is never applied here. +/// hand-rolled literal — summaries never carry config-driven reasoning +/// *effort* (see `apply_openrouter_mutations`, which the summary path never +/// calls). It spells the token limit `max_tokens` directly for the same +/// reason: the mutation that renames it is never applied here. fn openrouter_summary_body( effective_model: &str, system_prompt: &str, @@ -2264,7 +2282,7 @@ fn openrouter_summary_body( json!({ "model": effective_model, "stream": false, - "max_tokens": max_output_tokens.saturating_mul(2), + "max_tokens": summary_completion_cap(Provider::OpenRouter, max_output_tokens), "reasoning": { "max_tokens": max_output_tokens, "exclude": true,