From 69839dc24322f39e54d4eae034ebfb2a2d937b02 Mon Sep 17 00:00:00 2001
From: Tyson Thomas <tysonthomas9@users.noreply.github.com>
Date: Mon, 10 Nov 2025 23:11:04 -0800
Subject: [PATCH 1/2] fix litellm model issue

---
 .../agent_framework/ConfigurableAgentTool.ts  | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts b/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts
index a7c522eef9..6bcd6e252d 100644
--- a/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts
+++ b/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts
@@ -460,9 +460,10 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config
     const apiKey = callCtx.apiKey;
     const provider = callCtx.provider;
 
-    // Check if this provider requires an API key
-    // BrowserOperator doesn't require an API key (endpoint is hardcoded)
-    const requiresApiKey = provider !== 'browseroperator';
+    // Check if API key is required based on provider
+    // LiteLLM and BrowserOperator has optional API keys
+    // Other providers (OpenAI, Groq, OpenRouter) require API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       const errorResult = this.createErrorResult(`API key not configured for ${this.name}`, [], 'error');
@@ -499,15 +500,17 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config
     // Resolve model name from context or configuration
     let modelName: string;
     if (this.config.modelName === MODEL_SENTINELS.USE_MINI) {
-      if (!callCtx.miniModel) {
-        throw new Error(`Mini model not provided in context for agent '${this.name}'. Ensure context includes miniModel.`);
+      // Fall back to main model if mini model is not configured
+      modelName = callCtx.miniModel || callCtx.mainModel || callCtx.model || '';
+      if (!modelName) {
+        throw new Error(`Mini model not provided in context for agent '${this.name}'. Ensure context includes miniModel or mainModel.`);
       }
-      modelName = callCtx.miniModel;
     } else if (this.config.modelName === MODEL_SENTINELS.USE_NANO) {
-      if (!callCtx.nanoModel) {
-        throw new Error(`Nano model not provided in context for agent '${this.name}'. Ensure context includes nanoModel.`);
+      // Fall back through nano -> mini -> main model chain
+      modelName = callCtx.nanoModel || callCtx.miniModel || callCtx.mainModel || callCtx.model || '';
+      if (!modelName) {
+        throw new Error(`Nano model not provided in context for agent '${this.name}'. Ensure context includes nanoModel, miniModel, or mainModel.`);
       }
-      modelName = callCtx.nanoModel;
     } else if (typeof this.config.modelName === 'function') {
       modelName = this.config.modelName();
     } else if (this.config.modelName) {

From 76d53d00b3999cb5490999b9f19c9d63f49ba48d Mon Sep 17 00:00:00 2001
From: Tyson Thomas <tysonthomas9@users.noreply.github.com>
Date: Mon, 10 Nov 2025 23:32:47 -0800
Subject: [PATCH 2/2] update based on comments

---
 .../agent_framework/ConfigurableAgentTool.ts    | 17 +++++++++++++----
 .../ai_chat/tools/CombinedExtractionTool.ts     |  4 ++--
 .../FullPageAccessibilityTreeToMarkdownTool.ts  |  4 ++--
 .../panels/ai_chat/tools/HTMLToMarkdownTool.ts  |  4 ++--
 .../ai_chat/tools/SchemaBasedExtractorTool.ts   |  4 ++--
 .../tools/StreamlinedSchemaExtractorTool.ts     |  4 ++--
 front_end/panels/ai_chat/tools/Tools.ts         |  4 ++--
 7 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts b/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts
index 6bcd6e252d..a908677c3a 100644
--- a/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts
+++ b/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts
@@ -461,7 +461,7 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config
     const provider = callCtx.provider;
 
     // Check if API key is required based on provider
-    // LiteLLM and BrowserOperator has optional API keys
+    // LiteLLM and BrowserOperator have optional API keys
     // Other providers (OpenAI, Groq, OpenRouter) require API keys
     const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
@@ -528,16 +528,25 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config
     if (callCtx.model && !this.config.modelName) {
       modelName = callCtx.model;
     }
-    
+
+    // Update context with resolved fallback models for tools to use
+    // This ensures tools that check ctx.miniModel or ctx.nanoModel get the fallback
+    if (this.config.modelName === MODEL_SENTINELS.USE_MINI && !callCtx.miniModel) {
+      callCtx.miniModel = modelName;  // Use the resolved fallback
+    }
+    if (this.config.modelName === MODEL_SENTINELS.USE_NANO && !callCtx.nanoModel) {
+      callCtx.nanoModel = modelName;  // Use the resolved fallback
+    }
+
     // Validate required context
     if (!callCtx.provider) {
       throw new Error(`Provider not provided in context for agent '${this.name}'. Ensure context includes provider.`);
     }
-    
+
     const temperature = this.config.temperature ?? 0;
     const systemPrompt = this.config.systemPrompt;
     const tools = this.getToolInstances();
-    
+
     // Prepare initial messages
     const internalMessages = this.prepareInitialMessages(args);
     const runnerConfig: AgentRunnerConfig = {
diff --git a/front_end/panels/ai_chat/tools/CombinedExtractionTool.ts b/front_end/panels/ai_chat/tools/CombinedExtractionTool.ts
index 0f8699af8b..6478b61495 100644
--- a/front_end/panels/ai_chat/tools/CombinedExtractionTool.ts
+++ b/front_end/panels/ai_chat/tools/CombinedExtractionTool.ts
@@ -87,8 +87,8 @@ export class CombinedExtractionTool implements Tool<CombinedExtractionArgs, Comb
     // Get provider from context
     const provider = ctx?.provider;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return {
diff --git a/front_end/panels/ai_chat/tools/FullPageAccessibilityTreeToMarkdownTool.ts b/front_end/panels/ai_chat/tools/FullPageAccessibilityTreeToMarkdownTool.ts
index 014087a33e..e89c94bfc5 100644
--- a/front_end/panels/ai_chat/tools/FullPageAccessibilityTreeToMarkdownTool.ts
+++ b/front_end/panels/ai_chat/tools/FullPageAccessibilityTreeToMarkdownTool.ts
@@ -55,8 +55,8 @@ export class FullPageAccessibilityTreeToMarkdownTool implements Tool<Record<stri
     const provider = ctx.provider;
     const model = ctx.nanoModel;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return { error: 'API key not configured.' };
diff --git a/front_end/panels/ai_chat/tools/HTMLToMarkdownTool.ts b/front_end/panels/ai_chat/tools/HTMLToMarkdownTool.ts
index 815c5ff2ef..025406d96b 100644
--- a/front_end/panels/ai_chat/tools/HTMLToMarkdownTool.ts
+++ b/front_end/panels/ai_chat/tools/HTMLToMarkdownTool.ts
@@ -67,8 +67,8 @@ export class HTMLToMarkdownTool implements Tool<HTMLToMarkdownArgs, HTMLToMarkdo
     // Get provider from context
     const provider = ctx?.provider;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return {
diff --git a/front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts b/front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts
index f4c34aefe3..0de86cea20 100644
--- a/front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts
+++ b/front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts
@@ -108,8 +108,8 @@ Schema Examples:
     // Get provider from context
     const provider = ctx?.provider;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return {
diff --git a/front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts b/front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts
index 4e29c7fec8..ed8c2b0f24 100644
--- a/front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts
+++ b/front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts
@@ -113,8 +113,8 @@ export class StreamlinedSchemaExtractorTool implements Tool<StreamlinedSchemaExt
     // Get provider from context
     const provider = ctx?.provider;
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = provider !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = provider !== 'litellm' && provider !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {
       return {
diff --git a/front_end/panels/ai_chat/tools/Tools.ts b/front_end/panels/ai_chat/tools/Tools.ts
index 948f5b5f42..6178af79fb 100644
--- a/front_end/panels/ai_chat/tools/Tools.ts
+++ b/front_end/panels/ai_chat/tools/Tools.ts
@@ -2967,8 +2967,8 @@ Important guidelines:
       return { error: 'Missing LLM context (provider/model) for ObjectiveDrivenActionTool' };
     }
 
-    // BrowserOperator doesn't require API key
-    const requiresApiKey = providerForAction !== 'browseroperator';
+    // LiteLLM and BrowserOperator have optional API keys
+    const requiresApiKey = providerForAction !== 'litellm' && providerForAction !== 'browseroperator';
 
     if (requiresApiKey && !apiKey) {return { error: 'API key not configured.' };}
     if (typeof objective !== 'string' || objective.trim() === '') {