Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions front_end/panels/ai_chat/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ ts_library("unittests") {
"ui/__tests__/ChatViewAgentSessionsOrder.test.ts",
"ui/__tests__/ChatViewSequentialSessionsTransition.test.ts",
"ui/__tests__/ChatViewInputClear.test.ts",
"ui/__tests__/SettingsDialogOpenRouterCache.test.ts",
"ui/input/__tests__/InputBarClear.test.ts",
"ui/message/__tests__/MessageCombiner.test.ts",
"ui/message/__tests__/StructuredResponseController.test.ts",
Expand Down
41 changes: 36 additions & 5 deletions front_end/panels/ai_chat/agent_framework/AgentRunner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { sanitizeMessagesForModel } from '../LLM/MessageSanitizer.js';
const logger = createLogger('AgentRunner');

import { ConfigurableAgentTool, ToolRegistry, type ConfigurableAgentArgs, type ConfigurableAgentResult, type AgentRunTerminationReason, type HandoffConfig /* , HandoffContextTransform, ContextFilterRegistry*/ } from './ConfigurableAgentTool.js';
import { MODEL_SENTINELS } from '../core/Constants.js';

/**
* Configuration for the AgentRunner
Expand All @@ -33,6 +34,10 @@ export interface AgentRunnerConfig {
provider: LLMProvider;
/** Optional vision capability check. Defaults to false (no vision). */
getVisionCapability?: (modelName: string) => Promise<boolean> | boolean;
/** Mini model for smaller/faster operations */
miniModel?: string;
/** Nano model for smallest/fastest operations */
nanoModel?: string;
}

/**
Expand Down Expand Up @@ -218,6 +223,8 @@ export class AgentRunner {
parentSession?: AgentSession, // For natural nesting
defaultProvider?: LLMProvider,
defaultGetVisionCapability?: (modelName: string) => Promise<boolean> | boolean,
miniModel?: string, // Mini model for smaller/faster operations
nanoModel?: string, // Nano model for smallest/fastest operations
overrides?: { sessionId?: string; parentSessionId?: string; traceId?: string }
): Promise<ConfigurableAgentResult & { agentSession: AgentSession }> {
const targetAgentName = handoffConfig.targetAgentName;
Expand Down Expand Up @@ -286,12 +293,28 @@ export class AgentRunner {
// Enhance the target agent's system prompt with page context
const enhancedSystemPrompt = await enhancePromptWithPageContext(targetConfig.systemPrompt);

// Resolve model name for the target agent
let resolvedModelName: string;
if (typeof targetConfig.modelName === 'function') {
resolvedModelName = targetConfig.modelName();
} else if (targetConfig.modelName === MODEL_SENTINELS.USE_MINI) {
if (!miniModel) {
throw new Error(`Mini model not provided for handoff to agent '${targetAgentName}'. Ensure miniModel is passed in context.`);
}
resolvedModelName = miniModel;
} else if (targetConfig.modelName === MODEL_SENTINELS.USE_NANO) {
if (!nanoModel) {
throw new Error(`Nano model not provided for handoff to agent '${targetAgentName}'. Ensure nanoModel is passed in context.`);
}
resolvedModelName = nanoModel;
} else {
resolvedModelName = targetConfig.modelName || defaultModelName;
}

// Construct Runner Config & Hooks for the target agent
const targetRunnerConfig: AgentRunnerConfig = {
apiKey,
modelName: typeof targetConfig.modelName === 'function'
? targetConfig.modelName()
: (targetConfig.modelName || defaultModelName),
modelName: resolvedModelName,
systemPrompt: enhancedSystemPrompt,
tools: targetConfig.tools
.map(toolName => ToolRegistry.getRegisteredTool(toolName))
Expand All @@ -300,6 +323,8 @@ export class AgentRunner {
temperature: targetConfig.temperature ?? defaultTemperature,
provider: defaultProvider as LLMProvider,
getVisionCapability: defaultGetVisionCapability,
miniModel,
nanoModel,
};
const targetRunnerHooks: AgentRunnerHooks = {
prepareInitialMessages: undefined, // History already formed by transform or passthrough
Expand Down Expand Up @@ -845,6 +870,8 @@ export class AgentRunner {
currentSession, // Pass current session for natural nesting
config.provider,
config.getVisionCapability,
config.miniModel,
config.nanoModel,
{ sessionId: nestedSessionId, parentSessionId: currentSession.sessionId, traceId: getCurrentTracingContext()?.traceId }
);

Expand Down Expand Up @@ -947,11 +974,13 @@ export class AgentRunner {
}

try {
logger.info(`${agentName} Executing tool: ${toolToExecute.name} with args:`, toolArgs);
logger.info(`${agentName} Executing tool: ${toolToExecute.name}`);
const execTracingContext = getCurrentTracingContext();
toolResultData = await toolToExecute.execute(toolArgs as any, ({
provider: config.provider,
model: modelName,
miniModel: config.miniModel,
nanoModel: config.nanoModel,
getVisionCapability: config.getVisionCapability,
overrideSessionId: preallocatedChildId,
overrideParentSessionId: currentSession.sessionId,
Expand Down Expand Up @@ -1210,7 +1239,9 @@ export class AgentRunner {
undefined, // No llmToolArgs for max iterations handoff
currentSession, // Pass current session for natural nesting
config.provider,
config.getVisionCapability
config.getVisionCapability,
config.miniModel,
config.nanoModel
);
// Extract the result and session
const { agentSession: childSession, ...actualResult } = handoffResult;
Expand Down
71 changes: 60 additions & 11 deletions front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,30 @@

import { AgentService } from '../core/AgentService.js';
import type { Tool } from '../tools/Tools.js';
import { AIChatPanel } from '../ui/AIChatPanel.js';
import { ChatMessageEntity, type ChatMessage } from '../models/ChatTypes.js';
import { createLogger } from '../core/Logger.js';
import { getCurrentTracingContext } from '../tracing/TracingConfig.js';
import { MODEL_SENTINELS } from '../core/Constants.js';
import type { AgentSession } from './AgentSessionTypes.js';
import type { LLMProvider } from '../LLM/LLMTypes.js';

const logger = createLogger('ConfigurableAgentTool');

import { AgentRunner, type AgentRunnerConfig, type AgentRunnerHooks } from './AgentRunner.js';

// Context passed along with agent/tool calls
export interface CallCtx {
provider?: LLMProvider,
model?: string,
miniModel?: string,
nanoModel?: string,
mainModel?: string,
getVisionCapability?: (modelName: string) => Promise<boolean> | boolean,
overrideSessionId?: string,
overrideParentSessionId?: string,
overrideTraceId?: string,
}

/**
* Defines the possible reasons an agent run might terminate.
*/
Expand Down Expand Up @@ -412,27 +426,62 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config

// Initialize
const maxIterations = this.config.maxIterations || 10;
const modelName = typeof this.config.modelName === 'function'
? this.config.modelName()
: (this.config.modelName || AIChatPanel.instance().getSelectedModel());

// Parse execution context first
const callCtx = (_ctx || {}) as CallCtx;

// Resolve model name from context or configuration
let modelName: string;
if (this.config.modelName === MODEL_SENTINELS.USE_MINI) {
if (!callCtx.miniModel) {
throw new Error(`Mini model not provided in context for agent '${this.name}'. Ensure context includes miniModel.`);
}
modelName = callCtx.miniModel;
} else if (this.config.modelName === MODEL_SENTINELS.USE_NANO) {
if (!callCtx.nanoModel) {
throw new Error(`Nano model not provided in context for agent '${this.name}'. Ensure context includes nanoModel.`);
}
modelName = callCtx.nanoModel;
} else if (typeof this.config.modelName === 'function') {
modelName = this.config.modelName();
} else if (this.config.modelName) {
modelName = this.config.modelName;
} else {
// Use main model from context, or fallback to context model
const contextModel = callCtx.mainModel || callCtx.model;
if (!contextModel) {
throw new Error(`No model provided for agent '${this.name}'. Ensure context includes model or mainModel.`);
}
modelName = contextModel;
}

// Override with context model only if agent doesn't have its own model configuration
if (callCtx.model && !this.config.modelName) {
modelName = callCtx.model;
}

// Validate required context
if (!callCtx.provider) {
throw new Error(`Provider not provided in context for agent '${this.name}'. Ensure context includes provider.`);
}

const temperature = this.config.temperature ?? 0;

const systemPrompt = this.config.systemPrompt;
const tools = this.getToolInstances();

// Prepare initial messages
const internalMessages = this.prepareInitialMessages(args);

// Prepare runner config and hooks
const runnerConfig: AgentRunnerConfig = {
apiKey,
modelName,
systemPrompt,
tools,
maxIterations,
temperature,
provider: AIChatPanel.getProviderForModel(modelName),
getVisionCapability: (m: string) => AIChatPanel.isVisionCapable(m),
provider: callCtx.provider,
getVisionCapability: callCtx.getVisionCapability ?? (() => false),
miniModel: callCtx.miniModel,
nanoModel: callCtx.nanoModel,
};

const runnerHooks: AgentRunnerHooks = {
Expand All @@ -446,7 +495,7 @@ export class ConfigurableAgentTool implements Tool<ConfigurableAgentArgs, Config
};

// Run the agent
const ctx: any = _ctx || {};
const ctx: any = callCtx || {};
const result = await AgentRunner.run(
internalMessages,
args,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ import { BookmarkStoreTool } from '../../tools/BookmarkStoreTool.js';
import { DocumentSearchTool } from '../../tools/DocumentSearchTool.js';
import { NavigateURLTool, PerformActionTool, GetAccessibilityTreeTool, SearchContentTool, NavigateBackTool, NodeIDsToURLsTool, TakeScreenshotTool, ScrollPageTool } from '../../tools/Tools.js';
import { HTMLToMarkdownTool } from '../../tools/HTMLToMarkdownTool.js';
import { AIChatPanel } from '../../ui/AIChatPanel.js';
import { ChatMessageEntity, type ChatMessage } from '../../models/ChatTypes.js';
import {
ConfigurableAgentTool,
ToolRegistry, type AgentToolConfig, type ConfigurableAgentArgs
} from '../ConfigurableAgentTool.js';
import { WaitTool } from '../../tools/Tools.js';
import { MODEL_SENTINELS } from '../../core/Constants.js';
import { ThinkingTool } from '../../tools/ThinkingTool.js';
import type { Tool } from '../../tools/Tools.js';

Expand Down Expand Up @@ -70,7 +70,6 @@ If the page does not match the expected content, retry with a different URL patt
Remember: Always use navigate_url to actually go to the constructed URLs. Return easy-to-read markdown reports.`,
tools: ['navigate_url', 'get_page_content'],
maxIterations: 5,
modelName: () => AIChatPanel.instance().getSelectedModel(),
temperature: 0.1,
schema: {
type: 'object',
Expand Down Expand Up @@ -322,7 +321,7 @@ Remember: You gather data, content_writer_agent writes the report. Always hand o
'document_search'
],
maxIterations: 15,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0,
schema: {
type: 'object',
Expand Down Expand Up @@ -423,7 +422,7 @@ Your process should follow these steps:
The final output should be in markdown format, and it should be lengthy and detailed. Aim for 5-10 pages of content, at least 1000 words.`,
tools: [],
maxIterations: 3,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.3,
schema: {
type: 'object',
Expand Down Expand Up @@ -531,7 +530,7 @@ Conclusion: Fix the args format and retry with proper syntax: { "method": "fill"
'take_screenshot',
],
maxIterations: 10,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.5,
schema: {
type: 'object',
Expand Down Expand Up @@ -640,7 +639,7 @@ Remember that verification is time-sensitive - the page state might change durin
'take_screenshot'
],
maxIterations: 3,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.2,
schema: {
type: 'object',
Expand Down Expand Up @@ -725,7 +724,7 @@ When selecting an element to click, prioritize:
'node_ids_to_urls',
],
maxIterations: 5,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.7,
schema: {
type: 'object',
Expand Down Expand Up @@ -805,7 +804,7 @@ When selecting a form field to fill, prioritize:
'schema_based_extractor',
],
maxIterations: 5,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.7,
schema: {
type: 'object',
Expand Down Expand Up @@ -881,7 +880,7 @@ When selecting an element for keyboard input, prioritize:
'schema_based_extractor',
],
maxIterations: 5,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.7,
schema: {
type: 'object',
Expand Down Expand Up @@ -966,7 +965,7 @@ When selecting an element to hover over, prioritize:
'schema_based_extractor',
],
maxIterations: 5,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.7,
schema: {
type: 'object',
Expand Down Expand Up @@ -1048,7 +1047,7 @@ The accessibility tree includes information about scrollable containers. Look fo
'schema_based_extractor',
],
maxIterations: 5,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.7,
schema: {
type: 'object',
Expand Down Expand Up @@ -1283,7 +1282,6 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and
'thinking',
],
maxIterations: 15,
modelName: () => AIChatPanel.instance().getSelectedModel(),
temperature: 0.3,
schema: {
type: 'object',
Expand Down Expand Up @@ -1422,7 +1420,7 @@ Remember to adapt your analysis based on the product category - different attrib
'get_page_content',
],
maxIterations: 5,
modelName: () => AIChatPanel.getMiniModel(),
modelName: MODEL_SENTINELS.USE_MINI,
temperature: 0.2,
schema: {
type: 'object',
Expand Down
Loading