From 8a12edacc3bcd3266ed91c23e518df4456a27b59 Mon Sep 17 00:00:00 2001 From: Tyson Thomas Date: Sat, 1 Nov 2025 19:12:44 -0700 Subject: [PATCH 1/6] Add support for browser operator provider --- config/gni/devtools_grd_files.gni | 1 + front_end/panels/ai_chat/BUILD.gn | 2 + .../ai_chat/LLM/BrowserOperatorProvider.ts | 428 ++++++++++++++++++ front_end/panels/ai_chat/LLM/LLMClient.ts | 43 ++ front_end/panels/ai_chat/LLM/LLMTypes.ts | 12 +- .../ai_chat/agent_framework/AgentRunner.ts | 1 + .../agent_framework/ConfigurableAgentTool.ts | 9 +- .../implementation/agents/SearchAgent.ts | 4 +- front_end/panels/ai_chat/core/AgentNodes.ts | 7 +- front_end/panels/ai_chat/core/AgentService.ts | 19 +- .../ai_chat/core/LLMConfigurationManager.ts | 3 + .../ai_chat/tools/CombinedExtractionTool.ts | 8 +- ...FullPageAccessibilityTreeToMarkdownTool.ts | 12 +- .../ai_chat/tools/HTMLToMarkdownTool.ts | 10 +- .../ai_chat/tools/SchemaBasedExtractorTool.ts | 54 ++- .../tools/StreamlinedSchemaExtractorTool.ts | 24 +- front_end/panels/ai_chat/tools/Tools.ts | 5 +- front_end/panels/ai_chat/ui/AIChatPanel.ts | 70 ++- front_end/panels/ai_chat/ui/ChatView.ts | 13 +- front_end/panels/ai_chat/ui/SettingsDialog.ts | 129 +++++- .../__tests__/ChatViewAgentSessions.test.ts | 20 + front_end/panels/ai_chat/ui/input/InputBar.ts | 9 +- 22 files changed, 815 insertions(+), 68 deletions(-) create mode 100644 front_end/panels/ai_chat/LLM/BrowserOperatorProvider.ts diff --git a/config/gni/devtools_grd_files.gni b/config/gni/devtools_grd_files.gni index 05c2456566..a04a129c0a 100644 --- a/config/gni/devtools_grd_files.gni +++ b/config/gni/devtools_grd_files.gni @@ -688,6 +688,7 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/LLM/LiteLLMProvider.js", "front_end/panels/ai_chat/LLM/GroqProvider.js", "front_end/panels/ai_chat/LLM/OpenRouterProvider.js", + "front_end/panels/ai_chat/LLM/BrowserOperatorProvider.js", "front_end/panels/ai_chat/LLM/LLMClient.js", "front_end/panels/ai_chat/LLM/MessageSanitizer.js", "front_end/panels/ai_chat/tools/Tools.js", diff --git a/front_end/panels/ai_chat/BUILD.gn b/front_end/panels/ai_chat/BUILD.gn index 4ff6a6bd62..8d306a8fd3 100644 --- a/front_end/panels/ai_chat/BUILD.gn +++ b/front_end/panels/ai_chat/BUILD.gn @@ -76,6 +76,7 @@ devtools_module("ai_chat") { "LLM/LiteLLMProvider.ts", "LLM/GroqProvider.ts", "LLM/OpenRouterProvider.ts", + "LLM/BrowserOperatorProvider.ts", "LLM/MessageSanitizer.ts", "LLM/LLMClient.ts", "tools/Tools.ts", @@ -243,6 +244,7 @@ _ai_chat_sources = [ "LLM/LiteLLMProvider.ts", "LLM/GroqProvider.ts", "LLM/OpenRouterProvider.ts", + "LLM/BrowserOperatorProvider.ts", "LLM/MessageSanitizer.ts", "LLM/LLMClient.ts", "tools/Tools.ts", diff --git a/front_end/panels/ai_chat/LLM/BrowserOperatorProvider.ts b/front_end/panels/ai_chat/LLM/BrowserOperatorProvider.ts new file mode 100644 index 0000000000..c0c7881f2f --- /dev/null +++ b/front_end/panels/ai_chat/LLM/BrowserOperatorProvider.ts @@ -0,0 +1,428 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import type { LLMMessage, LLMResponse, LLMCallOptions, LLMProvider, ModelInfo } from './LLMTypes.js'; +import { LLMBaseProvider } from './LLMProvider.js'; +import { LLMRetryManager } from './LLMErrorHandler.js'; +import { LLMResponseParser } from './LLMResponseParser.js'; +import { createLogger } from '../core/Logger.js'; + +const logger = createLogger('BrowserOperatorProvider'); + +/** + * BrowserOperator provider implementation + * + * Connects to the BrowserOperator API server which acts as a unified proxy + * for multiple LLM providers (OpenAI, Cerebras, Groq). + * + * Features: + * - Agent-based semantic routing via X-Agent header + * - Model abstraction using main/mini/nano aliases + * - Built-in retry and fallback handled by API server + * - OpenAI-compatible API + */ +export class BrowserOperatorProvider extends LLMBaseProvider { + private static readonly DEFAULT_BASE_URL = 'https://api.browseroperator.io/v1'; + private static readonly CHAT_COMPLETIONS_PATH = '/chat/completions'; + private static readonly HEALTH_PATH = '/health'; + + readonly name: LLMProvider = 'browseroperator'; + + constructor( + private readonly apiKey: string | null, + private readonly baseUrl?: string // Optional override for testing only + ) { + super(); + } + + /** + * Constructs the full endpoint URL - hardcoded to localhost + */ + private getEndpoint(): string { + // Use provided baseUrl only for testing, otherwise use hardcoded default + const baseUrl = this.baseUrl || BrowserOperatorProvider.DEFAULT_BASE_URL; + const cleanBaseUrl = baseUrl.replace(/\/$/, ''); + return `${cleanBaseUrl}${BrowserOperatorProvider.CHAT_COMPLETIONS_PATH}`; + } + + /** + * Gets the health check endpoint URL + */ + private getHealthEndpoint(): string { + const baseUrl = this.baseUrl || BrowserOperatorProvider.DEFAULT_BASE_URL; + const cleanUrl = baseUrl.replace(/\/v1\/?$/, ''); + return `${cleanUrl}${BrowserOperatorProvider.HEALTH_PATH}`; + } + + /** + * Converts LLMMessage format to OpenAI-compatible format + */ + private convertMessagesToOpenAI(messages: LLMMessage[]): any[] { + return messages.map(msg => { + const baseMessage: any = { + role: msg.role, + content: msg.content + }; + + // Ensure tool call arguments are strings per OpenAI spec + if (msg.tool_calls && Array.isArray(msg.tool_calls)) { + baseMessage.tool_calls = msg.tool_calls.map(tc => { + const args = (tc.function as any).arguments; + const argsString = typeof args === 'string' ? args : JSON.stringify(args ?? {}); + return { + ...tc, + function: { + ...tc.function, + arguments: argsString, + }, + }; + }); + } + + // Add optional fields if present + if (msg.tool_call_id) { + baseMessage.tool_call_id = msg.tool_call_id; + } + if (msg.name) { + baseMessage.name = msg.name; + } + + // For tool role, content must be a string; stringify objects/arrays + if (msg.role === 'tool') { + if (typeof baseMessage.content !== 'string') { + baseMessage.content = JSON.stringify(baseMessage.content ?? ''); + } + } + + return baseMessage; + }); + } + + /** + * Makes a request to the BrowserOperator API server + */ + private async makeAPIRequest(payloadBody: any, agentName?: string): Promise { + try { + const endpoint = this.getEndpoint(); + + // Use agent name directly from calling agent, fallback to 'default' + const selectedAgent = agentName || 'default'; + + logger.info('=== BrowserOperator API Request ==='); + logger.info('Endpoint:', endpoint); + logger.info('Agent (X-Agent header):', selectedAgent); + logger.info('Model:', payloadBody.model); + logger.info('Message count:', payloadBody.messages?.length || 0); + logger.info('Has tools:', !!payloadBody.tools); + logger.info('Temperature:', payloadBody.temperature); + + // Log full request payload (useful for debugging) + logger.debug('Full request payload:', JSON.stringify(payloadBody, null, 2)); + + const requestHeaders = { + 'Content-Type': 'application/json', + 'X-Agent': selectedAgent, + ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}), + }; + + logger.debug('Request headers:', requestHeaders); + + const startTime = Date.now(); + const response = await fetch(endpoint, { + method: 'POST', + headers: requestHeaders, + body: JSON.stringify(payloadBody), + }); + const duration = Date.now() - startTime; + + logger.info(`Response status: ${response.status} ${response.statusText}`); + logger.info(`Response time: ${duration}ms`); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({ error: { message: 'Unknown error' } })); + logger.error('=== BrowserOperator API Error ==='); + logger.error(`Status: ${response.status} ${response.statusText}`); + logger.error('Error data: ' + JSON.stringify(errorData, null, 2)); + throw new Error(`BrowserOperator API error: ${response.statusText} - ${errorData?.error?.message || 'Unknown error'}`); + } + + const data = await response.json(); + + logger.info('=== BrowserOperator API Response ==='); + logger.info('Response time:', `${duration}ms`); + logger.info('Choices count:', data.choices?.length || 0); + + if (data.usage) { + logger.info('Token usage:', { + prompt: data.usage.prompt_tokens, + completion: data.usage.completion_tokens, + total: data.usage.total_tokens + }); + } + + // Log first choice content preview + if (data.choices?.[0]) { + const firstChoice = data.choices[0]; + if (firstChoice.message?.content) { + const contentPreview = firstChoice.message.content.substring(0, 200); + logger.info('Response preview:', contentPreview + (firstChoice.message.content.length > 200 ? '...' : '')); + } + if (firstChoice.message?.tool_calls) { + logger.info('Tool calls:', firstChoice.message.tool_calls.length); + } + } + + // Log full response in debug mode + logger.debug('Full response:', JSON.stringify(data, null, 2)); + + return data; + } catch (error) { + logger.error('=== BrowserOperator API Request Failed ==='); + logger.error('Error:', error); + throw error; + } + } + + /** + * Processes the BrowserOperator response and converts to LLMResponse format + */ + private processBrowserOperatorResponse(data: any): LLMResponse { + const result: LLMResponse = { + rawResponse: data + }; + + if (!data?.choices || data.choices.length === 0) { + throw new Error('No choices in BrowserOperator response'); + } + + const choice = data.choices[0]; + const message = choice.message; + + if (!message) { + throw new Error('No message in BrowserOperator choice'); + } + + // Check for tool calls + if (message.tool_calls && message.tool_calls.length > 0) { + const toolCall = message.tool_calls[0]; + if (toolCall.function) { + try { + result.functionCall = { + name: toolCall.function.name, + arguments: JSON.parse(toolCall.function.arguments) + }; + } catch (error) { + logger.error('Error parsing function arguments:', error); + result.functionCall = { + name: toolCall.function.name, + arguments: toolCall.function.arguments // Keep as string if parsing fails + }; + } + } + } else if (message.content) { + // Plain text response + result.text = message.content.trim(); + } + + return result; + } + + /** + * Call the BrowserOperator API with messages + */ + async callWithMessages( + modelName: string, + messages: LLMMessage[], + options?: LLMCallOptions + ): Promise { + return LLMRetryManager.simpleRetry(async () => { + logger.debug('Calling BrowserOperator with messages...', { model: modelName, messageCount: messages.length }); + + // Construct payload body in OpenAI Chat Completions format + const payloadBody: any = { + model: modelName, // Use model alias (main/mini/nano) + messages: this.convertMessagesToOpenAI(messages), + }; + + // Add temperature if provided + if (options?.temperature !== undefined) { + payloadBody.temperature = options.temperature; + } + + // Add tools if provided + if (options?.tools) { + // Ensure all tools have valid parameters + payloadBody.tools = options.tools.map(tool => { + if (tool.type === 'function' && tool.function) { + return { + ...tool, + function: { + ...tool.function, + parameters: tool.function.parameters || { type: 'object', properties: {} } + } + }; + } + return tool; + }); + } + + // Ensure tool_choice is set to 'auto' when tools are present unless explicitly provided + if (options?.tools && !options?.tool_choice) { + payloadBody.tool_choice = 'auto'; + } else if (options?.tool_choice) { + payloadBody.tool_choice = options.tool_choice; + } + + logger.info('Request payload:', payloadBody); + + // Extract agent name from options (set by AgentRunner) + const agentName = options?.agentName; + + const data = await this.makeAPIRequest(payloadBody, agentName); + return this.processBrowserOperatorResponse(data); + }, options?.retryConfig); + } + + /** + * Simple call method for backward compatibility + */ + async call( + modelName: string, + prompt: string, + systemPrompt: string, + options?: LLMCallOptions + ): Promise { + const messages: LLMMessage[] = []; + + if (systemPrompt) { + messages.push({ + role: 'system', + content: systemPrompt + }); + } + + messages.push({ + role: 'user', + content: prompt + }); + + return this.callWithMessages(modelName, messages, options); + } + + /** + * Parse response into standardized action structure + */ + parseResponse(response: LLMResponse): ReturnType { + return LLMResponseParser.parseResponse(response); + } + + /** + * Get all models supported by this provider + * Returns static list of model aliases - API server handles provider-specific mapping + */ + async getModels(): Promise { + return [ + { + id: 'main', + name: 'Auto', + provider: 'browseroperator' as LLMProvider, + capabilities: { + functionCalling: true, + reasoning: false, + vision: false, + structured: true + } + }, + { + id: 'mini', + name: 'Auto', + provider: 'browseroperator' as LLMProvider, + capabilities: { + functionCalling: true, + reasoning: false, + vision: false, + structured: true + } + }, + { + id: 'nano', + name: 'Auto', + provider: 'browseroperator' as LLMProvider, + capabilities: { + functionCalling: true, + reasoning: false, + vision: false, + structured: true + } + } + ]; + } + + /** + * Test the BrowserOperator connection with a health check + */ + async testConnection(modelName: string): Promise<{success: boolean, message: string}> { + logger.debug('Testing BrowserOperator connection...'); + + try { + const healthUrl = this.getHealthEndpoint(); + logger.debug('Health check URL:', healthUrl); + + const response = await fetch(healthUrl); + + if (!response.ok) { + return { + success: false, + message: `Health check failed: ${response.statusText}` + }; + } + + const data = await response.json(); + + // Also test a simple chat completion + const testPrompt = 'Please respond with "Connection successful!" to confirm the connection is working.'; + const testResponse = await this.call(modelName, testPrompt, '', { + temperature: 0.1, + }); + + if (testResponse.text?.toLowerCase().includes('connection')) { + return { + success: true, + message: `Successfully connected to BrowserOperator API server. Health: ${data.status}`, + }; + } + return { + success: true, + message: `Connected to BrowserOperator, but received unexpected response: ${testResponse.text || 'No response'}`, + }; + } catch (error) { + logger.error('BrowserOperator connection test failed:', error); + return { + success: false, + message: error instanceof Error ? error.message : 'Unknown error occurred', + }; + } + } + + /** + * Validate that required credentials are available for BrowserOperator + * No credentials needed - endpoint is hardcoded + */ + validateCredentials(): {isValid: boolean, message: string, missingItems?: string[]} { + return { + isValid: true, + message: `BrowserOperator configured with endpoint: ${BrowserOperatorProvider.DEFAULT_BASE_URL}. Agent routing is automatic.` + }; + } + + /** + * Get the storage keys this provider uses for credentials + * Returns empty object since endpoint and agent are hardcoded/automatic + */ + getCredentialStorageKeys(): {apiKey?: string} { + return { + apiKey: 'ai_chat_browseroperator_api_key' // Optional API key for authentication + }; + } +} diff --git a/front_end/panels/ai_chat/LLM/LLMClient.ts b/front_end/panels/ai_chat/LLM/LLMClient.ts index 95c22037a2..e04565303b 100644 --- a/front_end/panels/ai_chat/LLM/LLMClient.ts +++ b/front_end/panels/ai_chat/LLM/LLMClient.ts @@ -8,6 +8,7 @@ import { OpenAIProvider } from './OpenAIProvider.js'; import { LiteLLMProvider } from './LiteLLMProvider.js'; import { GroqProvider } from './GroqProvider.js'; import { OpenRouterProvider } from './OpenRouterProvider.js'; +import { BrowserOperatorProvider } from './BrowserOperatorProvider.js'; import { LLMResponseParser } from './LLMResponseParser.js'; import { createLogger } from '../core/Logger.js'; @@ -40,6 +41,7 @@ export interface LLMCallRequest { tools?: any[]; temperature?: number; retryConfig?: Partial; + agentName?: string; // Name of the calling agent for provider-specific routing } /** @@ -92,6 +94,12 @@ export class LLMClient { case 'openrouter': providerInstance = new OpenRouterProvider(providerConfig.apiKey); break; + case 'browseroperator': + providerInstance = new BrowserOperatorProvider( + providerConfig.apiKey || null, + providerConfig.providerURL // Optional override for testing + ); + break; default: logger.warn(`Unknown provider type: ${providerConfig.provider}`); continue; @@ -154,6 +162,10 @@ export class LLMClient { if (request.retryConfig) { options.retryConfig = request.retryConfig; } + // Forward agent name for provider-specific routing + if ((request as any).agentName) { + options.agentName = (request as any).agentName; + } return provider.callWithMessages(request.model, messages, options); } @@ -343,6 +355,34 @@ export class LLMClient { return provider.testConnection(modelName); } + /** + * Static method to test BrowserOperator connection (for UI use without initialization) + */ + static async testBrowserOperatorConnection(endpoint: string): Promise<{success: boolean, message: string}> { + try { + const healthUrl = endpoint.replace(/\/v1\/?$/, '') + '/health'; + const response = await fetch(healthUrl); + + if (!response.ok) { + return { + success: false, + message: `Health check failed: ${response.statusText}` + }; + } + + const data = await response.json(); + return { + success: true, + message: `Connected to BrowserOperator API server. Status: ${data.status}` + }; + } catch (error) { + return { + success: false, + message: error instanceof Error ? error.message : 'Unknown error occurred' + }; + } + } + /** * Static method to validate credentials for a specific provider */ @@ -364,6 +404,9 @@ export class LLMClient { case 'openrouter': provider = new OpenRouterProvider(''); break; + case 'browseroperator': + provider = new BrowserOperatorProvider(null, ''); + break; default: return { isValid: false, diff --git a/front_end/panels/ai_chat/LLM/LLMTypes.ts b/front_end/panels/ai_chat/LLM/LLMTypes.ts index 645de39ace..5f86739572 100644 --- a/front_end/panels/ai_chat/LLM/LLMTypes.ts +++ b/front_end/panels/ai_chat/LLM/LLMTypes.ts @@ -54,13 +54,16 @@ export interface UnifiedLLMOptions { // Tool usage (for function calling) tools?: any[]; tool_choice?: any; - + + // Agent context + agentName?: string; // Name of the calling agent for provider-specific routing + // Feature flags strictJsonMode?: boolean; // Enables strict JSON parsing with retries - + // Retry configuration override customRetryConfig?: Partial; - + // Legacy compatibility (deprecated - use customRetryConfig instead) maxRetries?: number; } @@ -139,7 +142,7 @@ export interface ExtendedRetryConfig extends ErrorRetryConfig { /** * LLM Provider types */ -export type LLMProvider = 'openai' | 'litellm' | 'groq' | 'openrouter'; +export type LLMProvider = 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator'; /** * Content types for multimodal messages (text + images + files) @@ -197,6 +200,7 @@ export interface LLMCallOptions { temperature?: number; reasoningLevel?: 'low' | 'medium' | 'high'; // For O-series models retryConfig?: Partial; + agentName?: string; // Name of the calling agent for provider-specific routing } /** diff --git a/front_end/panels/ai_chat/agent_framework/AgentRunner.ts b/front_end/panels/ai_chat/agent_framework/AgentRunner.ts index fc05dba479..0ba86d5565 100644 --- a/front_end/panels/ai_chat/agent_framework/AgentRunner.ts +++ b/front_end/panels/ai_chat/agent_framework/AgentRunner.ts @@ -666,6 +666,7 @@ export class AgentRunner { systemPrompt: currentSystemPrompt, tools: toolSchemas, temperature: temperature ?? 0, + agentName: agentName, // Pass agent identity for provider-specific routing }); // Complete the generation observation diff --git a/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts b/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts index 6ef74bf40a..99b77afec9 100644 --- a/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts +++ b/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts @@ -428,8 +428,13 @@ export class ConfigurableAgentTool implements Tool t.name) }); + // Resolve agent name for provider-specific routing + const agentName = agentDescriptor?.name || state.selectedAgentType || 'default'; + // Execute LLM call with retry logic const retryResult = await errorHandler.executeWithRetry( async () => { @@ -235,6 +238,7 @@ export function createAgentNode(modelName: string, provider: LLMProvider, temper } })), temperature: this.temperature, + agentName: agentName, }); // Parse the response @@ -1014,7 +1018,8 @@ export function createToolExecutorNode(state: AgentState, provider: LLMProvider, isError, toolCallId, // Link back to the tool call for OpenAI format ...(isError && { error: resultText }), - uiLane: isAgentTool ? 'agent' as const : 'chat', + // On errors, surface the tool result in the main chat lane so users see it + uiLane: (isAgentTool && !isError) ? 'agent' as const : 'chat', }; logger.debug('ToolExecutorNode: Adding tool result message with toolCallId:', { toolCallId, toolResultMessage }); diff --git a/front_end/panels/ai_chat/core/AgentService.ts b/front_end/panels/ai_chat/core/AgentService.ts index b244be4558..1a4b8b69ed 100644 --- a/front_end/panels/ai_chat/core/AgentService.ts +++ b/front_end/panels/ai_chat/core/AgentService.ts @@ -201,6 +201,14 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ }); } break; + case 'browseroperator': + // BrowserOperator doesn't require apiKey + // But we pass it if available for optional authentication + providers.push({ + provider: 'browseroperator' as const, + apiKey: apiKey || '' + }); + break; } if (providers.length === 0) { @@ -238,6 +246,8 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ providerName = 'Groq'; } else if (provider === 'openrouter') { providerName = 'OpenRouter'; + } else if (provider === 'browseroperator') { + providerName = 'BrowserOperator'; } throw new Error(`${providerName} API key is required for this configuration`); } @@ -787,14 +797,19 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ if (selectedProvider === 'openrouter') { return true; } - + + // BrowserOperator provider doesn't require an API key (endpoint is hardcoded) + if (selectedProvider === 'browseroperator') { + return false; + } + // For LiteLLM, only require API key if no endpoint is configured if (selectedProvider === 'litellm') { const hasLiteLLMEndpoint = Boolean(localStorage.getItem('ai_chat_litellm_endpoint')); // If we have an endpoint, API key is optional return !hasLiteLLMEndpoint; } - + // Default to requiring API key for any unknown provider return true; } catch (error) { diff --git a/front_end/panels/ai_chat/core/LLMConfigurationManager.ts b/front_end/panels/ai_chat/core/LLMConfigurationManager.ts index 2ce3aa2160..728e900812 100644 --- a/front_end/panels/ai_chat/core/LLMConfigurationManager.ts +++ b/front_end/panels/ai_chat/core/LLMConfigurationManager.ts @@ -33,6 +33,7 @@ const STORAGE_KEYS = { LITELLM_API_KEY: 'ai_chat_litellm_api_key', GROQ_API_KEY: 'ai_chat_groq_api_key', OPENROUTER_API_KEY: 'ai_chat_openrouter_api_key', + BROWSEROPERATOR_API_KEY: 'ai_chat_browseroperator_api_key', } as const; /** @@ -118,6 +119,8 @@ export class LLMConfigurationManager { return localStorage.getItem(STORAGE_KEYS.GROQ_API_KEY) || ''; case 'openrouter': return localStorage.getItem(STORAGE_KEYS.OPENROUTER_API_KEY) || ''; + case 'browseroperator': + return localStorage.getItem(STORAGE_KEYS.BROWSEROPERATOR_API_KEY) || ''; default: return ''; } diff --git a/front_end/panels/ai_chat/tools/CombinedExtractionTool.ts b/front_end/panels/ai_chat/tools/CombinedExtractionTool.ts index dee45efd7b..0f8699af8b 100644 --- a/front_end/panels/ai_chat/tools/CombinedExtractionTool.ts +++ b/front_end/panels/ai_chat/tools/CombinedExtractionTool.ts @@ -84,7 +84,13 @@ export class CombinedExtractionTool implements Tool { logger.debug('Executing with args', args); - + const { schema, instruction, reasoning } = args; const agentService = AgentService.getInstance(); const apiKey = agentService.getApiKey(); - if (!apiKey) { + // Get provider from context + const provider = ctx?.provider; + + // BrowserOperator doesn't require API key + const requiresApiKey = provider !== 'browseroperator'; + + if (requiresApiKey && !apiKey) { return { success: false, data: null, @@ -173,7 +179,7 @@ Schema Examples: instruction: instruction || 'Extract data according to schema', domContent: treeText, schema: transformedSchema, - apiKey, + apiKey: apiKey || '', // Use empty string for BrowserOperator ctx, }); @@ -185,13 +191,21 @@ Schema Examples: data: null, }; } + // Check if extraction returned a parsing error + if (initialExtraction.__parsing_failed__) { + return { + success: false, + error: initialExtraction.__error__ || 'JSON parsing failed during extraction', + data: null, + }; + } // 6. Refine Call const refinedData = await this.callRefinementLLM({ instruction: instruction || 'Refine the extracted data based on the original request', schema: transformedSchema, // Use the same transformed schema initialData: initialExtraction, - apiKey, + apiKey: apiKey || '', // Use empty string for BrowserOperator ctx, }); @@ -203,11 +217,19 @@ Schema Examples: data: null, }; } + // Check if refinement returned a parsing error + if (refinedData.__parsing_failed__) { + return { + success: false, + error: refinedData.__error__ || 'JSON parsing failed during refinement', + data: null, + }; + } // 7. LLM + Tool Call for URL Resolution - New approach const finalData = await this.resolveUrlsWithLLM({ data: refinedData, - apiKey, + apiKey: apiKey || '', // Use empty string for BrowserOperator schema, // Original schema to understand what fields are URLs }); @@ -229,7 +251,7 @@ Schema Examples: extractedData: finalData, // Use the final data with URLs for assessment domContent: treeText, // Pass the DOM content for context schema, // Pass the schema to understand what was requested - apiKey, + apiKey: apiKey || '', // Use empty string for BrowserOperator ctx, }); @@ -487,8 +509,16 @@ Only output the JSON object with real data from the accessibility tree.`; try { return LLMResponseParser.parseJSONWithFallbacks(response); } catch (e) { + const errorMsg = e instanceof Error ? e.message : String(e); logger.error('Failed to parse extraction JSON:', e); - return null; + logger.warn('Raw LLM response:', response.substring(0, 500)); + // Return error object with embedded raw response + return { + __parsing_failed__: true, + __error__: `JSON parsing failed during extraction: ${errorMsg}\n\nRaw LLM Response:\n${response}`, + __raw_response__: response, + __step__: 'extraction' + }; } } } catch (error) { @@ -579,8 +609,16 @@ Do not add any conversational text or explanations or thinking tags.`; try { return LLMResponseParser.parseJSONWithFallbacks(response); } catch (e) { + const errorMsg = e instanceof Error ? e.message : String(e); logger.error('Failed to parse refinement JSON:', e); - return null; + logger.warn('Raw LLM response:', response.substring(0, 500)); + // Return error object with embedded raw response + return { + __parsing_failed__: true, + __error__: `JSON parsing failed during refinement: ${errorMsg}\n\nRaw LLM Response:\n${response}`, + __raw_response__: response, + __step__: 'refinement' + }; } } } catch (error) { diff --git a/front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts b/front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts index 0cac2d2087..4e29c7fec8 100644 --- a/front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts +++ b/front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts @@ -82,7 +82,7 @@ export class StreamlinedSchemaExtractorTool implements Tool { try { - const context = await this.setupExecution(args); + const context = await this.setupExecution(args, ctx); if (context.success !== true) { return context as StreamlinedExtractionResult; } @@ -105,12 +105,18 @@ export class StreamlinedSchemaExtractorTool implements Tool { + private async setupExecution(args: StreamlinedSchemaExtractionArgs, ctx?: LLMContext): Promise { const { schema, instruction } = args; const agentService = AgentService.getInstance(); const apiKey = agentService.getApiKey(); - if (!apiKey) { + // Get provider from context + const provider = ctx?.provider; + + // BrowserOperator doesn't require API key + const requiresApiKey = provider !== 'browseroperator'; + + if (requiresApiKey && !apiKey) { return { success: false, data: null, @@ -141,7 +147,7 @@ export class StreamlinedSchemaExtractorTool implements Tool { return await this.extractWithJsonRetry( - context.schema, - context.treeText, - context.instruction, - context.apiKey, + context.schema, + context.treeText, + context.instruction, + context.apiKey || '', // Use empty string for BrowserOperator this.MAX_JSON_RETRIES, ctx ); @@ -191,7 +197,7 @@ export class StreamlinedSchemaExtractorTool implements Tool option.value === modelName); - const originalProvider = (modelOption?.type as 'openai' | 'litellm' | 'groq' | 'openrouter') || 'openai'; + const originalProvider = (modelOption?.type as 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator') || 'openai'; // Check if the model's original provider is available in the registry if (LLMProviderRegistry.hasProvider(originalProvider)) { @@ -386,15 +392,15 @@ export class AIChatPanel extends UI.Panel.Panel { // If the original provider isn't available, fall back to the currently selected provider const currentProvider = localStorage.getItem(PROVIDER_SELECTION_KEY) || 'openai'; logger.debug(`Provider ${originalProvider} not available for model ${modelName}, falling back to current provider: ${currentProvider}`); - return currentProvider as 'openai' | 'litellm' | 'groq' | 'openrouter'; + return currentProvider as 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator'; } /** * Gets the currently selected provider from localStorage * @returns The currently selected provider */ - static getCurrentProvider(): 'openai' | 'litellm' | 'groq' | 'openrouter' { - return (localStorage.getItem(PROVIDER_SELECTION_KEY) || 'openai') as 'openai' | 'litellm' | 'groq' | 'openrouter'; + static getCurrentProvider(): 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator' { + return (localStorage.getItem(PROVIDER_SELECTION_KEY) || 'openai') as 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator'; } /** @@ -467,7 +473,7 @@ export class AIChatPanel extends UI.Panel.Panel { * @param provider Optional provider to filter by * @returns Array of model options */ - static getModelOptions(provider?: 'openai' | 'litellm' | 'groq' | 'openrouter'): ModelOption[] { + static getModelOptions(provider?: 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator'): ModelOption[] { // Try to get from all_model_options first (comprehensive list) const allModelOptionsStr = localStorage.getItem('ai_chat_all_model_options'); if (allModelOptionsStr) { @@ -538,14 +544,16 @@ export class AIChatPanel extends UI.Panel.Panel { const existingLiteLLMModels = existingAllModels.filter((m: ModelOption) => m.type === 'litellm'); const existingGroqModels = existingAllModels.filter((m: ModelOption) => m.type === 'groq'); const existingOpenRouterModels = existingAllModels.filter((m: ModelOption) => m.type === 'openrouter'); - + const existingBrowserOperatorModels = existingAllModels.filter((m: ModelOption) => m.type === 'browseroperator'); + // Update models based on what type of models we're adding // Always use DEFAULT_OPENAI_MODELS for OpenAI to ensure we have the latest hardcoded list let updatedOpenAIModels = DEFAULT_OPENAI_MODELS; let updatedLiteLLMModels = existingLiteLLMModels; let updatedGroqModels = existingGroqModels; let updatedOpenRouterModels = existingOpenRouterModels; - + let updatedBrowserOperatorModels = existingBrowserOperatorModels; + // Replace models for the provider type we're updating if (providerModels.length > 0) { const firstModelType = providerModels[0].type; @@ -557,6 +565,8 @@ export class AIChatPanel extends UI.Panel.Panel { updatedOpenRouterModels = providerModels; } else if (firstModelType === 'openai') { updatedOpenAIModels = providerModels; + } else if (firstModelType === 'browseroperator') { + updatedBrowserOperatorModels = providerModels; } } @@ -565,7 +575,8 @@ export class AIChatPanel extends UI.Panel.Panel { ...updatedOpenAIModels, ...updatedLiteLLMModels, ...updatedGroqModels, - ...updatedOpenRouterModels + ...updatedOpenRouterModels, + ...updatedBrowserOperatorModels ]; // Save the comprehensive list to localStorage @@ -588,7 +599,7 @@ export class AIChatPanel extends UI.Panel.Panel { } } else if (selectedProvider === 'openrouter') { MODEL_OPTIONS = updatedOpenRouterModels; - + // Add placeholder if no OpenRouter models available if (MODEL_OPTIONS.length === 0) { MODEL_OPTIONS.push({ @@ -597,10 +608,21 @@ export class AIChatPanel extends UI.Panel.Panel { type: 'openrouter' as const }); } + } else if (selectedProvider === 'browseroperator') { + MODEL_OPTIONS = updatedBrowserOperatorModels; + + // Add placeholder if no BrowserOperator models available + if (MODEL_OPTIONS.length === 0) { + MODEL_OPTIONS.push({ + value: MODEL_PLACEHOLDERS.NO_MODELS, + label: 'BrowserOperator: Models not loaded', + type: 'browseroperator' as const + }); + } } else { // For LiteLLM provider, include custom models and fetched models MODEL_OPTIONS = updatedLiteLLMModels; - + // Add placeholder if needed for LiteLLM when we have no models if (hadWildcard && MODEL_OPTIONS.length === 0) { MODEL_OPTIONS.push({ @@ -633,7 +655,7 @@ export class AIChatPanel extends UI.Panel.Panel { * @param modelType Type of the model ('openai' or 'litellm') * @returns Updated model options */ - static addCustomModelOption(modelName: string, modelType: 'openai' | 'litellm' | 'groq' | 'openrouter' = 'litellm'): ModelOption[] { + static addCustomModelOption(modelName: string, modelType: 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator' = 'litellm'): ModelOption[] { // Get existing custom models const savedCustomModels = JSON.parse(localStorage.getItem('ai_chat_custom_models') || '[]'); @@ -973,7 +995,7 @@ export class AIChatPanel extends UI.Panel.Panel { const currentProvider = localStorage.getItem(PROVIDER_SELECTION_KEY) || 'openai'; const providerDefaults = DEFAULT_PROVIDER_MODELS[currentProvider] || DEFAULT_PROVIDER_MODELS.openai; - const availableModels = AIChatPanel.getModelOptions(currentProvider as 'openai' | 'litellm' | 'groq' | 'openrouter'); + const availableModels = AIChatPanel.getModelOptions(currentProvider as 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator'); let allValid = true; @@ -1463,24 +1485,24 @@ export class AIChatPanel extends UI.Panel.Panel { * @returns true if at least one provider has valid credentials */ #hasAnyProviderCredentials(): boolean { - + const selectedProvider = localStorage.getItem(PROVIDER_SELECTION_KEY) || 'openai'; - + // Check all providers except LiteLLM (unless LiteLLM is selected) - const providers = ['openai', 'groq', 'openrouter']; - + const providers = ['openai', 'groq', 'openrouter', 'browseroperator']; + // Only include LiteLLM if it's the selected provider if (selectedProvider === 'litellm') { providers.push('litellm'); } - + for (const provider of providers) { const validation = LLMClient.validateProviderCredentials(provider); if (validation.isValid) { return true; } } - + return false; } @@ -1524,6 +1546,9 @@ export class AIChatPanel extends UI.Panel.Panel { case 'openrouter': tempProvider = new OpenRouterProvider(''); break; + case 'browseroperator': + tempProvider = new BrowserOperatorProvider(null, ''); + break; default: logger.warn(`Unknown provider: ${provider}`); return {canProceed: false, apiKey: null}; @@ -2059,6 +2084,7 @@ export class AIChatPanel extends UI.Panel.Panel { isModelSelectorDisabled: this.#isProcessing, isInputDisabled: false, inputPlaceholder: this.#getInputPlaceholderText(), + currentProvider: localStorage.getItem(PROVIDER_SELECTION_KEY) || 'openai', // Add OAuth login state showOAuthLogin: (() => { if (BUILD_CONFIG.AUTOMATED_MODE) { diff --git a/front_end/panels/ai_chat/ui/ChatView.ts b/front_end/panels/ai_chat/ui/ChatView.ts index fa0ceb0bb4..4794f18d4e 100644 --- a/front_end/panels/ai_chat/ui/ChatView.ts +++ b/front_end/panels/ai_chat/ui/ChatView.ts @@ -67,6 +67,8 @@ export interface Props { // Add OAuth login related properties showOAuthLogin?: boolean; onOAuthLogin?: () => void; + // Add current provider for model selector behavior + currentProvider?: string; } @customElement('devtools-chat-view') @@ -97,6 +99,7 @@ export class ChatView extends HTMLElement { #onModelSelectorFocus?: () => void; #selectedAgentType?: string | null; #isModelSelectorDisabled = false; + #currentProvider?: string; // URL change listener #onInspectedURLChangedBound?: (event: Event) => void; #lastSuggestionHost: string | null = null; @@ -285,6 +288,7 @@ export class ChatView extends HTMLElement { this.#onModelSelectorFocus = data.onModelSelectorFocus; this.#selectedAgentType = data.selectedAgentType; this.#isModelSelectorDisabled = data.isModelSelectorDisabled || false; + this.#currentProvider = data.currentProvider; // Store input disabled state and placeholder this.#isInputDisabled = data.isInputDisabled || false; @@ -975,7 +979,13 @@ export class ChatView extends HTMLElement { // Render model selector via dedicated component #renderModelSelectorInline() { - if (!this.#modelOptions || !this.#modelOptions.length || !this.#selectedModel || !this.#onModelChanged) { + if ( + this.#currentProvider === 'browseroperator' || + !this.#modelOptions || + !this.#modelOptions.length || + !this.#selectedModel || + !this.#onModelChanged + ) { return ''; } return html` @@ -1010,6 +1020,7 @@ export class ChatView extends HTMLElement { .modelOptions=${this.#modelOptions} .selectedModel=${this.#selectedModel} .modelSelectorDisabled=${this.#isModelSelectorDisabled} + .currentProvider=${this.#currentProvider} .selectedPromptType=${this.#selectedPromptType} .agentButtonsHandler=${this.#handlePromptButtonClickBound} .centered=${centered} diff --git a/front_end/panels/ai_chat/ui/SettingsDialog.ts b/front_end/panels/ai_chat/ui/SettingsDialog.ts index 24f1e373ce..a822ae7ebf 100644 --- a/front_end/panels/ai_chat/ui/SettingsDialog.ts +++ b/front_end/panels/ai_chat/ui/SettingsDialog.ts @@ -21,7 +21,7 @@ const logger = createLogger('SettingsDialog'); interface ModelOption { value: string; label: string; - type: 'openai' | 'litellm' | 'groq' | 'openrouter'; + type: 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator'; } // Local storage keys @@ -113,6 +113,30 @@ const UIStrings = { *@description Fetch OpenRouter models button text */ fetchOpenRouterModelsButton: 'Fetch OpenRouter Models', + /** + *@description BrowserOperator provider option + */ + browseroperatorProvider: 'BrowserOperator', + /** + *@description BrowserOperator endpoint label + */ + browseroperatorEndpointLabel: 'BrowserOperator Endpoint', + /** + *@description BrowserOperator endpoint hint + */ + browseroperatorEndpointHint: 'URL for your BrowserOperator API server (e.g., http://localhost:8080/v1)', + /** + *@description BrowserOperator agent label + */ + browseroperatorAgentLabel: 'Default Agent', + /** + *@description BrowserOperator agent hint + */ + browseroperatorAgentHint: 'Agent for routing: deep-research (Cerebras), web-agent (OpenAI), code-assist (Groq), chat, fast, default', + /** + *@description Test BrowserOperator connection button + */ + testBrowseroperatorConnection: 'Test Connection', /** *@description OpenAI API Key label */ @@ -502,6 +526,8 @@ export class SettingsDialog { static #groqNanoModelSelect: any | null = null; static #openrouterMiniModelSelect: any | null = null; static #openrouterNanoModelSelect: any | null = null; + static #browseroperatorMiniModelSelect: any | null = null; + static #browseroperatorNanoModelSelect: any | null = null; static async show( selectedModel: string, @@ -572,7 +598,7 @@ export class SettingsDialog { providerSection.appendChild(providerHint); // Use the stored provider from localStorage - const currentProvider = (localStorage.getItem(PROVIDER_SELECTION_KEY) || 'openai') as 'openai' | 'litellm' | 'groq' | 'openrouter'; + const currentProvider = (localStorage.getItem(PROVIDER_SELECTION_KEY) || 'openai') as 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator'; // Create provider selection dropdown const providerSelect = document.createElement('select'); @@ -604,6 +630,12 @@ export class SettingsDialog { openrouterOption.selected = currentProvider === 'openrouter'; providerSelect.appendChild(openrouterOption); + const browseroperatorOption = document.createElement('option'); + browseroperatorOption.value = 'browseroperator'; + browseroperatorOption.textContent = i18nString(UIStrings.browseroperatorProvider); + browseroperatorOption.selected = currentProvider === 'browseroperator'; + providerSelect.appendChild(browseroperatorOption); + // Ensure the select's value reflects the computed currentProvider providerSelect.value = currentProvider; @@ -627,7 +659,12 @@ export class SettingsDialog { openrouterContent.className = 'provider-content openrouter-content'; openrouterContent.style.display = currentProvider === 'openrouter' ? 'block' : 'none'; contentDiv.appendChild(openrouterContent); - + + const browseroperatorContent = document.createElement('div'); + browseroperatorContent.className = 'provider-content browseroperator-content'; + browseroperatorContent.style.display = currentProvider === 'browseroperator' ? 'block' : 'none'; + contentDiv.appendChild(browseroperatorContent); + // Event listener for provider change providerSelect.addEventListener('change', async () => { const selectedProvider = providerSelect.value; @@ -637,7 +674,7 @@ export class SettingsDialog { litellmContent.style.display = selectedProvider === 'litellm' ? 'block' : 'none'; groqContent.style.display = selectedProvider === 'groq' ? 'block' : 'none'; openrouterContent.style.display = selectedProvider === 'openrouter' ? 'block' : 'none'; - + browseroperatorContent.style.display = selectedProvider === 'browseroperator' ? 'block' : 'none'; // If switching to LiteLLM, fetch the latest models if endpoint is configured if (selectedProvider === 'litellm') { @@ -1983,9 +2020,9 @@ export class SettingsDialog { // Generic helper function to get valid model for provider function getValidModelForProvider( - currentModel: string, - providerModels: ModelOption[], - provider: 'openai' | 'litellm' | 'groq' | 'openrouter', + currentModel: string, + providerModels: ModelOption[], + provider: 'openai' | 'litellm' | 'groq' | 'openrouter' | 'browseroperator', modelType: 'mini' | 'nano' ): string { // Check if current model is valid for this provider @@ -2539,7 +2576,71 @@ export class SettingsDialog { // Initialize OpenRouter model selectors await updateOpenRouterModelSelectors(); - + + // Setup BrowserOperator content + const browseroperatorSettingsSection = document.createElement('div'); + browseroperatorSettingsSection.className = 'settings-section'; + browseroperatorContent.appendChild(browseroperatorSettingsSection); + + // Info display - endpoint and agent routing are automatic + const browseroperatorInfoLabel = document.createElement('div'); + browseroperatorInfoLabel.className = 'settings-label'; + browseroperatorInfoLabel.textContent = 'Configuration'; + browseroperatorSettingsSection.appendChild(browseroperatorInfoLabel); + + const browseroperatorInfoText = document.createElement('div'); + browseroperatorInfoText.className = 'settings-hint'; + browseroperatorInfoText.innerHTML = ` + Endpoint: http://localhost:8080/v1 (hardcoded)
+ Agent Routing: Automatic based on calling agent (e.g., research_agent, action_agent) + `; + browseroperatorInfoText.style.marginBottom = '16px'; + browseroperatorSettingsSection.appendChild(browseroperatorInfoText); + + // Model selection section + const browseroperatorModelSection = document.createElement('div'); + browseroperatorModelSection.className = 'settings-section model-selection-section'; + browseroperatorContent.appendChild(browseroperatorModelSection); + + // Static model list for BrowserOperator + const browseroperatorModels: ModelOption[] = [ + { value: 'main', label: 'Auto', type: 'browseroperator' }, + { value: 'mini', label: 'Auto', type: 'browseroperator' }, + { value: 'nano', label: 'Auto', type: 'browseroperator' } + ]; + + // Add models to global model options + updateModelOptions(browseroperatorModels, false); + + const validMiniModel = getValidModelForProvider(miniModel, browseroperatorModels, 'browseroperator', 'mini'); + const validNanoModel = getValidModelForProvider(nanoModel, browseroperatorModels, 'browseroperator', 'nano'); + + SettingsDialog.#browseroperatorMiniModelSelect = createModelSelector( + browseroperatorModelSection, + i18nString(UIStrings.miniModelLabel), + i18nString(UIStrings.miniModelDescription), + 'browseroperator-mini-model-select', + browseroperatorModels, + validMiniModel, + i18nString(UIStrings.defaultMiniOption), + undefined + ); + // Disable the selector since BrowserOperator uses automatic routing + SettingsDialog.#browseroperatorMiniModelSelect.disabled = true; + + SettingsDialog.#browseroperatorNanoModelSelect = createModelSelector( + browseroperatorModelSection, + i18nString(UIStrings.nanoModelLabel), + i18nString(UIStrings.nanoModelDescription), + 'browseroperator-nano-model-select', + browseroperatorModels, + validNanoModel, + i18nString(UIStrings.defaultNanoOption), + undefined + ); + // Disable the selector since BrowserOperator uses automatic routing + SettingsDialog.#browseroperatorNanoModelSelect.disabled = true; + // Add Vector DB configuration section const vectorDBSection = document.createElement('div'); vectorDBSection.className = 'settings-section vector-db-section'; @@ -3292,7 +3393,9 @@ export class SettingsDialog { } else { localStorage.removeItem(OPENROUTER_API_KEY_STORAGE_KEY); } - + + // BrowserOperator settings are hardcoded - no need to save endpoint or agent + // Determine which mini/nano model selectors to use based on current provider let miniModelValue = ''; let nanoModelValue = ''; @@ -3329,6 +3432,14 @@ export class SettingsDialog { if (SettingsDialog.#openrouterNanoModelSelect) { nanoModelValue = SettingsDialog.#openrouterNanoModelSelect.value; } + } else if (selectedProvider === 'browseroperator') { + // Get values from BrowserOperator selectors + if (SettingsDialog.#browseroperatorMiniModelSelect) { + miniModelValue = SettingsDialog.#browseroperatorMiniModelSelect.value; + } + if (SettingsDialog.#browseroperatorNanoModelSelect) { + nanoModelValue = SettingsDialog.#browseroperatorNanoModelSelect.value; + } } // Save mini model if selected diff --git a/front_end/panels/ai_chat/ui/__tests__/ChatViewAgentSessions.test.ts b/front_end/panels/ai_chat/ui/__tests__/ChatViewAgentSessions.test.ts index eb173ed511..a36544a64c 100644 --- a/front_end/panels/ai_chat/ui/__tests__/ChatViewAgentSessions.test.ts +++ b/front_end/panels/ai_chat/ui/__tests__/ChatViewAgentSessions.test.ts @@ -344,6 +344,26 @@ describe('ChatView visibility rules: agent-managed tool calls/results are hidden document.body.removeChild(view); }); + it('shows configurable agent tool error in chat when execution fails', async () => { + const view = document.createElement('devtools-chat-view') as any; + document.body.appendChild(view); + const toolCallId = 'id-err'; + + // Simulate: agent-lane tool call (filtered from chat) + error tool result routed to chat lane + view.data = {messages: [ + makeUser('start'), + { entity: ChatMessageEntity.MODEL, action: 'tool', toolName: 'search_agent', toolCallId, isFinalAnswer: false, uiLane: 'agent' } as any, + { entity: ChatMessageEntity.TOOL_RESULT, toolName: 'search_agent', toolCallId, resultText: 'url.trim is not a function', isError: true, uiLane: 'chat' } as any, + ], state: 'idle', isTextInputEmpty: true, onSendMessage: () => {}, onPromptSelected: () => {}} as any; + await raf(); + + const shadow = view.shadowRoot!; + const errors = shadow.querySelectorAll('.tool-result-message.error'); + assert.isAtLeast(errors.length, 1); + assert.include((errors[0].textContent || ''), 'url.trim is not a function'); + document.body.removeChild(view); + }); + it('mixed: agent-managed tool hidden; regular tool visible', async () => { const view = document.createElement('devtools-chat-view') as any; document.body.appendChild(view); diff --git a/front_end/panels/ai_chat/ui/input/InputBar.ts b/front_end/panels/ai_chat/ui/input/InputBar.ts index cc68cb70fc..aafde41f54 100644 --- a/front_end/panels/ai_chat/ui/input/InputBar.ts +++ b/front_end/panels/ai_chat/ui/input/InputBar.ts @@ -24,6 +24,7 @@ export class InputBar extends HTMLElement { #modelOptions?: Array<{value: string, label: string}>; #selectedModel?: string; #modelSelectorDisabled = false; + #currentProvider?: string; #selectedPromptType?: string|null; #agentButtonsHandler: (event: Event) => void = () => {}; #centered = false; @@ -35,6 +36,7 @@ export class InputBar extends HTMLElement { set modelOptions(v: Array<{value: string, label: string}>|undefined) { this.#modelOptions = v; this.#render(); } set selectedModel(v: string|undefined) { this.#selectedModel = v; this.#render(); } set modelSelectorDisabled(v: boolean) { this.#modelSelectorDisabled = !!v; this.#render(); } + set currentProvider(v: string|undefined) { this.#currentProvider = v; this.#render(); } set selectedPromptType(v: string|null|undefined) { this.#selectedPromptType = v ?? null; this.#render(); } set agentButtonsHandler(fn: (event: Event) => void) { this.#agentButtonsHandler = fn || (() => {}); this.#render(); } set centered(v: boolean) { this.#centered = !!v; this.#render(); } @@ -106,7 +108,12 @@ export class InputBar extends HTMLElement { const agentButtons = BaseOrchestratorAgent.renderAgentTypeButtons(this.#selectedPromptType ?? null, this.#agentButtonsHandler, this.#centered); - const modelSelector = (this.#modelOptions && this.#modelOptions.length && this.#selectedModel) ? html` + const modelSelector = ( + this.#currentProvider !== 'browseroperator' && + this.#modelOptions && + this.#modelOptions.length && + this.#selectedModel + ) ? html` Date: Sun, 2 Nov 2025 22:26:36 -0800 Subject: [PATCH 2/6] More improvements and additional tools --- front_end/panels/ai_chat/BUILD.gn | 6 + .../ai_chat/agent_framework/AgentRunner.ts | 111 ++- .../agent_framework/AgentRunnerEventBus.ts | 2 +- .../implementation/ConfiguredAgents.ts | 4 + .../implementation/agents/ResearchAgent.ts | 2 + .../implementation/agents/SearchAgent.ts | 2 + .../implementation/agents/WebTaskAgent.ts | 2 + front_end/panels/ai_chat/core/AgentNodes.ts | 10 +- front_end/panels/ai_chat/core/AgentService.ts | 41 +- .../panels/ai_chat/core/PageInfoManager.ts | 23 + .../panels/ai_chat/tools/ExecuteCodeTool.ts | 123 ++++ .../ai_chat/tools/SchemaBasedExtractorTool.ts | 525 ++++++++++++-- .../ai_chat/tools/SequentialThinkingTool.ts | 16 +- front_end/panels/ai_chat/tools/Tools.ts | 43 +- .../panels/ai_chat/tools/UpdateTodoTool.ts | 129 ++++ .../ai_chat/tools/VisualIndicatorTool.ts | 639 ++++++++++++++++++ 16 files changed, 1591 insertions(+), 87 deletions(-) create mode 100644 front_end/panels/ai_chat/tools/ExecuteCodeTool.ts create mode 100644 front_end/panels/ai_chat/tools/UpdateTodoTool.ts create mode 100644 front_end/panels/ai_chat/tools/VisualIndicatorTool.ts diff --git a/front_end/panels/ai_chat/BUILD.gn b/front_end/panels/ai_chat/BUILD.gn index 8d306a8fd3..7e0a5ab4d1 100644 --- a/front_end/panels/ai_chat/BUILD.gn +++ b/front_end/panels/ai_chat/BUILD.gn @@ -99,11 +99,14 @@ devtools_module("ai_chat") { "tools/DeleteFileTool.ts", "tools/ReadFileTool.ts", "tools/ListFilesTool.ts", + "tools/UpdateTodoTool.ts", + "tools/ExecuteCodeTool.ts", "tools/SequentialThinkingTool.ts", "tools/ThinkingTool.ts", "tools/RenderWebAppTool.ts", "tools/GetWebAppDataTool.ts", "tools/RemoveWebAppTool.ts", + "tools/VisualIndicatorTool.ts", "agent_framework/ConfigurableAgentTool.ts", "agent_framework/AgentRunner.ts", "agent_framework/AgentRunnerEventBus.ts", @@ -267,11 +270,14 @@ _ai_chat_sources = [ "tools/DeleteFileTool.ts", "tools/ReadFileTool.ts", "tools/ListFilesTool.ts", + "tools/UpdateTodoTool.ts", + "tools/ExecuteCodeTool.ts", "tools/SequentialThinkingTool.ts", "tools/ThinkingTool.ts", "tools/RenderWebAppTool.ts", "tools/GetWebAppDataTool.ts", "tools/RemoveWebAppTool.ts", + "tools/VisualIndicatorTool.ts", "agent_framework/ConfigurableAgentTool.ts", "agent_framework/AgentRunner.ts", "agent_framework/AgentRunnerEventBus.ts", diff --git a/front_end/panels/ai_chat/agent_framework/AgentRunner.ts b/front_end/panels/ai_chat/agent_framework/AgentRunner.ts index 0ba86d5565..20f924642b 100644 --- a/front_end/panels/ai_chat/agent_framework/AgentRunner.ts +++ b/front_end/panels/ai_chat/agent_framework/AgentRunner.ts @@ -15,6 +15,7 @@ import { AgentErrorHandler } from '../core/AgentErrorHandler.js'; import { AgentRunnerEventBus } from './AgentRunnerEventBus.js'; import { callLLMWithTracing } from '../tools/LLMTracingWrapper.js'; import { sanitizeMessagesForModel } from '../LLM/MessageSanitizer.js'; +import { FileStorageManager } from '../tools/FileStorageManager.js'; const logger = createLogger('AgentRunner'); @@ -555,6 +556,24 @@ export class AgentRunner { // Check if execution has been aborted if (abortSignal?.aborted) { logger.info(`${agentName} execution aborted at iteration ${iteration + 1}/${maxIterations}`); + + // Complete session with abort + currentSession.status = 'error'; + currentSession.endTime = new Date(); + currentSession.terminationReason = 'error'; + + // Emit session completed event + if (AgentRunner.eventBus) { + AgentRunner.eventBus.emitProgress({ + type: 'session_completed', + sessionId: currentSession.sessionId, + parentSessionId: currentSession.parentSessionId, + agentName, + timestamp: new Date(), + data: { session: currentSession, reason: 'aborted' } + }); + } + const abortResult = createErrorResult('Execution was cancelled', messages, 'error'); return { ...abortResult, agentSession: currentSession }; } @@ -571,9 +590,25 @@ export class AgentRunner { - You are currently on step ${iteration + 1} of ${maxIterations} maximum steps. - Focus on making meaningful progress with each step.`; - // Enhance system prompt with iteration info and page context + // Inject todos into system prompt if they exist + let todosContext = ''; + try { + const fileManager = FileStorageManager.getInstance(); + const todosFile = await fileManager.readFile('todos.md'); + + if (todosFile?.content) { + todosContext = `\n\n## CURRENT TODO LIST\n${todosFile.content}\n\nUpdate the todo list using the 'update_todo' tool as you complete tasks. Mark completed items with [x].`; + } else { + todosContext = `\n\n## TODO LIST\nNo todo list exists yet. If this is a multi-step task, create a todo list using the 'update_todo' tool to track your progress.`; + } + } catch (error) { + logger.debug('Failed to read todos, skipping injection:', error); + // Continue without todos if reading fails + } + + // Enhance system prompt with iteration info, todos, and page context // This includes updating the accessibility tree inside enhancePromptWithPageContext - const currentSystemPrompt = await enhancePromptWithPageContext(systemPrompt + iterationInfo); + const currentSystemPrompt = await enhancePromptWithPageContext(systemPrompt + iterationInfo + todosContext); let llmResponse: LLMResponse; let generationId: string | undefined; // Declare in iteration scope for tool call access @@ -753,6 +788,18 @@ export class AgentRunner { agentSession.endTime = new Date(); agentSession.terminationReason = 'error'; + // Emit session completed event + if (AgentRunner.eventBus) { + AgentRunner.eventBus.emitProgress({ + type: 'session_completed', + sessionId: agentSession.sessionId, + parentSessionId: agentSession.parentSessionId, + agentName, + timestamp: new Date(), + data: { session: agentSession, reason: 'error' } + }); + } + // Use error hook with structured summary const result = createErrorResult(errorMsg, messages, 'error'); result.summary = { @@ -909,6 +956,18 @@ export class AgentRunner { agentSession.endTime = new Date(); agentSession.terminationReason = 'handed_off'; + // Emit session completed event + if (AgentRunner.eventBus) { + AgentRunner.eventBus.emitProgress({ + type: 'session_completed', + sessionId: agentSession.sessionId, + parentSessionId: agentSession.parentSessionId, + agentName, + timestamp: new Date(), + data: { session: agentSession, reason: 'handed_off' } + }); + } + return { ...handoffResult, agentSession }; } else if (toolToExecute) { @@ -1195,6 +1254,18 @@ export class AgentRunner { agentSession.endTime = new Date(); agentSession.terminationReason = 'final_answer'; + // Emit session completed event + if (AgentRunner.eventBus) { + AgentRunner.eventBus.emitProgress({ + type: 'session_completed', + sessionId: agentSession.sessionId, + parentSessionId: agentSession.parentSessionId, + agentName, + timestamp: new Date(), + data: { session: agentSession, reason: 'final_answer' } + }); + } + // Exit loop and return success with structured summary const result = createSuccessResult(answer, messages, 'final_answer'); result.summary = { @@ -1238,6 +1309,18 @@ export class AgentRunner { agentSession.endTime = new Date(); agentSession.terminationReason = 'error'; + // Emit session completed event + if (AgentRunner.eventBus) { + AgentRunner.eventBus.emitProgress({ + type: 'session_completed', + sessionId: agentSession.sessionId, + parentSessionId: agentSession.parentSessionId, + agentName, + timestamp: new Date(), + data: { session: agentSession, reason: 'error' } + }); + } + // Use error hook with structured summary const result = createErrorResult(errorMsg, messages, 'error'); result.summary = { @@ -1286,6 +1369,18 @@ export class AgentRunner { agentSession.endTime = new Date(); agentSession.terminationReason = 'handed_off'; + // Emit session completed event + if (AgentRunner.eventBus) { + AgentRunner.eventBus.emitProgress({ + type: 'session_completed', + sessionId: agentSession.sessionId, + parentSessionId: agentSession.parentSessionId, + agentName, + timestamp: new Date(), + data: { session: agentSession, reason: 'handed_off' } + }); + } + return { ...actualResult, agentSession }; // Return the result from the handoff target } } @@ -1298,6 +1393,18 @@ export class AgentRunner { agentSession.endTime = new Date(); agentSession.terminationReason = 'max_iterations'; + // Emit session completed event + if (AgentRunner.eventBus) { + AgentRunner.eventBus.emitProgress({ + type: 'session_completed', + sessionId: agentSession.sessionId, + parentSessionId: agentSession.parentSessionId, + agentName, + timestamp: new Date(), + data: { session: agentSession, reason: 'max_iterations' } + }); + } + // Generate summary of agent progress instead of generic error message const progressSummary = await this.summarizeAgentProgress(messages, maxIterations, agentName, modelName, 'max_iterations', config.provider, config.getVisionCapability); const result = createErrorResult('Agent reached maximum iterations', messages, 'max_iterations'); diff --git a/front_end/panels/ai_chat/agent_framework/AgentRunnerEventBus.ts b/front_end/panels/ai_chat/agent_framework/AgentRunnerEventBus.ts index 9f6187b906..3094835e71 100644 --- a/front_end/panels/ai_chat/agent_framework/AgentRunnerEventBus.ts +++ b/front_end/panels/ai_chat/agent_framework/AgentRunnerEventBus.ts @@ -5,7 +5,7 @@ import * as Common from '../../../core/common/common.js'; export interface AgentRunnerProgressEvent { - type: 'session_started' | 'tool_started' | 'tool_completed' | 'session_updated' | 'child_agent_started'; + type: 'session_started' | 'tool_started' | 'tool_completed' | 'session_updated' | 'child_agent_started' | 'session_completed'; sessionId: string; parentSessionId?: string; agentName: string; diff --git a/front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts b/front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts index b9050f7391..b79a7e64c2 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts @@ -9,6 +9,8 @@ import { StreamlinedSchemaExtractorTool } from '../../tools/StreamlinedSchemaExt import { BookmarkStoreTool } from '../../tools/BookmarkStoreTool.js'; import { DocumentSearchTool } from '../../tools/DocumentSearchTool.js'; import { NavigateURLTool, PerformActionTool, GetAccessibilityTreeTool, SearchContentTool, NavigateBackTool, NodeIDsToURLsTool, TakeScreenshotTool, ScrollPageTool, WaitTool, RenderWebAppTool, GetWebAppDataTool, RemoveWebAppTool, CreateFileTool, UpdateFileTool, DeleteFileTool, ReadFileTool, ListFilesTool } from '../../tools/Tools.js'; +import { UpdateTodoTool } from '../../tools/UpdateTodoTool.js'; +import { ExecuteCodeTool } from '../../tools/ExecuteCodeTool.js'; import { HTMLToMarkdownTool } from '../../tools/HTMLToMarkdownTool.js'; import { ConfigurableAgentTool, ToolRegistry } from '../ConfigurableAgentTool.js'; import { ThinkingTool } from '../../tools/ThinkingTool.js'; @@ -54,6 +56,8 @@ export function initializeConfiguredAgents(): void { ToolRegistry.registerToolFactory('delete_file', () => new DeleteFileTool()); ToolRegistry.registerToolFactory('read_file', () => new ReadFileTool()); ToolRegistry.registerToolFactory('list_files', () => new ListFilesTool()); + ToolRegistry.registerToolFactory('update_todo', () => new UpdateTodoTool()); + ToolRegistry.registerToolFactory('execute_code', () => new ExecuteCodeTool()); // Register webapp rendering tools ToolRegistry.registerToolFactory('render_webapp', () => new RenderWebAppTool()); diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts index 377dae7124..0cce995a37 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts @@ -158,6 +158,7 @@ Before handing off, ensure your latest findings are reflected in the shared file 'navigate_back', 'fetcher_tool', 'extract_data', + 'execute_code', 'node_ids_to_urls', 'bookmark_store', 'document_search', @@ -167,6 +168,7 @@ Before handing off, ensure your latest findings are reflected in the shared file 'delete_file', 'read_file', 'list_files', + 'update_todo', ], maxIterations: 15, modelName: MODEL_SENTINELS.USE_MINI, diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts index 1fea4960ba..4364c8363d 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts @@ -130,6 +130,7 @@ If you absolutely cannot find any reliable leads, return status "failed" with ga 'node_ids_to_urls', 'fetcher_tool', 'extract_data', + 'execute_code', 'scroll_page', 'action_agent', 'html_to_markdown', @@ -138,6 +139,7 @@ If you absolutely cannot find any reliable leads, return status "failed" with ga 'delete_file', 'read_file', 'list_files', + 'update_todo', ], maxIterations: 12, modelName: MODEL_SENTINELS.USE_MINI, diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts index d83646a5af..3805e0d72a 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts @@ -198,6 +198,7 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and 'navigate_back', 'action_agent', 'extract_data', + 'execute_code', 'node_ids_to_urls', 'direct_url_navigator_agent', 'scroll_page', @@ -212,6 +213,7 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and 'delete_file', 'read_file', 'list_files', + 'update_todo', ], maxIterations: 15, temperature: 0.3, diff --git a/front_end/panels/ai_chat/core/AgentNodes.ts b/front_end/panels/ai_chat/core/AgentNodes.ts index 358c8e9575..c24f3b607e 100644 --- a/front_end/panels/ai_chat/core/AgentNodes.ts +++ b/front_end/panels/ai_chat/core/AgentNodes.ts @@ -891,15 +891,19 @@ export function createToolExecutorNode(state: AgentState, provider: LLMProvider, } // Special handling for ConfigurableAgentTool results - if (selectedTool instanceof ConfigurableAgentTool && result && typeof result === 'object' && + if (selectedTool instanceof ConfigurableAgentTool && result && typeof result === 'object' && ('output' in result || 'error' in result || 'success' in result)) { // For ConfigurableAgentTool, only send the output/error fields to the LLM, never intermediateSteps const agentResult = result as any; // Cast to any to access ConfigurableAgentResult properties - resultText = agentResult.output || (agentResult.error ? `Error: ${agentResult.error}` : 'No output'); + // Prioritize summary.content (detailed LLM analysis), fallback to output/error + resultText = agentResult.summary?.content + || agentResult.output + || (agentResult.error ? `Error: ${agentResult.error}` : 'No output'); logger.debug(`Filtered ConfigurableAgentTool result for LLM:`, { toolName, originalResult: result, - filteredResult: resultText + filteredResult: resultText, + hasSummary: !!agentResult.summary?.content }); } else if (toolName === 'finalize_with_critique') { logger.debug('ToolExecutorNode: finalize_with_critique result:', result); diff --git a/front_end/panels/ai_chat/core/AgentService.ts b/front_end/panels/ai_chat/core/AgentService.ts index 1a4b8b69ed..637c1dedd7 100644 --- a/front_end/panels/ai_chat/core/AgentService.ts +++ b/front_end/panels/ai_chat/core/AgentService.ts @@ -23,6 +23,7 @@ import { AgentRunner } from '../agent_framework/AgentRunner.js'; import type { AgentSession, AgentMessage } from '../agent_framework/AgentSessionTypes.js'; import type { LLMProvider } from '../LLM/LLMTypes.js'; import { BUILD_CONFIG } from './BuildConfig.js'; +import { VisualIndicatorManager } from '../tools/VisualIndicatorTool.js'; // Cache break: 2025-09-17T17:54:00Z - Force rebuild with AUTOMATED_MODE bypass const logger = createLogger('AgentService'); @@ -36,6 +37,7 @@ export enum Events { AGENT_TOOL_STARTED = 'agent-tool-started', AGENT_TOOL_COMPLETED = 'agent-tool-completed', AGENT_SESSION_UPDATED = 'agent-session-updated', + AGENT_SESSION_COMPLETED = 'agent-session-completed', CHILD_AGENT_STARTED = 'child-agent-started', } @@ -48,6 +50,7 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ [Events.AGENT_TOOL_STARTED]: { session: AgentSession, toolCall: AgentMessage }, [Events.AGENT_TOOL_COMPLETED]: { session: AgentSession, toolResult: AgentMessage }, [Events.AGENT_SESSION_UPDATED]: AgentSession, + [Events.AGENT_SESSION_COMPLETED]: AgentSession, [Events.CHILD_AGENT_STARTED]: { parentSession: AgentSession, childAgentName: string, childSessionId: string }, }> { static instance: AgentService; @@ -120,10 +123,13 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ // Initialize AgentRunner event system AgentRunner.initializeEventBus(); - + // Subscribe to AgentRunner events AgentRunnerEventBus.getInstance().addEventListener('agent-progress', this.#handleAgentProgress.bind(this)); + // Initialize visual indicator system + VisualIndicatorManager.getInstance().initialize(); + // Subscribe to configuration changes this.#configManager.addChangeListener(this.#handleConfigurationChange.bind(this)); } @@ -857,6 +863,39 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ } } break; + case 'session_completed': + // Get the completed session from the event data or active sessions + const completedSession = progressEvent.data?.session || + this.#activeAgentSessions.get(progressEvent.sessionId); + + if (completedSession) { + logger.info('[AgentService] Session completed:', { + sessionId: progressEvent.sessionId, + status: completedSession.status, + terminationReason: completedSession.terminationReason + }); + + // Update the session in our tracking with the completed state + this.#activeAgentSessions.set(progressEvent.sessionId, completedSession); + + // Upsert the completed session to messages (shows final_answer in transcript) + this.#upsertAgentSessionInMessages(completedSession); + + // Dispatch completion event for UI components + this.dispatchEventToListeners(Events.AGENT_SESSION_COMPLETED, completedSession); + + // Also dispatch session updated for components listening to that + this.dispatchEventToListeners(Events.AGENT_SESSION_UPDATED, completedSession); + + // Trigger messages changed to update the chat transcript + this.dispatchEventToListeners(Events.MESSAGES_CHANGED, [...this.#state.messages]); + + // Clean up after a short delay (5 seconds) to allow UI to finish rendering + this.#cleanupCompletedSession(progressEvent.sessionId); + } else { + logger.warn('[AgentService] Session completed but session not found:', progressEvent.sessionId); + } + break; } } diff --git a/front_end/panels/ai_chat/core/PageInfoManager.ts b/front_end/panels/ai_chat/core/PageInfoManager.ts index 92b8126c65..7d1d58be14 100644 --- a/front_end/panels/ai_chat/core/PageInfoManager.ts +++ b/front_end/panels/ai_chat/core/PageInfoManager.ts @@ -5,6 +5,7 @@ import * as SDK from '../../../core/sdk/sdk.js'; import * as Utils from '../common/utils.js'; // Path relative to core/ assuming utils.ts will be in common/ later, this will be common/utils.js import { VisitHistoryManager } from '../tools/VisitHistoryManager.js'; // Path relative to core/ assuming VisitHistoryManager.ts will be in core/ +import { FileStorageManager } from '../tools/FileStorageManager.js'; import { createLogger } from './Logger.js'; const logger = createLogger('PageInfoManager'); @@ -189,6 +190,15 @@ export async function enhancePromptWithPageContext(basePrompt: string): Promise< const accessibilityTree = PageInfoManager.getInstance().getAccessibilityTree(); const iframeContent = PageInfoManager.getInstance().getIframeContent(); + // Get current session files + const fileManager = FileStorageManager.getInstance(); + let files: any[] = []; + try { + files = await fileManager.listFiles(); + } catch (error) { + logger.warn('Failed to fetch files for context:', error); + } + // If no page info is available, return the original prompt if (!pageInfo) { return basePrompt; @@ -220,6 +230,18 @@ ${iframe.contentSimplified} ).join('\n ')} ` : ''} + ${files.length > 0 ? + ` + + ${files.map((file) => + ` + ${file.fileName} + ${new Date(file.createdAt).toLocaleString()} + ${new Date(file.updatedAt).toLocaleString()} + ${file.size} characters + ` + ).join('\n ')} + ` : ''} Instructions: @@ -228,6 +250,7 @@ Instructions: - If you need the full page accessibility tree to answer the user's query, you have the ability to request it at any time. - Use the page title, URL, and partial accessibility tree to inform your answers. ${iframeContent && iframeContent.length > 0 ? '- The page contains embedded iframes with their own content, which is included above.' : ''} +${files.length > 0 ? `- ${files.length} file${files.length === 1 ? '' : 's'} ${files.length === 1 ? 'has' : 'have'} been created during this session. You can read, update, or reference ${files.length === 1 ? 'this file' : 'these files'} using the file management tools (read_file, update_file, create_file, delete_file, list_files).` : ''} - If the user asks about the page, refer to this context. - If the partial accessibility tree is present, use it to answer questions about visible page structure, elements, or accessibility. - If you need to extract any data from the entire page, you must always use the extract_data tool to do so. Do not attempt to extract data from the full page by any other means. diff --git a/front_end/panels/ai_chat/tools/ExecuteCodeTool.ts b/front_end/panels/ai_chat/tools/ExecuteCodeTool.ts new file mode 100644 index 0000000000..ff55982f6f --- /dev/null +++ b/front_end/panels/ai_chat/tools/ExecuteCodeTool.ts @@ -0,0 +1,123 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import * as SDK from '../../../core/sdk/sdk.js'; +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext } from './Tools.js'; + +const logger = createLogger('Tool:ExecuteCode'); + +/** + * Arguments for code execution + */ +export interface ExecuteCodeArgs { + code: string; + reasoning: string; +} + +/** + * Tool for executing arbitrary JavaScript code in the page context + * Useful for extracting data, getting links, running custom logic, etc. + */ +export class ExecuteCodeTool implements Tool { + name = 'execute_code'; + description = `Executes JavaScript code in the current page context and returns the raw result. + +Use this tool when you need to: +- Extract all links from the page: Array.from(document.links).map(a => ({text: a.textContent.trim(), href: a.href})) +- Get specific DOM elements with custom logic: document.querySelectorAll('.item').map(el => ({...})) +- Extract table data: Array.from(document.querySelectorAll('table tr')).map(row => [...]) +- Get computed styles: window.getComputedStyle(document.querySelector('.target')) +- Run custom JavaScript that doesn't fit schema-based extraction +- Check page state: document.readyState, window.location, etc. +- Extract images: Array.from(document.images).map(img => ({src: img.src, alt: img.alt})) +- Get metadata: {title: document.title, url: window.location.href, description: document.querySelector('meta[name="description"]')?.content} + +The code executes in the page's JavaScript context with full DOM API access. +The raw JavaScript return value is returned directly without any parsing or wrapping. + +IMPORTANT: +- The code should be a valid JavaScript expression or IIFE +- Use arrow functions or IIFEs for multi-line code: (() => { /* code */ })() +- Return values must be JSON-serializable (strings, numbers, objects, arrays) +- DOM nodes cannot be returned directly - extract their properties instead +- Avoid side effects unless intentional (e.g., modifying the page) +- The result is returned exactly as JavaScript produces it (no wrapper objects) + +Examples: +• Get all links: Array.from(document.links).map(a => ({text: a.textContent.trim(), href: a.href})) +• Extract product data: Array.from(document.querySelectorAll('.product')).map(p => ({name: p.querySelector('.name').textContent, price: p.querySelector('.price').textContent})) +• Get page metadata: ({title: document.title, url: location.href, images: document.images.length}) +• Check element existence: !!document.querySelector('#login-button') +• Get all headings: Array.from(document.querySelectorAll('h1, h2, h3')).map(h => ({level: h.tagName, text: h.textContent.trim()}))`; + + schema = { + type: 'object', + properties: { + code: { + type: 'string', + description: 'JavaScript code to execute in the page context. Must be a valid expression or IIFE that returns a value.' + }, + reasoning: { + type: 'string', + description: 'Explanation of what this code does and why you are executing it (shown to user)' + } + }, + required: ['code', 'reasoning'] + }; + + async execute(args: ExecuteCodeArgs, _ctx?: LLMContext): Promise { + const { code, reasoning } = args; + + if (typeof code !== 'string' || code.trim().length === 0) { + return { error: 'Code must be a non-empty string' }; + } + + logger.info(`Executing code with reasoning: ${reasoning}`); + logger.debug(`Code to execute: ${code.substring(0, 200)}${code.length > 200 ? '...' : ''}`); + + // Get the main target + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (!target) { + return { error: 'No page target available' }; + } + + try { + // Execute the code in the page context + const result = await target.runtimeAgent().invoke_evaluate({ + expression: code, + returnByValue: true, // Return the actual value, not a remote object reference + awaitPromise: true, // Wait for promises to resolve + timeout: 10000, // 10 second timeout + }); + + // Check for exceptions + if (result.exceptionDetails) { + const errorMessage = result.exceptionDetails.text || 'Unknown error'; + const errorStack = result.exceptionDetails.exception?.description || ''; + + logger.error(`Code execution failed: ${errorMessage}`); + logger.debug(`Exception details:`, result.exceptionDetails); + + return { + error: errorMessage, + exceptionDetails: errorStack + }; + } + + // Return the raw result value directly + const resultValue = result.result.value; + logger.info(`Code executed successfully, result type: ${result.result.type}`); + logger.debug(`Result preview: ${JSON.stringify(resultValue).substring(0, 200)}...`); + + return resultValue; + + } catch (error) { + logger.error('Error executing code:', error); + return { + error: error instanceof Error ? error.message : String(error) + }; + } + } +} diff --git a/front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts b/front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts index 831257dcb4..f4c34aefe3 100644 --- a/front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts +++ b/front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts @@ -15,6 +15,24 @@ import { NodeIDsToURLsTool, type Tool } from './Tools.js'; const logger = createLogger('Tool:SchemaBasedExtractor'); +// Chunking interfaces +interface ContentChunk { + id: number; + content: string; + tokenCount: number; + sectionInfo?: { + heading?: string; + level?: number; + startNodeId?: string; + }; +} + +interface ChunkExtractionResult { + chunkId: number; + data: any; + itemCount: number; +} + // Define the structure for the metadata LLM call's expected response interface ExtractionMetadata { progress: string; @@ -36,6 +54,11 @@ export interface SchemaExtractionResult { * Tool for extracting structured data from DOM based on schema definitions */ export class SchemaBasedExtractorTool implements Tool { + // Chunking configuration + private readonly CHUNK_TOKEN_LIMIT = 40000; // ~160k characters per chunk + private readonly CHARS_PER_TOKEN = 4; // Conservative estimate + private readonly TOKEN_LIMIT_FOR_CHUNKING = 65000; // Auto-chunk if tree exceeds this + name = 'extract_data'; description = `Extracts structured data from a web page's DOM using a user-provided JSON schema and natural language instruction. - The schema defines the exact structure and types of data to extract (e.g., text, numbers, URLs). @@ -171,85 +194,151 @@ Schema Examples: logger.debug('Processed Accessibility Tree Text (length):', treeText.length); // logger.debug('[SchemaBasedExtractorTool] Tree Text:', treeText); // Uncomment for full tree text - // ---- Start Multi-step LLM Process ---- + // Auto-detection: Check if we need to chunk + const estimatedTokens = this.estimateTokenCount(treeText); + logger.info(`Tree token count: ${estimatedTokens} (threshold: ${this.TOKEN_LIMIT_FOR_CHUNKING})`); + + let finalData: any; + + if (estimatedTokens > this.TOKEN_LIMIT_FOR_CHUNKING) { + // ---- Chunked Extraction Flow ---- + logger.info('Tree exceeds token limit, using chunked extraction'); + + // Create chunks (tries sections first, falls back to tokens) + const chunks = this.chunkBySections(treeText); + logger.info(`Created ${chunks.length} chunks`, chunks.map(c => ({ + id: c.id, + tokens: c.tokenCount, + heading: c.sectionInfo?.heading + }))); + + // Extract from each chunk + const chunkResults: any[] = []; + for (const chunk of chunks) { + logger.info(`Processing chunk ${chunk.id + 1}/${chunks.length}...`); + + try { + const extractedData = await this.extractFromChunk( + chunk, + transformedSchema, + instruction || 'Extract data according to schema', + apiKey || '', + ctx + ); + chunkResults.push(extractedData); + logger.info(`Chunk ${chunk.id + 1} extraction complete`); + } catch (error) { + logger.error(`Error extracting from chunk ${chunk.id}:`, error); + // Continue with other chunks even if one fails + } + } - // 5. Initial Extract Call - logger.debug('Starting initial LLM extraction...'); - const initialExtraction = await this.callExtractionLLM({ - instruction: instruction || 'Extract data according to schema', - domContent: treeText, - schema: transformedSchema, - apiKey: apiKey || '', // Use empty string for BrowserOperator - ctx, - }); + // Merge results using LLM + logger.info('Merging chunk results with LLM...'); + const mergedData = await this.callMergeLLM({ + chunkResults, + schema: transformedSchema, + instruction: instruction || 'Extract data according to schema', + apiKey: apiKey || '', + ctx + }); + + if (!mergedData) { + return { + success: false, + error: 'Failed to merge chunk results', + data: null + }; + } - logger.debug('Initial extraction result:', initialExtraction); - if (!initialExtraction) { // Check if initial extraction failed - return { - success: false, - error: 'Initial data extraction failed', - data: null, - }; - } - // Check if extraction returned a parsing error - if (initialExtraction.__parsing_failed__) { - return { - success: false, - error: initialExtraction.__error__ || 'JSON parsing failed during extraction', - data: null, - }; - } + finalData = mergedData; + logger.info('Chunk merging complete'); + + } else { + // ---- Standard Single-Pass Extraction Flow ---- + logger.info('Using standard single-pass extraction'); + + // 5. Initial Extract Call + logger.debug('Starting initial LLM extraction...'); + const initialExtraction = await this.callExtractionLLM({ + instruction: instruction || 'Extract data according to schema', + domContent: treeText, + schema: transformedSchema, + apiKey: apiKey || '', // Use empty string for BrowserOperator + ctx, + }); + + logger.debug('Initial extraction result:', initialExtraction); + if (!initialExtraction) { // Check if initial extraction failed + return { + success: false, + error: 'Initial data extraction failed', + data: null, + }; + } + // Check if extraction returned a parsing error + if (initialExtraction.__parsing_failed__) { + return { + success: false, + error: initialExtraction.__error__ || 'JSON parsing failed during extraction', + data: null, + }; + } - // 6. Refine Call - const refinedData = await this.callRefinementLLM({ - instruction: instruction || 'Refine the extracted data based on the original request', - schema: transformedSchema, // Use the same transformed schema - initialData: initialExtraction, - apiKey: apiKey || '', // Use empty string for BrowserOperator - ctx, - }); + // 6. Refine Call + const refinedData = await this.callRefinementLLM({ + instruction: instruction || 'Refine the extracted data based on the original request', + schema: transformedSchema, // Use the same transformed schema + initialData: initialExtraction, + apiKey: apiKey || '', // Use empty string for BrowserOperator + ctx, + }); + + logger.debug('Refinement result:', refinedData); + if (!refinedData) { // Check if refinement failed + return { + success: false, + error: 'Data refinement step failed', + data: null, + }; + } + // Check if refinement returned a parsing error + if (refinedData.__parsing_failed__) { + return { + success: false, + error: refinedData.__error__ || 'JSON parsing failed during refinement', + data: null, + }; + } - logger.debug('Refinement result:', refinedData); - if (!refinedData) { // Check if refinement failed - return { - success: false, - error: 'Data refinement step failed', - data: null, - }; - } - // Check if refinement returned a parsing error - if (refinedData.__parsing_failed__) { - return { - success: false, - error: refinedData.__error__ || 'JSON parsing failed during refinement', - data: null, - }; + finalData = refinedData; } - // 7. LLM + Tool Call for URL Resolution - New approach - const finalData = await this.resolveUrlsWithLLM({ - data: refinedData, + // ---- URL Resolution (common for both flows) ---- + logger.debug('Resolving URLs...'); + const dataWithUrls = await this.resolveUrlsWithLLM({ + data: finalData, apiKey: apiKey || '', // Use empty string for BrowserOperator schema, // Original schema to understand what fields are URLs }); logger.debug('Data after URL resolution:', - JSON.stringify(Array.isArray(finalData) ? finalData.slice(0, 2) : finalData, null, 2).substring(0, 500)); + JSON.stringify(Array.isArray(dataWithUrls) ? dataWithUrls.slice(0, 2) : dataWithUrls, null, 2).substring(0, 500)); - // 7a. Check if any URL fields still contain numeric node IDs + // Check if any URL fields still contain numeric node IDs let urlResolutionWarning: string | undefined; - const dataString = JSON.stringify(finalData); + const dataString = JSON.stringify(dataWithUrls); // Simple heuristic: if we have numbers where URLs are expected in common URL field names if (dataString.match(/"(url|link|href|website|webpage)"\s*:\s*\d+/i)) { urlResolutionWarning = 'Note: Some URL fields may contain unresolved node IDs instead of actual URLs.'; logger.warn('Detected potential unresolved node IDs in URL fields'); } - // 8. Metadata Call + // ---- Metadata Call (common for both flows) ---- const metadata = await this.callMetadataLLM({ instruction: instruction || 'Assess extraction completion', - extractedData: finalData, // Use the final data with URLs for assessment - domContent: treeText, // Pass the DOM content for context + extractedData: dataWithUrls, // Use the final data with URLs for assessment + domContent: treeText.substring(0, 3000), // Truncate for metadata call schema, // Pass the schema to understand what was requested apiKey: apiKey || '', // Use empty string for BrowserOperator ctx, @@ -257,22 +346,13 @@ Schema Examples: logger.debug('Metadata result:', metadata); if (!metadata) { // Check if metadata call failed - // Decide if this should be a hard failure or just return without metadata logger.warn('Metadata extraction step failed, proceeding without metadata.'); - // If metadata is critical, return failure: - // return { - // success: false, - // error: 'Metadata extraction step failed', - // data: null, - // }; } - // ---- End Multi-step LLM Process ---- - // Prepare the result const result: SchemaExtractionResult = { success: true, - data: finalData, + data: dataWithUrls, metadata: metadata || undefined, // Include metadata if successful, otherwise undefined }; @@ -888,6 +968,315 @@ Return ONLY a valid JSON object conforming to the required metadata schema.`; return data; // Return original data on error } } + + /** + * Estimates token count from text + */ + private estimateTokenCount(text: string): number { + return Math.ceil(text.length / this.CHARS_PER_TOKEN); + } + + /** + * Chunks content by detecting sections (headings in accessibility tree) + */ + private chunkBySections(treeText: string): ContentChunk[] { + const chunks: ContentChunk[] = []; + + // Split by heading patterns in accessibility tree + // Format: [nodeId] heading: Heading Text + const lines = treeText.split('\n'); + const sectionStarts: Array<{ index: number, heading: string, nodeId: string, level: number }> = []; + + // Find all headings + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + // Match patterns like: [123] heading: Some Heading Text + const headingMatch = line.match(/\[(\d+)\]\s+heading(?:\s+level (\d+))?:\s*(.+)/i); + if (headingMatch) { + const nodeId = headingMatch[1]; + const level = headingMatch[2] ? parseInt(headingMatch[2]) : 2; // Default to level 2 + const heading = headingMatch[3].trim(); + sectionStarts.push({ index: i, heading, nodeId, level }); + } + } + + logger.debug(`Found ${sectionStarts.length} section headings`); + + // If no headings found, fall back to token-based chunking + if (sectionStarts.length === 0) { + logger.warn('No section headings found, falling back to token-based chunking'); + return this.chunkByTokens(treeText); + } + + // Create chunks from sections + let chunkId = 0; + let currentChunkLines: string[] = []; + let currentChunkStart = 0; + + for (let i = 0; i < sectionStarts.length; i++) { + const section = sectionStarts[i]; + const nextSection = sectionStarts[i + 1]; + + // Extract lines for this section + const sectionEnd = nextSection ? nextSection.index : lines.length; + const sectionLines = lines.slice(section.index, sectionEnd); + + // Check if adding this section would exceed limit + const combinedLines = [...currentChunkLines, ...sectionLines]; + const combinedText = combinedLines.join('\n'); + const combinedTokens = this.estimateTokenCount(combinedText); + + if (combinedTokens > this.CHUNK_TOKEN_LIMIT && currentChunkLines.length > 0) { + // Create chunk from accumulated content + const chunkText = currentChunkLines.join('\n'); + chunks.push({ + id: chunkId++, + content: chunkText, + tokenCount: this.estimateTokenCount(chunkText), + sectionInfo: { + heading: sectionStarts[currentChunkStart]?.heading, + level: sectionStarts[currentChunkStart]?.level, + startNodeId: sectionStarts[currentChunkStart]?.nodeId + } + }); + + // Start new chunk with current section + currentChunkLines = sectionLines; + currentChunkStart = i; + } else { + // Add section to current chunk + currentChunkLines.push(...sectionLines); + } + } + + // Add final chunk if there's content + if (currentChunkLines.length > 0) { + const chunkText = currentChunkLines.join('\n'); + chunks.push({ + id: chunkId++, + content: chunkText, + tokenCount: this.estimateTokenCount(chunkText), + sectionInfo: { + heading: sectionStarts[currentChunkStart]?.heading, + level: sectionStarts[currentChunkStart]?.level, + startNodeId: sectionStarts[currentChunkStart]?.nodeId + } + }); + } + + return chunks; + } + + /** + * Chunks content by token count (fallback when no sections detected) + */ + private chunkByTokens(treeText: string): ContentChunk[] { + const chunks: ContentChunk[] = []; + const lines = treeText.split('\n'); + + let chunkId = 0; + let currentChunkLines: string[] = []; + let currentTokens = 0; + + for (const line of lines) { + const lineTokens = this.estimateTokenCount(line); + + if (currentTokens + lineTokens > this.CHUNK_TOKEN_LIMIT && currentChunkLines.length > 0) { + // Create chunk + const chunkText = currentChunkLines.join('\n'); + chunks.push({ + id: chunkId++, + content: chunkText, + tokenCount: currentTokens + }); + + // Start new chunk + currentChunkLines = [line]; + currentTokens = lineTokens; + } else { + currentChunkLines.push(line); + currentTokens += lineTokens; + } + } + + // Add final chunk + if (currentChunkLines.length > 0) { + chunks.push({ + id: chunkId++, + content: currentChunkLines.join('\n'), + tokenCount: currentTokens + }); + } + + return chunks; + } + + /** + * Extract data from a single chunk using LLM + */ + private async extractFromChunk( + chunk: ContentChunk, + schema: SchemaDefinition, + instruction: string, + apiKey: string, + ctx?: LLMContext + ): Promise { + const systemPrompt = `You are a structured data extraction agent. +Your task is to extract data from a CHUNK of a larger document based on a given schema. +This chunk is part ${chunk.id + 1} of a larger page. +${chunk.sectionInfo?.heading ? `This chunk covers the section: "${chunk.sectionInfo.heading}"` : ''} + +CRITICAL RULES: +1. ONLY extract data that exists in THIS chunk - do not hallucinate +2. If no relevant data exists in this chunk, return an empty result +3. For URL fields, extract the numeric accessibility node ID (not the URL string) +4. Return ONLY valid JSON matching the schema + +Focus on extracting any relevant data from this chunk. The results will be merged with other chunks.`; + + const extractionPrompt = ` +INSTRUCTION: ${instruction} + +SCHEMA: +\`\`\`json +${JSON.stringify(schema, null, 2)} +\`\`\` + +CHUNK CONTENT (Part ${chunk.id + 1}): +\`\`\` +${chunk.content} +\`\`\` + +Extract structured data from this chunk according to the schema. +Return ONLY the JSON object.`; + + try { + if (!ctx?.provider || !ctx.nanoModel) { + throw new Error('Missing LLM context for extraction'); + } + + const llmResponse = await callLLMWithTracing( + { + provider: ctx.provider, + model: ctx.nanoModel || ctx.model, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: extractionPrompt } + ], + systemPrompt, + temperature: 0.1, + options: { retryConfig: { maxRetries: 2, baseDelayMs: 1000 } } + }, + { + toolName: this.name, + operationName: 'extract_chunk', + context: `chunk_${chunk.id}`, + additionalMetadata: { + chunkId: chunk.id, + chunkTokens: chunk.tokenCount, + section: chunk.sectionInfo?.heading + } + } + ); + + const response = llmResponse.text || ''; + return LLMResponseParser.parseJSONWithFallbacks(response); + } catch (error) { + logger.error(`Error extracting from chunk ${chunk.id}:`, error); + throw error; + } + } + + /** + * LLM call to merge chunk results into final data + */ + private async callMergeLLM(options: { + chunkResults: any[], + schema: SchemaDefinition, + instruction: string, + apiKey: string, + ctx?: LLMContext, + }): Promise { + const { chunkResults, schema, instruction, apiKey } = options; + logger.debug('Calling Merge LLM to combine chunk results...'); + + const systemPrompt = `You are a data merging agent in a multi-agent system. +Your task is to intelligently merge multiple JSON extraction results from different chunks of the same page. + +CRITICAL RULES: +1. Merge all data into a single result conforming to the schema +2. Remove duplicates (same items appearing in multiple chunks) +3. Maintain numeric node IDs for URL fields - DO NOT convert to URLs +4. If the schema expects an array, combine all arrays and deduplicate +5. If the schema expects an object with arrays, merge each array property separately +6. Return ONLY valid JSON matching the schema + +Focus on creating a complete, deduplicated result from all chunks.`; + + const mergePrompt = ` +ORIGINAL INSTRUCTION: ${instruction} + +SCHEMA: +\`\`\`json +${JSON.stringify(schema, null, 2)} +\`\`\` + +CHUNK RESULTS (${chunkResults.length} chunks): +\`\`\`json +${JSON.stringify(chunkResults, null, 2)} +\`\`\` + +TASK: Merge all chunk results into a single result that conforms to the schema. +- Remove duplicate items (compare by content, not just IDs) +- Combine all arrays +- Keep numeric node IDs in URL fields +Return ONLY the merged JSON object.`; + + try { + if (!options.ctx?.provider || !(options.ctx.nanoModel || options.ctx.model)) { + throw new Error('Missing LLM context for merging'); + } + const provider = options.ctx.provider; + const model = options.ctx.nanoModel || options.ctx.model; + const llmResponse = await callLLMWithTracing( + { + provider, + model, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: mergePrompt } + ], + systemPrompt, + temperature: 0.1, + options: { retryConfig: { maxRetries: 3, baseDelayMs: 1500 } } + }, + { + toolName: this.name, + operationName: 'merge_chunks', + context: 'chunk_merging', + additionalMetadata: { + chunkCount: chunkResults.length, + instructionLength: instruction.length + } + } + ); + const response = llmResponse.text || ''; + try { + return LLMResponseParser.parseStrictJSON(response); + } catch { + try { + return LLMResponseParser.parseJSONWithFallbacks(response); + } catch (e) { + logger.error('Failed to parse merge JSON:', e); + logger.warn('Raw LLM response:', response.substring(0, 500)); + return null; + } + } + } catch (error) { + logger.error('Error in callMergeLLM:', error); + return null; + } + } } /** diff --git a/front_end/panels/ai_chat/tools/SequentialThinkingTool.ts b/front_end/panels/ai_chat/tools/SequentialThinkingTool.ts index 047848d25e..54fcf06e80 100644 --- a/front_end/panels/ai_chat/tools/SequentialThinkingTool.ts +++ b/front_end/panels/ai_chat/tools/SequentialThinkingTool.ts @@ -276,12 +276,22 @@ Based on the screenshot and current state, create a grounded sequential plan for throw new Error('No response from LLM'); } + // Store raw response for error reporting + const rawResponseText = response.text; + // This will throw if JSON parsing fails, triggering a retry - const parsedResult = LLMResponseParser.parseStrictJSON(response.text) as SequentialThinkingResult; - + let parsedResult: SequentialThinkingResult; + try { + parsedResult = LLMResponseParser.parseStrictJSON(rawResponseText) as SequentialThinkingResult; + } catch (parseError) { + // Include full raw response in error for debugging + const errorMsg = parseError instanceof Error ? parseError.message : String(parseError); + throw new Error(`Failed to parse response: ${errorMsg}. LLM returned: ${rawResponseText}`); + } + // Validate result structure if (!parsedResult.currentState || !parsedResult.nextSteps || !Array.isArray(parsedResult.nextSteps)) { - throw new Error('Invalid response structure from LLM - missing required fields'); + throw new Error(`Invalid response structure from LLM - missing required fields. LLM returned: ${rawResponseText}`); } return parsedResult; diff --git a/front_end/panels/ai_chat/tools/Tools.ts b/front_end/panels/ai_chat/tools/Tools.ts index 5da254e5f6..d69fda19de 100644 --- a/front_end/panels/ai_chat/tools/Tools.ts +++ b/front_end/panels/ai_chat/tools/Tools.ts @@ -197,6 +197,9 @@ export interface ScrollResult { x: number, y: number, }; + viewportHeight?: number; // Height of the viewport in pixels + scrollHeight?: number; // Total scrollable height of the document + scrolledPages?: number; // Number of pages scrolled (if using pages parameter) } /** @@ -1326,17 +1329,19 @@ export class SearchContentTool implements Tool<{ query: string, limit?: number } /** * Tool for scrolling the page */ -export class ScrollPageTool implements Tool<{ position?: { x: number, y: number }, direction?: string, amount?: number }, ScrollResult | ErrorResult> { +export class ScrollPageTool implements Tool<{ position?: { x: number, y: number }, direction?: string, amount?: number, pages?: number }, ScrollResult | ErrorResult> { name = 'scroll_page'; - description = 'Scrolls the page to a specific position or in a specific direction'; + description = 'Scrolls the page to a specific position, in a direction, or by viewport pages. Use pages parameter for predictable scrolling (e.g., pages: 1 scrolls down one full viewport height, pages: -1 scrolls up).'; - async execute(args: { position?: { x: number, y: number }, direction?: string, amount?: number }, _ctx?: LLMContext): Promise { + async execute(args: { position?: { x: number, y: number }, direction?: string, amount?: number, pages?: number }, _ctx?: LLMContext): Promise { const position = args.position; + const pages = args.pages; const direction = args.direction; const amount = args.amount || 300; // Default scroll amount - if (!position && !direction) { - return { error: 'Either position or direction must be provided' }; + // Priority: position > pages > direction + if (!position && pages === undefined && !direction) { + return { error: 'Either position, pages, or direction must be provided' }; } // Get the main target @@ -1356,6 +1361,14 @@ export class ScrollPageTool implements Tool<{ position?: { x: number, y: number top: ${position.y || 0}, behavior: 'smooth' });` : + pages !== undefined ? + `// Scroll by viewport heights + const viewportHeight = window.innerHeight; + const scrollAmount = viewportHeight * ${pages}; + window.scrollBy({ + top: scrollAmount, + behavior: 'smooth' + });` : `// Scroll in direction const direction = "${direction}"; const amount = ${amount}; @@ -1375,14 +1388,17 @@ export class ScrollPageTool implements Tool<{ position?: { x: number, y: number }` } - // Return current scroll position + // Return current scroll position with viewport info return { success: true, message: "Scroll operation completed", position: { x: window.pageXOffset, y: window.pageYOffset - } + }, + viewportHeight: window.innerHeight, + scrollHeight: document.documentElement.scrollHeight, + scrolledPages: ${pages !== undefined ? pages : 0} }; })()`, returnByValue: true, @@ -1411,14 +1427,18 @@ export class ScrollPageTool implements Tool<{ position?: { x: number, y: number }, }, }, + pages: { + type: 'number', + description: 'Number of viewport heights to scroll. Positive scrolls down, negative scrolls up. Examples: 1 (one page down), 0.5 (half page down), -1 (one page up), 2 (two pages down). This is the recommended way to scroll for content extraction workflows.', + }, direction: { type: 'string', - description: 'Direction to scroll (up, down, left, right, top, bottom)', + description: 'Direction to scroll (up, down, left, right, top, bottom). Use pages parameter instead for more predictable scrolling.', enum: ['up', 'down', 'left', 'right', 'top', 'bottom'], }, amount: { type: 'number', - description: 'Amount to scroll in pixels (default: 300)', + description: 'Amount to scroll in pixels when using direction (default: 300). Use pages parameter instead for viewport-relative scrolling.', }, }, }; @@ -3514,6 +3534,9 @@ export { GetWebAppDataTool } from './GetWebAppDataTool.js'; export type { GetWebAppDataArgs, GetWebAppDataResult } from './GetWebAppDataTool.js'; export { RemoveWebAppTool } from './RemoveWebAppTool.js'; export type { RemoveWebAppArgs, RemoveWebAppResult } from './RemoveWebAppTool.js'; + +// Export visual indicator manager +export { VisualIndicatorManager } from './VisualIndicatorTool.js'; export { CreateFileTool } from './CreateFileTool.js'; export type { CreateFileArgs, CreateFileResult } from './CreateFileTool.js'; export { UpdateFileTool } from './UpdateFileTool.js'; @@ -3524,6 +3547,8 @@ export { ReadFileTool } from './ReadFileTool.js'; export type { ReadFileArgs, ReadFileResult } from './ReadFileTool.js'; export { ListFilesTool } from './ListFilesTool.js'; export type { ListFilesArgs, ListFilesResult } from './ListFilesTool.js'; +export { ExecuteCodeTool } from './ExecuteCodeTool.js'; +export type { ExecuteCodeArgs } from './ExecuteCodeTool.js'; // Abortable sleep utility for tools that need delays/polling function abortableSleep(ms: number, signal?: AbortSignal): Promise { return new Promise((resolve, reject) => { diff --git a/front_end/panels/ai_chat/tools/UpdateTodoTool.ts b/front_end/panels/ai_chat/tools/UpdateTodoTool.ts new file mode 100644 index 0000000000..2cba3fb3cf --- /dev/null +++ b/front_end/panels/ai_chat/tools/UpdateTodoTool.ts @@ -0,0 +1,129 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext } from './Tools.js'; +import { FileStorageManager } from './FileStorageManager.js'; + +const logger = createLogger('Tool:UpdateTodo'); + +const TODO_FILENAME = 'todos.md'; + +export interface UpdateTodoArgs { + todoList: string; + reasoning: string; +} + +export interface UpdateTodoResult { + success: boolean; + message?: string; + todoCount?: number; + error?: string; +} + +export class UpdateTodoTool implements Tool { + name = 'update_todo'; + description = 'Updates the complete todo list for tracking long-term tasks. Agent sends the entire markdown checklist every time, marking completed items with [x]. Use "- [ ]" for incomplete tasks and "- [x]" for completed tasks.'; + + schema = { + type: 'object', + properties: { + todoList: { + type: 'string', + description: 'Complete markdown checklist of todos. Use "- [ ]" for incomplete items, "- [x]" for completed items. Send the ENTIRE list every time, even if only one item changed.' + }, + reasoning: { + type: 'string', + description: 'Explanation for why the todo list is being updated' + } + }, + required: ['todoList', 'reasoning'] + }; + + async execute(args: UpdateTodoArgs, _ctx?: LLMContext): Promise { + logger.info('Executing update todo', { reasoning: args.reasoning }); + const manager = FileStorageManager.getInstance(); + + try { + // Validate todo list format + const validation = this.validateTodoList(args.todoList); + if (!validation.valid) { + return { + success: false, + error: validation.error + }; + } + + // Check if file exists + const existingFile = await manager.readFile(TODO_FILENAME); + + let result; + if (existingFile) { + // Update existing file + result = await manager.updateFile(TODO_FILENAME, args.todoList, false); + } else { + // Create new file + result = await manager.createFile(TODO_FILENAME, args.todoList, 'text/markdown'); + } + + const todoCount = this.countTodos(args.todoList); + const completedCount = this.countCompleted(args.todoList); + + return { + success: true, + message: `Updated todo list: ${todoCount.total} tasks (${completedCount} completed, ${todoCount.incomplete} remaining)`, + todoCount: todoCount.total + }; + } catch (error: any) { + logger.error('Failed to update todo list', { error: error?.message }); + return { + success: false, + error: error?.message || 'Failed to update todo list.' + }; + } + } + + private validateTodoList(todoList: string): { valid: boolean; error?: string } { + if (!todoList || !todoList.trim()) { + return { valid: false, error: 'Todo list cannot be empty.' }; + } + + // Check if it contains at least one todo item + const todoPattern = /^[\s]*-\s+\[([ x])\]/m; + if (!todoPattern.test(todoList)) { + return { + valid: false, + error: 'Todo list must contain at least one checkbox item using markdown format: "- [ ]" or "- [x]"' + }; + } + + return { valid: true }; + } + + private countTodos(todoList: string): { total: number; incomplete: number; completed: number } { + const lines = todoList.split('\n'); + let total = 0; + let completed = 0; + + for (const line of lines) { + if (line.match(/^[\s]*-\s+\[([ x])\]/)) { + total++; + if (line.match(/^[\s]*-\s+\[x\]/)) { + completed++; + } + } + } + + return { + total, + completed, + incomplete: total - completed + }; + } + + private countCompleted(todoList: string): number { + const matches = todoList.match(/^[\s]*-\s+\[x\]/gm); + return matches ? matches.length : 0; + } +} diff --git a/front_end/panels/ai_chat/tools/VisualIndicatorTool.ts b/front_end/panels/ai_chat/tools/VisualIndicatorTool.ts new file mode 100644 index 0000000000..4b88414383 --- /dev/null +++ b/front_end/panels/ai_chat/tools/VisualIndicatorTool.ts @@ -0,0 +1,639 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import * as Common from '../../../core/common/common.js'; +import * as SDK from '../../../core/sdk/sdk.js'; +import { createLogger } from '../core/Logger.js'; +import { AgentRunnerEventBus, type AgentRunnerProgressEvent } from '../agent_framework/AgentRunnerEventBus.js'; + +const logger = createLogger('VisualIndicatorTool'); + +/** + * Format tool name from snake_case to Title Case + * Example: "scroll_to_selector" -> "Scroll To Selector" + */ +function formatToolName(toolName: string): string { + return toolName + .split('_') + .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) + .join(' '); +} + +/** + * Visual indicator manager that shows real-time feedback for AI agent actions + * in the inspected page. Displays: + * 1. Page glow effect (animated border around the page) + * 2. Thinking overlay (bottom toast showing current action and reasoning) + */ +export class VisualIndicatorManager { + private static instance: VisualIndicatorManager | null = null; + private eventBus: AgentRunnerEventBus; + private isActive = false; + private currentSessionId: string | null = null; + private currentAgentName: string | null = null; + private activeSessions = new Set(); + private currentToolInfo = new Map(); + private needsNavigationListenerSetup = false; + + private constructor() { + this.eventBus = AgentRunnerEventBus.getInstance(); + } + + static getInstance(): VisualIndicatorManager { + if (!this.instance) { + this.instance = new VisualIndicatorManager(); + } + return this.instance; + } + + /** + * Initialize the visual indicator system + */ + initialize(): void { + logger.info('Visual indicator system initialized'); + this.setupEventListeners(); + this.setupNavigationListener(); + } + + /** + * Setup event listeners for agent progress events + */ + private setupEventListeners(): void { + logger.info('Setting up event listeners for visual indicators'); + this.eventBus.addEventListener('agent-progress', this.handleProgressEvent.bind(this)); + } + + /** + * Setup listener for page navigation events to re-inject indicators + */ + private setupNavigationListener(): void { + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (!target) { + logger.warn('[VisualIndicator] No primary page target available for navigation listener'); + this.needsNavigationListenerSetup = true; + return; + } + + const resourceTreeModel = target.model(SDK.ResourceTreeModel.ResourceTreeModel); + if (!resourceTreeModel) { + logger.warn('[VisualIndicator] ResourceTreeModel not available for navigation listener'); + this.needsNavigationListenerSetup = true; + return; + } + + logger.info('[VisualIndicator] Setting up navigation listener successfully'); + resourceTreeModel.addEventListener( + SDK.ResourceTreeModel.Events.FrameNavigated, + this.handleFrameNavigated.bind(this) + ); + this.needsNavigationListenerSetup = false; + } + + /** + * Handle frame navigation events - re-inject indicators if active + */ + private async handleFrameNavigated(event: Common.EventTarget.EventTargetEvent): Promise { + const frame = event.data; + + // Only handle main frame navigations (ignore iframes) + if (!frame.isMainFrame()) { + return; + } + + // Only re-inject if indicators are currently active + if (!this.isActive) { + return; + } + + logger.info('[VisualIndicator] Main frame navigated, re-injecting indicators'); + + // Get current agent name (from stored state or default) + const agentName = this.currentAgentName || 'AI Agent'; + + // Re-inject indicators into the new page + await this.injectIndicatorsIntoPage(agentName); + + // Restore current tool info if available + if (this.currentSessionId) { + const toolInfo = this.currentToolInfo.get(this.currentSessionId); + if (toolInfo) { + logger.info('[VisualIndicator] Restoring tool info after navigation:', toolInfo); + // Create a synthetic progress event to update the overlay + const syntheticEvent: AgentRunnerProgressEvent = { + type: 'tool_started', + sessionId: this.currentSessionId, + agentName, + timestamp: new Date(), + data: { + toolCall: { + type: 'tool_call', + content: { + type: 'tool_call', + toolName: toolInfo.toolName.toLowerCase().replace(/ /g, '_'), + toolArgs: {}, + toolCallId: 'restored', + reasoning: toolInfo.reasoning + } + } + } + }; + await this.updateThinkingOverlay(syntheticEvent); + } + } + } + + /** + * Handle agent progress events and update visual indicators + */ + private async handleProgressEvent(event: Common.EventTarget.EventTargetEvent): Promise { + const progressEvent = event.data; + + logger.info('[VisualIndicator] Progress event received:', { + type: progressEvent.type, + sessionId: progressEvent.sessionId, + agentName: progressEvent.agentName, + activeSessions: Array.from(this.activeSessions), + isActive: this.isActive, + currentSessionId: this.currentSessionId, + hasData: !!progressEvent.data + }); + + switch (progressEvent.type) { + case 'session_started': + this.activeSessions.add(progressEvent.sessionId); + await this.showIndicators(progressEvent); + break; + case 'tool_started': + await this.updateThinkingOverlay(progressEvent); + break; + case 'tool_completed': + await this.updateThinkingOverlay(progressEvent); + break; + case 'session_updated': + await this.updateThinkingOverlay(progressEvent); + break; + case 'session_completed': + await this.onSessionCompleted(progressEvent.sessionId); + break; + } + } + + /** + * Notify that a session has completed (call from AgentService) + */ + async onSessionCompleted(sessionId: string): Promise { + logger.info('[VisualIndicator] Session completed:', sessionId); + this.activeSessions.delete(sessionId); + + // Clean up stored tool info for this session + this.currentToolInfo.delete(sessionId); + + // Hide indicators when no more active sessions (regardless of which one completed) + if (this.activeSessions.size === 0) { + await this.hideIndicators(); + } + } + + /** + * Show visual indicators (glow + overlay) + */ + private async showIndicators(event: AgentRunnerProgressEvent): Promise { + logger.info('[VisualIndicator] Showing indicators for session:', event.sessionId); + this.isActive = true; + this.currentSessionId = event.sessionId; + this.currentAgentName = event.agentName; + + // Retry navigation listener setup if it failed during initialization + if (this.needsNavigationListenerSetup) { + logger.info('[VisualIndicator] Retrying navigation listener setup'); + this.setupNavigationListener(); + } + + await this.injectIndicatorsIntoPage(event.agentName); + } + + /** + * Inject visual indicators (glow + overlay) into the page DOM + * This method can be called both initially and after page navigations + * Includes retry logic for when document.body is not yet available + */ + private async injectIndicatorsIntoPage(agentName: string, retryCount = 0): Promise { + const maxRetries = 5; + const retryDelay = Math.min(100 * Math.pow(2, retryCount), 2000); // 100ms, 200ms, 400ms, 800ms, 1600ms, 2000ms + + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (!target) { + logger.warn('[VisualIndicator] No primary page target available'); + return; + } + + try { + const runtimeAgent = target.runtimeAgent(); + + const result = await runtimeAgent.invoke_evaluate({ + expression: ` + (() => { + console.log('[VisualIndicator] Injecting visual indicators into page', { + hasBody: !!document.body, + readyState: document.readyState, + timestamp: Date.now() + }); + + // Check if DOM is ready (document.body exists) + if (!document.body) { + console.log('[VisualIndicator] document.body not ready yet'); + return { success: false, needsRetry: true, message: 'document.body not available' }; + } + + // Prevent duplicate injection + if (document.getElementById('devtools-agent-glow-style')) { + console.log('[VisualIndicator] Already injected, skipping'); + return { success: true, message: 'Already injected' }; + } + + // Inject glow CSS + const glowStyle = document.createElement('style'); + glowStyle.id = 'devtools-agent-glow-style'; + glowStyle.textContent = \` + @keyframes devtools-agent-glow { + 0%, 100% { + box-shadow: 0 0 20px 2px rgba(0, 164, 254, 0.4), + inset 0 0 20px 2px rgba(0, 164, 254, 0.2); + } + 50% { + box-shadow: 0 0 40px 4px rgba(0, 164, 254, 0.7), + inset 0 0 40px 4px rgba(0, 164, 254, 0.3); + } + } + + html.devtools-agent-active { + animation: devtools-agent-glow 2s ease-in-out infinite; + } + + #devtools-agent-indicator { + position: fixed; + bottom: 24px; + left: 50%; + transform: translateX(-50%) translateY(10px); + background: linear-gradient(135deg, rgba(0, 20, 40, 0.75) 0%, rgba(0, 10, 30, 0.70) 100%); + backdrop-filter: blur(16px) saturate(180%); + -webkit-backdrop-filter: blur(16px) saturate(180%); + color: white; + padding: 16px 28px 18px 28px; + border-radius: 12px; + border: 1px solid rgba(0, 164, 254, 0.25); + border-top: 2px solid rgba(0, 164, 254, 0.6); + box-shadow: + 0 8px 32px rgba(0, 0, 0, 0.4), + 0 2px 8px rgba(0, 0, 0, 0.2), + inset 0 1px 0 rgba(255, 255, 255, 0.1); + z-index: 999999; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', sans-serif; + font-size: 14px; + max-width: 620px; + min-width: 320px; + opacity: 0; + transition: all 0.4s cubic-bezier(0.4, 0, 0.2, 1); + pointer-events: none; + } + + #devtools-agent-indicator.visible { + opacity: 1; + transform: translateX(-50%) translateY(0); + } + + .devtools-agent-name { + display: flex; + align-items: center; + font-weight: 600; + font-size: 16px; + color: #4fc3f7; + margin-bottom: 8px; + padding-bottom: 8px; + border-bottom: 1px solid rgba(255, 255, 255, 0.08); + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.3); + letter-spacing: 0.3px; + } + + .devtools-agent-action { + font-weight: 500; + font-size: 14px; + margin-bottom: 6px; + color: rgba(255, 255, 255, 0.95); + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.2); + line-height: 1.5; + } + + .devtools-agent-reasoning { + font-size: 12px; + color: rgba(255, 255, 255, 0.7); + font-style: italic; + margin-top: 6px; + line-height: 1.5; + padding-left: 2px; + } + + .devtools-agent-reasoning:empty { + display: none; + } + + .devtools-agent-spinner { + display: inline-block; + width: 15px; + height: 15px; + border: 2.5px solid rgba(79, 195, 247, 0.25); + border-top-color: #4fc3f7; + border-radius: 50%; + animation: devtools-agent-spin 0.8s linear infinite; + margin-right: 10px; + flex-shrink: 0; + } + + @keyframes devtools-agent-spin { + to { transform: rotate(360deg); } + } + \`; + document.head.appendChild(glowStyle); + + // Add glow class to html + document.documentElement.classList.add('devtools-agent-active'); + + // Create thinking overlay + const overlay = document.createElement('div'); + overlay.id = 'devtools-agent-indicator'; + overlay.innerHTML = \` +
+ + +
+
Starting...
+
+ \`; + + // Set agent name via textContent (safe from XSS) + const titleEl = overlay.querySelector('.devtools-agent-title'); + if (titleEl) { + const formattedAgentName = ${JSON.stringify(agentName || 'AI Agent')}.split('_').map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()).join(' '); + titleEl.textContent = formattedAgentName; + } + + document.body.appendChild(overlay); + + // Fade in + requestAnimationFrame(() => { + overlay.classList.add('visible'); + console.log('[VisualIndicator] Overlay visible, glow active'); + }); + + return { success: true, message: 'Visual indicators shown' }; + })() + `, + returnByValue: true + }); + + // Check if we need to retry + const resultValue = result.result?.value; + if (resultValue?.needsRetry && retryCount < maxRetries) { + logger.info(`[VisualIndicator] DOM not ready, scheduling retry ${retryCount + 1}/${maxRetries} in ${retryDelay}ms`); + setTimeout(() => { + void this.injectIndicatorsIntoPage(agentName, retryCount + 1); + }, retryDelay); + return; + } + + if (resultValue?.success) { + logger.info('[VisualIndicator] Visual indicators injected successfully:', resultValue.message); + } else if (retryCount >= maxRetries) { + logger.warn('[VisualIndicator] Failed to inject indicators after max retries'); + } + } catch (error) { + logger.error('Error injecting visual indicators:', error); + + // Retry on exception if we haven't exceeded max retries + if (retryCount < maxRetries) { + logger.info(`[VisualIndicator] Error occurred, scheduling retry ${retryCount + 1}/${maxRetries} in ${retryDelay}ms`); + setTimeout(() => { + void this.injectIndicatorsIntoPage(agentName, retryCount + 1); + }, retryDelay); + } + } + } + + /** + * Update the thinking overlay with current action + */ + private async updateThinkingOverlay(event: AgentRunnerProgressEvent): Promise { + if (!this.isActive) { + return; + } + + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (!target) { + return; + } + + try { + const runtimeAgent = target.runtimeAgent(); + + // Extract agent name from event and update current state + const rawAgentName = event.agentName || this.currentAgentName || 'AI Agent'; + const agentName = formatToolName(rawAgentName); + if (event.agentName) { + this.currentAgentName = event.agentName; + } + + // Extract action and reasoning from event + let action = 'Working...'; + let reasoning = ''; + + if (event.type === 'tool_started' && event.data?.toolCall) { + const toolCall = event.data.toolCall; + const toolName = toolCall.content?.toolName || 'tool'; + const formattedToolName = formatToolName(toolName); + let toolReasoning = toolCall.content?.reasoning || ''; + + // Fallback: if the provider didn't return reasoning, try the latest session reasoning message + if (!toolReasoning && event.data?.session?.messages && Array.isArray(event.data.session.messages)) { + try { + const messages = event.data.session.messages as Array; + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i]; + if (m?.type === 'reasoning' && m?.content?.text) { + toolReasoning = String(m.content.text || '').trim(); + if (toolReasoning) break; + } + } + } catch { + // Ignore fallback errors silently + } + } + + // Log detailed tool call info for debugging + logger.info('[VisualIndicator] Tool started:', { + toolName, + formattedToolName, + reasoning: toolReasoning, + toolArgs: toolCall.content?.toolArgs, + fullData: event.data + }); + + // Store tool info for later use during completion + this.currentToolInfo.set(event.sessionId, { + toolName: formattedToolName, + reasoning: toolReasoning + }); + + action = `Running: ${formattedToolName}`; + reasoning = toolReasoning; + + } else if (event.type === 'tool_completed' && event.data?.toolResult) { + const toolResult = event.data.toolResult; + const storedInfo = this.currentToolInfo.get(event.sessionId); + + // Log detailed tool result info for debugging + logger.info('[VisualIndicator] Tool completed:', { + success: toolResult.content?.success, + error: toolResult.content?.error, + storedInfo, + fullData: event.data + }); + + // Get tool name from stored info or result + const formattedToolName = storedInfo?.toolName || + (toolResult.content?.toolName ? formatToolName(toolResult.content.toolName) : 'Tool'); + + // Show completion status with tool name + if (toolResult.content?.success) { + action = `${formattedToolName} completed ✓`; + } else if (toolResult.content?.error) { + action = `${formattedToolName} failed ✗`; + // Don't show technical errors as reasoning - they're already in the action text + } else { + action = `${formattedToolName} completed`; + } + + // Keep showing original reasoning if no error, otherwise show error + if (toolResult.content?.success) { + if (storedInfo?.reasoning) { + reasoning = storedInfo.reasoning; + } else if (event.data?.session?.messages && Array.isArray(event.data.session.messages)) { + // Fallback in case we missed the tool_started or it had no reasoning + try { + const messages = event.data.session.messages as Array; + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i]; + if (m?.type === 'reasoning' && m?.content?.text) { + const fallback = String(m.content.text || '').trim(); + if (fallback) { reasoning = fallback; break; } + } + } + } catch { + // Ignore fallback errors silently + } + } + } + } + + // Log what we're displaying + logger.info('[VisualIndicator] Updating overlay:', { agentName, action, reasoning }); + + await runtimeAgent.invoke_evaluate({ + expression: ` + (() => { + const overlay = document.getElementById('devtools-agent-indicator'); + if (!overlay) { + return { success: false, message: 'Overlay not found' }; + } + + // Update agent name in header + const titleEl = overlay.querySelector('.devtools-agent-title'); + if (titleEl) { + titleEl.textContent = ${JSON.stringify(agentName)}; + } + + // Update action and reasoning + const actionEl = overlay.querySelector('.devtools-agent-action'); + const reasoningEl = overlay.querySelector('.devtools-agent-reasoning'); + + if (actionEl) { + actionEl.textContent = ${JSON.stringify(action)}; + } + + if (reasoningEl) { + reasoningEl.textContent = ${JSON.stringify(reasoning)}; + // Hide reasoning div when empty + reasoningEl.style.display = ${JSON.stringify(reasoning ? 'block' : 'none')}; + } + + return { success: true, message: 'Overlay updated' }; + })() + `, + returnByValue: true + }); + + logger.info('Thinking overlay updated successfully'); + } catch (error) { + logger.error('Error updating thinking overlay:', error); + } + } + + /** + * Hide visual indicators (glow + overlay) + */ + async hideIndicators(): Promise { + if (!this.isActive) { + return; + } + + this.isActive = false; + this.currentSessionId = null; + + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (!target) { + return; + } + + try { + const runtimeAgent = target.runtimeAgent(); + + await runtimeAgent.invoke_evaluate({ + expression: ` + (() => { + // Remove glow class + document.documentElement.classList.remove('devtools-agent-active'); + + // Fade out overlay + const overlay = document.getElementById('devtools-agent-indicator'); + if (overlay) { + overlay.classList.remove('visible'); + setTimeout(() => { + overlay.remove(); + }, 300); + } + + // Remove glow style + const glowStyle = document.getElementById('devtools-agent-glow-style'); + if (glowStyle) { + glowStyle.remove(); + } + + return { success: true, message: 'Visual indicators hidden' }; + })() + `, + returnByValue: true + }); + + logger.info('Visual indicators hidden successfully'); + } catch (error) { + logger.error('Error hiding visual indicators:', error); + } + } + + /** + * Check if indicators are currently active + */ + isIndicatorActive(): boolean { + return this.isActive; + } +} From 9be49e18738f4403df809b0bfc99c974d3b569df Mon Sep 17 00:00:00 2001 From: Tyson Thomas Date: Tue, 4 Nov 2025 07:43:24 -0800 Subject: [PATCH 3/6] more improvement on visual indicator --- front_end/panels/ai_chat/core/AgentService.ts | 4 +- .../ai_chat/tools/VisualIndicatorTool.ts | 88 ++++++++++++++----- 2 files changed, 68 insertions(+), 24 deletions(-) diff --git a/front_end/panels/ai_chat/core/AgentService.ts b/front_end/panels/ai_chat/core/AgentService.ts index 637c1dedd7..f97e388cef 100644 --- a/front_end/panels/ai_chat/core/AgentService.ts +++ b/front_end/panels/ai_chat/core/AgentService.ts @@ -127,8 +127,8 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ // Subscribe to AgentRunner events AgentRunnerEventBus.getInstance().addEventListener('agent-progress', this.#handleAgentProgress.bind(this)); - // Initialize visual indicator system - VisualIndicatorManager.getInstance().initialize(); + // Initialize visual indicator system with reference to AgentService + VisualIndicatorManager.getInstance().initialize(this); // Subscribe to configuration changes this.#configManager.addChangeListener(this.#handleConfigurationChange.bind(this)); diff --git a/front_end/panels/ai_chat/tools/VisualIndicatorTool.ts b/front_end/panels/ai_chat/tools/VisualIndicatorTool.ts index 4b88414383..a061bb61a1 100644 --- a/front_end/panels/ai_chat/tools/VisualIndicatorTool.ts +++ b/front_end/panels/ai_chat/tools/VisualIndicatorTool.ts @@ -29,6 +29,7 @@ function formatToolName(toolName: string): string { export class VisualIndicatorManager { private static instance: VisualIndicatorManager | null = null; private eventBus: AgentRunnerEventBus; + private agentService: any | null = null; // AgentService reference for checking running state private isActive = false; private currentSessionId: string | null = null; private currentAgentName: string | null = null; @@ -48,10 +49,11 @@ export class VisualIndicatorManager { } /** - * Initialize the visual indicator system + * Initialize the visual indicator system with AgentService reference */ - initialize(): void { - logger.info('Visual indicator system initialized'); + initialize(agentService: any): void { + this.agentService = agentService; + logger.info('Visual indicator system initialized with AgentService'); this.setupEventListeners(); this.setupNavigationListener(); } @@ -179,6 +181,27 @@ export class VisualIndicatorManager { } } + /** + * Check if any agent is currently running (source of truth from AgentService) + */ + private hasAnyRunningAgent(): boolean { + if (!this.agentService) { + logger.warn('[VisualIndicator] No AgentService - cannot check running state'); + return false; + } + + const activeSessions = this.agentService.getActiveAgentSessions(); + const hasRunning = activeSessions.some((session: any) => session.status === 'running'); + + logger.info('[VisualIndicator] Running agent check:', { + totalSessions: activeSessions.length, + hasRunning, + sessionStates: activeSessions.map((s: any) => ({ id: s.sessionId, status: s.status })) + }); + + return hasRunning; + } + /** * Notify that a session has completed (call from AgentService) */ @@ -189,9 +212,12 @@ export class VisualIndicatorManager { // Clean up stored tool info for this session this.currentToolInfo.delete(sessionId); - // Hide indicators when no more active sessions (regardless of which one completed) - if (this.activeSessions.size === 0) { + // Check if any agent is still running (source of truth from AgentService) + if (!this.hasAnyRunningAgent()) { + logger.info('[VisualIndicator] No agents running - hiding indicators'); await this.hideIndicators(); + } else { + logger.info('[VisualIndicator] Other agents still running - keeping indicators visible'); } } @@ -268,6 +294,7 @@ export class VisualIndicatorManager { } html.devtools-agent-active { + min-height: 100vh; animation: devtools-agent-glow 2s ease-in-out infinite; } @@ -280,22 +307,23 @@ export class VisualIndicatorManager { backdrop-filter: blur(16px) saturate(180%); -webkit-backdrop-filter: blur(16px) saturate(180%); color: white; - padding: 16px 28px 18px 28px; - border-radius: 12px; + padding: 12px 20px 14px 20px; + border-radius: 10px; border: 1px solid rgba(0, 164, 254, 0.25); border-top: 2px solid rgba(0, 164, 254, 0.6); box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4), 0 2px 8px rgba(0, 0, 0, 0.2), inset 0 1px 0 rgba(255, 255, 255, 0.1); - z-index: 999999; + z-index: 2147483647; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', sans-serif; - font-size: 14px; - max-width: 620px; - min-width: 320px; + font-size: 12px; + max-width: 520px; + min-width: 280px; opacity: 0; transition: all 0.4s cubic-bezier(0.4, 0, 0.2, 1); - pointer-events: none; + pointer-events: auto; + cursor: default; } #devtools-agent-indicator.visible { @@ -303,14 +331,22 @@ export class VisualIndicatorManager { transform: translateX(-50%) translateY(0); } + #devtools-agent-indicator.visible:hover { + opacity: 0.15; + background: linear-gradient(135deg, rgba(0, 20, 40, 0.15) 0%, rgba(0, 10, 30, 0.1) 100%); + backdrop-filter: blur(4px) saturate(120%); + -webkit-backdrop-filter: blur(4px) saturate(120%); + transition: all 0.2s ease-out; + } + .devtools-agent-name { display: flex; align-items: center; font-weight: 600; - font-size: 16px; + font-size: 14px; color: #4fc3f7; - margin-bottom: 8px; - padding-bottom: 8px; + margin-bottom: 6px; + padding-bottom: 6px; border-bottom: 1px solid rgba(255, 255, 255, 0.08); text-shadow: 0 1px 2px rgba(0, 0, 0, 0.3); letter-spacing: 0.3px; @@ -318,18 +354,18 @@ export class VisualIndicatorManager { .devtools-agent-action { font-weight: 500; - font-size: 14px; - margin-bottom: 6px; + font-size: 12px; + margin-bottom: 4px; color: rgba(255, 255, 255, 0.95); text-shadow: 0 1px 2px rgba(0, 0, 0, 0.2); line-height: 1.5; } .devtools-agent-reasoning { - font-size: 12px; + font-size: 10px; color: rgba(255, 255, 255, 0.7); font-style: italic; - margin-top: 6px; + margin-top: 4px; line-height: 1.5; padding-left: 2px; } @@ -340,8 +376,8 @@ export class VisualIndicatorManager { .devtools-agent-spinner { display: inline-block; - width: 15px; - height: 15px; + width: 13px; + height: 13px; border: 2.5px solid rgba(79, 195, 247, 0.25); border-top-color: #4fc3f7; border-radius: 50%; @@ -439,6 +475,7 @@ export class VisualIndicatorManager { // Extract agent name from event and update current state const rawAgentName = event.agentName || this.currentAgentName || 'AI Agent'; const agentName = formatToolName(rawAgentName); + if (event.agentName) { this.currentAgentName = event.agentName; } @@ -451,7 +488,14 @@ export class VisualIndicatorManager { const toolCall = event.data.toolCall; const toolName = toolCall.content?.toolName || 'tool'; const formattedToolName = formatToolName(toolName); - let toolReasoning = toolCall.content?.reasoning || ''; + + // Extract reasoning from multiple sources (matching UI pattern) + // Priority: 1) LLM reasoning (O-models), 2) toolArgs.reasoning (most common), 3) fallback aliases + const toolArgs = toolCall.content?.toolArgs || {}; + const reasonFromArgs = toolArgs?.reasoning ?? toolArgs?.reason; + let toolReasoning = toolCall.content?.reasoning || + (reasonFromArgs !== undefined ? String(reasonFromArgs) : '') || + ''; // Fallback: if the provider didn't return reasoning, try the latest session reasoning message if (!toolReasoning && event.data?.session?.messages && Array.isArray(event.data.session.messages)) { From ef751cf1332ce041614983aaf3445cfb6c1c54f4 Mon Sep 17 00:00:00 2001 From: Tyson Thomas Date: Wed, 5 Nov 2025 21:55:57 -0800 Subject: [PATCH 4/6] More UI and agent improvements --- config/gni/devtools_grd_files.gni | 6 + front_end/panels/ai_chat/BUILD.gn | 6 + .../ai_chat/agent_framework/AgentRunner.ts | 148 ++- .../agent_framework/ConfigurableAgentTool.ts | 47 + .../implementation/agents/ActionAgent.ts | 49 +- .../agents/ContentWriterAgent.ts | 8 +- .../implementation/agents/ResearchAgent.ts | 364 +++++--- .../implementation/agents/SearchAgent.ts | 2 - .../implementation/agents/WebTaskAgent.ts | 4 +- front_end/panels/ai_chat/core/AgentService.ts | 31 +- front_end/panels/ai_chat/core/Version.ts | 4 +- .../test-cases/research-agent-tests.ts | 2 +- front_end/panels/ai_chat/tools/Tools.ts | 85 +- front_end/panels/ai_chat/ui/AIChatPanel.ts | 363 +++++--- front_end/panels/ai_chat/ui/ChatView.ts | 207 ++++- .../panels/ai_chat/ui/FileContentViewer.ts | 862 ++++++++++++++++++ .../panels/ai_chat/ui/FileListDisplay.ts | 646 +++++++++++++ .../panels/ai_chat/ui/TodoListDisplay.ts | 313 +++++++ front_end/panels/ai_chat/ui/chatView.css | 26 +- 19 files changed, 2774 insertions(+), 399 deletions(-) create mode 100644 front_end/panels/ai_chat/ui/FileContentViewer.ts create mode 100644 front_end/panels/ai_chat/ui/FileListDisplay.ts create mode 100644 front_end/panels/ai_chat/ui/TodoListDisplay.ts diff --git a/config/gni/devtools_grd_files.gni b/config/gni/devtools_grd_files.gni index a04a129c0a..7f68872504 100644 --- a/config/gni/devtools_grd_files.gni +++ b/config/gni/devtools_grd_files.gni @@ -658,6 +658,9 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/ui/mcp/MCPConnectorsCatalogDialog.js", "front_end/panels/ai_chat/ui/EvaluationDialog.js", "front_end/panels/ai_chat/ui/WebAppCodeViewer.js", + "front_end/panels/ai_chat/ui/TodoListDisplay.js", + "front_end/panels/ai_chat/ui/FileListDisplay.js", + "front_end/panels/ai_chat/ui/FileContentViewer.js", "front_end/panels/ai_chat/core/AgentService.js", "front_end/panels/ai_chat/core/State.js", "front_end/panels/ai_chat/core/Graph.js", @@ -715,6 +718,9 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/tools/DeleteFileTool.js", "front_end/panels/ai_chat/tools/ReadFileTool.js", "front_end/panels/ai_chat/tools/ListFilesTool.js", + "front_end/panels/ai_chat/tools/ExecuteCodeTool.js", + "front_end/panels/ai_chat/tools/UpdateTodoTool.js", + "front_end/panels/ai_chat/tools/VisualIndicatorTool.js", "front_end/panels/ai_chat/common/utils.js", "front_end/panels/ai_chat/common/log.js", "front_end/panels/ai_chat/common/context.js", diff --git a/front_end/panels/ai_chat/BUILD.gn b/front_end/panels/ai_chat/BUILD.gn index 7e0a5ab4d1..d0368fa2ac 100644 --- a/front_end/panels/ai_chat/BUILD.gn +++ b/front_end/panels/ai_chat/BUILD.gn @@ -43,6 +43,9 @@ devtools_module("ai_chat") { "ui/PromptEditDialog.ts", "ui/EvaluationDialog.ts", "ui/WebAppCodeViewer.ts", + "ui/TodoListDisplay.ts", + "ui/FileListDisplay.ts", + "ui/FileContentViewer.ts", "ai_chat_impl.ts", "models/ChatTypes.ts", "core/Graph.ts", @@ -212,6 +215,9 @@ _ai_chat_sources = [ "ui/SettingsDialog.ts", "ui/EvaluationDialog.ts", "ui/WebAppCodeViewer.ts", + "ui/TodoListDisplay.ts", + "ui/FileListDisplay.ts", + "ui/FileContentViewer.ts", "ui/mcp/MCPConnectionsDialog.ts", "ui/mcp/MCPConnectorsCatalogDialog.ts", "ai_chat_impl.ts", diff --git a/front_end/panels/ai_chat/agent_framework/AgentRunner.ts b/front_end/panels/ai_chat/agent_framework/AgentRunner.ts index 20f924642b..a39becdd0b 100644 --- a/front_end/panels/ai_chat/agent_framework/AgentRunner.ts +++ b/front_end/panels/ai_chat/agent_framework/AgentRunner.ts @@ -54,6 +54,8 @@ export interface AgentRunnerHooks { createSuccessResult: (output: string, intermediateSteps: ChatMessage[], reason: AgentRunTerminationReason) => ConfigurableAgentResult; /** Function to create an error result */ createErrorResult: (error: string, intermediateSteps: ChatMessage[], reason: AgentRunTerminationReason) => ConfigurableAgentResult; + /** Function to run after agent execution completes (optional) */ + afterExecute?: (result: ConfigurableAgentResult, agentSession: AgentSession) => Promise; } @@ -71,6 +73,33 @@ export class AgentRunner { AgentRunner.eventBus = AgentRunnerEventBus.getInstance(); } } + + /** + * Clears the todo list file if it exists and has content + * Called when an agent completes or fails to clean up state + * Only clears if the agent has access to the update_todo tool + */ + private static async clearTodoList(agentName: string, tools: Array>): Promise { + // Only clear todos if the agent has the update_todo tool + const hasUpdateTodoTool = tools.some(tool => tool.name === 'update_todo'); + if (!hasUpdateTodoTool) { + logger.debug(`Agent ${agentName} does not have update_todo tool, skipping todo list cleanup`); + return; + } + + try { + const fileManager = FileStorageManager.getInstance(); + const todosFile = await fileManager.readFile('todos.md'); + + if (todosFile?.content && todosFile.content.trim().length > 0) { + await fileManager.deleteFile('todos.md'); + logger.info(`Cleared non-empty todo list for ${agentName}`); + } + } catch (error) { + logger.debug(`Failed to clear todo list for ${agentName}:`, error); + } + } + /** * Helper function to convert ChatMessage[] to LLMMessage[] */ @@ -407,7 +436,7 @@ export class AgentRunner { const agentName = executingAgent?.name || 'Unknown'; logger.info(`Starting execution loop for agent: ${agentName}`); const { apiKey, modelName, systemPrompt, tools, maxIterations, temperature, agentDescriptor } = config; - const { prepareInitialMessages, createSuccessResult, createErrorResult } = hooks; + const { prepareInitialMessages, createSuccessResult, createErrorResult, afterExecute } = hooks; // Create session when agent starts (natural timing) @@ -574,7 +603,20 @@ export class AgentRunner { }); } + // Clear todo list on abort + await AgentRunner.clearTodoList(agentName, tools); + const abortResult = createErrorResult('Execution was cancelled', messages, 'error'); + + // Execute afterExecute hook if defined + if (afterExecute) { + try { + await afterExecute(abortResult, currentSession); + } catch (error) { + logger.warn(`afterExecute hook failed for ${agentName}:`, error); + } + } + return { ...abortResult, agentSession: currentSession }; } @@ -587,23 +629,29 @@ export class AgentRunner { // Prepare prompt and call LLM const iterationInfo = ` ## Current Progress -- You are currently on step ${iteration + 1} of ${maxIterations} maximum steps. +- You are currently on step ${iteration + 1} of ${maxIterations - 1} maximum steps. - Focus on making meaningful progress with each step.`; - // Inject todos into system prompt if they exist + // Inject todos into system prompt ONLY if agent has update_todo tool let todosContext = ''; - try { - const fileManager = FileStorageManager.getInstance(); - const todosFile = await fileManager.readFile('todos.md'); + const hasUpdateTodoTool = config.tools.some(tool => tool.name === 'update_todo'); - if (todosFile?.content) { - todosContext = `\n\n## CURRENT TODO LIST\n${todosFile.content}\n\nUpdate the todo list using the 'update_todo' tool as you complete tasks. Mark completed items with [x].`; - } else { - todosContext = `\n\n## TODO LIST\nNo todo list exists yet. If this is a multi-step task, create a todo list using the 'update_todo' tool to track your progress.`; + if (hasUpdateTodoTool) { + try { + const fileManager = FileStorageManager.getInstance(); + const todosFile = await fileManager.readFile('todos.md'); + + if (todosFile?.content) { + todosContext = `\n\n## CURRENT TODO LIST\n${todosFile.content}\n\nUpdate the todo list using the 'update_todo' tool as you complete tasks. Mark completed items with [x].`; + } else { + todosContext = `\n\n## TODO LIST\nNo todo list exists yet. If this is a multi-step task, create a todo list using the 'update_todo' tool to track your progress.`; + } + } catch (error) { + logger.debug('Failed to read todos, skipping injection:', error); + // Continue without todos if reading fails } - } catch (error) { - logger.debug('Failed to read todos, skipping injection:', error); - // Continue without todos if reading fails + } else { + logger.debug(`Skipping todo injection for ${agentName} - update_todo tool not available`); } // Enhance system prompt with iteration info, todos, and page context @@ -800,12 +848,25 @@ export class AgentRunner { }); } + // Clear todo list on error + await AgentRunner.clearTodoList(agentName, tools); + // Use error hook with structured summary const result = createErrorResult(errorMsg, messages, 'error'); result.summary = { type: 'error', content: errorSummary }; + + // Execute afterExecute hook if defined + if (afterExecute) { + try { + await afterExecute(result, agentSession); + } catch (error) { + logger.warn(`afterExecute hook failed for ${agentName}:`, error); + } + } + return { ...result, agentSession }; } @@ -1246,8 +1307,19 @@ export class AgentRunner { logger.info(`${agentName} LLM provided final answer.`); - // Generate summary of successful completion - const completionSummary = await this.summarizeAgentProgress(messages, maxIterations, agentName, modelName, 'final_answer', config.provider, config.getVisionCapability); + // Clear non-empty todo list when sending final message + await AgentRunner.clearTodoList(agentName, tools); + + // Conditionally generate and append summary based on agent configuration + let finalAnswer = answer; + if (executingAgent?.config?.includeSummaryInAnswer === true) { + logger.info(`Generating summary for ${agentName} (includeSummaryInAnswer=true)`); + const completionSummary = await this.summarizeAgentProgress(messages, maxIterations, agentName, modelName, 'final_answer', config.provider, config.getVisionCapability); + // Append summary to the answer with clear separator + finalAnswer = `${answer}\n\n---\n\n### Analysis of Agentic Conversation\n\n${completionSummary}`; + } else { + logger.info(`Skipping summary for ${agentName} (includeSummaryInAnswer not enabled)`); + } // Complete session naturally agentSession.status = 'completed'; @@ -1266,12 +1338,19 @@ export class AgentRunner { }); } - // Exit loop and return success with structured summary - const result = createSuccessResult(answer, messages, 'final_answer'); - result.summary = { - type: 'completion', - content: completionSummary - }; + // Exit loop and return success with final answer (summary appended if configured) + const result = createSuccessResult(finalAnswer, messages, 'final_answer'); + + // Execute afterExecute hook if defined + if (afterExecute) { + try { + await afterExecute(result, agentSession); + } catch (error) { + logger.warn(`afterExecute hook failed for ${agentName}:`, error); + // Continue and return result even if afterExecute fails + } + } + return { ...result, agentSession }; } else if (parsedAction.type === 'error') { @@ -1321,12 +1400,25 @@ export class AgentRunner { }); } + // Clear todo list on error + await AgentRunner.clearTodoList(agentName, tools); + // Use error hook with structured summary const result = createErrorResult(errorMsg, messages, 'error'); result.summary = { type: 'error', content: errorSummary }; + + // Execute afterExecute hook if defined + if (afterExecute) { + try { + await afterExecute(result, agentSession); + } catch (error) { + logger.warn(`afterExecute hook failed for ${agentName}:`, error); + } + } + return { ...result, agentSession }; } } @@ -1407,11 +1499,25 @@ export class AgentRunner { // Generate summary of agent progress instead of generic error message const progressSummary = await this.summarizeAgentProgress(messages, maxIterations, agentName, modelName, 'max_iterations', config.provider, config.getVisionCapability); + + // Clear todo list on max iterations + await AgentRunner.clearTodoList(agentName, tools); + const result = createErrorResult('Agent reached maximum iterations', messages, 'max_iterations'); result.summary = { type: 'timeout', content: progressSummary }; + + // Execute afterExecute hook if defined + if (afterExecute) { + try { + await afterExecute(result, agentSession); + } catch (error) { + logger.warn(`afterExecute hook failed for ${agentName}:`, error); + } + } + return { ...result, agentSession }; } diff --git a/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts b/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts index 99b77afec9..a7c522eef9 100644 --- a/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts +++ b/front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts @@ -181,6 +181,36 @@ export interface AgentToolConfig { * (both success and error results). Defaults to false (steps are omitted). */ includeIntermediateStepsOnReturn?: boolean; + + /** + * If true, generate a summary of the agent's execution and append it to the final answer. + * Summary includes: user request, agent decisions, and final outcome. + * Defaults to false (no summary generated). + * Use this for agents where understanding the execution process is valuable (e.g., web automation agents). + */ + includeSummaryInAnswer?: boolean; + + /** + * Optional lifecycle hook that runs before the agent starts executing. + * Use this for agent-specific pre-execution logic such as environment setup, + * page navigation, or prerequisite checks. + * + * @param callCtx - The call context containing API keys, models, and other execution context + * @returns Promise that resolves when pre-execution is complete + */ + beforeExecute?: (callCtx: CallCtx) => Promise; + + /** + * Optional lifecycle hook that runs after the agent completes execution. + * Use this for agent-specific post-execution logic such as saving results, + * cleanup operations, or data aggregation. + * + * @param result - The final agent execution result (success or error) + * @param agentSession - The complete agent session with all messages and tool calls + * @param callCtx - The call context containing API keys, models, and other execution context + * @returns Promise that resolves when post-execution is complete + */ + afterExecute?: (result: ConfigurableAgentResult, agentSession: AgentSession, callCtx: CallCtx) => Promise; } /** @@ -453,6 +483,16 @@ export class ConfigurableAgentTool implements Tool this.config.createErrorResult!(err, steps, reason, this.config) : (err, steps, reason) => this.createErrorResult(err, steps, reason), + // Wrap afterExecute to pass callCtx (AgentRunner doesn't have access to callCtx) + afterExecute: this.config.afterExecute + ? async (result, agentSession) => this.config.afterExecute!(result, agentSession, callCtx) + : undefined, }; // Run the agent @@ -543,6 +587,9 @@ export class ConfigurableAgentTool implements Tool => { + // Auto-navigate away from chrome:// URLs since action agent cannot interact with chrome:// pages + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (target) { + try { + const urlResult = await target.runtimeAgent().invoke_evaluate({ + expression: 'window.location.href', + returnByValue: true, + }); + + const currentUrl = urlResult.result?.value as string; + if (currentUrl && currentUrl.startsWith('chrome://')) { + logger.info(`Action agent invoked on chrome:// URL (${currentUrl}). Auto-navigating to Google...`); + + // Get navigate_url tool and execute + const navigateTool = ToolRegistry.getRegisteredTool('navigate_url'); + if (navigateTool) { + // Create LLMContext from CallCtx for tool execution + const llmContext = { + apiKey: callCtx.apiKey, + provider: callCtx.provider!, + model: callCtx.model || callCtx.mainModel || '', + getVisionCapability: callCtx.getVisionCapability, + miniModel: callCtx.miniModel, + nanoModel: callCtx.nanoModel, + abortSignal: callCtx.abortSignal + }; + await navigateTool.execute({ + url: 'https://google.com', + reasoning: 'Auto-navigation from chrome:// URL to enable action agent functionality' + }, llmContext); + logger.info('Auto-navigation to Google completed successfully'); + } else { + logger.warn('navigate_url tool not found, skipping auto-navigation'); + } + } + } catch (error) { + logger.warn('Failed to check/navigate away from chrome:// URL:', error); + // Continue with agent execution even if auto-navigation fails + } + } + }, }; } diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts index dd780f6a8a..033f131e9b 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts @@ -54,13 +54,13 @@ Your process should follow these steps: 9. **Conclusion**: Summary of key points and final thoughts 10. **References**: Properly formatted citations for all sources used -The final output should be in markdown format, and it should be lengthy and detailed. Aim for 5-10 pages of content, at least 1000 words.`, +The final output should be in markdown format, and it should be lengthy and detailed. Aim for 5-10 pages of content, at least 1000 words. + +IMPORTANT: YOU DO NOT HAVE ACCESS TO ANY TOOLS OTHER THAN 'read_file' AND 'list_files' DURING THIS AGENT'S EXECUTION. YOU HAVE ACCESS TO THE PREVIOUS AGENT'S RESEARCH DATA THROUGH MESSAGES AND FILES ONLY. DO NOT ATTEMPT TO USE ANY OTHER TOOLS. +`, tools: [ 'read_file', 'list_files', - 'create_file', - 'update_file', - 'delete_file', ], maxIterations: 3, modelName: MODEL_SENTINELS.USE_MINI, diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts index 0cce995a37..937244f9db 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts @@ -1,8 +1,14 @@ -import type { AgentToolConfig, ConfigurableAgentArgs } from "../../ConfigurableAgentTool.js"; +import type { AgentToolConfig, ConfigurableAgentArgs, ConfigurableAgentResult, CallCtx } from "../../ConfigurableAgentTool.js"; import type { ChatMessage } from "../../../models/ChatTypes.js"; +import type { AgentSession } from "../../AgentSessionTypes.js"; import { ChatMessageEntity } from "../../../models/ChatTypes.js"; import { MODEL_SENTINELS } from "../../../core/Constants.js"; import { AGENT_VERSION } from "./AgentVersion.js"; +import { createLogger } from "../../../core/Logger.js"; +import { FileStorageManager } from "../../../tools/FileStorageManager.js"; +import type { FetcherToolResult } from "../../../tools/FetcherTool.js"; + +const logger = createLogger('ResearchAgent'); /** * Create the configuration for the Research Agent @@ -11,166 +17,93 @@ export function createResearchAgentConfig(): AgentToolConfig { return { name: 'research_agent', version: AGENT_VERSION, - description: 'Performs in-depth research on a specific query autonomously using multiple steps and internal tool calls (navigation, fetching, extraction). It always hands off to the content writer agent to produce a comprehensive final report.', + description: 'Performs in-depth research on a specific query autonomously using multiple steps and internal tool calls (navigation, fetching, extraction). Returns comprehensive research findings with proper citations and structured data.', ui: { displayName: 'Research Agent', avatar: '🔍', color: '#3b82f6', backgroundColor: '#f8fafc' }, - systemPrompt: `You are a research subagent working as part of a team. You have been given a specific research task with clear requirements. Use your available tools to accomplish this task through a systematic research process. - -## Understanding Your Task - -You will receive: -- **task**: The specific research objective to accomplish -- **reasoning**: Why this research is being conducted (shown to the user) -- **context**: Additional details about constraints or focus areas (optional) -- **scope**: Whether this is a focused, comprehensive, or exploratory investigation -- **priority_sources**: Specific sources to prioritize if provided - -Adapt your research approach based on the scope: -- **Focused**: 3-5 tool calls, quick specific answers -- **Comprehensive**: 5-10 tool calls, in-depth analysis from multiple sources -- **Exploratory**: 10-15 tool calls, broad investigation of the topic landscape - -## Research Process - -### 1. Planning Phase -First, think through the task thoroughly: -- Review the task requirements and any provided context -- Note the scope (focused/comprehensive/exploratory) to determine effort level -- Check for priority_sources to guide your search strategy -- Determine your research budget based on scope: - - Focused scope: 5-10 tool calls for quick, specific answers - - Comprehensive scope: 10-15 tool calls for detailed analysis - - Exploratory scope: 15-30 tool calls for broad investigation -- Identify which tools are most relevant for the task - -### 2. Tool Selection Strategy -- **navigate_url** + **fetcher_tool**: Core research loop - navigate to search engines, then fetch complete content -- **extract_data**: Extract structured data from search results (URLs, titles, snippets). Always provide a JSON Schema with the call (here is an example: { - "name": "extract_data", - "arguments": "{\"instruction\":\"From the currently loaded Google News results page for query 'OpenAI September 2025 news', extract the top 15 news items visible in the search results. For each item extract: title (string), snippet (string), url (string, format:url), source (string), and publishDate (string). Return a JSON object with property 'results' which is an array of these items.\",\"reasoning\":\"Collect structured list of recent news articles about OpenAI in September 2025 so we can batch-fetch the full content for comprehensive research.\",\"schema\":{\"type\":\"object\",\"properties\":{\"results\":{\"type\":\"array\",\"items\":{\"type\":\"object\",\"properties\":{\"title\":{\"type\":\"string\"},\"snippet\":{\"type\":\"string\"},\"url\":{\"type\":\"string\",\"format\":\"url\"},\"source\":{\"type\":\"string\"},\"publishDate\":{\"type\":\"string\"}},\"required\":[\"title\",\"url\",\"source\"]}}},\"required\":[\"results\"]}}" -}) -- **html_to_markdown**: Use when you need high-quality page text in addition to (not instead of) structured extractions. -- **fetcher_tool**: BATCH PROCESS multiple URLs at once - accepts an array of URLs to save tool calls - -### 3. Workspace Coordination -- Treat the file management tools as your shared scratchpad with other agents in the session. -- Start each iteration by calling 'list_files' and 'read_file' on any artifacts relevant to your task so you understand existing progress. -- Persist work products incrementally with 'create_file'/'update_file'. Use descriptive names (e.g. 'research/-sources.json') and include agent name, timestamp, query used, and quality notes so others can audit or extend the work. -- Append to existing files when adding new findings; only delete files if they are obsolete AND all valuable information is captured elsewhere. -- Record open questions or follow-ups in dedicated tracking files so parallel subtasks avoid duplicating effort. - -**CRITICAL - Batch URL Fetching**: -- The fetcher_tool accepts an ARRAY of URLs: {urls: [url1, url2, url3], reasoning: "..."} -- ALWAYS batch multiple URLs together instead of calling fetcher_tool multiple times -- Example: After extracting 5 URLs from search results, call fetcher_tool ONCE with all 5 URLs -- This dramatically reduces tool calls and improves efficiency - -### 4. Research Loop (OODA) -Execute an excellent Observe-Orient-Decide-Act loop: - -**Observe**: What information has been gathered? What's still needed? -**Orient**: What tools/queries would best gather needed information? -**Decide**: Make informed decisions on specific tool usage -**Act**: Execute the tool call - -**Efficient Research Workflow**: -1. Use navigate_url to search for your topic -2. Use extract_data to collect ALL URLs from search results -3. Call fetcher_tool ONCE with the array of all extracted URLs -4. Analyze the batch results and determine if more searches are needed -5. Repeat with different search queries if necessary - -- Execute a MINIMUM of 10 distinct tool calls for comprehensive research -- Maximum of 30 tool calls to prevent system overload -- Batch processing URLs counts as ONE tool call, making research much more efficient -- NEVER repeat the same query - adapt based on findings -- If hitting diminishing returns, complete the task immediately - -### 5. Source Quality Evaluation -Think critically about sources: -- Distinguish facts from speculation (watch for "could", "may", future tense) -- Identify problematic sources (aggregators vs. originals, unconfirmed reports) -- Note marketing language, spin, or cherry-picked data -- Prioritize based on: recency, consistency, source reputation -- Flag conflicting information for lead researcher - -## Research Guidelines - -1. **Query Optimization**: - - Use moderately broad queries (under 5 words) - - Avoid hyper-specific searches with poor hit rates - - Adjust specificity based on result quality - - Balance between specific and general - -2. **Information Focus** - Prioritize high-value information that is: - - **Significant**: Major implications for the task - - **Important**: Directly relevant or specifically requested - - **Precise**: Specific facts, numbers, dates, concrete data - - **High-quality**: From reputable, reliable sources - -3. **Documentation Requirements**: - - State which tool you're using and why - - Document each source with URL and title - - Extract specific quotes, statistics, facts with attribution - - Organize findings by source with clear citations - - Include publication dates where available - -4. **Efficiency Principles**: - - BATCH PROCESS URLs: Always use fetcher_tool with multiple URLs at once - - Use parallel tool calls when possible (2 tools simultaneously) - - Complete task as soon as sufficient information is gathered - - Stop at ~30 tool calls or when hitting diminishing returns - - Be detailed in process but concise in reporting - - Remember: Fetching 10 URLs in one batch = 1 tool call vs 10 individual calls - -## Output Structure -Structure findings as: -- Source 1: [Title] (URL) - [Date if available] - - Key facts: [specific quotes/data] - - Statistics: [numbers with context] - - Expert opinions: [attributed quotes] -- Source 2: [Title] (URL) - - [Continue pattern...] - -## Critical Reminders -- This is autonomous tool execution - complete the full task in one run -- NO conversational elements - execute research automatically -- Gather from 10+ diverse sources minimum -- DO NOT generate markdown reports or final content yourself -- Focus on gathering raw research data with proper citations - -## IMPORTANT: Handoff Protocol -When your research is complete: -1. NEVER generate markdown content or final reports yourself -2. Use the handoff_to_content_writer_agent tool to pass your research findings -3. The handoff tool expects: {query: "research topic", reasoning: "explanation for user"} -4. The content_writer_agent will create the final report from your research data - -Remember: You gather data, content_writer_agent writes the report. Always hand off when research is complete. - -Before handing off, ensure your latest findings are reflected in the shared files (e.g. summaries, raw notes, structured datasets). This enables the orchestrator and content writer to understand what has been completed, reuse your artifacts, and avoid redundant rework.`, + systemPrompt: `You are a research agent. Execute systematic research using your tools autonomously. + +## Task Inputs & Scope +- **query**: Research objective +- **context**: Constraints/focus areas (optional) +- **scope**: focused (5-10 tool calls), comprehensive (10-15), exploratory (15-30) + +## CRITICAL REQUIREMENT +**YOU MUST CREATE FILES BEFORE COMPLETING**: Never return a final answer until you have successfully created both required files ([topic]_research.md and [topic]_sources.json). The files are the PRIMARY deliverable - your final answer is just a confirmation that files exist. + +## Core Research Workflow +1. Navigate to search engines (navigate_url) +2. Extract URLs from results (extract_data - provide JSON schema) +3. **CRITICAL**: Batch fetch all URLs at once (fetcher_tool with array: {urls: [url1, url2, ...]} - NEVER fetch individually) +4. Analyze content and iterate with different queries +5. Target 10+ sources minimum, max 30 tool calls total +6. **MANDATORY**: Create both required files before returning final answer + +## Key Tools +- **navigate_url + fetcher_tool**: Primary research loop +- **extract_data**: Structured data extraction with JSON schema +- **html_to_markdown**: Clean page text extraction +- **create_file/update_file/read_file/list_files**: Persist and track findings across iterations + +## Quality Standards +- Prioritize reputable, recent sources over aggregators +- Distinguish facts from speculation ("could", "may" indicate speculation) +- Use moderately broad queries (under 5 words) +- Cite URLs, publication dates, authors for all findings +- Extract specific quotes, statistics, concrete data + +## File Output (MANDATORY - DO NOT SKIP) +**CRITICAL**: You MUST create files BEFORE returning your final answer. Files are the PRIMARY deliverable. + +Create descriptive file names based on your research topic. Use format: topic-slug_type.ext + +**REQUIRED FILES (Both must be created)**: + +1. **[topic]_research.md** (5000+ words) - REQUIRED + - Executive summary (2-3 paragraphs) + - Detailed findings organized by theme + - Source citations with quotes, statistics, analysis + - Data quality assessment and limitations + - Comprehensive conclusions + - Methodology section: search strategy, tools used, confidence level, suggested follow-up + +2. **[topic]_sources.json** - REQUIRED + - Structured metadata: url, title, author, publishDate, credibilityScore, keyFindings, quotes + - Include totalSources, searchStrategy, completedAt + +Example for "AI trends in 2025": ai-trends-2025_research.md, ai-trends-2025_sources.json + +**VERIFICATION CHECKLIST** (Complete before final answer): +- [ ] Created [topic]_research.md with 5000+ words of detailed findings +- [ ] Created [topic]_sources.json with structured source metadata +- [ ] Both files use descriptive, topic-based names +- [ ] Files contain all research data gathered + +## Final Answer Format +**ONLY AFTER FILES ARE CREATED**, return BRIEF confirmation (2-3 sentences): +"Research completed on [topic]. Created 'filename_research.md' with [N] sources in 'filename_sources.json'. Key finding: [one sentence summary]." + +**IMPORTANT**: +- DO NOT return final answer until BOTH files are created +- Use descriptive, unique file names based on topic +- All detailed content goes in FILES, not in final answer +- Your final answer should reference the actual file names you created`, tools: [ 'navigate_url', 'navigate_back', 'fetcher_tool', 'extract_data', - 'execute_code', 'node_ids_to_urls', - 'bookmark_store', - 'document_search', 'html_to_markdown', 'create_file', 'update_file', - 'delete_file', 'read_file', 'list_files', - 'update_todo', ], - maxIterations: 15, + maxIterations: 30, modelName: MODEL_SENTINELS.USE_MINI, temperature: 0, schema: { @@ -209,15 +142,142 @@ ${args.scope ? `The scope of research expected: ${args.scope}` : ''} `, }]; }, - handoffs: [ - { - targetAgentName: 'content_writer_agent', - trigger: 'llm_tool_call' - }, - { - targetAgentName: 'content_writer_agent', - trigger: 'max_iterations' + handoffs: [], + afterExecute: async (result: ConfigurableAgentResult, agentSession: AgentSession, _callCtx: CallCtx): Promise => { + logger.info('===== ResearchAgent afterExecute hook started ====='); + logger.info(`Agent session has ${agentSession.messages.length} messages`); + + try { + const fileManager = FileStorageManager.getInstance(); + let savedCount = 0; + let fetcherToolCount = 0; + + // Iterate through all messages in the session to find fetcher_tool results + for (const message of agentSession.messages) { + // Type narrow to ToolResultMessage + if (message.type !== 'tool_result') { + continue; + } + + const toolResult = message.content as { type: 'tool_result'; toolName: string; result?: any }; + + // Check if this is a fetcher_tool result + if (toolResult.toolName === 'fetcher_tool' && toolResult.result) { + fetcherToolCount++; + logger.info(`Found fetcher_tool result #${fetcherToolCount}`); + const fetcherResult = toolResult.result as FetcherToolResult; + + // Process each source in the fetcher result + if (fetcherResult.sources && Array.isArray(fetcherResult.sources)) { + for (const source of fetcherResult.sources) { + // Only save successful fetches with content + if (source.success && source.markdownContent && source.markdownContent.trim().length > 0) { + try { + // Create a sanitized filename from the URL + const filename = sanitizeUrlToFilename(source.url); + + // Create file content with metadata header + const fileContent = `# ${source.title || 'Untitled'} + +**Source URL:** ${source.url} +**Fetched:** ${new Date().toISOString()} + +--- + +${source.markdownContent}`; + + // Save to the research/ subdirectory + try { + await fileManager.createFile(`research-${filename}`, fileContent, 'text/markdown'); + logger.info(`✓ Created file: research-${filename} (${source.url})`); + } catch (createError: any) { + // If file exists, try to update it instead + if (createError.message?.includes('already exists')) { + await fileManager.updateFile(`research-${filename}`, fileContent); + logger.info(`✓ Updated file: research-${filename} (${source.url})`); + } else { + throw createError; + } + } + savedCount++; + } catch (error) { + logger.warn(`Failed to save fetcher result for ${source.url}:`, error); + } + } + } + } + } + } + + logger.info('===== ResearchAgent afterExecute summary ====='); + logger.info(`Found ${fetcherToolCount} fetcher_tool calls`); + logger.info(`Successfully saved ${savedCount} files`); + + if (savedCount > 0) { + logger.info(`✓ ResearchAgent afterExecute: Saved ${savedCount} fetched sources to files`); + } else { + if (fetcherToolCount === 0) { + logger.warn('⚠ No fetcher_tool results found in session messages'); + } else { + logger.warn('⚠ Found fetcher_tool results but no files were saved (check for errors above)'); + } + } + } catch (error: any) { + logger.error('❌ ResearchAgent afterExecute: Failed to save fetcher results:', error); + logger.error('Error details:', { message: error.message, stack: error.stack }); + // Don't throw - we don't want to break the agent execution } - ], + }, }; } + +/** + * Sanitize a URL to create a safe filename + */ +function sanitizeUrlToFilename(url: string): string { + try { + const urlObj = new URL(url); + + // Extract domain and path + let domain = urlObj.hostname.replace(/^www\./, ''); + let path = urlObj.pathname.replace(/^\//, '').replace(/\/$/, ''); + + // Create a base name from domain and path + let baseName = domain; + if (path) { + // Take first 2 path segments for readability + const pathParts = path.split('/').filter(p => p.length > 0); + if (pathParts.length > 0) { + baseName += '-' + pathParts.slice(0, 2).join('-'); + } + } + + // Remove special characters and limit length + baseName = baseName + .replace(/[^a-zA-Z0-9-_]/g, '-') + .replace(/-+/g, '-') + .substring(0, 80); + + // Add a short hash of the full URL to prevent collisions + const hash = simpleHash(url).substring(0, 8); + + return `${baseName}-${hash}.md`; + } catch (error) { + // Fallback for invalid URLs + const hash = simpleHash(url); + return `source-${hash}.md`; + } +} + +/** + * Simple hash function for generating short unique identifiers + */ +function simpleHash(str: string): string { + let hash = 0; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32-bit integer + } + return Math.abs(hash).toString(36); +} diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts index 4364c8363d..1fea4960ba 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts @@ -130,7 +130,6 @@ If you absolutely cannot find any reliable leads, return status "failed" with ga 'node_ids_to_urls', 'fetcher_tool', 'extract_data', - 'execute_code', 'scroll_page', 'action_agent', 'html_to_markdown', @@ -139,7 +138,6 @@ If you absolutely cannot find any reliable leads, return status "failed" with ga 'delete_file', 'read_file', 'list_files', - 'update_todo', ], maxIterations: 12, modelName: MODEL_SENTINELS.USE_MINI, diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts index 3805e0d72a..7accf1ef69 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts @@ -198,7 +198,6 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and 'navigate_back', 'action_agent', 'extract_data', - 'execute_code', 'node_ids_to_urls', 'direct_url_navigator_agent', 'scroll_page', @@ -215,7 +214,7 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and 'list_files', 'update_todo', ], - maxIterations: 15, + maxIterations: 30, temperature: 0.3, schema: { type: 'object', @@ -247,5 +246,6 @@ Execute this web task autonomously`, }]; }, handoffs: [], + includeSummaryInAnswer: true, // Enable summary for web automation tasks to provide execution insights }; } diff --git a/front_end/panels/ai_chat/core/AgentService.ts b/front_end/panels/ai_chat/core/AgentService.ts index f97e388cef..bb1eb0f8a4 100644 --- a/front_end/panels/ai_chat/core/AgentService.ts +++ b/front_end/panels/ai_chat/core/AgentService.ts @@ -59,6 +59,10 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ #graph?: CompiledGraph; #apiKey: string|null = null; #isInitialized = false; + /** + * Active async generator for current execution. Set when execution starts, + * cleared only after the final message is dispatched to listeners. + */ #runningGraphStatePromise?: AsyncGenerator; #abortController?: AbortController; #executionId?: string; @@ -102,6 +106,19 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ return AgentService.activeExecutions.get(executionId); } + /** + * Check if the agent is currently executing. + * Returns true from when execution starts until after the final message has been + * sent to listeners. This ensures UI can show "running" state while the response + * is being streamed to the user, and only transitions to "stopped" after completion. + * + * @returns true if agent execution is in progress (including final message delivery), + * false if agent is idle and ready for new input + */ + isRunning(): boolean { + return this.#runningGraphStatePromise !== undefined; + } + constructor() { super(); @@ -396,11 +413,16 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ throw new Error('Empty message. Please enter some text.'); } - // In AUTOMATED_MODE, ensure the graph is initialized even without API key - if (BUILD_CONFIG.AUTOMATED_MODE && !this.#graph) { + // Auto-initialize graph if it's not ready (handles race conditions automatically) + if (!this.#graph) { + logger.info('Graph not initialized, initializing now...'); const config = this.#configManager.getConfiguration(); - // Initialize with empty API key in AUTOMATED_MODE - will be overridden by request - await this.initialize('', config.mainModel, config.miniModel || '', config.nanoModel || ''); + await this.initialize( + this.#apiKey, + config.mainModel, + config.miniModel || '', + config.nanoModel || '' + ); } // Create a user message @@ -618,6 +640,7 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ this.#executionId = undefined; } this.#abortController = undefined; + // Clear running state - final message has already been sent to listeners at line 583 this.#runningGraphStatePromise = undefined; // Return the most recent message (could be final answer, tool call, or error) diff --git a/front_end/panels/ai_chat/core/Version.ts b/front_end/panels/ai_chat/core/Version.ts index fae99706c5..47c47daeb5 100644 --- a/front_end/panels/ai_chat/core/Version.ts +++ b/front_end/panels/ai_chat/core/Version.ts @@ -3,8 +3,8 @@ // found in the LICENSE file. export const VERSION_INFO = { - version: '0.3.4', - buildDate: '2025-09-19', + version: '0.5.0', + buildDate: '2025-10-05', channel: 'stable' } as const; diff --git a/front_end/panels/ai_chat/evaluation/test-cases/research-agent-tests.ts b/front_end/panels/ai_chat/evaluation/test-cases/research-agent-tests.ts index a6770c5724..5699b88457 100644 --- a/front_end/panels/ai_chat/evaluation/test-cases/research-agent-tests.ts +++ b/front_end/panels/ai_chat/evaluation/test-cases/research-agent-tests.ts @@ -39,7 +39,7 @@ export const basicResearchTest: TestCase = { 'At least 3-5 different sources were consulted', 'Information is factually accurate and up-to-date', 'Research demonstrates understanding of the topic evolution', - 'Handoff to content_writer_agent occurred with comprehensive data' + 'Final answer provides comprehensive research summary with proper citations' ], temperature: 0 } diff --git a/front_end/panels/ai_chat/tools/Tools.ts b/front_end/panels/ai_chat/tools/Tools.ts index d69fda19de..948f5b5f42 100644 --- a/front_end/panels/ai_chat/tools/Tools.ts +++ b/front_end/panels/ai_chat/tools/Tools.ts @@ -2143,7 +2143,7 @@ export class PerformActionTool implements Tool<{ method: string, nodeId: number const response = await llmClient.call({ provider, model, - systemPrompt: 'You are a DOM verification assistant. Analyze page content to determine if actions succeeded.', + systemPrompt: 'You are a DOM verification assistant. Analyze page content and tree diff data to determine if actions succeeded.', messages: [ { role: 'user', @@ -2157,16 +2157,29 @@ ACTION DETAILS: - Reasoning: ${reasoning} ${verificationMessage ? `- Verification status: ${verificationMessage}` : ''} +OBJECTIVE PAGE CHANGE EVIDENCE: +${treeDiff ? `- Tree Changes Detected: ${treeDiff.hasChanges ? 'YES' : 'NO'} +- Change Summary: ${treeDiff.summary} +- Added Elements: ${treeDiff.added.length} (first few: ${JSON.stringify(treeDiff.added.slice(0, 25))}) +- Removed Elements: ${treeDiff.removed.length} (first few: ${JSON.stringify(treeDiff.removed.slice(0, 25))}) +- Modified Elements: ${treeDiff.modified.length} (first few: ${JSON.stringify(treeDiff.modified.slice(0, 25))})` : 'Tree diff not available'} + CURRENT PAGE CONTENT (after action): ${afterContent} -Based on the page content and action details, please describe: -- What changes occurred in the page content -- Whether the action appears to have succeeded -- Any error messages or unexpected behavior -- Your overall assessment of the action's success +IMPORTANT VERIFICATION RULES: +1. If Tree Changes Detected = YES with significant modifications (e.g., 100+ modified elements, root node changed), the action was SUCCESSFUL +2. Trust the objective pageChange data over subjective DOM analysis +3. For navigation actions: Changed root node IDs indicate successful page navigation +4. For click actions: Many DOM modifications suggest the action triggered UI changes + +Based on the objective evidence and page content, please describe: +- What changes occurred according to the tree diff +- Whether the OBJECTIVE evidence shows the action succeeded +- Any error messages or unexpected behavior in the page content +- Your assessment based primarily on the tree change metrics -Provide a clear, concise response about what happened.` +Provide a clear, concise response that prioritizes objective metrics.` } ], temperature: 0 @@ -2211,7 +2224,7 @@ Provide a clear, concise response about what happened.` const response = await llmClient.call({ provider, model, - systemPrompt: 'You are a visual verification assistant. Compare before/after screenshots and page context to determine if actions succeeded.', + systemPrompt: 'You are a visual verification assistant. Compare before/after screenshots and tree diff data to determine if actions succeeded. Always prioritize objective tree change metrics over subjective visual analysis.', messages: [ { role: 'user', @@ -2227,19 +2240,31 @@ ACTION DETAILS: - Arguments: ${JSON.stringify(actionArgsArray)} - Reasoning: ${reasoning} +OBJECTIVE PAGE CHANGE EVIDENCE: +${treeDiff ? `- Tree Changes Detected: ${treeDiff.hasChanges ? 'YES' : 'NO'} +- Change Summary: ${treeDiff.summary} +- Added Elements: ${treeDiff.added.length} (first few: ${JSON.stringify(treeDiff.added.slice(0, 3))}) +- Removed Elements: ${treeDiff.removed.length} (first few: ${JSON.stringify(treeDiff.removed.slice(0, 3))}) +- Modified Elements: ${treeDiff.modified.length} (first few: ${JSON.stringify(treeDiff.modified.slice(0, 3))})` : 'Tree diff not available'} + CURRENT PAGE CONTENT (visible elements): ${currentPageContent} -Please compare the before and after screenshots and describe: -- What visual changes occurred between the two images -- Whether these changes indicate the action was successful -- Any error messages, validation warnings, or unexpected behavior you notice -- Loading states, navigation changes, or form submissions that occurred -- Your overall assessment of whether the action achieved its intended result +IMPORTANT VERIFICATION RULES: +1. If Tree Changes Detected = YES with significant modifications (e.g., 100+ modified elements), the action was SUCCESSFUL +2. Trust the objective tree change metrics over subjective visual interpretation +3. For navigation: Changed root node IDs indicate successful page navigation even if screenshots look similar +4. Visual similarities don't mean failure - focus on the objective tree diff data + +Please analyze and describe: +- What the objective tree diff shows (this is the PRIMARY evidence) +- What visual changes you observe in the screenshots (secondary evidence) +- Your assessment based PRIMARILY on the tree change metrics +- Whether the action succeeded based on objective evidence The first image shows the page BEFORE the action, the second image shows the page AFTER the action. -Provide a clear, descriptive response about what happened and whether the action appears to have succeeded.` +Provide a clear response that prioritizes objective tree metrics over visual interpretation.` }, { type: 'image_url', @@ -2285,7 +2310,7 @@ Provide a clear, descriptive response about what happened and whether the action const response = await llmClient.call({ provider, model, - systemPrompt: 'You are a visual verification assistant. Analyze screenshots and page context to determine if actions succeeded.', + systemPrompt: 'You are a visual verification assistant. Analyze screenshots and tree diff data to determine if actions succeeded. Always prioritize objective tree change metrics over subjective visual analysis.', messages: [ { role: 'user', @@ -2301,19 +2326,31 @@ ACTION DETAILS: - Arguments: ${JSON.stringify(actionArgsArray)} - Reasoning: ${reasoning} +OBJECTIVE PAGE CHANGE EVIDENCE: +${treeDiff ? `- Tree Changes Detected: ${treeDiff.hasChanges ? 'YES' : 'NO'} +- Change Summary: ${treeDiff.summary} +- Added Elements: ${treeDiff.added.length} (first few: ${JSON.stringify(treeDiff.added.slice(0, 3))}) +- Removed Elements: ${treeDiff.removed.length} (first few: ${JSON.stringify(treeDiff.removed.slice(0, 3))}) +- Modified Elements: ${treeDiff.modified.length} (first few: ${JSON.stringify(treeDiff.modified.slice(0, 3))})` : 'Tree diff not available'} + CURRENT PAGE CONTENT (visible elements): ${currentPageContent} -Please examine the screenshot and page content to describe: -- What the current state of the page shows -- Any visible indicators that suggest the action succeeded or failed -- Error messages, validation warnings, or unexpected behavior you notice -- Loading states, navigation changes, or form submissions that may have occurred -- Your assessment of whether the action achieved its intended result +IMPORTANT VERIFICATION RULES: +1. If Tree Changes Detected = YES with significant modifications, the action was SUCCESSFUL +2. Trust the objective tree change metrics as the PRIMARY indicator +3. The screenshot provides additional context but is SECONDARY to tree diff data +4. For navigation: Changed root node IDs indicate successful page navigation + +Please examine and describe: +- What the objective tree diff shows (PRIMARY evidence) +- What the screenshot reveals (secondary context) +- Your assessment based PRIMARILY on the tree change metrics +- Whether the action succeeded according to objective evidence -Note: Only the after-action screenshot is available for analysis. +Note: Only the after-action screenshot is available for visual analysis. -Provide a clear, descriptive response about what you observe and whether the action appears to have succeeded.` +Provide a clear response that prioritizes objective tree metrics.` }, { type: 'image_url', diff --git a/front_end/panels/ai_chat/ui/AIChatPanel.ts b/front_end/panels/ai_chat/ui/AIChatPanel.ts index 7e4f6766ba..dc399d39af 100644 --- a/front_end/panels/ai_chat/ui/AIChatPanel.ts +++ b/front_end/panels/ai_chat/ui/AIChatPanel.ts @@ -6,10 +6,7 @@ import type * as Common from '../../../core/common/common.js'; import * as Host from '../../../core/host/host.js'; import * as i18n from '../../../core/i18n/i18n.js'; import * as SDK from '../../../core/sdk/sdk.js'; -import * as Buttons from '../../../ui/components/buttons/buttons.js'; import * as UI from '../../../ui/legacy/legacy.js'; -import * as Lit from '../../../ui/lit/lit.js'; -import * as VisualLogging from '../../../ui/visual_logging/visual_logging.js'; import {AgentService, Events as AgentEvents} from '../core/AgentService.js'; import { LLMClient } from '../LLM/LLMClient.js'; import { LLMConfigurationManager } from '../core/LLMConfigurationManager.js'; @@ -84,7 +81,7 @@ import chatViewStyles from './chatView.css.js'; import { ChatView } from './ChatView.js'; import { type ChatMessage, ChatMessageEntity, type ImageInputData, type ModelChatMessage, State as ChatViewState } from '../models/ChatTypes.js'; import { HelpDialog } from './HelpDialog.js'; -import { SettingsDialog, isVectorDBEnabled } from './SettingsDialog.js'; +import { SettingsDialog } from './SettingsDialog.js'; import { EvaluationDialog } from './EvaluationDialog.js'; import { MODEL_PLACEHOLDERS } from '../core/Constants.js'; import * as Snackbars from '../../../ui/components/snackbars/snackbars.js'; @@ -94,7 +91,6 @@ import { getMCPConfig } from '../mcp/MCPConfig.js'; import { onMCPConfigChange } from '../mcp/MCPConfig.js'; import { MCPConnectorsCatalogDialog } from './mcp/MCPConnectorsCatalogDialog.js'; -const {html} = Lit; // Model type definition export interface ModelOption { @@ -223,106 +219,6 @@ const UIStrings = { const str_ = i18n.i18n.registerUIStrings('panels/ai_chat/ui/AIChatPanel.ts', UIStrings); const i18nString = i18n.i18n.getLocalizedString.bind(undefined, str_); -interface ToolbarViewInput { - onNewChatClick: () => void; - onHistoryClick: (event: MouseEvent) => void; - onDeleteClick: () => void; - onHelpClick: () => void; - onMCPConnectorsClick: () => void; - onSettingsClick: () => void; - onEvaluationTestClick: () => void; - onBookmarkClick: () => void; - isDeleteHistoryButtonVisible: boolean; - isCenteredView: boolean; - isVectorDBEnabled: boolean; -} - -function toolbarView(input: ToolbarViewInput): Lit.LitTemplate { - // clang-format off - // Add history button when history feature is implemented - // - //
- return html` - - `; - // clang-format on -} - let aiChatPanelInstance: AIChatPanel|null = null; // For testing purposes - allows resetting the singleton instance @@ -753,7 +649,15 @@ export class AIChatPanel extends UI.Panel.Panel { #miniModel = ''; // Mini model selection #nanoModel = ''; // Nano model selection #canSendMessages = false; // Add flag to track if we can send messages (has required credentials) - #settingsButton: HTMLElement | null = null; // Reference to the settings button + + // Native UI.Toolbar instances and buttons + #leftToolbar!: UI.Toolbar.Toolbar; + #rightToolbar!: UI.Toolbar.Toolbar; + #newChatButton!: UI.Toolbar.ToolbarButton; + #deleteButton!: UI.Toolbar.ToolbarButton; + #bookmarkButton!: UI.Toolbar.ToolbarButton; + #settingsMenuButton!: UI.Toolbar.ToolbarMenuButton; + #closeButton!: UI.Toolbar.ToolbarButton; #liteLLMApiKey: string | null = null; // LiteLLM API key #liteLLMEndpoint: string | null = null; // LiteLLM endpoint #apiKey: string | null = null; // Regular API key @@ -811,8 +715,78 @@ export class AIChatPanel extends UI.Panel.Panel { // Create container for the toolbar this.#toolbarContainer = document.createElement('div'); + this.#toolbarContainer.classList.add('toolbar-container'); + this.#toolbarContainer.setAttribute('role', 'toolbar'); + this.#toolbarContainer.style.cssText = 'display: flex; justify-content: space-between; width: 100%; padding: 0 4px; box-sizing: border-box; margin: 0 0 10px 0;'; this.contentElement.appendChild(this.#toolbarContainer); + // Create left toolbar using DOM method (not constructor) + this.#leftToolbar = this.#toolbarContainer.createChild('devtools-toolbar', 'ai-chat-left-toolbar') as UI.Toolbar.Toolbar; + + // Create right toolbar using DOM method (not constructor) + this.#rightToolbar = this.#toolbarContainer.createChild('devtools-toolbar', 'ai-chat-right-toolbar') as UI.Toolbar.Toolbar; + this.#rightToolbar.style.cssText = 'overflow: visible;'; + + // Create toolbar buttons ONCE + this.#newChatButton = new UI.Toolbar.ToolbarButton( + i18nString(UIStrings.newChat), + 'plus', + undefined, + 'ai-chat.new-chat' + ); + this.#newChatButton.addEventListener( + UI.Toolbar.ToolbarButton.Events.CLICK, + this.#onNewChatClick, + this + ); + + this.#deleteButton = new UI.Toolbar.ToolbarButton( + i18nString(UIStrings.deleteChat), + 'bin', + undefined, + 'ai-chat.delete' + ); + this.#deleteButton.addEventListener( + UI.Toolbar.ToolbarButton.Events.CLICK, + this.#onDeleteClick, + this + ); + + this.#bookmarkButton = new UI.Toolbar.ToolbarButton( + i18nString(UIStrings.bookmarkPage), + 'download', + undefined, + 'ai-chat.bookmark-page' + ); + this.#bookmarkButton.addEventListener( + UI.Toolbar.ToolbarButton.Events.CLICK, + this.#onBookmarkClick, + this + ); + + this.#settingsMenuButton = this.#createSettingsMenuButton(); + + this.#closeButton = new UI.Toolbar.ToolbarButton( + 'Close Chat Window', + 'cross', + undefined, + 'ai-chat.close-devtools' + ); + this.#closeButton.addEventListener( + UI.Toolbar.ToolbarButton.Events.CLICK, + () => Host.InspectorFrontendHost.InspectorFrontendHostInstance.closeWindow(), + this + ); + + // Add buttons to toolbars ONCE (order matters for right toolbar) + this.#leftToolbar.appendToolbarItem(this.#newChatButton); + + this.#rightToolbar.appendSeparator(); + this.#rightToolbar.appendToolbarItem(this.#deleteButton); + this.#rightToolbar.appendToolbarItem(this.#bookmarkButton); + this.#rightToolbar.appendToolbarItem(this.#settingsMenuButton); + this.#rightToolbar.appendToolbarItem(this.#closeButton); + // Create container for the chat view this.#chatViewContainer = document.createElement('div'); this.#chatViewContainer.style.flex = '1'; @@ -824,7 +798,7 @@ export class AIChatPanel extends UI.Panel.Panel { this.#chatView.style.flexGrow = '1'; this.#chatView.style.overflow = 'auto'; this.#chatViewContainer.appendChild(this.#chatView); - + // Add event listener for manual setup requests from ChatView this.#chatView.addEventListener('manual-setup-requested', this.#handleManualSetupRequest.bind(this)); } @@ -1680,14 +1654,11 @@ export class AIChatPanel extends UI.Panel.Panel { * Update the settings button highlight based on credentials state */ #updateSettingsButtonHighlight(): void { - if (!this.#canSendMessages && !this.#settingsButton) { - // Try to find the settings button after rendering - this.#settingsButton = this.#toolbarContainer.querySelector('.ai-chat-right-toolbar devtools-button[title="Settings"]'); - + if (!this.#canSendMessages) { // Add pulsating animation to draw attention to settings - if (this.#settingsButton) { + if (this.#settingsMenuButton && this.#settingsMenuButton.element) { // Add CSS animation to make it glow/pulse - this.#settingsButton.classList.add('settings-highlight'); + this.#settingsMenuButton.element.classList.add('settings-highlight'); // Add the style to the document head if it doesn't exist yet const styleId = 'settings-highlight-style'; @@ -1699,7 +1670,7 @@ export class AIChatPanel extends UI.Panel.Panel { animation: pulse 2s infinite; position: relative; } - + @keyframes pulse { 0% { box-shadow: 0 0 0 0 rgba(var(--color-primary-rgb), 0.7); @@ -1715,10 +1686,9 @@ export class AIChatPanel extends UI.Panel.Panel { document.head.appendChild(style); } } - } else if (this.#canSendMessages && this.#settingsButton) { + } else if (this.#canSendMessages && this.#settingsMenuButton && this.#settingsMenuButton.element) { // Remove the highlight if we now have an API key - this.#settingsButton.classList.remove('settings-highlight'); - this.#settingsButton = null; + this.#settingsMenuButton.element.classList.remove('settings-highlight'); } } @@ -2040,25 +2010,56 @@ export class AIChatPanel extends UI.Panel.Panel { this.#updateChatViewState(); } + /** + * Creates the settings menu button with dropdown items + */ + #createSettingsMenuButton(): UI.Toolbar.ToolbarMenuButton { + const menuButton = new UI.Toolbar.ToolbarMenuButton( + (contextMenu) => { + // Add menu items + contextMenu.defaultSection().appendItem( + 'Settings', + () => this.#onSettingsClick(), + {jslogContext: 'settings'} + ); + contextMenu.defaultSection().appendItem( + 'Help', + () => this.#onHelpClick(), + {jslogContext: 'help'} + ); + contextMenu.defaultSection().appendItem( + 'Evaluations', + () => this.#onEvaluationTestClick(), + {jslogContext: 'evaluations'} + ); + contextMenu.defaultSection().appendItem( + 'Connectors', + () => this.#onMCPConnectorsClick(), + {jslogContext: 'connectors'} + ); + }, + true, // isIconDropdown + true, // useSoftMenu + 'ai-chat.settings-menu', // jslogContext + 'dots-vertical' // iconName + ); + + menuButton.setTitle('Settings Menu'); + return menuButton; + } + /** * Updates the toolbar UI */ #updateToolbar(): void { - const isCenteredView = this.#chatView?.isCenteredView ?? false; - - Lit.render(toolbarView({ - onNewChatClick: this.#onNewChatClick.bind(this), - onHistoryClick: this.#onHistoryClick.bind(this), - onDeleteClick: this.#onDeleteClick.bind(this), - onHelpClick: this.#onHelpClick.bind(this), - onMCPConnectorsClick: this.#onMCPConnectorsClick.bind(this), - onSettingsClick: this.#onSettingsClick.bind(this), - onEvaluationTestClick: this.#onEvaluationTestClick.bind(this), - onBookmarkClick: this.#onBookmarkClick.bind(this), - isDeleteHistoryButtonVisible: this.#messages.length > 1, - isCenteredView, - isVectorDBEnabled: isVectorDBEnabled(), - }), this.#toolbarContainer, { host: this }); + // Update button visibility based on current state + // Delete button is only visible when there are messages to delete + this.#deleteButton.setVisible(this.#messages.length > 1); + + // Bookmark button visibility can be controlled here if needed + // this.#bookmarkButton.setVisible(someCondition); + + // All other buttons (New Chat, Settings Menu, Close) are always visible } /** @@ -2094,6 +2095,8 @@ export class AIChatPanel extends UI.Panel.Panel { return !hasCredentials; })(), onOAuthLogin: this.#handleOAuthLogin.bind(this), + // Add example prompt model switching + onExamplePromptModelSwitch: this.#handleExamplePromptModelSwitch.bind(this), }; } catch (error) { logger.error('Error updating ChatView state:', error); @@ -2115,6 +2118,93 @@ export class AIChatPanel extends UI.Panel.Panel { } } + /** + * Handles model switching when example prompts with model preferences are selected + * Applies the provided model preferences to main, mini, and nano models + */ + #handleExamplePromptModelSwitch(modelPreferences: { main?: string; mini?: string; nano?: string }): void { + logger.info('=== HANDLE EXAMPLE PROMPT MODEL SWITCH ==='); + logger.info('Model preferences received:', modelPreferences); + logger.info('Current provider:', localStorage.getItem(PROVIDER_SELECTION_KEY)); + logger.info('Current MODEL_OPTIONS count:', MODEL_OPTIONS.length); + logger.info('Sample available models:', MODEL_OPTIONS.slice(0, 10).map(m => `${m.value} (${m.type})`)); + + let modelsApplied = false; + + // Apply main model + if (modelPreferences.main) { + const exists = MODEL_OPTIONS.some(opt => opt.value === modelPreferences.main); + logger.info(`Main model "${modelPreferences.main}" exists in MODEL_OPTIONS:`, exists); + + if (exists) { + logger.info(`Previous main model: "${this.#selectedModel}"`); + this.#selectedModel = modelPreferences.main; + localStorage.setItem(MODEL_SELECTION_KEY, this.#selectedModel); + modelsApplied = true; + logger.info('✅ Applied main model:', modelPreferences.main); + } else { + logger.warn('❌ Main model not found in MODEL_OPTIONS:', modelPreferences.main); + logger.warn('First 10 available models:', MODEL_OPTIONS.slice(0, 10).map(m => m.value)); + logger.warn('Hint: Make sure OpenRouter models are fetched before clicking example prompts'); + } + } + + // Apply mini model + if (modelPreferences.mini) { + const exists = MODEL_OPTIONS.some(opt => opt.value === modelPreferences.mini); + logger.info(`Mini model "${modelPreferences.mini}" exists in MODEL_OPTIONS:`, exists); + + if (exists) { + logger.info(`Previous mini model: "${this.#miniModel}"`); + this.#miniModel = modelPreferences.mini; + localStorage.setItem(MINI_MODEL_STORAGE_KEY, this.#miniModel); + modelsApplied = true; + logger.info('✅ Applied mini model:', modelPreferences.mini); + } else { + logger.warn('❌ Mini model not found in MODEL_OPTIONS:', modelPreferences.mini); + } + } + + // Apply nano model + if (modelPreferences.nano) { + const exists = MODEL_OPTIONS.some(opt => opt.value === modelPreferences.nano); + logger.info(`Nano model "${modelPreferences.nano}" exists in MODEL_OPTIONS:`, exists); + + if (exists) { + logger.info(`Previous nano model: "${this.#nanoModel}"`); + this.#nanoModel = modelPreferences.nano; + localStorage.setItem(NANO_MODEL_STORAGE_KEY, this.#nanoModel); + modelsApplied = true; + logger.info('✅ Applied nano model:', modelPreferences.nano); + } else { + logger.warn('❌ Nano model not found in MODEL_OPTIONS:', modelPreferences.nano); + } + } + + // Update UI and reinitialize agent if any models were applied + if (modelsApplied) { + logger.info('✅ Model switch complete!'); + logger.info('New model selection:', { + main: this.#selectedModel, + mini: this.#miniModel, + nano: this.#nanoModel + }); + logger.info('Updating UI and reinitializing agent service with new models...'); + + this.performUpdate(); + this.#initializeAgentService(); // ✅ Immediately reinitialize with new models + + logger.info('✅ Agent service reinitialization triggered'); + } else { + logger.error('❌ No models were applied!'); + logger.error('Possible reasons:'); + logger.error('1. Models not loaded in MODEL_OPTIONS (try fetching OpenRouter models in Settings)'); + logger.error('2. Provider not set to openrouter'); + logger.error('3. Model IDs in example prompt config don\'t match actual OpenRouter model IDs'); + logger.error('Current MODEL_OPTIONS:', MODEL_OPTIONS.map(m => m.value)); + } + } + #onNewChatClick(): void { this.#agentService.clearConversation(); this.#messages = this.#agentService.getMessages(); @@ -2128,15 +2218,6 @@ export class AIChatPanel extends UI.Panel.Panel { UI.ARIAUtils.LiveAnnouncer.alert(i18nString(UIStrings.newChatCreated)); } - /** - * Handles history button click - * @param event Mouse event - */ - #onHistoryClick(_event: MouseEvent): void { - // Not yet implemented - logger.info('History feature not yet implemented'); - } - #onDeleteClick(): void { this.#onNewChatClick(); UI.ARIAUtils.LiveAnnouncer.alert(i18nString(UIStrings.chatDeleted)); diff --git a/front_end/panels/ai_chat/ui/ChatView.ts b/front_end/panels/ai_chat/ui/ChatView.ts index 4794f18d4e..53eed08f42 100644 --- a/front_end/panels/ai_chat/ui/ChatView.ts +++ b/front_end/panels/ai_chat/ui/ChatView.ts @@ -30,6 +30,8 @@ import './input/InputBar.js'; import './model_selector/ModelSelector.js'; import { combineMessages } from './message/MessageCombiner.js'; import { StructuredResponseController } from './message/StructuredResponseController.js'; +import './TodoListDisplay.js'; +import './FileListDisplay.js'; // Shared chat types import type { ChatMessage, ModelChatMessage, ToolResultMessage, AgentSessionMessage, ImageInputData } from '../models/ChatTypes.js'; @@ -44,6 +46,105 @@ const {customElement} = Decorators; // Markdown rendering moved to ui/markdown/MarkdownRenderers.ts +// Example prompt configuration with optional model preferences +export interface ExamplePromptConfig { + displayText: string; // Text shown in UI button + promptText?: string; // Actual prompt sent to agent (if omitted, uses displayText) + agentType?: string; + modelPreferences?: { + [provider: string]: { + main?: string; + mini?: string; + nano?: string; + }; + }; +} + +// Centralized example prompts configuration +export const EXAMPLE_PROMPTS = { + DEEP_RESEARCH_AI_AGENTS: { + displayText: '🔬 Deep research AI agents', + promptText: 'Deep research latest breakthroughs in AI agents and provide a comprehensive analysis with citations', + agentType: BaseOrchestratorAgent.BaseOrchestratorAgentType.DEEP_RESEARCH, + modelPreferences: { + openrouter: { + main: 'z-ai/glm-4.6:exacto', + mini: 'x-ai/grok-4-fast', + nano: 'google/gemini-2.5-flash-lite' + } + } + }, + STAR_BROWSER_OPERATOR_REPO: { + displayText: '⭐ Star on GitHub', + promptText: 'Go to github.com/BrowserOperator/browser-operator-core to star the repo. If user is not logged in, ask them to log in first.', + // No agentType - uses default behavior + modelPreferences: { + openrouter: { + main: 'anthropic/claude-sonnet-4.5', + mini: 'google/gemini-2.5-flash', + nano: 'google/gemini-2.5-flash-lite' + } + } + }, + FIND_CONTENT_WRITERS: { + displayText: 'Find content writers', + promptText: 'Find content writers in Seattle, WA with their contact information and portfolio links', + agentType: BaseOrchestratorAgent.BaseOrchestratorAgentType.SEARCH, + modelPreferences: { + openrouter: { + main: 'z-ai/glm-4.6:exacto', + mini: 'x-ai/grok-4-fast', + nano: 'google/gemini-2.5-flash-lite' + } + } + }, + APPLE_STOCKS_ANALYSIS: { + displayText: '📊 Apple stocks analysis', + promptText: 'Provide me detailed analysis of Apple stocks with current price, trends, and expert opinions', + agentType: BaseOrchestratorAgent.BaseOrchestratorAgentType.DEEP_RESEARCH, + modelPreferences: { + openrouter: { + main: 'z-ai/glm-4.6:exacto', + mini: 'x-ai/grok-4-fast', + nano: 'google/gemini-2.5-flash-lite' + } + } + }, + IPHONE_REVIEWS: { + displayText: '📱 iPhone 17 reviews', + promptText: 'What are the reviews of iPhone 17? Include specs, pricing, and expert opinions', + agentType: BaseOrchestratorAgent.BaseOrchestratorAgentType.DEEP_RESEARCH, + modelPreferences: { + openrouter: { + main: 'z-ai/glm-4.6:exacto', + mini: 'google/gemini-2.5-flash', + nano: 'google/gemini-2.5-flash-lite' + } + } + }, + SUMMARIZE_NEWS: { + displayText: '📰 Summarize today\'s news', + promptText: 'Summarize today\'s top news stories across different categories', + agentType: BaseOrchestratorAgent.BaseOrchestratorAgentType.DEEP_RESEARCH, + modelPreferences: { + openrouter: { + main: 'z-ai/glm-4.6:exacto', + mini: 'google/gemini-2.5-flash', + nano: 'google/gemini-2.5-flash-lite' + } + } + }, + SUMMARIZE_PAGE: { + displayText: 'Summarize this page' + // No agentType - uses default, no model preferences + }, + EXTRACT_LINKS: { + displayText: 'Extract all links', + promptText: 'Extract all links and titles from this page in a structured format' + // No agentType - uses default, no model preferences + } +} as const; + export interface Props { messages: ChatMessage[]; onSendMessage: (text: string, imageInput?: ImageInputData) => void; @@ -69,6 +170,8 @@ export interface Props { onOAuthLogin?: () => void; // Add current provider for model selector behavior currentProvider?: string; + // Callback for switching models when specific example prompts are selected + onExamplePromptModelSwitch?: (modelPreferences: { main?: string; mini?: string; nano?: string }) => void; } @customElement('devtools-chat-view') @@ -85,6 +188,7 @@ export class ChatView extends HTMLElement { #onSendMessage?: (text: string, imageInput?: ImageInputData) => void; #onImageInputClear?: () => void; #onPromptSelected?: (promptType: string | null) => void; + #onExamplePromptModelSwitch?: (modelPreferences: { main?: string; mini?: string; nano?: string }) => void; // input is handled by #markdownRenderer = new MarkdownRenderer(); #isFirstMessageView = true; // Track if we're in the centered first-message view @@ -281,6 +385,7 @@ export class ChatView extends HTMLElement { this.#onSendMessage = data.onSendMessage; this.#onImageInputClear = data.onImageInputClear; this.#onPromptSelected = data.onPromptSelected; + this.#onExamplePromptModelSwitch = data.onExamplePromptModelSwitch; // Add model selection properties this.#modelOptions = data.modelOptions; this.#selectedModel = data.selectedModel; @@ -809,8 +914,10 @@ export class ChatView extends HTMLElement { onRetry: () => this.dispatchEvent(new CustomEvent('retry', { bubbles: true })) }) : Lit.nothing} + + ${this.#renderInputBar(false)} - + `, this.#shadow, {host: this}); } @@ -833,35 +940,71 @@ export class ChatView extends HTMLElement {
Try one of these
- ${examples.map(ex => html` - - `)} + ${examples.map(ex => { + const tooltipText = ex.promptText || ex.displayText; + return html` + + `; + })}
`; } // On suggestion click, fill input field and focus it - #handleExampleClick(text: string, agentType?: string): void { + #handleExampleClick(promptConfig: ExamplePromptConfig): void { + // Get the actual prompt text to send (falls back to displayText if promptText not provided) + const promptText = promptConfig.promptText || promptConfig.displayText; + + logger.info('=== EXAMPLE PROMPT CLICKED ==='); + logger.info('Display text:', promptConfig.displayText); + logger.info('Prompt text:', promptText); + logger.info('Agent type:', promptConfig.agentType || 'default (none)'); + const bar = this.#shadow.querySelector('ai-input-bar') as (HTMLElement & { setInputValue?: (t: string) => void }) | null; if (bar && typeof (bar as any).setInputValue === 'function') { - (bar as any).setInputValue(text); + (bar as any).setInputValue(promptText); } else { // Fallback: try to set directly on ai-chat-input if present const input = bar?.querySelector('ai-chat-input') as (HTMLElement & { value?: string, focusInput?: () => void }) | null; if (input) { - (input as any).value = text; + (input as any).value = promptText; if (typeof (input as any).focusInput === 'function') { (input as any).focusInput(); } // Bubble change up so parent state updates - bar?.dispatchEvent(new CustomEvent('inputchange', { bubbles: true, detail: { value: text }})); + bar?.dispatchEvent(new CustomEvent('inputchange', { bubbles: true, detail: { value: promptText }})); } } // Auto-select agent type if provided - if (agentType) { - this.#autoSelectAgent(agentType); + if (promptConfig.agentType) { + this.#autoSelectAgent(promptConfig.agentType); + } + + // Trigger model switch if this prompt has model preferences for the current provider + logger.info('=== MODEL SWITCH CHECK ==='); + logger.info('Has modelPreferences?', !!promptConfig.modelPreferences); + logger.info('Current provider:', this.#currentProvider); + logger.info('Has callback?', !!this.#onExamplePromptModelSwitch); + + if (promptConfig.modelPreferences && this.#currentProvider && this.#onExamplePromptModelSwitch) { + const providerPreferences = promptConfig.modelPreferences[this.#currentProvider]; + logger.info(`Provider preferences for "${this.#currentProvider}":`, providerPreferences); + + if (providerPreferences) { + logger.info('✅ Calling onExamplePromptModelSwitch with:', providerPreferences); + this.#onExamplePromptModelSwitch(providerPreferences); + } else { + logger.warn(`❌ No model preferences found for provider: ${this.#currentProvider}`); + logger.info('Available providers in config:', Object.keys(promptConfig.modelPreferences)); + } + } else { + logger.warn('❌ Model switch conditions not met:', { + hasModelPreferences: !!promptConfig.modelPreferences, + currentProvider: this.#currentProvider, + hasCallback: !!this.#onExamplePromptModelSwitch + }); } } @@ -891,10 +1034,10 @@ export class ChatView extends HTMLElement { } // Build example suggestions (generic + page-specific if URL is present) - #getExampleSuggestions(): Array<{ text: string; agentType?: string }> { - const generic: Array<{ text: string; agentType?: string }> = [ - { text: 'Summarize this page' }, - { text: 'Extract all links and titles from this page' }, + #getExampleSuggestions(): ExamplePromptConfig[] { + const generic: ExamplePromptConfig[] = [ + EXAMPLE_PROMPTS.SUMMARIZE_PAGE, + EXAMPLE_PROMPTS.EXTRACT_LINKS, ]; const url = this.#getCurrentPageURL(); @@ -911,15 +1054,13 @@ export class ChatView extends HTMLElement { if (isChromeInternalPage) { // Provide mixed examples for all Chrome internal pages - const researchAgent = BaseOrchestratorAgent.BaseOrchestratorAgentType.DEEP_RESEARCH; - const searchAgent = BaseOrchestratorAgent.BaseOrchestratorAgentType.SEARCH; - const searchExamples: Array<{ text: string; agentType?: string }> = [ - { text: 'Deep research latest breakthroughs in AI agents', agentType: researchAgent }, - { text: 'Find content writers in Seattle, WA', agentType: searchAgent }, - { text: 'Provide me analysis of Apple stocks?', agentType: researchAgent }, - { text: 'Summarize today\'s news', agentType: researchAgent }, + return [ + EXAMPLE_PROMPTS.DEEP_RESEARCH_AI_AGENTS, + EXAMPLE_PROMPTS.FIND_CONTENT_WRITERS, + EXAMPLE_PROMPTS.APPLE_STOCKS_ANALYSIS, + EXAMPLE_PROMPTS.SUMMARIZE_NEWS, + EXAMPLE_PROMPTS.STAR_BROWSER_OPERATOR_REPO, ]; - return searchExamples; } // Detect common search engines @@ -927,23 +1068,21 @@ export class ChatView extends HTMLElement { if (isSearchSite) { // Provide deep-research oriented examples and pre-select the deep research agent on click - const researchAgent = BaseOrchestratorAgent.BaseOrchestratorAgentType.DEEP_RESEARCH; - const searchExamples: Array<{ text: string; agentType?: string }> = [ - { text: 'Deep research latest breakthroughs in AI agents', agentType: researchAgent }, - { text: 'What are the reviews of iPhone 17?', agentType: researchAgent }, - { text: 'Provide me analysis of Apple stocks?', agentType: researchAgent }, - { text: 'Summarize today\'s news', agentType: researchAgent }, + return [ + EXAMPLE_PROMPTS.DEEP_RESEARCH_AI_AGENTS, + EXAMPLE_PROMPTS.IPHONE_REVIEWS, + EXAMPLE_PROMPTS.APPLE_STOCKS_ANALYSIS, + EXAMPLE_PROMPTS.SUMMARIZE_NEWS, ]; - return searchExamples; } - const specific: Array<{ text: string; agentType?: string }> = [ - { text: `What do you think about ${hostname ? hostname + ' ' : ''}page?` }, + const specific: ExamplePromptConfig[] = [ + { displayText: `What do you think about ${hostname ? hostname + ' ' : ''}page?` }, ]; - // Merge, de-duplicate by text, cap to concise set - const map = new Map(); - [...specific, ...generic].forEach(item => { if (!map.has(item.text)) map.set(item.text, item); }); + // Merge, de-duplicate by displayText, cap to concise set + const map = new Map(); + [...specific, ...generic].forEach(item => { if (!map.has(item.displayText)) map.set(item.displayText, item); }); return Array.from(map.values()).slice(0, 6); } diff --git a/front_end/panels/ai_chat/ui/FileContentViewer.ts b/front_end/panels/ai_chat/ui/FileContentViewer.ts new file mode 100644 index 0000000000..613a805bbb --- /dev/null +++ b/front_end/panels/ai_chat/ui/FileContentViewer.ts @@ -0,0 +1,862 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import { createLogger } from '../core/Logger.js'; +import type { FileSummary } from '../tools/FileStorageManager.js'; +import * as Marked from '../../../third_party/marked/marked.js'; + +const logger = createLogger('FileContentViewer'); + +type FileType = 'code' | 'json' | 'markdown' | 'text' | 'html' | 'css'; + +/** + * FileContentViewer - Full-screen file viewer using RenderWebAppTool + * + * Displays file content in a professional full-screen iframe with: + * - Syntax-aware formatting + * - Copy and download functionality + * - Clean, modern design + */ +export class FileContentViewer { + /** + * Display file content in full-screen view + */ + static async show(file: FileSummary, content: string): Promise { + try { + // Import RenderWebAppTool + const { RenderWebAppTool } = await import('../tools/RenderWebAppTool.js'); + + // Build viewer components + const viewerHTML = await FileContentViewer.buildHTML(file, content); + const viewerCSS = FileContentViewer.buildCSS(); + const viewerJS = FileContentViewer.buildJS(file.fileName, content); + + // Use RenderWebAppTool to display full-screen viewer + const tool = new RenderWebAppTool(); + const result = await tool.execute({ + html: viewerHTML, + css: viewerCSS, + js: viewerJS, + reasoning: `Display file content: ${file.fileName}` + } as any); + + if ('error' in result) { + logger.error('Failed to open file viewer:', result.error); + } else { + logger.info('File viewer opened successfully', { fileName: file.fileName }); + } + } catch (error) { + logger.error('Error opening file viewer:', error); + throw error; + } + } + + /** + * Detect file type based on extension + */ + private static detectFileType(fileName: string): FileType { + const ext = fileName.toLowerCase().split('.').pop() || ''; + + const typeMap: Record = { + 'json': 'json', + 'md': 'markdown', + 'markdown': 'markdown', + 'js': 'code', + 'ts': 'code', + 'jsx': 'code', + 'tsx': 'code', + 'py': 'code', + 'java': 'code', + 'cpp': 'code', + 'c': 'code', + 'go': 'code', + 'rs': 'code', + 'html': 'html', + 'htm': 'html', + 'css': 'css', + 'scss': 'css', + 'sass': 'css', + 'less': 'css', + }; + + return typeMap[ext] || 'text'; + } + + /** + * Get file type icon + */ + private static getFileIcon(fileType: FileType): string { + const iconMap: Record = { + 'code': '💻', + 'json': '📋', + 'markdown': '📝', + 'text': '📄', + 'html': '🌐', + 'css': '🎨', + }; + return iconMap[fileType]; + } + + /** + * Get file type label + */ + private static getFileTypeLabel(fileType: FileType): string { + const labelMap: Record = { + 'code': 'Code', + 'json': 'JSON', + 'markdown': 'Markdown', + 'text': 'Text', + 'html': 'HTML', + 'css': 'CSS', + }; + return labelMap[fileType]; + } + + /** + * Format content based on file type + */ + private static formatContent(content: string, fileType: FileType): string { + if (fileType === 'json') { + try { + const parsed = JSON.parse(content); + return JSON.stringify(parsed, null, 2); + } catch { + return content; + } + } + return content; + } + + /** + * Format file size + */ + private static formatSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + } + + /** + * Format timestamp + */ + private static formatTimestamp(timestamp: number): string { + const date = new Date(timestamp); + return date.toLocaleString(); + } + + /** + * Escape HTML for safe embedding in HTML context + */ + private static escapeHTML(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + } + + /** + * Sanitize HTML to prevent XSS attacks + * Removes dangerous tags and attributes from rendered markdown + */ + private static sanitizeHTML(html: string): string { + // Remove dangerous tags entirely + const dangerousTags = [ + 'script', 'iframe', 'object', 'embed', 'link', 'style', + 'form', 'input', 'button', 'textarea', 'select', + 'base', 'meta', 'title' + ]; + + let sanitized = html; + + // Remove dangerous tags and their content + for (const tag of dangerousTags) { + // Case-insensitive removal of opening and closing tags + const tagRegex = new RegExp(`<${tag}[^>]*>.*?`, 'gis'); + sanitized = sanitized.replace(tagRegex, ''); + // Also remove self-closing variants + const selfClosingRegex = new RegExp(`<${tag}[^>]*/>`, 'gi'); + sanitized = sanitized.replace(selfClosingRegex, ''); + // Remove opening tags without closing tags + const openingRegex = new RegExp(`<${tag}[^>]*>`, 'gi'); + sanitized = sanitized.replace(openingRegex, ''); + } + + // Remove dangerous event handler attributes + const eventHandlers = [ + 'onload', 'onerror', 'onclick', 'ondblclick', 'onmousedown', 'onmouseup', + 'onmouseover', 'onmousemove', 'onmouseout', 'onmouseenter', 'onmouseleave', + 'onfocus', 'onblur', 'onkeydown', 'onkeyup', 'onkeypress', + 'onsubmit', 'onchange', 'oninput', 'onreset', 'onselect', + 'onabort', 'oncanplay', 'oncanplaythrough', 'ondurationchange', + 'onemptied', 'onended', 'onloadeddata', 'onloadedmetadata', + 'onloadstart', 'onpause', 'onplay', 'onplaying', 'onprogress', + 'onratechange', 'onseeked', 'onseeking', 'onstalled', 'onsuspend', + 'ontimeupdate', 'onvolumechange', 'onwaiting', + 'onanimationstart', 'onanimationend', 'onanimationiteration', + 'ontransitionend', 'ontoggle', 'onwheel', 'oncopy', 'oncut', 'onpaste' + ]; + + for (const handler of eventHandlers) { + // Remove event handlers with various quote styles and spacing + const handlerRegex = new RegExp(`\\s+${handler}\\s*=\\s*["'][^"']*["']`, 'gi'); + sanitized = sanitized.replace(handlerRegex, ''); + const handlerRegexNoQuotes = new RegExp(`\\s+${handler}\\s*=\\s*[^\\s>]+`, 'gi'); + sanitized = sanitized.replace(handlerRegexNoQuotes, ''); + } + + // Remove javascript: URLs from href and src attributes + sanitized = sanitized.replace(/href\s*=\s*["']javascript:[^"']*["']/gi, 'href="#"'); + sanitized = sanitized.replace(/src\s*=\s*["']javascript:[^"']*["']/gi, 'src=""'); + + // Remove data: URLs from src attributes (can be used for XSS) + sanitized = sanitized.replace(/src\s*=\s*["']data:text\/html[^"']*["']/gi, 'src=""'); + + return sanitized; + } + + /** + * Render markdown content to HTML + */ + private static async renderMarkdownToHTML(content: string): Promise { + try { + // Use Marked's built-in parser to convert markdown to HTML strings + const html = await Marked.Marked.parse(content); + return html; + } catch (error) { + logger.error('Failed to render markdown:', error); + // Fallback to escaped plain text + return content + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); + } + } + + /** + * Build HTML structure + */ + private static async buildHTML(file: FileSummary, content: string): Promise { + const fileType = FileContentViewer.detectFileType(file.fileName); + const icon = FileContentViewer.getFileIcon(fileType); + const typeLabel = FileContentViewer.getFileTypeLabel(fileType); + const formattedContent = FileContentViewer.formatContent(content, fileType); + const size = FileContentViewer.formatSize(file.size); + const created = FileContentViewer.formatTimestamp(file.createdAt); + + // Render markdown as formatted HTML or escape for code display + let contentHTML: string; + if (fileType === 'markdown') { + // Render markdown to HTML and sanitize to prevent XSS + const renderedHTML = await FileContentViewer.renderMarkdownToHTML(formattedContent); + const sanitizedHTML = FileContentViewer.sanitizeHTML(renderedHTML); + // For markdown: hidden div with original source + visible rendered HTML + contentHTML = ` + +
${sanitizedHTML}
+ `; + } else { + // For code files: use escapeHTML helper and add id + const safeContent = FileContentViewer.escapeHTML(formattedContent); + contentHTML = `
${safeContent}
`; + } + + return ` +
+ +
+
+ ${icon} +
+

${file.fileName}

+
+ ${typeLabel} + • + ${size} + • + Created ${created} +
+
+
+
+ + +
+
+ + +
+ ${contentHTML} +
+
+ `; + } + + /** + * Build CSS styles + */ + private static buildCSS(): string { + return ` + * { + margin: 0; + padding: 0; + box-sizing: border-box; + } + + body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarell', sans-serif; + overflow: hidden; + background: #f5f7fa; + } + + .file-viewer { + width: 100vw; + height: 100vh; + display: grid; + grid-template-rows: auto 1fr; + background: linear-gradient(135deg, #f5f7fa 0%, #e8ecf1 100%); + } + + /* Header - Glassmorphic */ + .viewer-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 20px 32px; + background: linear-gradient(135deg, rgba(255, 255, 255, 0.98) 0%, rgba(248, 249, 250, 0.95) 100%); + backdrop-filter: blur(10px); + border-bottom: 1px solid rgba(0, 0, 0, 0.08); + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.08); + position: relative; + z-index: 10; + } + + .file-info { + display: flex; + align-items: center; + gap: 16px; + flex: 1; + min-width: 0; + } + + .file-icon { + font-size: 32px; + line-height: 1; + } + + .file-details { + flex: 1; + min-width: 0; + } + + .file-name { + font-size: 18px; + font-weight: 600; + color: #202124; + margin-bottom: 4px; + word-break: break-word; + } + + .file-meta { + display: flex; + align-items: center; + gap: 8px; + font-size: 13px; + color: #5f6368; + flex-wrap: wrap; + } + + .meta-separator { + opacity: 0.5; + } + + .header-actions { + display: flex; + gap: 8px; + } + + .action-btn { + display: flex; + align-items: center; + gap: 6px; + padding: 10px 16px; + border: 1px solid rgba(0, 0, 0, 0.08); + border-radius: 12px; + background: rgba(255, 255, 255, 0.9); + backdrop-filter: blur(10px); + cursor: pointer; + font-size: 13px; + font-weight: 500; + color: #202124; + transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); + white-space: nowrap; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.04); + } + + .action-btn svg { + width: 16px; + height: 16px; + transition: transform 0.3s cubic-bezier(0.4, 0, 0.2, 1); + } + + .action-btn:hover { + background: rgba(255, 255, 255, 1); + border-color: #1976d2; + color: #1976d2; + box-shadow: 0 4px 12px rgba(25, 118, 210, 0.15); + transform: translateY(-1px); + } + + .action-btn:hover svg { + transform: scale(1.1); + } + + .action-btn:active { + transform: translateY(0); + } + + .copy-btn.copied { + background: #4caf50; + border-color: #4caf50; + color: white; + box-shadow: 0 4px 12px rgba(76, 175, 80, 0.3); + } + + .download-btn { + background: linear-gradient(135deg, #1976d2, #1565c0); + border-color: transparent; + color: white; + box-shadow: 0 4px 12px rgba(25, 118, 210, 0.3); + } + + .download-btn:hover { + background: linear-gradient(135deg, #1565c0, #0d47a1); + box-shadow: 0 6px 16px rgba(25, 118, 210, 0.4); + } + + /* Content */ + .content-container { + overflow: auto; + background: transparent; + position: relative; + scroll-behavior: smooth; + } + + /* Code file styling */ + .file-content { + padding: 32px; + font-family: 'SF Mono', 'Monaco', 'Menlo', 'Consolas', 'Courier New', monospace; + font-size: 14px; + line-height: 1.6; + color: #202124; + white-space: pre-wrap; + word-wrap: break-word; + background: rgba(255, 255, 255, 0.7); + backdrop-filter: blur(10px); + border: none; + margin: 24px; + border-radius: 16px; + box-shadow: 0 2px 12px rgba(0, 0, 0, 0.06); + } + + .file-content code { + font-family: inherit; + font-size: inherit; + background: none; + padding: 0; + } + + /* Markdown document styling */ + .markdown-content { + max-width: 900px; + margin: 0 auto; + padding: 48px 64px; + background: rgba(255, 255, 255, 0.95); + backdrop-filter: blur(10px); + min-height: calc(100vh - 80px); + border-radius: 20px; + box-shadow: 0 4px 24px rgba(0, 0, 0, 0.08); + margin: 24px auto; + line-height: 1.7; + font-size: 15px; + color: #202124; + } + + .markdown-content h1, .markdown-content h2, .markdown-content h3, + .markdown-content h4, .markdown-content h5, .markdown-content h6 { + margin-top: 24px; + margin-bottom: 16px; + font-weight: 600; + line-height: 1.3; + color: #1a1a1a; + } + + .markdown-content h1 { + font-size: 32px; + border-bottom: 2px solid rgba(25, 118, 210, 0.2); + padding-bottom: 12px; + margin-top: 0; + } + + .markdown-content h2 { + font-size: 26px; + border-bottom: 1px solid rgba(0, 0, 0, 0.08); + padding-bottom: 8px; + } + + .markdown-content h3 { + font-size: 22px; + } + + .markdown-content p { + margin: 16px 0; + line-height: 1.7; + } + + .markdown-content ul, .markdown-content ol { + margin: 16px 0; + padding-left: 28px; + } + + .markdown-content li { + margin: 8px 0; + line-height: 1.6; + } + + .markdown-content code { + background: rgba(25, 118, 210, 0.08); + padding: 2px 6px; + border-radius: 4px; + font-family: 'SF Mono', 'Monaco', 'Menlo', 'Consolas', monospace; + font-size: 13px; + color: #1976d2; + } + + .markdown-content pre { + background: rgba(0, 0, 0, 0.04); + padding: 16px; + border-radius: 12px; + overflow-x: auto; + margin: 16px 0; + border: 1px solid rgba(0, 0, 0, 0.08); + } + + .markdown-content pre code { + background: none; + padding: 0; + color: #202124; + font-size: 13px; + } + + .markdown-content blockquote { + border-left: 4px solid #1976d2; + margin: 16px 0; + padding: 12px 20px; + background: rgba(25, 118, 210, 0.04); + border-radius: 0 8px 8px 0; + color: #5f6368; + font-style: italic; + } + + .markdown-content a { + color: #1976d2; + text-decoration: none; + border-bottom: 1px solid rgba(25, 118, 210, 0.3); + transition: all 0.2s ease; + } + + .markdown-content a:hover { + border-bottom-color: #1976d2; + color: #1565c0; + } + + .markdown-content hr { + border: none; + border-top: 1px solid rgba(0, 0, 0, 0.1); + margin: 32px 0; + } + + .markdown-content table { + border-collapse: collapse; + width: 100%; + margin: 16px 0; + border-radius: 8px; + overflow: hidden; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.06); + } + + .markdown-content th, .markdown-content td { + padding: 12px 16px; + text-align: left; + border-bottom: 1px solid rgba(0, 0, 0, 0.08); + } + + .markdown-content th { + background: rgba(25, 118, 210, 0.08); + font-weight: 600; + color: #1976d2; + } + + .markdown-content tr:hover { + background: rgba(0, 0, 0, 0.02); + } + + /* Scrollbar - Modern thin style */ + .content-container::-webkit-scrollbar { + width: 8px; + height: 8px; + } + + .content-container::-webkit-scrollbar-track { + background: transparent; + } + + .content-container::-webkit-scrollbar-thumb { + background: rgba(0, 0, 0, 0.2); + border-radius: 4px; + } + + .content-container::-webkit-scrollbar-thumb:hover { + background: rgba(0, 0, 0, 0.3); + } + + /* Dark mode support */ + @media (prefers-color-scheme: dark) { + body { + background: #1a1d23; + } + + .file-viewer { + background: linear-gradient(135deg, #1a1d23 0%, #252931 100%); + } + + .viewer-header { + background: linear-gradient(135deg, rgba(41, 42, 45, 0.98) 0%, rgba(32, 33, 36, 0.95) 100%); + border-bottom-color: rgba(255, 255, 255, 0.08); + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3); + } + + .file-name { + color: #e8eaed; + } + + .file-meta { + color: #9aa0a6; + } + + .action-btn { + background: rgba(60, 64, 67, 0.9); + border-color: rgba(255, 255, 255, 0.08); + color: #e8eaed; + } + + .action-btn:hover { + background: rgba(80, 84, 87, 1); + border-color: #1976d2; + box-shadow: 0 4px 12px rgba(25, 118, 210, 0.3); + } + + .download-btn { + background: linear-gradient(135deg, #1976d2, #1565c0); + } + + .download-btn:hover { + background: linear-gradient(135deg, #1565c0, #0d47a1); + } + + .content-container { + background: transparent; + } + + .file-content { + color: #e8eaed; + background: rgba(41, 42, 45, 0.7); + } + + .file-content code { + color: #e8eaed; + } + + .markdown-content { + background: rgba(41, 42, 45, 0.95); + color: #e8eaed; + box-shadow: 0 4px 24px rgba(0, 0, 0, 0.4); + } + + .markdown-content h1, + .markdown-content h2, + .markdown-content h3, + .markdown-content h4, + .markdown-content h5, + .markdown-content h6 { + color: #f1f3f4; + } + + .markdown-content h1 { + border-bottom-color: rgba(25, 118, 210, 0.3); + } + + .markdown-content h2 { + border-bottom-color: rgba(255, 255, 255, 0.08); + } + + .markdown-content code { + background: rgba(25, 118, 210, 0.15); + color: #64b5f6; + } + + .markdown-content pre { + background: rgba(0, 0, 0, 0.3); + border-color: rgba(255, 255, 255, 0.08); + } + + .markdown-content pre code { + color: #e8eaed; + } + + .markdown-content blockquote { + background: rgba(25, 118, 210, 0.1); + color: #9aa0a6; + border-left-color: #1976d2; + } + + .markdown-content a { + color: #64b5f6; + border-bottom-color: rgba(100, 181, 246, 0.3); + } + + .markdown-content a:hover { + color: #90caf9; + border-bottom-color: #64b5f6; + } + + .markdown-content hr { + border-top-color: rgba(255, 255, 255, 0.1); + } + + .markdown-content table { + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.4); + } + + .markdown-content th { + background: rgba(25, 118, 210, 0.15); + color: #64b5f6; + } + + .markdown-content td { + border-bottom-color: rgba(255, 255, 255, 0.08); + } + + .markdown-content tr:hover { + background: rgba(255, 255, 255, 0.03); + } + + .content-container::-webkit-scrollbar-thumb { + background: rgba(255, 255, 255, 0.2); + } + + .content-container::-webkit-scrollbar-thumb:hover { + background: rgba(255, 255, 255, 0.3); + } + } + `; + } + + /** + * Build JavaScript functionality + */ + private static buildJS(fileName: string, content: string): string { + return ` + const FILE_NAME = ${JSON.stringify(fileName)}; + + // Attach functions to window for global access (RenderWebAppTool wraps JS in IIFE) + window.copyContent = async function(event) { + event.preventDefault(); + const btn = event.currentTarget; + const textSpan = btn.querySelector('.btn-text'); + const originalText = textSpan.textContent; + + try { + const content = document.getElementById('file-content').textContent; + + // Try modern Clipboard API first + if (navigator.clipboard && navigator.clipboard.writeText) { + await navigator.clipboard.writeText(content); + } else { + // Fallback to execCommand for iframe/non-secure contexts + const textarea = document.createElement('textarea'); + textarea.value = content; + textarea.style.position = 'fixed'; + textarea.style.left = '-999999px'; + textarea.style.top = '-999999px'; + document.body.appendChild(textarea); + textarea.focus(); + textarea.select(); + + const successful = document.execCommand('copy'); + document.body.removeChild(textarea); + + if (!successful) { + throw new Error('execCommand copy failed'); + } + } + + // Show success feedback + btn.classList.add('copied'); + textSpan.textContent = 'Copied!'; + + setTimeout(() => { + btn.classList.remove('copied'); + textSpan.textContent = originalText; + }, 2000); + } catch (error) { + console.error('Failed to copy:', error); + textSpan.textContent = 'Failed'; + setTimeout(() => { + textSpan.textContent = originalText; + }, 2000); + } + }; + + window.downloadFile = function(event) { + event.preventDefault(); + try { + const content = document.getElementById('file-content').textContent; + const blob = new Blob([content], { type: 'text/plain;charset=utf-8' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = FILE_NAME; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } catch (error) { + console.error('Failed to download:', error); + alert('Failed to download file'); + } + }; + + // Prevent default drag and drop + document.addEventListener('dragover', (e) => e.preventDefault()); + document.addEventListener('drop', (e) => e.preventDefault()); + `; + } +} diff --git a/front_end/panels/ai_chat/ui/FileListDisplay.ts b/front_end/panels/ai_chat/ui/FileListDisplay.ts new file mode 100644 index 0000000000..84701931d0 --- /dev/null +++ b/front_end/panels/ai_chat/ui/FileListDisplay.ts @@ -0,0 +1,646 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import * as ComponentHelpers from '../../../ui/components/helpers/helpers.js'; +import * as Lit from '../../../ui/lit/lit.js'; +import { FileStorageManager, type FileSummary } from '../tools/FileStorageManager.js'; +import { createLogger } from '../core/Logger.js'; +import { AgentService } from '../core/AgentService.js'; + +const logger = createLogger('FileListDisplay'); + +const {html, nothing} = Lit; + +/** + * Component that displays a list of files created during the agent session + */ +export class FileListDisplay extends HTMLElement { + static readonly litTagName = Lit.StaticHtml.literal`ai-file-list-display`; + readonly #shadow = this.attachShadow({mode: 'open'}); + readonly #boundRender = this.#render.bind(this); + + #files: FileSummary[] = []; + #isCollapsed = false; + #viewingFile: FileSummary | null = null; + #viewingFileContent: string | null = null; + #refreshInterval?: number; + #boundHandleKeyDown = this.#handleKeyDown.bind(this); + + get files(): FileSummary[] { + return this.#files; + } + + set files(value: FileSummary[]) { + this.#files = value; + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + + get isCollapsed(): boolean { + return this.#isCollapsed; + } + + set isCollapsed(value: boolean) { + this.#isCollapsed = value; + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + + get viewingFile(): FileSummary | null { + return this.#viewingFile; + } + + set viewingFile(value: FileSummary | null) { + this.#viewingFile = value; + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + + get viewingFileContent(): string | null { + return this.#viewingFileContent; + } + + set viewingFileContent(value: string | null) { + this.#viewingFileContent = value; + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + + connectedCallback(): void { + // Load collapsed state from localStorage + const savedState = localStorage.getItem('ai_chat_files_collapsed'); + this.#isCollapsed = savedState === 'true'; + + this.#loadFiles(); + // Poll for updates every 2 seconds + this.#refreshInterval = window.setInterval(() => this.#loadFiles(), 2000); + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + + disconnectedCallback(): void { + if (this.#refreshInterval) { + clearInterval(this.#refreshInterval); + } + } + + async #loadFiles(): Promise { + try { + const manager = FileStorageManager.getInstance(); + const files = await manager.listFiles(); + + // Only update if files have actually changed (avoid unnecessary re-renders) + const filesChanged = files.length !== this.#files.length || + files.some((file, index) => + file.fileName !== this.#files[index]?.fileName || + file.updatedAt !== this.#files[index]?.updatedAt + ); + + if (filesChanged) { + this.#files = files; + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + } catch (error) { + logger.error('Failed to load files', error); + } + } + + #toggleCollapse(): void { + this.#isCollapsed = !this.#isCollapsed; + localStorage.setItem('ai_chat_files_collapsed', String(this.#isCollapsed)); + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + + async #handleViewFile(file: FileSummary): Promise { + try { + const manager = FileStorageManager.getInstance(); + const storedFile = await manager.readFile(file.fileName); + if (!storedFile) { + logger.error('File not found', file.fileName); + return; + } + + // Check if agent is running + const agentRunning = AgentService.instance?.isRunning() ?? false; + + if (!agentRunning) { + // Full-screen viewer (browser iframe) when agent is idle + logger.info('Opening full-screen file viewer (agent idle)', { fileName: file.fileName }); + const { FileContentViewer } = await import('./FileContentViewer.js'); + await FileContentViewer.show(file, storedFile.content); + } else { + // Current modal behavior when agent is running (don't interrupt) + logger.info('Opening modal file viewer (agent running)', { fileName: file.fileName }); + this.#viewingFile = file; + this.#viewingFileContent = storedFile.content; + // Add ESC key listener when modal opens + document.addEventListener('keydown', this.#boundHandleKeyDown); + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + } catch (error) { + logger.error('Failed to read file', error); + } + } + + #handleKeyDown(event: KeyboardEvent): void { + if (event.key === 'Escape' && this.#viewingFile) { + this.#closeModal(); + } + } + + #handleDownloadFile(file: FileSummary): void { + FileStorageManager.getInstance().readFile(file.fileName).then(storedFile => { + if (!storedFile) { + logger.error('File not found', file.fileName); + return; + } + + const blob = new Blob([storedFile.content], { type: storedFile.mimeType }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = file.fileName; + a.click(); + URL.revokeObjectURL(url); + }).catch(error => { + logger.error('Failed to download file', error); + }); + } + + #closeModal(): void { + this.#viewingFile = null; + this.#viewingFileContent = null; + // Remove ESC key listener when modal closes + document.removeEventListener('keydown', this.#boundHandleKeyDown); + void ComponentHelpers.ScheduledRender.scheduleRender(this, this.#boundRender); + } + + #formatFileSize(bytes: number): string { + if (bytes < 1024) { + return `${bytes} B`; + } + if (bytes < 1024 * 1024) { + return `${(bytes / 1024).toFixed(1)} KB`; + } + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; + } + + #formatTimestamp(timestamp: number): string { + const date = new Date(timestamp); + const now = new Date(); + const diff = now.getTime() - date.getTime(); + + // Less than 1 minute + if (diff < 60000) { + return 'just now'; + } + // Less than 1 hour + if (diff < 3600000) { + const minutes = Math.floor(diff / 60000); + return `${minutes}m ago`; + } + // Less than 1 day + if (diff < 86400000) { + const hours = Math.floor(diff / 3600000); + return `${hours}h ago`; + } + // Otherwise show date + return date.toLocaleDateString(); + } + + #render(): void { + if (this.#files.length === 0) { + Lit.render(nothing, this.#shadow, {host: this}); + return; + } + + Lit.render(html` + + +
+
this.#toggleCollapse()}> +
+ 📁 + Session Files + • ${this.#files.length} +
+
+ ▼ +
+
+ + ${!this.#isCollapsed ? html` +
+
+ ${this.#files.map(file => html` +
this.#handleViewFile(file)}> + 📄 + ${file.fileName} +
+ `)} +
+
+ ` : nothing} +
+ + ${this.#viewingFile && this.#viewingFileContent ? html` +
{ + if (e.target === e.currentTarget) { + this.#closeModal(); + } + }}> +
e.stopPropagation()}> +
+
+
${this.#viewingFile.fileName}
+
+ ${this.#formatFileSize(this.#viewingFile.size)} + Created ${this.#formatTimestamp(this.#viewingFile.createdAt)} +
+
+
+ + +
+
+
${this.#viewingFileContent}
+
+
+ ` : nothing} + `, this.#shadow, {host: this}); + } + + /** + * Public method to refresh the file list + */ + async refresh(): Promise { + await this.#loadFiles(); + } +} + +customElements.define('ai-file-list-display', FileListDisplay); + +declare global { + interface HTMLElementTagNameMap { + 'ai-file-list-display': FileListDisplay; + } +} diff --git a/front_end/panels/ai_chat/ui/TodoListDisplay.ts b/front_end/panels/ai_chat/ui/TodoListDisplay.ts new file mode 100644 index 0000000000..2c826bd0a6 --- /dev/null +++ b/front_end/panels/ai_chat/ui/TodoListDisplay.ts @@ -0,0 +1,313 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import * as Lit from '../../../ui/lit/lit.js'; +import { FileStorageManager } from '../tools/FileStorageManager.js'; +import { createLogger } from '../core/Logger.js'; + +const logger = createLogger('TodoListDisplay'); +const {html, Decorators, render} = Lit; +const {customElement} = Decorators; + +interface TodoItem { + completed: boolean; + text: string; +} + +interface ParsedTodos { + items: TodoItem[]; + total: number; + completed: number; +} + +@customElement('ai-todo-list') +export class TodoListDisplay extends HTMLElement { + static readonly litTagName = Lit.StaticHtml.literal`ai-todo-list`; + readonly #shadow = this.attachShadow({mode: 'open'}); + + #collapsed = false; + #todos = ''; + #refreshInterval?: number; + + connectedCallback(): void { + // Load collapsed state from localStorage + const savedState = localStorage.getItem('ai_chat_todo_collapsed'); + this.#collapsed = savedState === 'true'; + + this.#loadTodos(); + // Poll for updates every 2 seconds + this.#refreshInterval = window.setInterval(() => this.#loadTodos(), 2000); + } + + disconnectedCallback(): void { + if (this.#refreshInterval) { + clearInterval(this.#refreshInterval); + } + } + + async #loadTodos(): Promise { + try { + const file = await FileStorageManager.getInstance().readFile('todos.md'); + const newContent = file?.content || ''; + + if (newContent !== this.#todos) { + this.#todos = newContent; + this.#render(); + } + } catch (error) { + logger.debug('Failed to load todos:', error); + } + } + + #toggleCollapse(): void { + this.#collapsed = !this.#collapsed; + localStorage.setItem('ai_chat_todo_collapsed', String(this.#collapsed)); + this.#render(); + } + + #parseTodos(markdown: string): ParsedTodos { + const lines = markdown.split('\n'); + const items = lines + .filter(line => /^[\s]*-\s+\[([ x])\]/i.test(line)) + .map(line => ({ + completed: /\[x\]/i.test(line), + text: line.replace(/^[\s]*-\s+\[([ x])\]\s*/i, '').trim() + })) + .filter(item => item.text.length > 0); + + return { + items, + total: items.length, + completed: items.filter(i => i.completed).length + }; + } + + #render(): void { + // Don't render if no todos exist + if (!this.#todos || this.#todos.trim().length === 0) { + render(html``, this.#shadow, {host: this}); + return; + } + + const todoItems = this.#parseTodos(this.#todos); + + // Don't render if no valid todo items + if (todoItems.total === 0) { + render(html``, this.#shadow, {host: this}); + return; + } + + render(html` + + +
+
this.#toggleCollapse()}> +
+ 📋 + Tasks + ${todoItems.completed}/${todoItems.total} +
+ ▼ +
+ ${!this.#collapsed ? html` +
+ ${todoItems.items.map(item => html` +
+ ${item.completed ? '✓' : ''} + ${item.text} +
+ `)} +
+ ` : ''} +
+ `, this.#shadow, {host: this}); + } +} + +declare global { + interface HTMLElementTagNameMap { + 'ai-todo-list': TodoListDisplay; + } +} diff --git a/front_end/panels/ai_chat/ui/chatView.css b/front_end/panels/ai_chat/ui/chatView.css index 71d96e4354..f61eff8ab2 100644 --- a/front_end/panels/ai_chat/ui/chatView.css +++ b/front_end/panels/ai_chat/ui/chatView.css @@ -37,19 +37,23 @@ color-scheme: light dark; } -/* Ensure custom input bar expands with container width */ -ai-input-bar { - display: block; +/* Flexbox layout for proper space allocation */ +ai-message-list { + flex: 1 1 auto; + min-height: 0; /* Important: allows flex child to shrink below content size */ + overflow: hidden; +} + +ai-todo-list { + flex-shrink: 0; + flex-grow: 0; width: 100%; } -/* Float input bar above content; messages scroll behind */ -.chat-view-container > ai-input-bar { - position: absolute; - left: 0; - right: 0; - bottom: 0; - z-index: 5; +ai-input-bar { + flex-shrink: 0; + flex-grow: 0; + width: 100%; } /* Center the input bar in first-message (centered) view */ @@ -103,7 +107,7 @@ ai-input-bar { /* In expanded (conversation) view, reserve bottom space so the floating input bar does not overlap the last messages. */ .chat-view-container.expanded-view { - padding-bottom: 150px; /* tune if input height changes */ + padding-bottom: 0; /* tune if input height changes */ } @keyframes expandToFull { From bc8ffa16c8175fb8aed691fb3d8fca7d05e19abd Mon Sep 17 00:00:00 2001 From: Tyson Thomas Date: Thu, 6 Nov 2025 22:59:38 -0800 Subject: [PATCH 5/6] Update docs --- CONTRIBUTING.md | 179 +++++++++++++++++++++++++++++++++-- README.md | 236 +++++++++------------------------------------- assets/Banner.png | Bin 0 -> 800433 bytes 3 files changed, 216 insertions(+), 199 deletions(-) create mode 100644 assets/Banner.png diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3661d1f026..d6c1be1408 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,11 +1,172 @@ -# Contributing to Chromium DevTools +# Contributing to Browser Operator -Contributions to Chromium DevTools include code, documentation, and responding -to user questions. +Thank you for your interest in contributing to Browser Operator! We welcome contributions from the community, whether it's bug reports, feature requests, documentation improvements, or code contributions. -See the [Chrome DevTools Contribution Guide](./docs/contributing/README.md) -for details on how you can contribute. Also checkout the [Contributing to -Chromium](https://chromium.googlesource.com/chromium/src/+/main/docs/contributing.md) -and [Contributing to V8](https://v8.dev/docs/contribute) documents for details -how to contribute to the Chromium / V8 code base in general, which is relevant -when working on the DevTools back-end. +## 🚀 Getting Started + +### Prerequisites + +- Node.js and npm +- Git +- macOS 10.15+ or Windows 10 (64-bit)+ +- 8GB RAM (16GB recommended) + +### Setup + +1. **Fork the repository** on GitHub +2. **Clone your fork:** + ```bash + git clone https://github.com/YOUR_USERNAME/browser-operator-core.git + cd browser-operator-core + ``` +3. **Set up your development environment:** + See our [build instructions](https://docs.browseroperator.io) for detailed setup steps + +## 💻 Making Changes + +### 1. Create a Branch + +```bash +git checkout -b feature/your-feature-name +# or +git checkout -b fix/bug-description +``` + +### 2. Make Your Changes + +- Write clean, readable code +- Follow the existing code style +- Add tests for new features +- Update documentation as needed + +### 3. Test Your Changes + +```bash +# Run linters +npm run lint + +# Run tests +npm run test + +# Run web tests +npm run webtest + +# Build the project +npm run build +``` + +### 4. Commit Your Changes + +Write clear, descriptive commit messages: + +```bash +git commit -m "Add feature: description of what you added" +# or +git commit -m "Fix: description of what you fixed" +``` + +**Add license header to new files:** +```typescript +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +``` + +## 📬 Submitting Your Changes + +### 1. Push to Your Fork + +```bash +git push origin feature/your-feature-name +``` + +### 2. Open a Pull Request + +1. Go to the [Browser Operator repository](https://github.com/BrowserOperator/browser-operator-core) +2. Click "New Pull Request" +3. Select your fork and branch +4. Fill out the PR template with: + - Clear description of changes + - Reference to related issues (if any) + - Screenshots/videos for UI changes + - Testing steps + +### 3. Review Process + +- A maintainer will review your PR +- Address any feedback or requested changes +- Once approved, your PR will be merged + +## 🎯 Contribution Areas + +### Bug Reports + +Found a bug? [Open an issue](https://github.com/BrowserOperator/browser-operator-core/issues) with: +- Clear description of the bug +- Steps to reproduce +- Expected vs actual behavior +- Screenshots if applicable +- Browser version and OS + +### Feature Requests + +Have an idea? [Open an issue](https://github.com/BrowserOperator/browser-operator-core/issues) with: +- Clear description of the feature +- Use case and benefits +- Any relevant examples or mockups + +### Documentation + +Help improve our docs: +- Fix typos or unclear explanations +- Add examples or tutorials +- Translate documentation +- Update outdated information + +### Code Contributions + +Areas where you can contribute code: +- **New Tools**: Add new tools for the agent framework +- **New Agents**: Create specialized agents for specific tasks +- **UI Improvements**: Enhance the user interface +- **Bug Fixes**: Fix reported issues +- **Performance**: Optimize existing code +- **Tests**: Add or improve test coverage + +### Agent Framework Development + +Contributing new agents or tools? See: +- [Agent Framework Documentation](front_end/panels/ai_chat/agent_framework/Readme.md) +- [Creating Custom Agents](front_end/panels/ai_chat/core/Readme.md) +- Browse existing tools in [front_end/panels/ai_chat/tools/](front_end/panels/ai_chat/tools/) + +## 📝 Code Style + +- **TypeScript**: Follow existing patterns and conventions +- **Linting**: Code must pass `npm run lint` +- **Testing**: Add tests for new features +- **Comments**: Document complex logic +- **Types**: Use proper TypeScript types + +## 🤝 Community Guidelines + +- Be respectful and inclusive +- Help others learn and grow +- Provide constructive feedback +- Treat everyone with respect + +### Communication Channels + +- **Discord**: [Join our community](https://discord.gg/fp7ryHYBSY) +- **GitHub Issues**: For bugs and features +- **GitHub Discussions**: For questions and ideas + +## 📄 License + +By contributing to Browser Operator, you agree that your contributions will be licensed under the [BSD-3-Clause License](LICENSE). + +--- + +**Questions?** Join our [Discord community](https://discord.gg/fp7ryHYBSY) or open a [GitHub Discussion](https://github.com/BrowserOperator/browser-operator-core/discussions). + +Thank you for contributing to Browser Operator! 🎉 diff --git a/README.md b/README.md index 91c3258731..5309f339ad 100644 --- a/README.md +++ b/README.md @@ -1,220 +1,74 @@ -# Browser Operator - Open Source Agentic Browser - -![GitHub Release](https://img.shields.io/github/v/release/tysonthomas9/browser-operator-devtools-frontend) -![Platform](https://img.shields.io/badge/platform-macOS-blue) -[![License](https://img.shields.io/badge/license-BSD--3--Clause-green)](LICENSE) - -**The first open-source, privacy-friendly AI browser that transforms how you work on the web. Your intelligent partner for research, analysis, and automation - all running locally in your browser.** - -![Live Demo](front_end/panels/ai_chat/docs/demo.gif) - -## 🚀 Download & Get Started - -**[⬇️ Download Browser Operator for macOS](https://github.com/tysonthomas9/browser-operator-devtools-frontend/releases)** - -Or build from source: [Developer Setup Guide](front_end/panels/ai_chat/Readme.md) - -## 🎬 See It In Action - -### Deep Research & Analysis -Watch Browser Operator synthesize information from multiple sources, creating comprehensive research reports without manual copying and pasting. - - -https://github.com/user-attachments/assets/225319db-c5a0-4834-9f37-5787fb646d16 - - -### Smart Shopping Assistant -See how it automatically compares products, analyzes reviews, and helps you make informed purchasing decisions. - -https://github.com/user-attachments/assets/c478b18e-0342-400d-98ab-222c93eecd7a - -### Professional Research -Discover how businesses use Browser Operator for talent search, competitive analysis, and market research. - -https://github.com/user-attachments/assets/90150f0e-e8c8-4b53-b6a6-c739f143f4a0 - -## ✨ Key Features - -### 🤖 Intelligent Automation -- **Multi-Agent Framework**: Specialized agents work together to handle complex tasks -- **Autonomous Navigation**: Understands and interacts with any website -- **Smart Actions**: Click, fill forms, extract data, and navigate without manual scripting -- **Adaptive Learning**: Improves task execution based on patterns and feedback - -### 🔒 Privacy First (Use local LLM) -- **Local Processing**: Your data never leaves your machine -- **No Cloud Dependencies**: Full functionality without sending data to external servers -- **Secure Sessions**: Works with your existing browser authentication -- **Open Source**: Complete transparency in how your data is handled - -### 🧩 Extensible Platform -- **100+ AI Models**: Support for OpenAI, Claude, Gemini, Llama, and more -- **Custom Workflows**: Build your own automation sequences -- **Plugin Architecture**: Extend functionality with custom agents -- **API Integration**: Connect with your existing tools and services - -## 💡 What Can You Build? - - - - - - -
- -**Personal Productivity** -- 📚 Literature reviews and research papers -- 🛍️ Price tracking and comparison shopping -- 📰 News aggregation and summarization -- 📊 Data collection and analysis -- ✈️ Travel planning and booking research - - - -**Business Intelligence** -- 🔍 Competitive analysis and monitoring -- 👥 Talent sourcing and recruitment -- 📈 Market research and trends -- 🏢 Lead generation and qualification -- 📋 Compliance and audit automation - -
- -## 🛠️ Technical Architecture - -Browser Operator combines a Chromium-based browser with an advanced agentic framework: - -``` -┌─────────────────────────────────────────────────┐ -│ Browser Operator UI │ -├─────────────────────────────────────────────────┤ -│ Multi-Agent Orchestrator │ -├──────────────┬────────────────┬─────────────────┤ -│ Research │ Navigation │ Analysis │ -│ Agent │ Agent │ Agent │ -├──────────────┴────────────────┴─────────────────┤ -│ Chromium Browser Engine │ -└─────────────────────────────────────────────────┘ -``` - -### Core Components -- **Orchestrator Agent**: Coordinates multi-agent workflows and task distribution -- **Navigation Engine**: Handles web interactions and page understanding -- **Tool Registry**: Extensible system for adding new capabilities -- **State Management**: Maintains context across complex workflows - -[Full Technical Documentation →](front_end/panels/ai_chat/Readme.md) - -## ⚙️ Quick Setup - -### For Users: Pre-built Application - -1. [Download the latest release](https://github.com/tysonthomas9/browser-operator-devtools-frontend/releases) -2. Open Browser Operator -3. Configure your AI provider (see below) -4. Start automating! - -### For Developers: Build from Source +
-```bash -# Clone the repository -git clone https://github.com/tysonthomas9/browser-operator-devtools-frontend.git +![Browser Operator Banner](https://raw.githubusercontent.com/BrowserOperator/browser-operator-core/main/assets/Banner.png) -# Follow the detailed build instructions -cd browser-operator-devtools-frontend -# See front_end/panels/ai_chat/Readme.md for complete setup -``` +![GitHub Release](https://img.shields.io/github/v/release/BrowserOperator/browser-operator-core) +![Platform](https://img.shields.io/badge/platform-macOS%20%7C%20Windows-blue) +[![License](https://img.shields.io/badge/license-BSD--3--Clause-green)](LICENSE) -### AI Provider Configuration +**Open-source, privacy-focused AI browser that transforms how you work on the web. An intelligent platform for research, analysis, and automation—running locally in your browser.** -
-Option 1: OpenAI (Recommended for beginners) +[Download](https://github.com/BrowserOperator/browser-operator-core/releases) • [Docs](https://docs.browseroperator.io) • [Community](https://discord.gg/fp7ryHYBSY) -1. Get an API key from [OpenAI Platform](https://platform.openai.com) -2. Open Browser Operator settings -3. Select "OpenAI" as provider -4. Enter your API key -5. Choose a model (GPT-4.1 recommended) -6. Save and start using! +
- +--- -
-Option 2: LiteLLM (For multiple providers) +## 🚀 Getting Started -Perfect for using multiple AI providers or self-hosted models: +**[⬇️ Download for macOS](https://github.com/BrowserOperator/browser-operator-core/releases)** | **[⬇️ Download for Windows](https://github.com/BrowserOperator/browser-operator-core/releases)** -1. Set up your LiteLLM proxy server -2. Select "LiteLLM Provider" in settings -3. Enter proxy URL and API key -4. Click "Fetch Models" to verify connection -5. Select your preferred model +**System Requirements:** macOS 10.15+ or Windows 10 (64-bit)+ • 8GB RAM (16GB recommended) • 2GB free disk space -[LiteLLM Setup Video →](https://github.com/user-attachments/assets/579dcfdc-71c8-4664-87b8-c2b68cc5c1ce) +### AI Provider Setup -
+| Provider | Best For | Setup | +|----------|----------|-------| +| **[OpenRouter](https://openrouter.ai)** | Beginners, 400+ models (Claude, GPT, Gemini, Llama) | Sign in through browser | +| **[OpenAI](https://platform.openai.com)** | GPT models | API key required | +| **[Groq](https://console.groq.com)** | Ultra-fast inference | API key required | +| **[LiteLLM](https://docs.litellm.ai)** | Local models, privacy, advanced users | Proxy + Ollama setup | -
-Option 3: Local Models (Maximum privacy) +**Quick Setup:** Settings → Select provider → Enter credentials → Choose model → Save -Run completely offline with Ollama: +--- -1. Install Ollama on your system -2. Pull your preferred model (e.g., `ollama pull llama3`) -3. Configure Browser Operator to use local endpoint -4. Enjoy private, offline automation +## 🎬 What You Can Do -
+**Multi-Agent Automation** – Specialized AI agents work together to handle complex web tasks autonomously -## 🗺️ Roadmap +**Privacy-First** – All processing happens locally on your machine. Use local models with Ollama for complete offline operation -### ✅ Released -- Multi-agent workflow engine -- Support for 100+ AI models -- macOS application -- Core automation capabilities +**Extensible** – Compatible with 100+ AI models through OpenAI, Claude, Gemini, Llama, and more via LiteLLM -### 🚧 In Development -- Windows and Linux support -- Enhanced memory system -- Custom agent builder +### Use Cases -### 🔮 Planned Features -- MCP (Model Context Protocol) support -- Visual workflow designer -- Team collaboration features -- Advanced scheduling system +**Research & Analysis** – Literature reviews, data collection, competitive intelligence, market research -## 👥 Community & Support +https://github.com/user-attachments/assets/225319db-c5a0-4834-9f37-5787fb646d16 -### Get Help -- 📖 [Documentation](front_end/panels/ai_chat/Readme.md) -- 💬 [Discord Community](https://discord.gg/fp7ryHYBSY) -- 🐛 [Report Issues](https://github.com/tysonthomas9/browser-operator-devtools-frontend/issues) -- 🐦 [Follow Updates](https://x.com/BrowserOperator) +**Shopping & Price Tracking** – Product comparisons, review analysis, price monitoring -### Contributing +https://github.com/user-attachments/assets/c478b18e-0342-400d-98ab-222c93eecd7a -We welcome contributions! Here's how you can help: +**Business Automation** – Talent sourcing, lead generation, compliance audits -- **🐛 Report Bugs**: Help us identify and fix issues -- **✨ Request Features**: Share your ideas for new capabilities -- **📝 Improve Docs**: Help others get started -- **💻 Submit PRs**: Contribute code improvements +https://github.com/user-attachments/assets/90150f0e-e8c8-4b53-b6a6-c739f143f4a0 -See our [Contributing Guide](CONTRIBUTING.md) for details. +--- -## 📚 Documentation +## 👥 Community & Contributing -- [Getting Started Guide](front_end/panels/ai_chat/docs/GettingStarted.md) -- [Agent Framework](front_end/panels/ai_chat/agent_framework/Readme.md) -- [Creating Custom Agents](front_end/panels/ai_chat/core/Readme.md) -- [Architecture Overview](front_end/panels/ai_chat/docs/Architecture.md) -- [Build Instructions](front_end/panels/ai_chat/docs/PreBuilt.md) +**Get Help** +- 📖 [Documentation](https://docs.browseroperator.io) – Comprehensive guides and API references +- 💬 [Discord](https://discord.gg/fp7ryHYBSY) – Join our community for support and discussions +- 🐛 [GitHub Issues](https://github.com/BrowserOperator/browser-operator-core/issues) – Report bugs or request features -## 🙏 Acknowledgments +**Contribute** +- Submit pull requests, report bugs, improve docs, or share ideas +- See [Contributing Guide](CONTRIBUTING.md) and [build instructions](https://docs.browseroperator.io) in our docs -Browser Operator is built on top of Chromium and integrates with numerous open-source projects. Special thanks to all contributors and the open-source community. +--- ## 📄 License @@ -224,6 +78,8 @@ Browser Operator is released under the [BSD-3-Clause License](LICENSE).
-**⭐ Star this repo to support open-source AI development!** +**⭐ Star this repository to support open-source AI browser development** + +[browseroperator.io](https://browseroperator.io) -
\ No newline at end of file + diff --git a/assets/Banner.png b/assets/Banner.png new file mode 100644 index 0000000000000000000000000000000000000000..4fa9ac87b872335af75e30115bcb447b5d5e0368 GIT binary patch literal 800433 zcmeFYXIN9)w>^v%rHF+tMJ$L^>76JMQ6W+jnxRSv9i*2K5dj4$inLH79TIvdN>>C_ z2rYyTQIQ&YAe0dDa_+tV^E>B$f4|-RDOpdR?6udPYmPa_9P72=V_kL@eikMsCiaI9 z?in*NvAkhoI&paB^wEgAd&LS9n^NGzdv{F(k1f+#D;4Guq&+%Xb-p9=`tI@TH)7sD zWP1O=D8~MLUH$_$sq=MBG4*Z{<{|E>S-fqqB$ZiA;=0F;WRU8~0N124Q@8UayBE8y znHo372OG5ItSbWCEw0?Kx_YbZ&NoiSjjQMLAHue8$iWB>fl#AG{o=FRbc59tep8~uCa`Ri-9 z|9bp?T(`YX-v3{JB^uu*_K#m2eQf)Bf%nwEhtgDz=ly$x>5{H3=f8(+fB)a4nEr32 zj)LO%=QRRq@@CrydZ7Mhn=7|JV)=z%J{5uJ>nMU$z zq5}OwHoh*~08+;QdoDG7S`~MSXTy6+*`9WqpOou3ktHXYS@J8 ziGSak;7w${&&5giY~f!K?h__`5}>p>a-9UdKs$R4wWsKuQIL*Or96RM=qZgwrqF9T z?!XVz-9+j?-R{A(9(tD29Q44Cq}bnCnzO`O5(zGu@4`lAO!d3h8bwCY!DP3?Gn9Gx z|8-BN3GiiJu;CUu6d#c*RBr0C)j_l18ZQN=x?oz)`Od|4%b1=n(=JFksHr)-FCmf8 z8M(@BgKbAwv0zVA3KzR8;e+l(4aAw)H8T-{+7-)FhEjZ_CJ@c=6u5Ntj(M{OJP1E0Q+r0l_wGn9d_wb`q8gO=|0-bRa97#j`!NQ7Hd5l-Y-=)cH?oYqeA zn%6#MWQ7s8jMC-O2bbd(%)X(!u6s2I2-W5bABKqmbNuROKU%ifCfT}fgsYkJ$9yC| z`>1HWhYDjLl+);0fM7Sk*0}Az=kRm+)6(%gz)s^p7*$V4PQK!51LyfN?F_KVmY-vP zPt!i^iJOwvb?}|;Uy`Pmw^vKg6+%P?uR2+tQYF7VLsS8y^^WJID0tg6o#&Q_g9uA@ z5tdx?h*wIphf9~LZV)bFNZW&SoQR zjB5wzFgrewM`kN?w)THTKyE0v+xzwBk1srdci?I% z-;cRp4i|ghu;vUljKgabe5@iBlf?5`JMLcEtBinN;Fgj!_l7Xq0iuOuN8aZY!%MQ0j*@trS(|hW3>FEiXqZFQ-XvM#3ssCKI7e$46=}4pq z?xsymuEk!pHp~*N0LyW+)N;OW9o}act&ClD*YWFz6X${3&n_I6HrxWF&Z)=t!_3d* z7p{FY%vY>9Q?Rsi-d=Ii961TFV1DdSqhQI8K@Rc^8i~Egu@gx^J-$;zI)K194zQR0 z*Jh0N91I^o)1TECgsMx(C*hjJ;VBBoY$ESb9-;N{ovB>+aJ<;LvDAz(85?P9oZ2^w zgH@5CG}zswHs{Jp!wC!qe<$Pnx)g{mjw=2+Kj2S_gDu&w8hq=_2h1H{HxmkK2GoIn#BiR(%Bx!*s2%?Iz_I~VKpnsQdll-p(6HIAnS4sbahFIzB{meE#3k_(Q1CacgI^pIT7hX@!YmZA(Y!kxOiYiD83|&ws8tDLx`6$GNZWzpfPXxcmlCM_%Yw3W8l7l>ZCJ2xlXbt^QqHf2w1#Z-V3u+m+Z0Of zpn|~oCQz^*3d`|O#{e5uaK-72BNt&)o~uIKMdvxST3SW3~}@aS|_0cXZU2@5JmDvN*wrA&!(a@R{l}Pdj@s zJA;W&2S~F&^O+@p^pG*TC%dM_8F^;rzTC^>J}~5JiV1Be?B_=pxkDR(j|*eOCXTwD z>J(IK6jx-%TlS(<dP!JVkhW+jo+vuM=FHN9JmdGzE*&I9x+e2dla>%bS&~7Exk)- z!iR@Fq_8lWi??}u*lOj3@NV1V6Ze>&PxZGzX@M(b;lQ8aS!-lPjj201zeq?1Jp$Ft z{Eq-^lr3pnkU6l1*|ZwI|5_>$#$ENj)J;p1U0eJs-w#YW7G>4v@OF{x_@L%odT>Wy z11VYVD_=K9aOYrq3TmSZ2S+Z!BCVYKjZdo>7{nej4o9+ZRyIM#e;#*VI4gTy@liJu zhA7gT_MVwtZ>{M2Kw^q2l%;KM4ZoxnXcOA9*R;VAX&7ycZhcCvXnDtXE?WCuJt-_? z`kpM4E&tG;Ch7|9OiX$1QenV!P?6^TLf$`N=G;gX=v!B-JJ@77+e4DYdsyU$RmMT)X&X~GdLtE~^0-J|hfIbLEa;=Ggamw;s1y6# zF(o|+)N}QurGf>3RS!>`XS}x?TuN35=i7`a2wIjOu2lt&A*M1zHdL}XeolFcy+|*5 zxLbWDmb;y?XcfN_fsUcZxH{=D4cSj!Z+=l}cT2;Py^41yczH=J2X~McYuC5s15Vm@ zyg|I$X36YY@hu19$t6NG&~*3zb$#5g>$8`6(?vm^y<-eT_s2S1a&&Ij6my~qz^dKRVVN+$Km|y}JQ@>L zk@37m157?zZC(Yieiv!W$Rd!!~m7`joQ>+lP zBHr-p3Nbbpx26P7rtdlV;ArIipj_zXqT z=oAoq53x0%`=3B!ViJh@e)2y0;4vgh_=ainZHnj|Q2`-IiL3H(X<>=66dZiQiMi(t zF7qkLFsD1Pir|-K4Yx~NTn^z{BWt1bCRgx&&yo;#M2zDp!){V5*?na6yECk)s0wvD z(lR1ikLNV(;l0|`wVtJ)PGg;+s^vLCURq5ST0Jh}nAcXN%KWi!aSDH292L#t_B@dj zUO$_Hf|=TW89y+!8^ATSw(LxYlqmc|lxho4)3@E~xP%o)V$SYs2%w_*lB+YNUIM`- z@)WOCKdo_~jaw+)f3O z#c9*n4@-OG_~o0{NK}H4QNR2!{>x3f;iWtn7Rm8l#&Ot$euf*<7Q&tux zbN@=3o6{7qj271X$w#=DQ=pb;u_JM-(s`O08>NiJMs((~I7^zK_>06f)EM3x$b(U` zT*!=slRql5;Z?~EyZ&d_Im5E>!U0XrhvX7wwRSGT!r76J3Q-TcH#-9N0%h_a%@{*! z*_u-DLp0w&pY4<=E8vA6?9Q@-dongb8!_pTC$^68+R%}c>D-WoTU3~!XLh#3xqnQU z=I3^4qc--lw(+bS!{Tt}SU=|Hf^nN1789|bV18Ys8a}3e}k2GKgqWn-b+Np;P+`QD5p6vhpkZ053%-B$&GEKh zy9>Gi7Y>c}=W(E}4*{*TDgQ3PfIoeN%3?y?S+(JXjkgJ**M(DMOOOplzna$&6@Mqt zBk^b|LqBOtmeudl)-bqZ^k=i9WsD!-_l4sis+qVLmUXTyaJknYGokh?4A~>|*23mW6;39qQ zVChp#TRtSj&R81fMIxRh0dN=?#un5UGL4gy6vw7)@o&Mv2}a@ zqxyJGPkU@T^sYY-71HM>GRlWFEN}A6i zO{I`uMAbCbAR&sfHVmP6iOx{)5Wq?iY!hEG9h%OPXI0dOZ|4BGIq){V5McU%md(gD z6En>*(BwS6GQlC}$|JP@JAG6>{SzGgJ>|j&G=?mq_;CjTR>hJHykgs4c~Yh~ z;vA8s7~Un&AkXK4v}6F`o?|I4$+4-X=V=Q5Q)=Ik{lVpOKvg{&P)9IV{$)L@iVcgZ zZgU>{8mj6sf>y5vY*Y7Y@PN`bYy$ArwCTa`dHlPW9K8ZPhl?!Gw3KJ2FQm+#lH5M-AIxHt}EtzsHxf>#DwI+|ajMDqUfYLkUy4{AigALx0`e;8| zRf*^PVgT#L;v}{DPe_Lc6i=}t&`S&unDKIgbM}(|-*Ec;Q1lXS*~~l0NgYiCJ~AfV zqBJAfARJvLO&s}0E*QFt6w~bzP z^pu=He{U+4NSEfgdA1&cCyw9=4gu3ge3FgjjlcWh?olDR#049SbOAS7>C!`wihz}N z;X|((-(mp9OZk4HUi;AcUu1NaXH@gU`2F_bj)2HakH(ccqxQ!r|2P9-cgjZzeA3PA zz%Tp#2mK8yrlf8Czqp>!Fh|*?77w@Tuvg59H6D}RU4qPfugJK=kXObW=yLZc!zee7 zac)#6h&~)#hz1eLT+j|~W>7d^@zR{(1Y4{cxLzpFA_$MvZ1A{-(p7_O4frwgHwHkw z<65ZrxE4+LWY_rG8~ll>}!1vbrl+MKAN0~TegU8OEP|;rVihIB}wdp#~#fDaXSHZvYOq=h^ zkCUQ%eA_0BgX@fRm^!w9UK_4}yiuZJY1ZEJZYigV%@ZPD#9}K7Y7Oa>AF5!ce1Lyz z&(x)KeVVcLZ2tB2aJaE|Y}N#qnP4lptjtp~v=~BXa zx9rb+_bnnm(i9*Q-kCnxF2!&Z9TbU6qV!=;b~U9HoLZLn!iWmxCk&txubpk#(FNOZ=UL1M=lIGQa-R z%jk2QgZ%uM0Ubt5xL%}ecF-K(<)vQrwPj7uPV?55HkmA&m=g8v7dY8+K=D6X;9I$$TAd9iBY0kKa z=?&UvX5rpTbk%|Q67<6Xzf#*5&B zBZrT=DAG}-Gk<5Nw8C_dj?nqCCEpag0dwYC*fRw|UkLWb7v#H}RHLUt}mX| zV`Pe_-=}o`0a`fta347JWaN#^+-xo4oqi}u`!wV!qt+n6y=_}GZMM4w^~|>TR(-wB zl6uSBf~fFk_73n2k#pF@LIaD5v=mGMiGDmJ9;tIBgO&XcpTO6&7P}PWzBzUFvrf|B zxh6b~e#f99@2lE_x|=Sh9IxP?HWxXufqA~PlUP|G#ygm5_U^JaomcMA;bl^keuNo( z;(fI07O(o&4Cj*Jt}gi@>5&7%-G@$^_jP|nv0YGV070?aWqJL((OO#~69uxzAroid zo?jAlg4xCx#Q@qgcS?ee1KteJgH}3qKH9W!QVF!PQcr5xPRtipRwxB|1dru2Q1eTS zoIVP)f6G;CA?hN2%^i`T&PuWDf|l_>+jIY3&^B?x**D@w7Il3>-7e`CY_k-#G(Ic_ zrIU{DvFX7j=`_i?!=`M?aLIlcdDFPcXtfU(q6%QTJV&fhI#j)T8Ct3|PPc>$=&)0_CxPHm_zHaxAJT2)Uq_*0m8q9LX zM^--ra?$_OmN)IEzKF#s^&N&lKj_x(f^DbI0DbgrD&2Y^rF%2Q-G6-@<&Yi`{0N|MCn}1M+B0r;0SG&Wa2zQ_9;J<4%4h zkEiaD3&a&xBj#wM6Rn=CjcnQMnpC$h5q`iEy%Qf033KgrJAOxqu+tlV8O4qFIdzi5 zJKNJLxV>Fkyg(?}%ar_axD^q#{C5#{n?335n_v%FN~Htw971rZ6lEX-Ff!w1cT8G= zD0j5j^L_><3vIGZZ-rjq%)-aO7PvSyQH;EK8c(0HGkTPEOps{z`%5485a~jXBMZt9gm8uapiws)(@vg+$S+S4^OvR%1^~@CHTRG)GfX7K&>WT`9gGGF6Eu7Qi z{-bw!cDdw6{SAvA@@GzbO@u8*pN{H0Bf!o+$Yue90$nYc0|fJ*=-y2Ojl5NwC`NrO z;Rsug*i-trW}Bo9h<;)*#14hH@Q0zJ&e@Qc%OmiO#e?&oSKI|+59V0iU1`6SVatuq zZZ_Ey`7@m&rv8(2YJLw63gZz%z=M#jZ6l7PxWx@O21YP=sSK|{I%~934tsp}mbdFciqDkU2bTTiO^yJ*8M$Js; z5HK&yq${VtTA0i`H6ra%_jiA*jp(Qi8jNlT+PN-(GUEkh*e5vpkY|0szw}+gxVZp$ zY;;ruUoaLM)BEGYLX{Bax@FY83Uw!nZY8z8-yG9+AINvy zNjs^V^rJ{<^i3?m|6Rx4?Ff27fWU+S>6yJQ<*kYq#MbXHjSHX{#_}$>xR!V0=U$(h z^kt1Ju3%n25s=GvC>Ca+7{P;02o@uf)9nMIUgoZTnirZ#c8{(OM(4Ia%KjB$=lsIRfz=iL#VJ)$M-7Po zx!yRdg%3zV?D`G@IHI^)3mkPRu!Y!3hf=H~4#47i1VDch;y}05xB0PrNk1ofxQlbr3;z%EmW|EFrFBT_?&0ThR7EwK3e7 ztEMNHn<9BOrqfsJYjp#>*eNSjlGwz`I%`B;7X#l9aC)$-BXTWL!{2U#r->^?{ZC;L zSJd)&=0ai{oYX&?W>?li-yiMLf+42-Mxb$Pu5|US!taVE3&QM3de0>8&=3NfAi>d#(o1zmje#g z+v_9{5;p$c5baAWmwnGxv7ddu&E6;GrJmA4=f9|6oU{d^Uf5j7Ge|b7PzijHtdr_I zwptyuD~k#uCF7DpMy-^4-v7|>A6r@7uhGJ*SYTR;(p@z3JzP`C&hL%}zWu8^=J`U+m9POe zM0#{4*X!{-4KE7?K*ds4*B%A#`B4iejBu|JMnEg;Nr9QsS_bo)pN_=U8kVw@WBnTN z^kkwc!t`BC-rkv)nAf}JQ5FvGe(SR!=h{_2J@xn264Iq?s4A;hqgyNz;C(h)wFzN+ zU$yodKWpv9?cJ^(3k{XzDChn~R5Q=u5c?W@XbQYFudfBLPrk5yrG!}*YqV1m_fjX| zE>CH+!m)$HomQmN-D{_gMJbt0{^@Ujb_>r8{_V=5g4x6hAu!)=>E{PkIeWSGoHs^? zIcL)1fmVhfx}bz%-ml`bUi2-y{g1RJ;V5ed6+Dn}MH?sx)ZG0UQRhCv#v9*kL-b$j zV&4wv36prx^j&tQAKnPiqq&s@sHLpflNz!A;QU_O>H4+`Snb>#@fQ|NANprXgROmM z_&^-7+#3>XkL^r(W$xU_*jXPeIf9vLNv9Nr zS!=cikBDiBtZ`#ILfXMrrR#+sY>qc`FprkD0bER)_!rO`NsdPGp8c4-YqU$As?~rn z@`3hphY4-lGjkM4QHL352rkbiT~h}WO+e^QXfq^U5%;Y0MFMneIrPDtd+CYtI$rMW zxfP&Vo~&A=HRn^_i(y9Qvy!IImh5{BqhAO#s30kLWwT%BPVo(WZJ66J55E?vE)=p{ z)#<@`xN!NLv6pv};WY2vuJ_Dh6PU2VC-_j|rP{WDasCZsaf+zIjN=8A`<>%X*f4_)Q70rH{()t zu=q|Oq~@$%4JMEsZ&#Q8T)b-(35=dC-r~la3+Vmiga1bN@8W@3Ki4k~4N$EcJvhaNP^yR$YsHl?l-iAkyS=JU2 z6`4xDhe5cnYlyXw5&yiOx+G_lHov3~y@{1D6M*vBF!r zATZ}=OSi!c)~`&D*9rbgpHSo%0zGA0VOz5Oa0))j`F`>8>V`W!sd7?Ds&6Le7zV^% zw%oYpHCtWhUKK}$($YjZURJPB_q?sb7pvm4#+Bg=Qtgl8cpl&hKmKeFP76hSjw5Y= zo;ugj?~tX16bm1yxmx3jLbq_N>JiFj?oVhvnc8b8V#xqTmHb@9?3MYsPgOl2jMY~c z6}rD!XQYv46#6oTRT)A>4x{})i=z$tmel^E#_Jb743@ygLh!9t#G#NQ;r2)TaIzMV z@41GW@g<7R4Wm={z@V+|k*x8?)|;b^*22r63XN1G=FDqu(5De8(g;|`VXXAP-dw@G zvN;gz;@?uLZOME`(~E!UJbX;u^R!mV>ya9k5;IBmpj})T{ZI8+H{`V4JRxwVwu`Fk zd6pXO8BdK%eY`T#T3$%bLmXh24#k8kN(FQ?9Jvk1zvkqBNEBP=WY@7!S-WL%mYem!_ z!zUq5->=oEU2(P?UhxcV9)nO87AvDvzOv*2m*q+&phspR=9=Dx9QVzmu8fet*YW%x z#Zij+R|8X>pX~6t-$YGobvu&6)!5C5nzfU|?l;v|;fLbInf%7d4SgV0egJ~w6P-E{ z;(0=C)ipg!(ZRHrs4?PQK;BWolSR62*RL6FCHnoxevX!+S%;V~liFb%kn=v^pp|?ieASdl@ zLg;VlTV+RY%;PonOF>WAM)yr>tW6B{J1JvqHfe7jHx{Z<9l2Bl_NqbgKmZS8V&Cy- zN+Mp^HU2y{To-#F#G)_LlQ8gfS=4Ueee_Xr%FfARq*Ljaa|b=AW!DR8PX%HH%BvRx zci7A7A{C=fHr6zkV$~)hwewKrT9kUdpZ?$Wi=Kb~OQImVepC9%bvZz=X0bt7A1mdN zfzjoVt;wAqjXLcXz{k7&b5yiT2LNL$JADfpK>*fKvPvn)*r01na zk}K$#WjP|@^^zHB?FPT+iWzNsXy)z?kd{62^v#+#0y~$mq3L0hvyekp`G2)6(-rxQ zSEZOa3kbRA-48S*niso|et2mM5a#Ro&>t5lpgCGm= z(2t+R78g%%S7M};0BEk9{vJ0GNqz^Yk*^37>djMAlO3TjDVtt&k;9o9q<3lo{I62r#X0j8rE7*8W}@w-+a+b#ux34 zMjCRorUeBF5`ynrs$Rn{jd-8uoHSa$W0xQj{?@(!qsHOxjB7^Miai{x9~W`l2N4ap zKKje_Lg_=&`4?tSSH55k;@p$Gd8;MCQ0(`9mP}w!!2qZh-zWkS4HidbxTzud_d&xj~o&G`Ok|=Kx%?oC%d3QBz}KYMq#y)?U;l{@ftUf zIt85HIS&GNHI8=Tg(noYR6h8HJn*3cD5 z`8)xVqc6z0wz7?!RAf%5C9IUkN4%=3=P{lm` zwO|lGPq4W{5WbY{e#$YtVE{Q<31{x~u;NVE{+h*w_EO-79JZn(kBdu_!x*Uec<$l8 zwhz^jLe;`gPa-5U68hdTIoBm3*olEqm8zy(OmN6*APl*=(naml3i;f!J!m9LdiNYI zX;M1SgiI#o1hf~_p3GzD=*>&8}!6Y{=+(7XUJ3|*j_eNdfAZo z@v$`b!To?@J0m{+M2)W*$IoRkACDe9{ABP-jz}F7!Dl4M#@#w)NYi&6nQ1%k00vpF z&TBo9n%GJdACnuday!Zljc0T?$_oEu8Ib&|Gy1TqTbwQ3iHAZRjs5U+>#0M!1Xycx z*epE@HvO{6>`;H{`Cq;rI^Pdl7|gG>mpA20n#I&m(~T*|Sv?DL%r2h;Q6Di78al9r zfJEuy;NNK?p4Kh!m==L@BEHvYaV;H{BO{)ejRMnErD(`^$_P*5%#oS?`$0tN;~2x` z02=_3{tQ-hv#3L*v&P=_qP0|ex#OSYrUjwJ<4s?lHdKc0Oa)9F$@?g?yO)jMdPz34 z0q*O3%Mt04m08Bx0FMhzxTLhWRyx5WD`fYl>QUq7M9Q#Hn&$`}?=PFrKdz)Lg^>8= z;WKsVlv~ux=eI!)QSZ5O1+-}`h3=o$lyYY}N5i{X=#x}`XRCDU~?XPJQ8~ro( zq{}KwId>HPCy%!M+Dv%`(cCv5seslFo?>bU<^VKV5h-=CNt;GR8+o!9v(uXnt}BIS*)>Lj5DD=k@HP|CRCDX;KkChAG1*-sr% zSG*3-dbH^?S5~%d_BZ}}?cU#nvCv)NZ=fjC*k*Ehqh%u>zc_e(|H$J*S_&OEC4Gn+ zrn|p%YreQzSSNU$a2WNR-FWo}hbb)H0)MI^Hjncd)f?%^Mh3h5m0A~6dZ=d!u!kRO&RBB)`fh&uZZ{@F7BZDT0|9g4C7yTYqu>wrp{Gp==Uw$X zMM@r^ApH31{CCb5_3P`NY3kF2H|v`u?532Y|*&0r{_W$AA78 z_&kyF75e>FSIA=TN@#RdX=Crme1kqgv3=I3btz^0GLzQ>U;Mpe1GBC-_o3;rK=Gkd zIERc^X(=0$?XfII;uo#vd1Af(=!<+e^xf%e4UU`IEn%h3hwZAV^SpqnuN|tqjO=kp zNga3tK7|**!WT2Q00g_&6kkLY8PFB6$j)a1OQH*ee#cTRVQfEqgAi?j=sZzK{3xk( zN;nCVx3?=32_<{PI1d&jz&=a&sD~attFZ~$bfn1uyg!{u%+BuZS_{;Axag}Rytrwh z*&534{c%VicW4;6T-D{zZO$)Is#NpNgfxfCV9Qd9bRLf6;_M&D>kBubU~a)H7lmV8 zsAk9dK&U+r*2pb-gkKUS#Zn81wQxD436EKqn9O)hxYx&Bd^Gh{wn$}FgUPIi5 zF`&!iUC8BLG{6&R*npIu7-ev*%r_uHNQhyjdP)n~b)pLX%3YEG9iU=o52_Lnl7PCS zv`Xo}EE6@Ik1IzvwAac=wY$5AOiUL3S7?9aFD~wAaOHQXe-Mo|F>wt;^g5PFeEmJb zYLcL}4-fP3hVGP-n4Uuk-fmOgMk8N*m-(@d_H4>W2}JvCkbYHZn}y?Lc?p-7;wMpD zf?|K1)007?x-=OrpQplMzbzVPKm*fzDdgoH#o$ey{##M`^OT-5!vj5 zi8fYjji1xhOa~zM6pqd$$kB*>txsDI#!{CzWXSk*e7qC)?|~*H>KN{sR?3Nu_od8M z4pR!6+d3R*=~0Q#t=RqrbDpN?)8cjBzosNSsm+&jSA6y(9)qQoiQFpV3N<;;{}d96 z?=1Y6TOt>DXX3xk-DZ7kr*vdNcoY}LK;5o#k!swOGWzu zcP>q+!Pv&)>w}af|K~{O^Y*%jI{nhaP)}9XGR!fie*dyamb_@kBQHu*AsoWUR!S1$ z!PrOTFYf-9x8q(Wr&iSJGPZBf8e6H=2!^Y5wfgikH^3f!WPGA^zClg{^CNHIG43ia z>PFQ!jhB3-F4>w={tf53w>L0f&^m!ciPytu3%5e+_hUWUa!qnu=~NuQzZeQ7^CDB{ zj-v3q#t`4i=0PMYCA;}Hai>`>ZF@o z$I2UHD?nDGkLjzIc(+|ju`gmXl+KdFzkz^ULTMIUCod05u84wXIX?{eAQ!vTV zPpxQKz_DHgqhjqQ3Id=S9}83ji?8W*hgx+b9w=_Qm(7@Uj2r4EYm3kJvBtQ!cHu7|S(*rsGsi$S#wTLQ@-i@cXk*zBMTL@k1 zYQ425X$LwoQL77isXI?siSwguc7wHIQ-6pqm3chU;)qQ-+ndV*m%YEQED`eIyfe`^ zbW8)mJ!to}nzoPA*(IJRcD%NYd8~%?)N1e#Iz0H|T*+gQOXsi&{2WG)=wdxf(NUyjduT)y6z<1M29{b%4q5;hrhNPmP*i?uSS(mRkbZ~V|M<{ihl-fZB`Yr1rg0j9n=WO(YeZmTK0AGkM0*ND{%Q>!I6P=4Rkm)+pleA7ojM`=#x<#?+dfK+ z(!nH_pcMLW-|s5Ug`Tb(kaj8YNbX<#l4mm^$=bw=bHhtfOqXF{ENZ8vg*-V1y0EcS_C}k(NwI{}IPO!Z>w-_+dw00@cmn>fG zv47)cvyV5y+kz+mD5;GH?B>b{(OR$&a{2M`2HMWBm1phI$rkv`_FljEqjP?>%w