diff --git a/config/gni/devtools_grd_files.gni b/config/gni/devtools_grd_files.gni index 7392c90203c..b672463989f 100644 --- a/config/gni/devtools_grd_files.gni +++ b/config/gni/devtools_grd_files.gni @@ -656,6 +656,7 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/ui/mcp/MCPConnectionsDialog.js", "front_end/panels/ai_chat/ui/mcp/MCPConnectorsCatalogDialog.js", "front_end/panels/ai_chat/ui/EvaluationDialog.js", + "front_end/panels/ai_chat/ui/WebAppCodeViewer.js", "front_end/panels/ai_chat/core/AgentService.js", "front_end/panels/ai_chat/core/State.js", "front_end/panels/ai_chat/core/Graph.js", @@ -676,6 +677,7 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/core/AgentDescriptorRegistry.js", "front_end/panels/ai_chat/core/Version.js", "front_end/panels/ai_chat/core/VersionChecker.js", + "front_end/panels/ai_chat/core/LLMConfigurationManager.js", "front_end/panels/ai_chat/LLM/LLMTypes.js", "front_end/panels/ai_chat/LLM/LLMProvider.js", "front_end/panels/ai_chat/LLM/LLMProviderRegistry.js", @@ -702,6 +704,15 @@ grd_files_bundled_sources = [ "front_end/panels/ai_chat/tools/BookmarkStoreTool.js", "front_end/panels/ai_chat/tools/DocumentSearchTool.js", "front_end/panels/ai_chat/tools/ThinkingTool.js", + "front_end/panels/ai_chat/tools/RenderWebAppTool.js", + "front_end/panels/ai_chat/tools/GetWebAppDataTool.js", + "front_end/panels/ai_chat/tools/RemoveWebAppTool.js", + "front_end/panels/ai_chat/tools/FileStorageManager.js", + "front_end/panels/ai_chat/tools/CreateFileTool.js", + "front_end/panels/ai_chat/tools/UpdateFileTool.js", + "front_end/panels/ai_chat/tools/DeleteFileTool.js", + "front_end/panels/ai_chat/tools/ReadFileTool.js", + "front_end/panels/ai_chat/tools/ListFilesTool.js", "front_end/panels/ai_chat/common/utils.js", "front_end/panels/ai_chat/common/log.js", "front_end/panels/ai_chat/common/context.js", diff --git a/front_end/panels/ai_chat/BUILD.gn b/front_end/panels/ai_chat/BUILD.gn index 464fde0fa98..4ff6a6bd626 100644 --- a/front_end/panels/ai_chat/BUILD.gn +++ b/front_end/panels/ai_chat/BUILD.gn @@ -42,6 +42,7 @@ devtools_module("ai_chat") { "ui/SettingsDialog.ts", "ui/PromptEditDialog.ts", "ui/EvaluationDialog.ts", + "ui/WebAppCodeViewer.ts", "ai_chat_impl.ts", "models/ChatTypes.ts", "core/Graph.ts", @@ -91,8 +92,17 @@ devtools_module("ai_chat") { "tools/VectorDBClient.ts", "tools/BookmarkStoreTool.ts", "tools/DocumentSearchTool.ts", + "tools/FileStorageManager.ts", + "tools/CreateFileTool.ts", + "tools/UpdateFileTool.ts", + "tools/DeleteFileTool.ts", + "tools/ReadFileTool.ts", + "tools/ListFilesTool.ts", "tools/SequentialThinkingTool.ts", "tools/ThinkingTool.ts", + "tools/RenderWebAppTool.ts", + "tools/GetWebAppDataTool.ts", + "tools/RemoveWebAppTool.ts", "agent_framework/ConfigurableAgentTool.ts", "agent_framework/AgentRunner.ts", "agent_framework/AgentRunnerEventBus.ts", @@ -197,6 +207,7 @@ _ai_chat_sources = [ "ui/PromptEditDialog.ts", "ui/SettingsDialog.ts", "ui/EvaluationDialog.ts", + "ui/WebAppCodeViewer.ts", "ui/mcp/MCPConnectionsDialog.ts", "ui/mcp/MCPConnectorsCatalogDialog.ts", "ai_chat_impl.ts", @@ -248,8 +259,17 @@ _ai_chat_sources = [ "tools/VectorDBClient.ts", "tools/BookmarkStoreTool.ts", "tools/DocumentSearchTool.ts", + "tools/FileStorageManager.ts", + "tools/CreateFileTool.ts", + "tools/UpdateFileTool.ts", + "tools/DeleteFileTool.ts", + "tools/ReadFileTool.ts", + "tools/ListFilesTool.ts", "tools/SequentialThinkingTool.ts", "tools/ThinkingTool.ts", + "tools/RenderWebAppTool.ts", + "tools/GetWebAppDataTool.ts", + "tools/RemoveWebAppTool.ts", "agent_framework/ConfigurableAgentTool.ts", "agent_framework/AgentRunner.ts", "agent_framework/AgentRunnerEventBus.ts", @@ -393,6 +413,7 @@ ts_library("unittests") { "ui/__tests__/ChatViewSequentialSessionsTransition.test.ts", "ui/__tests__/ChatViewInputClear.test.ts", "ui/__tests__/SettingsDialogOpenRouterCache.test.ts", + "ui/__tests__/WebAppCodeViewer.test.ts", "ui/input/__tests__/InputBarClear.test.ts", "ui/message/__tests__/MessageCombiner.test.ts", "ui/message/__tests__/StructuredResponseController.test.ts", @@ -413,6 +434,11 @@ ts_library("unittests") { "ui/__tests__/AIChatPanel.test.ts", "ui/__tests__/LiveAgentSessionComponent.test.ts", "ui/message/__tests__/MessageList.test.ts", + "tools/__tests__/CreateFileTool.test.ts", + "tools/__tests__/UpdateFileTool.test.ts", + "tools/__tests__/ReadFileTool.test.ts", + "tools/__tests__/ListFilesTool.test.ts", + "tools/__tests__/FileStorageManager.test.ts", ] deps = [ diff --git a/front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts b/front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts index ceb55c4ab36..b9050f7391a 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts @@ -8,7 +8,7 @@ import { SchemaBasedExtractorTool } from '../../tools/SchemaBasedExtractorTool.j import { StreamlinedSchemaExtractorTool } from '../../tools/StreamlinedSchemaExtractorTool.js'; import { BookmarkStoreTool } from '../../tools/BookmarkStoreTool.js'; import { DocumentSearchTool } from '../../tools/DocumentSearchTool.js'; -import { NavigateURLTool, PerformActionTool, GetAccessibilityTreeTool, SearchContentTool, NavigateBackTool, NodeIDsToURLsTool, TakeScreenshotTool, ScrollPageTool, WaitTool } from '../../tools/Tools.js'; +import { NavigateURLTool, PerformActionTool, GetAccessibilityTreeTool, SearchContentTool, NavigateBackTool, NodeIDsToURLsTool, TakeScreenshotTool, ScrollPageTool, WaitTool, RenderWebAppTool, GetWebAppDataTool, RemoveWebAppTool, CreateFileTool, UpdateFileTool, DeleteFileTool, ReadFileTool, ListFilesTool } from '../../tools/Tools.js'; import { HTMLToMarkdownTool } from '../../tools/HTMLToMarkdownTool.js'; import { ConfigurableAgentTool, ToolRegistry } from '../ConfigurableAgentTool.js'; import { ThinkingTool } from '../../tools/ThinkingTool.js'; @@ -49,7 +49,17 @@ export function initializeConfiguredAgents(): void { ToolRegistry.registerToolFactory('scroll_page', () => new ScrollPageTool()); ToolRegistry.registerToolFactory('wait_for_page_load', () => new WaitTool()); ToolRegistry.registerToolFactory('thinking', () => new ThinkingTool()); - + ToolRegistry.registerToolFactory('create_file', () => new CreateFileTool()); + ToolRegistry.registerToolFactory('update_file', () => new UpdateFileTool()); + ToolRegistry.registerToolFactory('delete_file', () => new DeleteFileTool()); + ToolRegistry.registerToolFactory('read_file', () => new ReadFileTool()); + ToolRegistry.registerToolFactory('list_files', () => new ListFilesTool()); + + // Register webapp rendering tools + ToolRegistry.registerToolFactory('render_webapp', () => new RenderWebAppTool()); + ToolRegistry.registerToolFactory('get_webapp_data', () => new GetWebAppDataTool()); + ToolRegistry.registerToolFactory('remove_webapp', () => new RemoveWebAppTool()); + // Register bookmark and document search tools ToolRegistry.registerToolFactory('bookmark_store', () => new BookmarkStoreTool()); ToolRegistry.registerToolFactory('document_search', () => new DocumentSearchTool()); diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgent.ts index 7edc91fe60e..6cabdc6045a 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgent.ts @@ -91,6 +91,14 @@ Conclusion: Fix the args format and retry with proper syntax: { "method": "fill" 'node_ids_to_urls', 'scroll_page', 'take_screenshot', + 'render_webapp', + 'get_webapp_data', + 'remove_webapp', + 'create_file', + 'update_file', + 'delete_file', + 'read_file', + 'list_files', ], maxIterations: 10, modelName: MODEL_SENTINELS.USE_MINI, diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts index fd3d67ddb85..dd780f6a8aa 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts @@ -25,6 +25,12 @@ You are specifically designed to collaborate with the research_agent. When you r - Collected research data, which may include web content, extractions, analysis, and other information - Your job is to organize this information into a comprehensive, well-structured report +Use the session file workspace as your shared knowledge base: +- Immediately call 'list_files' to discover research artifacts (notes, structured datasets, outstanding questions) created earlier in the session. +- Read the relevant files before outlining to understand what has already been captured, current confidence levels, and any gaps that remain. +- If the handoff references specific files, open them with 'read_file' and incorporate their contents, citing source filenames or URLs when appropriate. +- Persist your outline, intermediate synthesis, and final report with 'create_file'/'update_file' so future revisions or downstream agents can reuse the material. + Your process should follow these steps: 1. Carefully analyze all the research data provided during the handoff 2. Identify key themes, findings, and important information from the data @@ -49,7 +55,13 @@ Your process should follow these steps: 10. **References**: Properly formatted citations for all sources used The final output should be in markdown format, and it should be lengthy and detailed. Aim for 5-10 pages of content, at least 1000 words.`, - tools: [], + tools: [ + 'read_file', + 'list_files', + 'create_file', + 'update_file', + 'delete_file', + ], maxIterations: 3, modelName: MODEL_SENTINELS.USE_MINI, temperature: 0.3, diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts index 67182068429..377dae71244 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts @@ -56,13 +56,20 @@ First, think through the task thoroughly: - **html_to_markdown**: Use when you need high-quality page text in addition to (not instead of) structured extractions. - **fetcher_tool**: BATCH PROCESS multiple URLs at once - accepts an array of URLs to save tool calls +### 3. Workspace Coordination +- Treat the file management tools as your shared scratchpad with other agents in the session. +- Start each iteration by calling 'list_files' and 'read_file' on any artifacts relevant to your task so you understand existing progress. +- Persist work products incrementally with 'create_file'/'update_file'. Use descriptive names (e.g. 'research/-sources.json') and include agent name, timestamp, query used, and quality notes so others can audit or extend the work. +- Append to existing files when adding new findings; only delete files if they are obsolete AND all valuable information is captured elsewhere. +- Record open questions or follow-ups in dedicated tracking files so parallel subtasks avoid duplicating effort. + **CRITICAL - Batch URL Fetching**: - The fetcher_tool accepts an ARRAY of URLs: {urls: [url1, url2, url3], reasoning: "..."} - ALWAYS batch multiple URLs together instead of calling fetcher_tool multiple times - Example: After extracting 5 URLs from search results, call fetcher_tool ONCE with all 5 URLs - This dramatically reduces tool calls and improves efficiency -### 3. Research Loop (OODA) +### 4. Research Loop (OODA) Execute an excellent Observe-Orient-Decide-Act loop: **Observe**: What information has been gathered? What's still needed? @@ -83,7 +90,7 @@ Execute an excellent Observe-Orient-Decide-Act loop: - NEVER repeat the same query - adapt based on findings - If hitting diminishing returns, complete the task immediately -### 4. Source Quality Evaluation +### 5. Source Quality Evaluation Think critically about sources: - Distinguish facts from speculation (watch for "could", "may", future tense) - Identify problematic sources (aggregators vs. originals, unconfirmed reports) @@ -143,7 +150,9 @@ When your research is complete: 3. The handoff tool expects: {query: "research topic", reasoning: "explanation for user"} 4. The content_writer_agent will create the final report from your research data -Remember: You gather data, content_writer_agent writes the report. Always hand off when research is complete.`, +Remember: You gather data, content_writer_agent writes the report. Always hand off when research is complete. + +Before handing off, ensure your latest findings are reflected in the shared files (e.g. summaries, raw notes, structured datasets). This enables the orchestrator and content writer to understand what has been completed, reuse your artifacts, and avoid redundant rework.`, tools: [ 'navigate_url', 'navigate_back', @@ -152,7 +161,12 @@ Remember: You gather data, content_writer_agent writes the report. Always hand o 'node_ids_to_urls', 'bookmark_store', 'document_search', - 'html_to_markdown' + 'html_to_markdown', + 'create_file', + 'update_file', + 'delete_file', + 'read_file', + 'list_files', ], maxIterations: 15, modelName: MODEL_SENTINELS.USE_MINI, diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts index dd5c459a695..55bdd77eb5e 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts @@ -23,6 +23,7 @@ export function createSearchAgentConfig(): AgentToolConfig { ## Operating Principles - Stay laser-focused on the requested objective; avoid broad reports or narrative summaries. - Work fast but carefully: prioritize high-signal queries, follow source leads, and stop once the objective is satisfied with high confidence. +- Use the session file workspace to coordinate: list existing files before launching new queries, read relevant artifacts, record harvested leads or verified results with 'create_file'/'update_file', and append incremental progress instead of creating overlapping files. - Never fabricate data. Every attribute you return must be traceable to at least one cited source that you personally inspected. ## Search Workflow @@ -32,6 +33,7 @@ export function createSearchAgentConfig(): AgentToolConfig { - Use navigate_url to reach the most relevant search entry point (search engines, directories, LinkedIn public results, company pages, press releases). - Use extract_data with an explicit JSON schema every time you capture structured search results. Prefer capturing multiple leads in one call. - Batch follow-up pages with fetcher_tool, and use html_to_markdown when you need to confirm context inside long documents. + - After each significant batch of new leads or fetcher_tool response, immediately persist the harvested candidates (including query, timestamp, and confidence notes) by appending to a coordination file via 'create_file'/'update_file'. This keeps other subtasks aligned and prevents redundant scraping. 4. **Mandatory Pagination Loop (ENFORCED)**: - Harvest target per task: collect 30–50 unique candidates before enrichment (unless the user specifies otherwise). Absolute minimum 25 when the request requires it. - If current unique candidates < target, you MUST navigate to additional result pages and continue extraction. @@ -47,6 +49,7 @@ export function createSearchAgentConfig(): AgentToolConfig { 5. **Verify**: - Cross-check critical attributes (e.g. confirm an email’s domain matches the company, confirm a title with two independent sources when possible). - Flag low-confidence findings explicitly in the output. + - Document verification status in the appropriate coordination file so other agents can see what has been confirmed and which leads still require attention. 6. **Decide completeness**: Stop once required attributes are filled for the requested number of entities or additional searching would be duplicative. ## Tooling Rules @@ -57,6 +60,7 @@ export function createSearchAgentConfig(): AgentToolConfig { }) - Use html_to_markdown when you need high-quality page text in addition to (not instead of) structured extractions. - Never call extract_data or fetcher_tool without a clear plan for how the results will fill gaps in the objective. +- Before starting new queries, call 'list_files'/'read_file' to review previous batches and avoid duplicating work; always append incremental findings to the existing coordination file for the current objective. ### Pagination and Next Page Handling - Prefer loading additional results directly in the SERP: @@ -128,7 +132,12 @@ If you absolutely cannot find any reliable leads, return status "failed" with ga 'extract_data', 'scroll_page', 'action_agent', - 'html_to_markdown' + 'html_to_markdown', + 'create_file', + 'update_file', + 'delete_file', + 'read_file', + 'list_files', ], maxIterations: 12, modelName: MODEL_SENTINELS.USE_MINI, diff --git a/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts b/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts index 23b49ad328d..d83646a5af0 100644 --- a/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts +++ b/front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts @@ -204,6 +204,14 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and 'take_screenshot', 'wait_for_page_load', 'thinking', + 'render_webapp', + 'get_webapp_data', + 'remove_webapp', + 'create_file', + 'update_file', + 'delete_file', + 'read_file', + 'list_files', ], maxIterations: 15, temperature: 0.3, diff --git a/front_end/panels/ai_chat/core/BaseOrchestratorAgent.ts b/front_end/panels/ai_chat/core/BaseOrchestratorAgent.ts index 52399d0e3f2..c30f7a57f6f 100644 --- a/front_end/panels/ai_chat/core/BaseOrchestratorAgent.ts +++ b/front_end/panels/ai_chat/core/BaseOrchestratorAgent.ts @@ -24,7 +24,16 @@ import { NodeIDsToURLsTool, GetVisitsByDomainTool, GetVisitsByKeywordTool, - SearchVisitHistoryTool, type Tool + SearchVisitHistoryTool, + RenderWebAppTool, + GetWebAppDataTool, + RemoveWebAppTool, + CreateFileTool, + UpdateFileTool, + DeleteFileTool, + ReadFileTool, + ListFilesTool, + type Tool } from '../tools/Tools.js'; // Imports from their own files @@ -48,6 +57,7 @@ Always delegate investigative work to the 'search_agent' tool so it can gather v - Launch search_agent with a clear objective, attribute list, filters, and quantity requirement. - Review the JSON output, double-check confidence values and citations, and surface the most credible findings. +- Use the file management tools ('create_file', 'update_file', 'read_file', 'list_files') to coordinate multi-step fact-finding. Persist subtask outputs as you go, read existing files before launching overlapping searches, and append incremental findings rather than duplicating effort. - If the user pivots into broad synthesis or long-form reporting, switch to the 'research_agent'. - Keep responses concise, cite the strongest sources, and present the structured findings provided by the agent. @@ -117,6 +127,12 @@ Based on query type, develop a specific research plan: - Synthesizing findings - Identifying gaps and deploying additional agents as needed +**Coordinate through session files:** +- Before launching a new subtask, call 'list_files' to inspect existing outputs and avoid duplication. +- Persist each subtask's plan, raw notes, and structured results with 'create_file'/'update_file'. Include timestamps and ownership so other agents can build on the work. +- Encourage sub-agents to read relevant files ('read_file') before acting, and to append updates instead of overwriting unless the instructions explicitly call for replacement. +- Use file summaries to track progress, surface blockers, and keep an audit trail for the final synthesis. + **Clear instructions to research agents must include:** - Specific research objectives (ideally one core objective per agent) - Expected output format with emphasis on collecting detailed, comprehensive data @@ -299,6 +315,14 @@ export const AGENT_CONFIGS: {[key: string]: AgentConfig} = { ToolRegistry.getToolInstance('research_agent') || (() => { throw new Error('research_agent tool not found'); })(), new FinalizeWithCritiqueTool(), new SearchVisitHistoryTool(), + new RenderWebAppTool(), + new GetWebAppDataTool(), + new RemoveWebAppTool(), + new CreateFileTool(), + new UpdateFileTool(), + new DeleteFileTool(), + new ReadFileTool(), + new ListFilesTool(), ] }, [BaseOrchestratorAgentType.DEEP_RESEARCH]: { @@ -315,6 +339,14 @@ export const AGENT_CONFIGS: {[key: string]: AgentConfig} = { ToolRegistry.getToolInstance('bookmark_store') || (() => { throw new Error('bookmark_store tool not found'); })(), ToolRegistry.getToolInstance('search_agent') || (() => { throw new Error('search_agent tool not found'); })(), new FinalizeWithCritiqueTool(), + new RenderWebAppTool(), + new GetWebAppDataTool(), + new RemoveWebAppTool(), + new CreateFileTool(), + new UpdateFileTool(), + new DeleteFileTool(), + new ReadFileTool(), + new ListFilesTool(), ] }, // [BaseOrchestratorAgentType.SHOPPING]: { @@ -500,6 +532,11 @@ export function getAgentTools(agentType: string): Array> { ToolRegistry.getToolInstance('research_agent') || (() => { throw new Error('research_agent tool not found'); })(), new FinalizeWithCritiqueTool(), new SearchVisitHistoryTool(), + new CreateFileTool(), + new UpdateFileTool(), + new DeleteFileTool(), + new ReadFileTool(), + new ListFilesTool(), ]; } diff --git a/front_end/panels/ai_chat/tools/CreateFileTool.ts b/front_end/panels/ai_chat/tools/CreateFileTool.ts new file mode 100644 index 00000000000..12ffabbe565 --- /dev/null +++ b/front_end/panels/ai_chat/tools/CreateFileTool.ts @@ -0,0 +1,73 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext } from './Tools.js'; +import { FileStorageManager, type StoredFile } from './FileStorageManager.js'; + +const logger = createLogger('Tool:CreateFile'); + +export interface CreateFileArgs { + fileName: string; + content: string; + mimeType?: string; + reasoning: string; +} + +export interface CreateFileResult { + success: boolean; + fileId?: string; + fileName?: string; + message?: string; + error?: string; +} + +export class CreateFileTool implements Tool { + name = 'create_file'; + description = 'Creates a new file in the current session storage. Fails if the file already exists.'; + + schema = { + type: 'object', + properties: { + fileName: { + type: 'string', + description: 'Unique name of the file to create (no path separators)' + }, + content: { + type: 'string', + description: 'Content to write to the file' + }, + mimeType: { + type: 'string', + description: 'Optional MIME type describing the content (default: text/plain)' + }, + reasoning: { + type: 'string', + description: 'Explanation for why this file is being created for the user' + } + }, + required: ['fileName', 'content', 'reasoning'] + }; + + async execute(args: CreateFileArgs, _ctx?: LLMContext): Promise { + logger.info('Executing create file', { fileName: args.fileName }); + const manager = FileStorageManager.getInstance(); + + try { + const file: StoredFile = await manager.createFile(args.fileName, args.content, args.mimeType); + return { + success: true, + fileId: file.id, + fileName: file.fileName, + message: `Created file "${file.fileName}" (${file.size} bytes).` + }; + } catch (error: any) { + logger.error('Failed to create file', { fileName: args.fileName, error: error?.message }); + return { + success: false, + error: error?.message || 'Failed to create file.' + }; + } + } +} diff --git a/front_end/panels/ai_chat/tools/DeleteFileTool.ts b/front_end/panels/ai_chat/tools/DeleteFileTool.ts new file mode 100644 index 00000000000..67434977585 --- /dev/null +++ b/front_end/panels/ai_chat/tools/DeleteFileTool.ts @@ -0,0 +1,59 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext } from './Tools.js'; +import { FileStorageManager } from './FileStorageManager.js'; + +const logger = createLogger('Tool:DeleteFile'); + +export interface DeleteFileArgs { + fileName: string; + reasoning: string; +} + +export interface DeleteFileResult { + success: boolean; + message?: string; + error?: string; +} + +export class DeleteFileTool implements Tool { + name = 'delete_file'; + description = 'Deletes a file from the current session storage.'; + + schema = { + type: 'object', + properties: { + fileName: { + type: 'string', + description: 'Name of the file to delete' + }, + reasoning: { + type: 'string', + description: 'Explanation for why the file can be safely deleted' + } + }, + required: ['fileName', 'reasoning'] + }; + + async execute(args: DeleteFileArgs, _ctx?: LLMContext): Promise { + logger.info('Executing delete file', { fileName: args.fileName }); + const manager = FileStorageManager.getInstance(); + + try { + await manager.deleteFile(args.fileName); + return { + success: true, + message: `Deleted file "${args.fileName}".` + }; + } catch (error: any) { + logger.error('Failed to delete file', { fileName: args.fileName, error: error?.message }); + return { + success: false, + error: error?.message || 'Failed to delete file.' + }; + } + } +} diff --git a/front_end/panels/ai_chat/tools/FileStorageManager.ts b/front_end/panels/ai_chat/tools/FileStorageManager.ts new file mode 100644 index 00000000000..b11c3d7f1d3 --- /dev/null +++ b/front_end/panels/ai_chat/tools/FileStorageManager.ts @@ -0,0 +1,281 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import { createLogger } from '../core/Logger.js'; + +const logger = createLogger('FileStorageManager'); + +const DATABASE_NAME = 'ai_chat_agent_files'; +const DATABASE_VERSION = 1; +const OBJECT_STORE_NAME = 'files'; +const INDEX_SESSION_ID = 'sessionId'; +const INDEX_FILE_NAME = 'fileName'; +const INDEX_CREATED_AT = 'createdAt'; +const INDEX_SESSION_FILE_NAME = 'sessionId_fileName'; + +export interface StoredFile { + id: string; + sessionId: string; + fileName: string; + content: string; + mimeType: string; + createdAt: number; + updatedAt: number; + size: number; +} + +export interface FileSummary { + fileName: string; + size: number; + mimeType: string; + createdAt: number; + updatedAt: number; +} + +interface ValidationResult { + valid: boolean; + error?: string; +} + +/** + * Manages IndexedDB-backed file storage scoped to the current DevTools session. + */ +export class FileStorageManager { + private static instance: FileStorageManager | null = null; + + private readonly sessionId: string; + private db: IDBDatabase | null = null; + private dbInitializationPromise: Promise | null = null; + + private constructor() { + this.sessionId = this.generateUUID(); + logger.info('Initialized FileStorageManager with session', { sessionId: this.sessionId }); + } + + static getInstance(): FileStorageManager { + if (!FileStorageManager.instance) { + FileStorageManager.instance = new FileStorageManager(); + } + return FileStorageManager.instance; + } + + async createFile(fileName: string, content: string, mimeType = 'text/plain'): Promise { + const validation = this.validateFileName(fileName); + if (!validation.valid) { + throw new Error(validation.error || 'Invalid file name'); + } + + const db = await this.ensureDatabase(); + + if (await this.fileExists(fileName)) { + throw new Error(`File "${fileName}" already exists in the current session.`); + } + + const now = Date.now(); + const file: StoredFile = { + id: this.generateFileId(), + sessionId: this.sessionId, + fileName, + content, + mimeType, + createdAt: now, + updatedAt: now, + size: this.calculateSize(content), + }; + + const transaction = db.transaction(OBJECT_STORE_NAME, 'readwrite'); + const store = transaction.objectStore(OBJECT_STORE_NAME); + + await this.requestToPromise(store.add(file)); + await this.transactionComplete(transaction); + + logger.info('Created file', { fileName, fileId: file.id, size: file.size }); + return file; + } + + async updateFile(fileName: string, content: string, append = false): Promise { + const db = await this.ensureDatabase(); + const existing = await this.getFileRecord(fileName); + if (!existing) { + throw new Error(`File "${fileName}" was not found in the current session.`); + } + + const newContent = append ? `${existing.content}${content}` : content; + const updated: StoredFile = { + ...existing, + content: newContent, + updatedAt: Date.now(), + size: this.calculateSize(newContent), + }; + + const transaction = db.transaction(OBJECT_STORE_NAME, 'readwrite'); + const store = transaction.objectStore(OBJECT_STORE_NAME); + await this.requestToPromise(store.put(updated)); + await this.transactionComplete(transaction); + + logger.info('Updated file', { fileName, fileId: existing.id, append }); + return updated; + } + + async deleteFile(fileName: string): Promise { + const db = await this.ensureDatabase(); + const existing = await this.getFileRecord(fileName); + if (!existing) { + throw new Error(`File "${fileName}" was not found in the current session.`); + } + + const transaction = db.transaction(OBJECT_STORE_NAME, 'readwrite'); + const store = transaction.objectStore(OBJECT_STORE_NAME); + await this.requestToPromise(store.delete(existing.id)); + await this.transactionComplete(transaction); + + logger.info('Deleted file', { fileName, fileId: existing.id }); + } + + async readFile(fileName: string): Promise { + const record = await this.getFileRecord(fileName); + return record || null; + } + + async listFiles(): Promise { + const db = await this.ensureDatabase(); + const transaction = db.transaction(OBJECT_STORE_NAME, 'readonly'); + const store = transaction.objectStore(OBJECT_STORE_NAME); + const index = store.index(INDEX_SESSION_ID); + + const request = index.getAll(IDBKeyRange.only(this.sessionId)); + const files = await this.requestToPromise(request); + await this.transactionComplete(transaction); + + const sorted = (files || []).sort((a, b) => b.createdAt - a.createdAt); + return sorted.map(file => ({ + fileName: file.fileName, + size: file.size, + mimeType: file.mimeType, + createdAt: file.createdAt, + updatedAt: file.updatedAt, + })); + } + + private async fileExists(fileName: string): Promise { + const record = await this.getFileRecord(fileName); + return Boolean(record); + } + + private validateFileName(fileName: string): ValidationResult { + if (!fileName || !fileName.trim()) { + return { valid: false, error: 'File name cannot be empty.' }; + } + if (/[/\\]/.test(fileName)) { + return { valid: false, error: 'File name cannot contain path separators ("/" or "\\").' }; + } + if (fileName.length > 255) { + return { valid: false, error: 'File name must be 255 characters or fewer.' }; + } + return { valid: true }; + } + + private async getFileRecord(fileName: string): Promise { + const db = await this.ensureDatabase(); + const transaction = db.transaction(OBJECT_STORE_NAME, 'readonly'); + const store = transaction.objectStore(OBJECT_STORE_NAME); + const index = store.index(INDEX_SESSION_FILE_NAME); + const request = index.get([this.sessionId, fileName]); + const file = await this.requestToPromise(request); + await this.transactionComplete(transaction); + return file; + } + + private async ensureDatabase(): Promise { + if (this.db) { + return this.db; + } + if (!('indexedDB' in globalThis)) { + throw new Error('IndexedDB is not supported in this environment.'); + } + if (this.dbInitializationPromise) { + this.db = await this.dbInitializationPromise; + return this.db; + } + this.dbInitializationPromise = this.openDatabase(); + try { + this.db = await this.dbInitializationPromise; + return this.db; + } catch (error) { + this.dbInitializationPromise = null; + logger.error('Failed to open IndexedDB database', { error }); + throw error; + } + } + + private openDatabase(): Promise { + return new Promise((resolve, reject) => { + const request = indexedDB.open(DATABASE_NAME, DATABASE_VERSION); + + request.onupgradeneeded = () => { + const db = request.result; + logger.info('Initializing file storage database'); + if (!db.objectStoreNames.contains(OBJECT_STORE_NAME)) { + const store = db.createObjectStore(OBJECT_STORE_NAME, { keyPath: 'id' }); + store.createIndex(INDEX_SESSION_ID, 'sessionId', { unique: false }); + store.createIndex(INDEX_FILE_NAME, 'fileName', { unique: false }); + store.createIndex(INDEX_CREATED_AT, 'createdAt', { unique: false }); + store.createIndex(INDEX_SESSION_FILE_NAME, ['sessionId', 'fileName'], { unique: true }); + } + }; + + request.onsuccess = () => { + resolve(request.result); + }; + + request.onerror = () => { + reject(request.error || new Error('Failed to open IndexedDB')); + }; + + request.onblocked = () => { + logger.warn('File storage database open request was blocked.'); + }; + }); + } + + private requestToPromise(request: IDBRequest): Promise { + return new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error || new Error('IndexedDB request failed')); + }); + } + + private transactionComplete(transaction: IDBTransaction): Promise { + return new Promise((resolve, reject) => { + transaction.oncomplete = () => resolve(); + transaction.onerror = () => reject(transaction.error || new Error('IndexedDB transaction failed')); + transaction.onabort = () => reject(transaction.error || new Error('IndexedDB transaction aborted')); + }); + } + + private calculateSize(content: string): number { + try { + return new TextEncoder().encode(content).length; + } catch (error) { + logger.warn('Falling back to length-based size calculation', { error }); + return content.length; + } + } + + private generateFileId(): string { + return this.generateUUID(); + } + + private generateUUID(): string { + if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') { + return crypto.randomUUID(); + } + const template = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'; + return template.replace(/[xy]/g, c => { + const r = Math.random() * 16 | 0; + const v = c === 'x' ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); + } +} diff --git a/front_end/panels/ai_chat/tools/GetWebAppDataTool.ts b/front_end/panels/ai_chat/tools/GetWebAppDataTool.ts new file mode 100644 index 00000000000..3fc778c8072 --- /dev/null +++ b/front_end/panels/ai_chat/tools/GetWebAppDataTool.ts @@ -0,0 +1,261 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import * as SDK from '../../../core/sdk/sdk.js'; +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext, ErrorResult } from './Tools.js'; + +const logger = createLogger('GetWebAppDataTool'); + +/** + * Arguments for retrieving webapp data + */ +export interface GetWebAppDataArgs { + webappId: string; + reasoning: string; + waitForSubmit?: boolean; + timeout?: number; +} + +/** + * Result of webapp data retrieval + */ +export interface GetWebAppDataResult { + success: boolean; + formData: Record; + message: string; +} + +/** + * Tool for retrieving data from rendered webapp iframe + * Extracts values from input, select, textarea, checkbox, and radio elements + * within the webapp iframe. + */ +export class GetWebAppDataTool implements Tool { + name = 'get_webapp_data'; + description = 'Retrieves data from form elements within a previously rendered webapp iframe. Can optionally wait for form submission before retrieving data. Returns an object with field names as keys and their values. Supports text inputs, emails, selects, textareas, checkboxes, and radio buttons.'; + + async execute(args: GetWebAppDataArgs, _ctx?: LLMContext): Promise { + logger.info('Retrieving webapp data', { + webappId: args.webappId, + reasoning: args.reasoning, + waitForSubmit: args.waitForSubmit, + timeout: args.timeout + }); + + const { webappId, reasoning, waitForSubmit = false, timeout = 30000 } = args; + + // Validate required arguments + if (!webappId || typeof webappId !== 'string') { + return { error: 'webappId is required and must be a string' }; + } + + if (!reasoning || typeof reasoning !== 'string') { + return { error: 'Reasoning is required and must be a string' }; + } + + // Get the primary page target + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (!target) { + logger.error('No primary page target available'); + return { error: 'No page target available' }; + } + + try { + const runtimeAgent = target.runtimeAgent(); + + // Wait for form submission if requested + if (waitForSubmit) { + const startTime = Date.now(); + const pollInterval = 500; // Poll every 500ms + + logger.info('Waiting for webapp form submission', { webappId, timeout }); + + while (Date.now() - startTime < timeout) { + const checkResult = await runtimeAgent.invoke_evaluate({ + expression: ` + (() => { + const iframe = document.getElementById(${JSON.stringify(webappId)}); + if (!iframe) { + return { found: false }; + } + const iframeDoc = iframe.contentDocument || iframe.contentWindow.document; + if (!iframeDoc) { + return { found: false }; + } + return { + found: true, + submitted: iframeDoc.body.getAttribute('data-submitted') === 'true' + }; + })() + `, + returnByValue: true, + }); + + const checkData = checkResult.result.value as { found: boolean; submitted?: boolean }; + + if (!checkData.found) { + logger.error('Webapp iframe not found while waiting for submission'); + return { error: `Webapp iframe not found with ID: ${webappId}` }; + } + + if (checkData.submitted) { + logger.info('Webapp form submission detected', { webappId }); + break; + } + + // Wait before next poll + await new Promise(resolve => setTimeout(resolve, pollInterval)); + } + + // Check if we timed out + if (Date.now() - startTime >= timeout) { + logger.warn('Timeout waiting for webapp form submission', { webappId, timeout }); + return { error: `Timeout waiting for webapp form submission after ${timeout}ms` }; + } + } + + // Execute data extraction script in page context + const result = await runtimeAgent.invoke_evaluate({ + expression: ` + (() => { + // Find the webapp iframe + const iframe = document.getElementById(${JSON.stringify(webappId)}); + if (!iframe) { + return { + success: false, + error: 'Webapp iframe not found with ID: ${webappId}' + }; + } + + // Access iframe document + const iframeDoc = iframe.contentDocument || iframe.contentWindow.document; + if (!iframeDoc) { + return { + success: false, + error: 'Cannot access webapp iframe content' + }; + } + + // Collect form data + const formData = {}; + + // Find all form elements within the iframe + const inputs = iframeDoc.querySelectorAll('input, select, textarea'); + + inputs.forEach((element) => { + // Get the field identifier (prefer name, fallback to id) + const fieldName = element.name || element.id; + + // Skip elements without identifiers + if (!fieldName) { + return; + } + + // Extract value based on element type + if (element.tagName.toLowerCase() === 'input') { + const inputType = element.type ? element.type.toLowerCase() : 'text'; + + if (inputType === 'checkbox') { + // For checkboxes, check if we already have this field name + if (fieldName in formData) { + // Multiple checkboxes with same name - create array + if (!Array.isArray(formData[fieldName])) { + formData[fieldName] = [formData[fieldName]]; + } + if (element.checked) { + formData[fieldName].push(element.value || true); + } + } else { + // First checkbox with this name + formData[fieldName] = element.checked ? (element.value || true) : false; + } + } else if (inputType === 'radio') { + // For radio buttons, only store if checked + if (element.checked) { + formData[fieldName] = element.value || true; + } + } else { + // Text, email, number, password, etc. + formData[fieldName] = element.value || ''; + } + } else if (element.tagName.toLowerCase() === 'select') { + // For select elements, get selected value + if (element.multiple) { + // Multiple select - get array of selected values + const selectedOptions = Array.from(element.selectedOptions || []); + formData[fieldName] = selectedOptions.map(opt => opt.value); + } else { + // Single select + formData[fieldName] = element.value || ''; + } + } else if (element.tagName.toLowerCase() === 'textarea') { + // For textareas + formData[fieldName] = element.value || ''; + } + }); + + return { + success: true, + formData: formData, + message: 'Webapp data retrieved successfully' + }; + })() + `, + returnByValue: true, + }); + + // Check for evaluation errors + if (result.exceptionDetails) { + const errorMsg = result.exceptionDetails.text || 'Unknown evaluation error'; + logger.error('Webapp data retrieval failed with exception:', errorMsg); + return { error: `Webapp data retrieval failed: ${errorMsg}` }; + } + + // Extract result + const retrievalResult = result.result.value as GetWebAppDataResult | { success: false; error: string }; + + if (!retrievalResult.success) { + const error = 'error' in retrievalResult ? retrievalResult.error : 'Unknown error'; + logger.error('Webapp data retrieval script returned error:', error); + return { error }; + } + + logger.info('Successfully retrieved webapp data', { + webappId, + fieldCount: Object.keys((retrievalResult as GetWebAppDataResult).formData).length + }); + + return retrievalResult as GetWebAppDataResult; + + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + logger.error('Failed to retrieve webapp data:', errorMsg); + return { error: `Failed to retrieve webapp data: ${errorMsg}` }; + } + } + + schema = { + type: 'object', + properties: { + webappId: { + type: 'string', + description: 'The unique webapp ID returned from the render_webapp tool. Used to identify which webapp to retrieve data from.', + }, + reasoning: { + type: 'string', + description: 'Required explanation for why this data is being retrieved (e.g., "Collecting submitted user information")', + }, + waitForSubmit: { + type: 'boolean', + description: 'If true, waits for form submission before retrieving data. The tool will poll until the form is submitted or timeout is reached. Default: false', + }, + timeout: { + type: 'number', + description: 'Maximum time to wait for form submission in milliseconds. Only used when waitForSubmit is true. Default: 30000 (30 seconds)', + }, + }, + required: ['webappId', 'reasoning'], + }; +} diff --git a/front_end/panels/ai_chat/tools/ListFilesTool.ts b/front_end/panels/ai_chat/tools/ListFilesTool.ts new file mode 100644 index 00000000000..a6a02cb9a8b --- /dev/null +++ b/front_end/panels/ai_chat/tools/ListFilesTool.ts @@ -0,0 +1,56 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext } from './Tools.js'; +import { FileStorageManager, type FileSummary } from './FileStorageManager.js'; + +const logger = createLogger('Tool:ListFiles'); + +export interface ListFilesArgs { + reasoning: string; +} + +export interface ListFilesResult { + success: boolean; + files?: FileSummary[]; + count?: number; + error?: string; +} + +export class ListFilesTool implements Tool { + name = 'list_files'; + description = 'Lists all files created during the current session along with their metadata.'; + + schema = { + type: 'object', + properties: { + reasoning: { + type: 'string', + description: 'Explanation for why the file list is needed' + } + }, + required: ['reasoning'] + }; + + async execute(_args: ListFilesArgs, _ctx?: LLMContext): Promise { + logger.info('Executing list files'); + const manager = FileStorageManager.getInstance(); + + try { + const files = await manager.listFiles(); + return { + success: true, + files, + count: files.length + }; + } catch (error: any) { + logger.error('Failed to list files', { error: error?.message }); + return { + success: false, + error: error?.message || 'Failed to list files.' + }; + } + } +} diff --git a/front_end/panels/ai_chat/tools/ReadFileTool.ts b/front_end/panels/ai_chat/tools/ReadFileTool.ts new file mode 100644 index 00000000000..12e0ff8d5a0 --- /dev/null +++ b/front_end/panels/ai_chat/tools/ReadFileTool.ts @@ -0,0 +1,76 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext } from './Tools.js'; +import { FileStorageManager, type StoredFile } from './FileStorageManager.js'; + +const logger = createLogger('Tool:ReadFile'); + +export interface ReadFileArgs { + fileName: string; + reasoning: string; +} + +export interface ReadFileResult { + success: boolean; + fileName?: string; + content?: string; + mimeType?: string; + size?: number; + createdAt?: number; + updatedAt?: number; + error?: string; +} + +export class ReadFileTool implements Tool { + name = 'read_file'; + description = 'Reads the full content and metadata for a file stored in the current session.'; + + schema = { + type: 'object', + properties: { + fileName: { + type: 'string', + description: 'Name of the file to read' + }, + reasoning: { + type: 'string', + description: 'Explanation for why the file needs to be read' + } + }, + required: ['fileName', 'reasoning'] + }; + + async execute(args: ReadFileArgs, _ctx?: LLMContext): Promise { + logger.info('Executing read file', { fileName: args.fileName }); + const manager = FileStorageManager.getInstance(); + + try { + const file: StoredFile | null = await manager.readFile(args.fileName); + if (!file) { + return { + success: false, + error: `File "${args.fileName}" was not found in the current session.` + }; + } + + return { + success: true, + fileName: file.fileName, + content: file.content, + mimeType: file.mimeType, + size: file.size, + createdAt: file.createdAt, + updatedAt: file.updatedAt, + }; + } catch (error: any) { + logger.error('Failed to read file', { fileName: args.fileName, error: error?.message }); + return { + success: false, + error: error?.message || 'Failed to read file.' + }; + } + } +} diff --git a/front_end/panels/ai_chat/tools/RemoveWebAppTool.ts b/front_end/panels/ai_chat/tools/RemoveWebAppTool.ts new file mode 100644 index 00000000000..0789e55bcb1 --- /dev/null +++ b/front_end/panels/ai_chat/tools/RemoveWebAppTool.ts @@ -0,0 +1,131 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import * as SDK from '../../../core/sdk/sdk.js'; +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext, ErrorResult } from './Tools.js'; + +const logger = createLogger('RemoveWebAppTool'); + +/** + * Arguments for removing webapp + */ +export interface RemoveWebAppArgs { + webappId: string; + reasoning: string; +} + +/** + * Result of webapp removal + */ +export interface RemoveWebAppResult { + success: boolean; + removed: string[]; + message: string; +} + +/** + * Tool for removing a rendered webapp iframe from the page + * Cleans up the iframe element and all associated resources. + */ +export class RemoveWebAppTool implements Tool { + name = 'remove_webapp'; + description = 'Removes a previously rendered webapp iframe from the page. Cleans up the full-screen iframe and releases resources. Use this after data collection is complete or when the webapp is no longer needed.'; + + async execute(args: RemoveWebAppArgs, _ctx?: LLMContext): Promise { + logger.info('Removing webapp', { + webappId: args.webappId, + reasoning: args.reasoning + }); + + const { webappId, reasoning } = args; + + // Validate required arguments + if (!webappId || typeof webappId !== 'string') { + return { error: 'webappId is required and must be a string' }; + } + + if (!reasoning || typeof reasoning !== 'string') { + return { error: 'Reasoning is required and must be a string' }; + } + + // Get the primary page target + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (!target) { + logger.error('No primary page target available'); + return { error: 'No page target available' }; + } + + try { + const runtimeAgent = target.runtimeAgent(); + + // Execute removal script in page context + const result = await runtimeAgent.invoke_evaluate({ + expression: ` + (() => { + const removed = []; + + // Remove the webapp iframe + const iframe = document.getElementById(${JSON.stringify(webappId)}); + if (iframe) { + iframe.remove(); + removed.push('iframe'); + } + + return { + success: true, + removed: removed, + message: removed.length > 0 ? + 'Webapp removed: ' + removed.join(', ') : + 'No webapp elements found to remove' + }; + })() + `, + returnByValue: true, + }); + + // Check for evaluation errors + if (result.exceptionDetails) { + const errorMsg = result.exceptionDetails.text || 'Unknown evaluation error'; + logger.error('Webapp removal failed with exception:', errorMsg); + return { error: `Webapp removal failed: ${errorMsg}` }; + } + + // Extract result + const removalResult = result.result.value as RemoveWebAppResult; + + if (!removalResult || !removalResult.success) { + logger.error('Webapp removal script returned unsuccessful result'); + return { error: 'Webapp removal script failed to execute properly' }; + } + + logger.info('Successfully removed webapp', { + webappId, + removed: removalResult.removed + }); + + return removalResult; + + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + logger.error('Failed to remove webapp:', errorMsg); + return { error: `Failed to remove webapp: ${errorMsg}` }; + } + } + + schema = { + type: 'object', + properties: { + webappId: { + type: 'string', + description: 'The unique webapp ID returned from the render_webapp tool. Used to identify which webapp to remove.', + }, + reasoning: { + type: 'string', + description: 'Required explanation for why this webapp is being removed (e.g., "User completed form submission, cleaning up")', + }, + }, + required: ['webappId', 'reasoning'], + }; +} diff --git a/front_end/panels/ai_chat/tools/RenderWebAppTool.ts b/front_end/panels/ai_chat/tools/RenderWebAppTool.ts new file mode 100644 index 00000000000..ff00a71cc76 --- /dev/null +++ b/front_end/panels/ai_chat/tools/RenderWebAppTool.ts @@ -0,0 +1,240 @@ +// Copyright 2025 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import * as SDK from '../../../core/sdk/sdk.js'; +import { createLogger } from '../core/Logger.js'; +import type { Tool, LLMContext, ErrorResult } from './Tools.js'; + +const logger = createLogger('RenderWebAppTool'); + +/** + * Arguments for webapp rendering + */ +export interface RenderWebAppArgs { + html: string; + css?: string; + js?: string; + reasoning: string; +} + +/** + * Result of webapp rendering + */ +export interface RenderWebAppResult { + success: boolean; + webappId: string; + message: string; +} + +/** + * Tool for rendering a full-screen webapp using an iframe + * This enables AI agents to render dynamic UI applications (forms, dialogs, interactive apps) + * in an isolated full-screen iframe for user interaction and data collection. + */ +export class RenderWebAppTool implements Tool { + name = 'render_webapp'; + description = 'Renders a full-screen webapp in an isolated iframe. Creates an interactive application with HTML, CSS, and JavaScript for collecting user data or showing dynamic content. The webapp runs in a sandboxed iframe with full viewport coverage. Returns a unique webappId for later data retrieval and cleanup.'; + + async execute(args: RenderWebAppArgs, _ctx?: LLMContext): Promise { + logger.info('Rendering webapp', { + htmlLength: args.html.length, + hasCss: !!args.css, + hasJs: !!args.js, + reasoning: args.reasoning + }); + + const { html, css, js, reasoning } = args; + + // Validate required arguments + if (!html || typeof html !== 'string') { + return { error: 'HTML content is required and must be a string' }; + } + + if (!reasoning || typeof reasoning !== 'string') { + return { error: 'Reasoning is required and must be a string' }; + } + + // Get the primary page target + const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget(); + if (!target) { + logger.error('No primary page target available'); + return { error: 'No page target available' }; + } + + // Navigate to blank page first for clean canvas + logger.info('Navigating to blank page before rendering webapp'); + const pageAgent = target.pageAgent(); + if (pageAgent) { + try { + const navResult = await pageAgent.invoke_navigate({ url: 'about:blank' }); + if (navResult.getError()) { + logger.warn(`Navigation to blank page failed: ${navResult.getError()}, continuing anyway`); + } else { + // Wait briefly for blank page to load (should be instant) + await new Promise(resolve => setTimeout(resolve, 300)); + logger.info('Navigated to blank page successfully'); + } + } catch (navError) { + logger.warn('Error navigating to blank page, continuing anyway:', navError); + } + } + + try { + const runtimeAgent = target.runtimeAgent(); + + // Execute webapp rendering script in page context + const result = await runtimeAgent.invoke_evaluate({ + expression: ` + (() => { + // Generate unique webapp ID + const webappId = 'devtools-webapp-' + Date.now() + '-' + Math.random().toString(36).substr(2, 9); + + // Create full-screen iframe + const iframe = document.createElement('iframe'); + iframe.id = webappId; + iframe.setAttribute('data-devtools-webapp', 'true'); + iframe.setAttribute('data-reasoning', ${JSON.stringify(reasoning)}); + + // Style iframe for full-screen coverage + iframe.style.position = 'fixed'; + iframe.style.top = '0'; + iframe.style.left = '0'; + iframe.style.width = '100vw'; + iframe.style.height = '100vh'; + iframe.style.border = 'none'; + iframe.style.zIndex = '999999'; + iframe.style.backgroundColor = 'white'; + + // Build complete HTML document for iframe + const fullHTML = '' + + '' + + '' + + '' + + '' + + (${JSON.stringify(css || '')} ? '' : '') + + '' + + '' + + ${JSON.stringify(html)} + + '' + + ''; + + // Set iframe content using srcdoc + iframe.srcdoc = fullHTML; + + // Append iframe to body + document.body.appendChild(iframe); + + // Wait for iframe to load, then inject JavaScript and submit detection + iframe.addEventListener('load', function() { + try { + const iframeDoc = iframe.contentDocument || iframe.contentWindow.document; + + // Inject JavaScript with automatic submit detection + const script = iframeDoc.createElement('script'); + script.setAttribute('data-devtools-injected', 'true'); + + // Build script content + let scriptContent = '(function() {'; + + // Add automatic submit detection for forms + scriptContent += 'const forms = document.querySelectorAll("form");'; + scriptContent += 'forms.forEach(function(form) {'; + scriptContent += ' form.addEventListener("submit", function(e) {'; + scriptContent += ' e.preventDefault();'; + scriptContent += ' document.body.setAttribute("data-submitted", "true");'; + scriptContent += ' document.body.setAttribute("data-submit-time", Date.now().toString());'; + scriptContent += ' });'; + scriptContent += '});'; + + // Add button click detection as fallback + scriptContent += 'const buttons = document.querySelectorAll("button");'; + scriptContent += 'buttons.forEach(function(btn) {'; + scriptContent += ' if (btn.type === "submit" || btn.textContent.toLowerCase().includes("submit")) {'; + scriptContent += ' btn.addEventListener("click", function(e) {'; + scriptContent += ' setTimeout(function() {'; + scriptContent += ' document.body.setAttribute("data-submitted", "true");'; + scriptContent += ' document.body.setAttribute("data-submit-time", Date.now().toString());'; + scriptContent += ' }, 100);'; + scriptContent += ' });'; + scriptContent += ' }'; + scriptContent += '});'; + + // Add custom JavaScript if provided + if (${JSON.stringify(js || '')}) { + scriptContent += 'try {'; + scriptContent += ${JSON.stringify(js || '')}; + scriptContent += '} catch (error) {'; + scriptContent += ' console.error("Error in custom webapp script:", error);'; + scriptContent += '}'; + } + + scriptContent += '})();'; + script.textContent = scriptContent; + iframeDoc.body.appendChild(script); + } catch (scriptError) { + console.error('Failed to inject script into iframe:', scriptError); + } + }); + + return { + success: true, + webappId: webappId, + message: 'Webapp rendered successfully in full-screen iframe' + }; + })() + `, + returnByValue: true, + }); + + // Check for evaluation errors + if (result.exceptionDetails) { + const errorMsg = result.exceptionDetails.text || 'Unknown evaluation error'; + logger.error('Webapp rendering failed with exception:', errorMsg); + return { error: `Webapp rendering failed: ${errorMsg}` }; + } + + // Extract result + const renderResult = result.result.value as RenderWebAppResult; + + if (!renderResult || !renderResult.success) { + logger.error('Webapp rendering script returned unsuccessful result'); + return { error: 'Webapp rendering script failed to execute properly' }; + } + + logger.info('Successfully rendered webapp', { + webappId: renderResult.webappId + }); + + return renderResult; + + } catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + logger.error('Failed to render webapp:', errorMsg); + return { error: `Failed to render webapp: ${errorMsg}` }; + } + } + + schema = { + type: 'object', + properties: { + html: { + type: 'string', + description: 'The HTML content for the webapp body (e.g., form, dialog, interactive UI). Will be inserted into the iframe body.', + }, + css: { + type: 'string', + description: 'Optional CSS styles for the webapp. Will be added to the iframe head as a