AI Chat Panel with Multi-Agent Framework

.gitignore

@@ -54,3 +54,6 @@ test/perf/.generated
 .eslintcache
 .stylelintcache
 **/tsconfig.tsbuildinfo
+
+# Dependencies
+node_modules/

.gitmodules

@@ -25,4 +25,4 @@
 [submodule "third_party/inspector_protocol"]
 	path = third_party/inspector_protocol
 	url = https://chromium.googlesource.com/deps/inspector_protocol
-	gclient-condition = build_with_chromium == False
+	gclient-condition = build_with_chromium == False

config/gni/devtools_grd_files.gni

@@ -597,6 +597,41 @@ grd_files_release_sources = [
   "front_end/panels/accessibility/accessibility.js",
   "front_end/panels/ai_assistance/ai_assistance-meta.js",
   "front_end/panels/ai_assistance/ai_assistance.js",
+  "front_end/panels/ai_chat/ui/AIChatPanel.js",
+  "front_end/panels/ai_chat/ui/ChatView.js",
+  "front_end/panels/ai_chat/ui/chatView.css.js",
+  "front_end/panels/ai_chat/core/AgentService.js",
+  "front_end/panels/ai_chat/core/State.js",
+  "front_end/panels/ai_chat/core/Graph.js",
+  "front_end/panels/ai_chat/core/Types.js",
+  "front_end/panels/ai_chat/core/ConfigurableGraph.js",
+  "front_end/panels/ai_chat/core/GraphConfigs.js",
+  "front_end/panels/ai_chat/core/OpenAIClient.js",
+  "front_end/panels/ai_chat/core/BaseOrchestratorAgent.js",
+  "front_end/panels/ai_chat/core/PageInfoManager.js",
+  "front_end/panels/ai_chat/core/AgentNodes.js",
+  "front_end/panels/ai_chat/core/ChatOpenAI.js",
+  "front_end/panels/ai_chat/core/GraphHelpers.js",
+  "front_end/panels/ai_chat/core/StateGraph.js",
+  "front_end/panels/ai_chat/tools/Tools.js",
+  "front_end/panels/ai_chat/tools/CombinedExtractionTool.js",
+  "front_end/panels/ai_chat/tools/CritiqueTool.js",
+  "front_end/panels/ai_chat/tools/FetcherTool.js",
+  "front_end/panels/ai_chat/tools/FinalizeWithCritiqueTool.js",
+  "front_end/panels/ai_chat/tools/HTMLToMarkdownTool.js",
+  "front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.js",
+  "front_end/panels/ai_chat/tools/VisitHistoryManager.js",
+  "front_end/panels/ai_chat/tools/FullPageAccessibilityTreeToMarkdownTool.js",
+  "front_end/panels/ai_chat/common/utils.js",
+  "front_end/panels/ai_chat/common/log.js",
+  "front_end/panels/ai_chat/common/context.js",
+  "front_end/panels/ai_chat/common/page.js",
+  "front_end/panels/ai_chat/ai_chat-meta.js",
+  "front_end/panels/ai_chat/ai_chat.js",
+  "front_end/panels/ai_chat/ai_chat_impl.js",
+  "front_end/panels/ai_chat/agent_framework/AgentRunner.js",
+  "front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.js",
+  "front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.js",
   "front_end/panels/animation/animation-meta.js",
   "front_end/panels/animation/animation.js",
   "front_end/panels/application/application-meta.js",

front_end/entrypoints/devtools_app/BUILD.gn

@@ -10,6 +10,7 @@ devtools_entrypoint("entrypoint") {
   deps = [
     "../../entrypoints/inspector_main:meta",
     "../../panels/accessibility:meta",
+    "../../panels/ai_chat:meta",
     "../../panels/animation:meta",
     "../../panels/application:meta",
     "../../panels/autofill:meta",

front_end/entrypoints/devtools_app/devtools_app.ts

@@ -27,6 +27,7 @@ import '../../panels/webauthn/webauthn-meta.js';
 import '../../panels/layer_viewer/layer_viewer-meta.js';
 import '../../panels/recorder/recorder-meta.js';
 import '../../panels/whats_new/whats_new-meta.js';
+import '../../panels/ai_chat/ai_chat-meta.js';
 
 import * as Root from '../../core/root/root.js';
 import * as Main from '../main/main.js';

front_end/models/logs/BUILD.gn

@@ -31,6 +31,7 @@ devtools_entrypoint("bundle") {
   visibility = [
     ":*",
     "../../entrypoints/main/*",
+    "../../panels/ai_chat/*",
     "../../panels/application/*",
     "../../panels/console/*",
     "../../panels/explain/*",

front_end/panels/ai_chat/BUILD.gn

@@ -0,0 +1,180 @@
+# Copyright 2025 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("../../../scripts/build/ninja/devtools_entrypoint.gni")
+import("../../../scripts/build/ninja/devtools_module.gni")
+import("../../../scripts/build/ninja/generate_css.gni")
+import("../../../scripts/build/typescript/typescript.gni")
+import("../visibility.gni")
+
+generate_css("css_files") {
+  sources = [
+    "ui/chatView.css",
+  ]
+}
+
+devtools_module("ai_chat") {
+  sources = [
+    "ui/AIChatPanel.ts",
+    "ui/ChatView.ts",
+    "ai_chat_impl.ts",
+    "core/Graph.ts",
+    "core/State.ts",
+    "core/Types.ts",
+    "core/AgentService.ts",
+    "core/GraphConfigs.ts",
+    "core/OpenAIClient.ts",
+    "core/ConfigurableGraph.ts",
+    "core/BaseOrchestratorAgent.ts",
+    "core/PageInfoManager.ts",
+    "core/AgentNodes.ts",
+    "core/ChatOpenAI.ts",
+    "core/GraphHelpers.ts",
+    "core/StateGraph.ts",
+    "tools/Tools.ts",
+    "tools/CritiqueTool.ts",
+    "tools/FetcherTool.ts",
+    "tools/FinalizeWithCritiqueTool.ts",
+    "tools/VisitHistoryManager.ts",
+    "tools/HTMLToMarkdownTool.ts",
+    "tools/SchemaBasedExtractorTool.ts",
+    "tools/CombinedExtractionTool.ts",
+    "tools/FullPageAccessibilityTreeToMarkdownTool.ts",
+    "agent_framework/ConfigurableAgentTool.ts",
+    "agent_framework/AgentRunner.ts",
+    "agent_framework/implementation/ConfiguredAgents.ts",
+    "common/utils.ts",
+    "common/log.ts",
+    "common/context.ts",
+    "common/page.ts",
+  ]
+
+  deps = [
+    ":css_files",
+    "../../core/common:bundle",
+    "../../core/i18n:bundle",
+    "../../core/sdk:bundle",
+    "../../generated:protocol",
+    "../../models/logs:bundle",
+    "../../ui/components/helpers:bundle",
+    "../../ui/components/markdown_view:bundle",
+    "../../ui/components/text_editor:bundle",
+    "../../ui/legacy:bundle",
+    "../../ui/lit:bundle",
+    "../../ui/visual_logging:bundle",
+  ]
+}
+
+# List of source files also used to determine JS outputs for metadata
+_ai_chat_sources = [
+    "ui/AIChatPanel.ts",
+    "ui/ChatView.ts",
+    "ai_chat_impl.ts",
+    "core/Graph.ts",
+    "core/State.ts",
+    "core/Types.ts",
+    "core/AgentService.ts",
+    "core/GraphConfigs.ts",
+    "core/OpenAIClient.ts",
+    "core/ConfigurableGraph.ts",
+    "core/BaseOrchestratorAgent.ts",
+    "core/PageInfoManager.ts",
+    "core/AgentNodes.ts",
+    "core/ChatOpenAI.ts",
+    "core/GraphHelpers.ts",
+    "core/StateGraph.ts",
+    "tools/Tools.ts",
+    "tools/CritiqueTool.ts",
+    "tools/FetcherTool.ts",
+    "tools/FinalizeWithCritiqueTool.ts",
+    "tools/VisitHistoryManager.ts",
+    "tools/HTMLToMarkdownTool.ts",
+    "tools/SchemaBasedExtractorTool.ts",
+    "tools/CombinedExtractionTool.ts",
+    "tools/FullPageAccessibilityTreeToMarkdownTool.ts",
+    "agent_framework/ConfigurableAgentTool.ts",
+    "agent_framework/AgentRunner.ts",
+    "agent_framework/implementation/ConfiguredAgents.ts",
+    "common/utils.ts",
+    "common/log.ts",
+    "common/context.ts",
+    "common/page.ts",
+]
+
+# Construct the expected JS output paths for the metadata
+_ai_chat_js_outputs_for_metadata = []
+_ai_chat_module_target_gen_dir = get_label_info(":ai_chat", "target_gen_dir")
+
+foreach(src, _ai_chat_sources) {
+  _extension = get_path_info(src, "extension")
+  _relative_file_name = rebase_path(src, ".")
+  _fileName = get_path_info(_relative_file_name, "dir") + "/" +
+              get_path_info(_relative_file_name, "name")
+
+  # Only consider .ts files that aren't definition files (.d.ts) and .js files
+  if ((_extension == "ts" &&
+       get_path_info(get_path_info(src, "name"), "extension") != "d") ||
+      _extension == "js") {
+    _ai_chat_js_outputs_for_metadata +=
+        [ "$_ai_chat_module_target_gen_dir/${_fileName}.js" ]
+  }
+}
+
+# This target generates the metadata needed for the release GRD check
+generated_file("ai_chat_release_js_metadata") {
+  outputs = [ "$target_gen_dir/ai_chat_release_js_metadata-tsconfig.json" ]
+  contents = "{\"files\":[],\"compilerOptions\":{\"composite\":true}}"
+
+  # Make sure the actual JS files are built first
+  deps = [ ":ai_chat" ]
+
+  metadata = {
+    # These paths should match the format ts_library would produce in debug
+    grd_files = _ai_chat_js_outputs_for_metadata
+  }
+}
+
+# Metadata target for the CSS file, needed for release GRD check
+_css_module_target_gen_dir = get_label_info(":css_files", "target_gen_dir")
+_css_outputs_for_metadata = [
+  "$_css_module_target_gen_dir/ui/chatView.css.js",
+]
+
+generated_file("ai_chat_release_css_metadata") {
+  outputs = [ "$target_gen_dir/ai_chat_release_css_metadata-tsconfig.json" ]
+  contents = "{\"files\":[],\"compilerOptions\":{\"composite\":true}}"
+
+  # Make sure the actual CSS file is built first
+  deps = [ ":css_files" ]
+
+  metadata = {
+    grd_files = _css_outputs_for_metadata
+  }
+}
+
+devtools_entrypoint("bundle") {
+  entrypoint = "ai_chat.ts"
+
+  deps = [
+    ":ai_chat",
+    ":css_files",
+    ":ai_chat_release_js_metadata",  # Add dependency on the metadata target
+    ":ai_chat_release_css_metadata", # Add dependency on the CSS metadata target
+  ]
+
+  visibility = [
+    ":*",
+    "../../entrypoints/*",
+  ]
+
+  visibility += devtools_panels_visibility
+}
+
+devtools_entrypoint("meta") {
+  entrypoint = "ai_chat-meta.ts"
+
+  deps = [ ":bundle" ]
+
+  visibility = [ "../../entrypoints/*" ]
+}

front_end/panels/ai_chat/Readme.md

@@ -0,0 +1,60 @@
+# AI Chat Panel
+
+This panel is a Multi-Agent Framework that allows a user to connect an existing LLM with the chromium browser.
+
+### Setup and Development
+
+1. Build the project and use watch mode
+```sh
+npm run build -- --watch
+```
+
+2. Serve the content of out/Default/gen/front_end on a web server, e.g. via python -m http.server.
+
+```sh
+cd out/Default/gen/front_end
+
+python3 -m http.server
+```
+
+3. Use the AI Chat panel.
+
+```sh
+<path-to-devtools-frontend>/third_party/chrome/chrome-<platform>/chrome --disable-infobars --custom-devtools-frontend=http://localhost:8000/
+```
+
+
+### Agent Architecture
+
+The AI Chat Panel uses the multi-agent framework with the following components:
+
+1. **State Management**: Tracks conversation history, user context, and DevTools state
+2. **Tools**: Provides capabilities for DOM inspection, network analysis, and code execution
+3. **Workflow**: Defines the agent's reasoning process and decision-making flow
+
+### Implementation Details
+
+The agent implementation is located in `front_end/panels/ai_chat/agent/` and consists of:
+
+- `AgentService.ts`: Main service that interfaces between UI and Graph Workflow
+- `State.ts`: Defines the agent's state schema
+- `Tools.ts`: Implements DevTools-specific tools
+- `Graph.ts`: Defines the agent's workflow and decision-making process
+
+### Usage Example
+
+```typescript
+import { AgentService } from './agent/AgentService.js';
+
+// Initialize the agent service
+const agentService = new AgentService();
+
+// Send a message to the agent
+async function sendMessage(text: string) {
+  const response = await agentService.sendMessage(text);
+  // Update UI with response
+}
+```
+
+## Reference
+https://chromium.googlesource.com/devtools/devtools-frontend/+/main/docs/get_the_code.md#Standalone-checkout-Checking-out-source