diff --git a/MODEL-CONFIGS.md b/MODEL-CONFIGS.md
new file mode 100644
index 00000000000..02eb8ebdaa7
--- /dev/null
+++ b/MODEL-CONFIGS.md
@@ -0,0 +1,450 @@
+# LLM Model Configuration Architecture
+
+## Overview
+
+This document outlines the comprehensive architecture for programmatic LLM model provider and configuration management in the browser-operator-core AI chat system. The system supports both manual user configuration and automated evaluation configuration with per-request overrides and persistent configuration updates.
+
+## Current System Analysis
+
+### Provider Support
+- **4 LLM Providers**: LiteLLM, OpenAI, Groq, OpenRouter
+- **3-Tier Model Selection**:
+  - **Main Model**: Primary model for agent execution
+  - **Mini Model**: Smaller/faster model for lightweight operations
+  - **Nano Model**: Smallest/fastest model for simple tasks
+
+### Current Configuration Flow
+1. **Provider Selection** (localStorage: `ai_chat_provider`)
+2. **Model Selection** (localStorage: `ai_chat_model_selection`, `ai_chat_mini_model`, `ai_chat_nano_model`)
+3. **API Configuration** (localStorage: provider-specific API keys and endpoints)
+4. **Context Propagation**: Configuration flows through AgentService → Graph → ConfigurableAgentTool → AgentRunner
+5. **Model Usage**: Models selected via `AIChatPanel.getMiniModel()`, `AIChatPanel.getNanoModel()` static methods
+
+### Current localStorage Keys
+- `ai_chat_provider`: Selected provider ('openai', 'litellm', 'groq', 'openrouter')
+- `ai_chat_model_selection`: Main model name
+- `ai_chat_mini_model`: Mini model name
+- `ai_chat_nano_model`: Nano model name
+- `ai_chat_api_key`: OpenAI API key
+- `ai_chat_litellm_endpoint`: LiteLLM server endpoint
+- `ai_chat_litellm_api_key`: LiteLLM API key
+- `ai_chat_groq_api_key`: Groq API key
+- `ai_chat_openrouter_api_key`: OpenRouter API key
+
+## Proposed Architecture: LLMConfigurationManager
+
+### Core Design Principles
+- **Single Source of Truth**: Centralized configuration management
+- **Override System**: Automated mode can override without affecting localStorage
+- **Persistent Configuration**: New eval API to set localStorage values programmatically
+- **Backwards Compatible**: Existing localStorage-based code continues to work
+- **Tab Behavior**:
+  - **Manual Mode**: Shared configuration across tabs via localStorage
+  - **Automated Mode**: Per-tab override capability for request multiplexing
+
+### Two Configuration Modes
+
+#### 1. Manual Mode
+- User configures through Settings UI
+- Configuration persisted to localStorage
+- Changes shared across all tabs
+- Traditional user experience maintained
+
+#### 2. Automated Mode
+Two sub-modes available:
+
+**Per-Request Override (Temporary)**
+- Configuration override for individual evaluation requests
+- No localStorage changes
+- Each tab can have different overrides simultaneously
+- Clean up after request completion
+
+**Persistent Configuration (New)**
+- Programmatic equivalent of Settings UI
+- Updates localStorage values
+- Changes apply to all tabs
+- Useful for evaluation setup/teardown
+
+## Implementation
+
+### 1. LLMConfigurationManager Singleton
+
+```typescript
+interface LLMConfig {
+  provider: 'openai' | 'litellm' | 'groq' | 'openrouter';
+  apiKey?: string;
+  endpoint?: string; // For LiteLLM
+  mainModel: string;
+  miniModel?: string;
+  nanoModel?: string;
+}
+
+class LLMConfigurationManager {
+  private static instance: LLMConfigurationManager;
+  private overrideConfig?: Partial<LLMConfig>; // Override for automated mode
+
+  static getInstance(): LLMConfigurationManager;
+
+  // Configuration retrieval with override fallback
+  getProvider(): string;
+  getMainModel(): string;
+  getMiniModel(): string;
+  getNanoModel(): string;
+  getApiKey(): string;
+  getEndpoint(): string | undefined;
+
+  // Override management (per-request automated mode)
+  setOverride(config: Partial<LLMConfig>): void;
+  clearOverride(): void;
+
+  // Persistence (manual mode & persistent automated mode)
+  saveConfiguration(config: LLMConfig): void;
+  loadConfiguration(): LLMConfig;
+}
+```
+
+#### Configuration Precedence
+1. **Override configuration** (if set) - for per-request automated mode
+2. **localStorage values** (fallback) - for manual mode and persistent automated mode
+
+### 2. Protocol Extension for Persistent Configuration
+
+#### Add to EvaluationProtocol.ts
+
+```typescript
+// New JSON-RPC method for persistent LLM configuration
+export interface LLMConfigurationRequest {
+  jsonrpc: '2.0';
+  method: 'configure_llm';
+  params: LLMConfigurationParams;
+  id: string;
+}
+
+export interface LLMConfigurationParams {
+  provider: 'openai' | 'litellm' | 'groq' | 'openrouter';
+  apiKey?: string;
+  endpoint?: string; // For LiteLLM
+  models: {
+    main: string;
+    mini?: string;
+    nano?: string;
+  };
+  // Optional: only update specific fields
+  partial?: boolean;
+}
+
+export interface LLMConfigurationResponse {
+  jsonrpc: '2.0';
+  result: {
+    status: 'success';
+    message: string;
+    appliedConfig: {
+      provider: string;
+      models: {
+        main: string;
+        mini: string;
+        nano: string;
+      };
+    };
+  };
+  id: string;
+}
+
+// Type guard
+export function isLLMConfigurationRequest(msg: any): msg is LLMConfigurationRequest;
+
+// Helper function
+export function createLLMConfigurationRequest(
+  id: string,
+  params: LLMConfigurationParams
+): LLMConfigurationRequest;
+```
+
+#### Update EvaluationParams for Per-Request Override
+
+```typescript
+export interface EvaluationParams {
+  evaluationId: string;
+  name: string;
+  url: string;
+  tool: string;
+  input: any;
+  model?: {
+    main_model?: string;
+    mini_model?: string;
+    nano_model?: string;
+    provider?: string;
+    api_key?: string;    // New: per-request API key
+    endpoint?: string;   // New: per-request endpoint (LiteLLM)
+  };
+  timeout: number;
+  metadata: {
+    tags: string[];
+    retries: number;
+    priority?: 'low' | 'normal' | 'high';
+  };
+}
+```
+
+### 3. Integration Points
+
+#### Manual Mode (Settings UI)
+- Settings dialog calls `LLMConfigurationManager.getInstance().saveConfiguration()`
+- All components read through manager methods
+- Changes propagate across tabs via storage events
+- User expectations: changes in one tab affect all tabs
+
+#### Automated Mode - Per-Request Override
+```typescript
+// In EvaluationAgent.executeChatEvaluation()
+const configManager = LLMConfigurationManager.getInstance();
+
+// Set override if provided in request
+if (input.provider || input.main_model || input.api_key) {
+  configManager.setOverride({
+    provider: input.provider,
+    mainModel: input.main_model,
+    miniModel: input.mini_model,
+    nanoModel: input.nano_model,
+    apiKey: input.api_key,
+    endpoint: input.endpoint
+  });
+}
+
+try {
+  // Execute with override
+  const result = await agentService.sendMessage(input.message);
+  return result;
+} finally {
+  // Clean up override
+  configManager.clearOverride();
+}
+```
+
+#### Automated Mode - Persistent Configuration
+```typescript
+// In EvaluationAgent.handleLLMConfigurationRequest()
+private async handleLLMConfigurationRequest(request: LLMConfigurationRequest): Promise<void> {
+  const configManager = LLMConfigurationManager.getInstance();
+
+  // Save to localStorage (same as Settings UI)
+  configManager.saveConfiguration({
+    provider: params.provider,
+    apiKey: params.apiKey,
+    endpoint: params.endpoint,
+    mainModel: params.models.main,
+    miniModel: params.models.mini,
+    nanoModel: params.models.nano
+  });
+
+  // Reinitialize AgentService with new configuration
+  const agentService = AgentService.getInstance();
+  await agentService.refreshCredentials();
+
+  // Send success response
+}
+```
+
+### 4. Component Updates Required
+
+1. **Replace localStorage access** in:
+   - `AgentService.ts` - Use manager for provider/key retrieval
+   - `AIChatPanel.ts` - Replace static model methods with manager calls
+   - `AgentNodes.ts` - Use manager for graph configuration
+   - `ConfigurableAgentTool.ts` - Use manager for CallCtx building
+
+2. **Add configuration refresh mechanisms**:
+   - AgentService reinitialize LLMClient when config changes
+   - Graph recreation with new model configuration
+   - Storage event listeners for cross-tab synchronization
+
+3. **Update EvaluationAgent**:
+   - Add `configure_llm` method handler
+   - Update per-request override logic
+   - Add configuration validation
+
+## Usage Examples
+
+### Per-Request Override (Temporary)
+```typescript
+// eval-server sends evaluation with custom config
+{
+  "jsonrpc": "2.0",
+  "method": "evaluate",
+  "params": {
+    "evaluationId": "eval-123",
+    "tool": "chat",
+    "input": { "message": "Hello" },
+    "model": {
+      "provider": "openai",
+      "main_model": "gpt-4",
+      "api_key": "sk-temp-key"
+    }
+  },
+  "id": "req-456"
+}
+```
+
+### Persistent Configuration
+```typescript
+// eval-server sets persistent configuration
+{
+  "jsonrpc": "2.0",
+  "method": "configure_llm",
+  "params": {
+    "provider": "openai",
+    "apiKey": "sk-persistent-key",
+    "models": {
+      "main": "gpt-4",
+      "mini": "gpt-4-mini",
+      "nano": "gpt-3.5-turbo"
+    }
+  },
+  "id": "config-789"
+}
+```
+
+## Benefits
+
+### Manual Mode
+- ✅ Maintains current UX - shared configuration across tabs
+- ✅ Settings persist in localStorage
+- ✅ No breaking changes for existing users
+
+### Automated Mode - Per-Request Override
+- ✅ Per-request configuration without side effects
+- ✅ Multiple evaluations with different configs in different tabs
+- ✅ Clean separation from manual configuration
+- ✅ Tab isolation for request multiplexing
+
+### Automated Mode - Persistent Configuration
+- ✅ Programmatic equivalent of Settings UI
+- ✅ Evaluation setup/teardown capabilities
+- ✅ Cross-tab configuration updates
+- ✅ Integration with existing localStorage system
+
+### Technical
+- ✅ Single source of truth for all configuration
+- ✅ Backwards compatible with existing code
+- ✅ Simple mental model: override if present, localStorage otherwise
+- ✅ No complex mode management needed
+- ✅ Flexible evaluation server capabilities
+
+## Implementation Status
+
+### ✅ Phase 1: Core Infrastructure (COMPLETED)
+1. ✅ **Create LLMConfigurationManager** singleton class - `/front_end/panels/ai_chat/core/LLMConfigurationManager.ts`
+2. ✅ **Update AgentService** to use manager instead of localStorage
+3. ✅ **Update AIChatPanel** model selection methods
+4. ✅ **Add configuration refresh mechanisms**
+
+### ✅ Phase 2: Per-Request Override Support (COMPLETED)
+5. ✅ **Update EvaluationAgent** per-request override logic
+6. ✅ **Update Graph/AgentNodes** configuration passing via LLMConfigurationManager
+7. ✅ **Per-request override functionality** implemented
+
+### ✅ Phase 3: Persistent Configuration API (COMPLETED)
+8. ✅ **Extend EvaluationProtocol** with `configure_llm` method
+9. ✅ **Implement `configure_llm` handler** in EvaluationAgent
+10. 🔄 **Add eval-server support** for persistent configuration (server-side implementation needed)
+11. ✅ **Add configuration validation** and error handling
+
+### 🔄 Phase 4: Testing & Documentation (NEXT)
+12. ✅ **Maintain backwards compatibility** during transition
+13. 🔄 **Add comprehensive tests** for both modes
+14. ✅ **Update documentation** and usage examples
+15. 🔄 **Performance testing** with multiple tabs and configurations
+
+## Key Implemented Features
+
+### LLMConfigurationManager (`/front_end/panels/ai_chat/core/LLMConfigurationManager.ts`)
+- ✅ Singleton pattern with override support
+- ✅ Configuration validation
+- ✅ Change listeners for real-time updates
+- ✅ localStorage persistence for manual mode
+- ✅ Override system for automated mode
+
+### Per-Request Override Support
+- ✅ EvaluationAgent supports model overrides via request parameters
+- ✅ Automatic cleanup after request completion
+- ✅ Configuration fallback hierarchy (override → localStorage)
+
+### Persistent Configuration API
+- ✅ `configure_llm` JSON-RPC method in EvaluationProtocol
+- ✅ Handler implementation in EvaluationAgent
+- ✅ Real-time AgentService reinitialization
+- ✅ Cross-tab configuration synchronization
+
+### Integration Points Updated
+- ✅ AgentService: Uses LLMConfigurationManager instead of direct localStorage
+- ✅ AIChatPanel: Static model methods updated to use manager
+- ✅ AgentNodes: Tool execution context uses manager configuration
+- ✅ EvaluationAgent: Supports both override and persistent modes
+
+## Usage Examples
+
+### Manual Mode (Settings UI)
+```typescript
+// Settings dialog usage
+const panel = AIChatPanel.instance();
+panel.setLLMConfiguration({
+  provider: 'openai',
+  apiKey: 'sk-...',
+  mainModel: 'gpt-4',
+  miniModel: 'gpt-4-mini',
+  nanoModel: 'gpt-3.5-turbo'
+});
+```
+
+### Automated Mode - Per-Request Override
+```typescript
+// Evaluation request with model override
+{
+  "jsonrpc": "2.0",
+  "method": "evaluate",
+  "params": {
+    "tool": "chat",
+    "input": { "message": "Hello" },
+    "model": {
+      "provider": "openai",
+      "main_model": "gpt-4",
+      "api_key": "sk-temp-key"
+    }
+  }
+}
+```
+
+### Automated Mode - Persistent Configuration
+```typescript
+// Set persistent configuration
+{
+  "jsonrpc": "2.0",
+  "method": "configure_llm",
+  "params": {
+    "provider": "openai",
+    "apiKey": "sk-persistent-key",
+    "models": {
+      "main": "gpt-4",
+      "mini": "gpt-4-mini",
+      "nano": "gpt-3.5-turbo"
+    }
+  }
+}
+```
+
+## Migration Strategy
+
+### Backwards Compatibility
+- Existing localStorage-based code continues to work during transition
+- Gradual migration of components to use LLMConfigurationManager
+- Fallback mechanisms for missing configuration values
+- No breaking changes to existing evaluation requests
+
+### Testing Strategy
+- Unit tests for LLMConfigurationManager override logic
+- Integration tests for Settings UI compatibility
+- End-to-end tests for evaluation server scenarios
+- Cross-tab synchronization testing
+- Performance testing with multiple concurrent evaluations
+
+This architecture provides a comprehensive solution for both manual user configuration and programmatic evaluation server control, while maintaining backwards compatibility and enabling powerful new automation capabilities.
\ No newline at end of file
diff --git a/eval-server/nodejs/.env.example b/eval-server/nodejs/.env.example
new file mode 100644
index 00000000000..a19f3a1cdf5
--- /dev/null
+++ b/eval-server/nodejs/.env.example
@@ -0,0 +1,45 @@
+# Evaluation Server Configuration
+# Copy this file to .env and configure your settings
+
+# Server Configuration
+PORT=8080
+HOST=127.0.0.1
+
+# LLM Provider API Keys
+# Configure one or more providers for evaluation
+
+# OpenAI Configuration
+OPENAI_API_KEY=sk-your-openai-api-key-here
+
+# LiteLLM Configuration (if using a LiteLLM server)
+LITELLM_ENDPOINT=http://localhost:4000
+LITELLM_API_KEY=your-litellm-api-key-here
+
+# Groq Configuration
+GROQ_API_KEY=gsk_your-groq-api-key-here
+
+# OpenRouter Configuration
+OPENROUTER_API_KEY=sk-or-v1-your-openrouter-api-key-here
+
+# Default LLM Configuration for Evaluations
+# These will be used as fallbacks when not specified in evaluation requests
+DEFAULT_PROVIDER=openai
+DEFAULT_MAIN_MODEL=gpt-4
+DEFAULT_MINI_MODEL=gpt-4-mini
+DEFAULT_NANO_MODEL=gpt-3.5-turbo
+
+# Logging Configuration
+LOG_LEVEL=info
+LOG_DIR=./logs
+
+# Client Configuration
+CLIENTS_DIR=./clients
+EVALS_DIR=./evals
+
+# RPC Configuration
+RPC_TIMEOUT=30000
+
+# Security
+# Set this to enable authentication for client connections
+# Leave empty to disable authentication
+AUTH_SECRET_KEY=
\ No newline at end of file
diff --git a/eval-server/nodejs/CLAUDE.md b/eval-server/nodejs/CLAUDE.md
index 5db83421a3f..ba84f31b0b3 100644
--- a/eval-server/nodejs/CLAUDE.md
+++ b/eval-server/nodejs/CLAUDE.md
@@ -22,6 +22,16 @@ bo-eval-server is a WebSocket-based evaluation server for LLM agents that implem
 - `OPENAI_API_KEY` - OpenAI API key for LLM judge functionality
 - `PORT` - WebSocket server port (default: 8080)
 
+### LLM Provider Configuration (Optional)
+- `GROQ_API_KEY` - Groq API key for Groq provider support
+- `OPENROUTER_API_KEY` - OpenRouter API key for OpenRouter provider support
+- `LITELLM_ENDPOINT` - LiteLLM server endpoint URL
+- `LITELLM_API_KEY` - LiteLLM API key for LiteLLM provider support
+- `DEFAULT_PROVIDER` - Default LLM provider (openai, groq, openrouter, litellm)
+- `DEFAULT_MAIN_MODEL` - Default main model name
+- `DEFAULT_MINI_MODEL` - Default mini model name
+- `DEFAULT_NANO_MODEL` - Default nano model name
+
 ## Architecture
 
 ### Core Components
@@ -33,10 +43,11 @@ bo-eval-server is a WebSocket-based evaluation server for LLM agents that implem
 - Handles bidirectional RPC communication
 
 **RPC Client** (`src/rpc-client.js`)
-- Implements JSON-RPC 2.0 protocol for server-to-client calls
+- Implements JSON-RPC 2.0 protocol for bidirectional communication
 - Manages request/response correlation with unique IDs
 - Handles timeouts and error conditions
 - Calls `Evaluate(request: String) -> String` method on connected agents
+- Supports `configure_llm` method for dynamic LLM provider configuration
 
 **LLM Evaluator** (`src/evaluator.js`)
 - Integrates with OpenAI API for LLM-as-a-judge functionality
@@ -78,7 +89,10 @@ logs/                  # Log files (created automatically)
 ### Key Features
 
 - **Bidirectional RPC**: Server can call methods on connected clients
-- **LLM-as-a-Judge**: Automated evaluation of agent responses using GPT-4
+- **Multi-Provider LLM Support**: Support for OpenAI, Groq, OpenRouter, and LiteLLM providers
+- **Dynamic LLM Configuration**: Runtime configuration via `configure_llm` JSON-RPC method
+- **Per-Client Configuration**: Each connected client can have different LLM settings
+- **LLM-as-a-Judge**: Automated evaluation of agent responses using configurable LLM providers
 - **Concurrent Evaluations**: Support for multiple agents and parallel evaluations
 - **Structured Logging**: All interactions logged as JSON for analysis
 - **Interactive CLI**: Built-in CLI for testing and server management
@@ -93,6 +107,79 @@ Agents must implement:
 - `Evaluate(task: string) -> string` method
 - "ready" message to signal availability for evaluations
 
+### Model Configuration Schema
+
+The server uses a canonical nested model configuration format that allows per-tier provider and API key settings:
+
+#### Model Configuration Structure
+
+```typescript
+interface ModelTierConfig {
+  provider: string;  // "openai" | "groq" | "openrouter" | "litellm"
+  model: string;     // Model name (e.g., "gpt-4", "llama-3.1-8b-instant")
+  api_key: string;   // API key for this tier
+}
+
+interface ModelConfig {
+  main_model: ModelTierConfig;  // Primary model for complex tasks
+  mini_model: ModelTierConfig;  // Secondary model for simpler tasks
+  nano_model: ModelTierConfig;  // Tertiary model for basic tasks
+}
+```
+
+#### Example: Evaluation with Model Configuration
+
+```json
+{
+  "jsonrpc": "2.0",
+  "method": "evaluate",
+  "params": {
+    "tool": "chat",
+    "input": {"message": "Hello"},
+    "model": {
+      "main_model": {
+        "provider": "openai",
+        "model": "gpt-4",
+        "api_key": "sk-main-key"
+      },
+      "mini_model": {
+        "provider": "openai",
+        "model": "gpt-4-mini",
+        "api_key": "sk-mini-key"
+      },
+      "nano_model": {
+        "provider": "groq",
+        "model": "llama-3.1-8b-instant",
+        "api_key": "gsk-nano-key"
+      }
+    }
+  }
+}
+```
+
+### Dynamic LLM Configuration
+
+The server supports runtime LLM configuration via the `configure_llm` JSON-RPC method:
+
+```json
+{
+  "jsonrpc": "2.0",
+  "method": "configure_llm",
+  "params": {
+    "provider": "openai|groq|openrouter|litellm",
+    "apiKey": "your-api-key",
+    "endpoint": "endpoint-url-for-litellm",
+    "models": {
+      "main": "main-model-name",
+      "mini": "mini-model-name",
+      "nano": "nano-model-name"
+    },
+    "partial": false
+  },
+  "id": "config-request-id"
+}
+```
+
 ### Configuration
 
 All configuration is managed through environment variables and `src/config.js`. Key settings:
diff --git a/eval-server/nodejs/README.md b/eval-server/nodejs/README.md
index dab2614fe72..d29f9bc2b16 100644
--- a/eval-server/nodejs/README.md
+++ b/eval-server/nodejs/README.md
@@ -145,7 +145,23 @@ server.onConnect(async client => {
       message: "Your question here"
     },
     timeout: 30000,             // Optional timeout (ms)
-    model: {},                  // Optional model config
+    model: {                    // Optional nested model config
+      main_model: {
+        provider: "openai",
+        model: "gpt-4",
+        api_key: "sk-..."
+      },
+      mini_model: {
+        provider: "openai",
+        model: "gpt-4-mini",
+        api_key: "sk-..."
+      },
+      nano_model: {
+        provider: "groq",
+        model: "llama-3.1-8b-instant",
+        api_key: "gsk-..."
+      }
+    },
     metadata: {                 // Optional metadata
       tags: ['api', 'test']
     }
diff --git a/eval-server/nodejs/examples/library-usage.js b/eval-server/nodejs/examples/library-usage.js
index 45da6081540..cfb3ffdf23a 100644
--- a/eval-server/nodejs/examples/library-usage.js
+++ b/eval-server/nodejs/examples/library-usage.js
@@ -7,6 +7,7 @@
 // Simple example demonstrating the programmatic API usage
 
 import { EvalServer } from '../src/lib/EvalServer.js';
+import { CONFIG } from '../src/config.js';
 
 console.log('🔧 Creating server...');
 const server = new EvalServer({
@@ -31,20 +32,57 @@ server.onConnect(async client => {
   console.log('   - Client tabId:', client.tabId);
   console.log('   - Client info:', client.getInfo());
 
+  // Check available LLM providers
+  console.log('\n🔑 Available LLM Providers:');
+  const availableProviders = [];
+  if (CONFIG.providers.openai.apiKey) {
+    availableProviders.push('openai');
+    console.log('   ✅ OpenAI configured');
+  }
+  if (CONFIG.providers.groq.apiKey) {
+    availableProviders.push('groq');
+    console.log('   ✅ Groq configured');
+  }
+  if (CONFIG.providers.openrouter.apiKey) {
+    availableProviders.push('openrouter');
+    console.log('   ✅ OpenRouter configured');
+  }
+  if (CONFIG.providers.litellm.apiKey && CONFIG.providers.litellm.endpoint) {
+    availableProviders.push('litellm');
+    console.log('   ✅ LiteLLM configured');
+  }
+
+  if (availableProviders.length === 0) {
+    console.log('   ❌ No providers configured. Add API keys to .env file.');
+    console.log('   ℹ️  Example: OPENAI_API_KEY=sk-your-key-here');
+  }
+
   try {
-    console.log('🔄 Starting evaluation...');
+    // Demonstrate basic evaluation first
+    console.log('\n🔄 Starting basic evaluation...');
     let response = await client.evaluate({
-      id: "test_eval",
-      name: "Capital of France", 
-      description: "Simple test evaluation",
+      id: "basic_eval",
+      name: "Capital of France",
+      description: "Basic test evaluation",
       tool: "chat",
       input: {
         message: "What is the capital of France?"
       }
     });
-    
-    console.log('✅ Evaluation completed!');
+
+    console.log('✅ Basic evaluation completed!');
     console.log('📊 Response:', JSON.stringify(response, null, 2));
+
+    // Demonstrate explicit model selection if OpenAI is available
+    if (CONFIG.providers.openai.apiKey) {
+      await demonstrateModelSelection(client);
+    }
+
+    // Demonstrate LLM configuration if providers are available
+    if (availableProviders.length > 0) {
+      await demonstrateLLMConfiguration(client, availableProviders);
+    }
+
   } catch (error) {
     console.log('❌ Evaluation failed:', error.message);
   }
@@ -54,6 +92,150 @@ server.onDisconnect(clientInfo => {
   console.log('👋 CLIENT DISCONNECTED:', clientInfo);
 });
 
+// Function to demonstrate explicit model selection within OpenAI
+async function demonstrateModelSelection(client) {
+  console.log('\n🤖 Demonstrating Model Selection (OpenAI)...');
+
+  const modelTests = [
+    {
+      model: 'gpt-4',
+      task: 'Complex reasoning',
+      message: 'Solve this step by step: If a train travels 60 mph for 2.5 hours, how far does it go?'
+    },
+    {
+      model: 'gpt-4-mini',
+      task: 'Simple question',
+      message: 'What is 2 + 2?'
+    },
+    {
+      model: 'gpt-3.5-turbo',
+      task: 'Creative writing',
+      message: 'Write a one-sentence story about a cat.'
+    }
+  ];
+
+  for (const test of modelTests) {
+    console.log(`\n🔧 Testing ${test.model} for ${test.task}...`);
+
+    try {
+      const response = await client.evaluate({
+        id: `model_test_${test.model.replace(/[^a-z0-9]/g, '_')}`,
+        name: `${test.model} ${test.task}`,
+        tool: "chat",
+        input: {
+          message: test.message
+        },
+        model: {
+          main_model: {
+            provider: "openai",
+            model: test.model,
+            api_key: CONFIG.providers.openai.apiKey
+          }
+        }
+      });
+
+      console.log(`   ✅ ${test.model} completed successfully`);
+      console.log(`   📊 Response: ${JSON.stringify(response.output).substring(0, 100)}...`);
+
+      // Wait between tests
+      await new Promise(resolve => setTimeout(resolve, 1500));
+
+    } catch (error) {
+      console.log(`   ❌ ${test.model} failed: ${error.message}`);
+    }
+  }
+
+  console.log('\n✨ Model selection demonstration completed!');
+}
+
+// Function to demonstrate LLM configuration
+async function demonstrateLLMConfiguration(client, availableProviders) {
+  console.log('\n🧪 Demonstrating LLM Configuration...');
+
+  for (const provider of availableProviders.slice(0, 2)) { // Test up to 2 providers
+    console.log(`\n🔧 Configuring ${provider.toUpperCase()} provider...`);
+
+    try {
+      // Configure different models based on provider
+      let models;
+      switch (provider) {
+        case 'openai':
+          models = {
+            main: 'gpt-4',
+            mini: 'gpt-4-mini',
+            nano: 'gpt-3.5-turbo'
+          };
+          break;
+        case 'groq':
+          models = {
+            main: 'llama-3.1-8b-instant',
+            mini: 'llama-3.1-8b-instant',
+            nano: 'llama-3.1-8b-instant'
+          };
+          break;
+        case 'openrouter':
+          models = {
+            main: 'anthropic/claude-3-sonnet',
+            mini: 'anthropic/claude-3-haiku',
+            nano: 'anthropic/claude-3-haiku'
+          };
+          break;
+        case 'litellm':
+          models = {
+            main: 'claude-3-sonnet-20240229',
+            mini: 'claude-3-haiku-20240307',
+            nano: 'claude-3-haiku-20240307'
+          };
+          break;
+      }
+
+      console.log(`   📦 Models: main=${models.main}, mini=${models.mini}, nano=${models.nano}`);
+
+      // Run evaluation with specific provider configuration
+      const response = await client.evaluate({
+        id: `${provider}_config_eval`,
+        name: `${provider.toUpperCase()} Configuration Test`,
+        description: `Test evaluation using ${provider} provider`,
+        tool: "chat",
+        input: {
+          message: `Hello! This is a test using the ${provider} provider. Please respond with a brief confirmation.`
+        },
+        model: {
+          main_model: {
+            provider: provider,
+            model: models.main,
+            api_key: CONFIG.providers[provider].apiKey,
+            endpoint: CONFIG.providers[provider].endpoint
+          },
+          mini_model: {
+            provider: provider,
+            model: models.mini,
+            api_key: CONFIG.providers[provider].apiKey,
+            endpoint: CONFIG.providers[provider].endpoint
+          },
+          nano_model: {
+            provider: provider,
+            model: models.nano,
+            api_key: CONFIG.providers[provider].apiKey,
+            endpoint: CONFIG.providers[provider].endpoint
+          }
+        }
+      });
+
+      console.log(`   ✅ ${provider.toUpperCase()} evaluation completed successfully`);
+      console.log(`   📊 Response preview: ${JSON.stringify(response.output).substring(0, 100)}...`);
+
+      // Wait between provider tests
+      await new Promise(resolve => setTimeout(resolve, 2000));
+
+    } catch (error) {
+      console.log(`   ❌ ${provider.toUpperCase()} configuration test failed:`, error.message);
+    }
+  }
+
+  console.log('\n✨ LLM configuration demonstration completed!');
+}
+
 console.log('🔧 Starting server...');
 await server.start();
 console.log('✅ Server started successfully on ws://127.0.0.1:8080');
diff --git a/eval-server/nodejs/examples/multiple-evals.js b/eval-server/nodejs/examples/multiple-evals.js
index cd5ee980a1f..b65522f2528 100755
--- a/eval-server/nodejs/examples/multiple-evals.js
+++ b/eval-server/nodejs/examples/multiple-evals.js
@@ -9,11 +9,12 @@
 
 import { EvalServer } from '../src/lib/EvalServer.js';
 import { EvaluationStack } from '../src/lib/EvaluationStack.js';
+import { CONFIG } from '../src/config.js';
 
 console.log('🔧 Creating evaluation stack...');
 const evalStack = new EvaluationStack();
 
-// Create multiple diverse evaluations for the stack
+// Create multiple diverse evaluations for the stack with different LLM configurations
 const evaluations = [
   {
     id: "math_eval",
@@ -22,25 +23,49 @@ const evaluations = [
     tool: "chat",
     input: {
       message: "What is 15 * 7 + 23? Please show your calculation steps."
-    }
+    },
+    // Use OpenAI if available, otherwise default
+    model: CONFIG.providers.openai.apiKey ? {
+      main_model: {
+        provider: 'openai',
+        model: 'gpt-4',
+        api_key: CONFIG.providers.openai.apiKey
+      }
+    } : {}
   },
   {
-    id: "geography_eval", 
+    id: "geography_eval",
     name: "Capital of France",
     description: "Geography knowledge test",
     tool: "chat",
     input: {
       message: "What is the capital of France?"
-    }
+    },
+    // Use Groq if available, otherwise default
+    model: CONFIG.providers.groq.apiKey ? {
+      main_model: {
+        provider: 'groq',
+        model: 'llama-3.1-8b-instant',
+        api_key: CONFIG.providers.groq.apiKey
+      }
+    } : {}
   },
   {
     id: "creative_eval",
     name: "Creative Writing",
     description: "Short creative writing task",
-    tool: "chat", 
+    tool: "chat",
     input: {
       message: "Write a two-sentence story about a robot discovering friendship."
-    }
+    },
+    // Use OpenRouter if available, otherwise default
+    model: CONFIG.providers.openrouter.apiKey ? {
+      main_model: {
+        provider: 'openrouter',
+        model: 'anthropic/claude-3-sonnet',
+        api_key: CONFIG.providers.openrouter.apiKey
+      }
+    } : {}
   },
   {
     id: "tech_eval",
@@ -49,7 +74,16 @@ const evaluations = [
     tool: "chat",
     input: {
       message: "Explain what HTTP stands for and what it's used for in simple terms."
-    }
+    },
+    // Use LiteLLM if available, otherwise default
+    model: (CONFIG.providers.litellm.apiKey && CONFIG.providers.litellm.endpoint) ? {
+      main_model: {
+        provider: 'litellm',
+        model: 'claude-3-haiku-20240307',
+        api_key: CONFIG.providers.litellm.apiKey,
+        endpoint: CONFIG.providers.litellm.endpoint
+      }
+    } : {}
   }
 ];
 
@@ -57,7 +91,8 @@ const evaluations = [
 console.log('📚 Adding evaluations to stack...');
 evaluations.forEach((evaluation, index) => {
   evalStack.push(evaluation);
-  console.log(`   ${index + 1}. ${evaluation.name} (${evaluation.id})`);
+  const providerInfo = evaluation.model?.main_model?.provider ? ` [${evaluation.model.main_model.provider}]` : ' [default]';
+  console.log(`   ${index + 1}. ${evaluation.name} (${evaluation.id})${providerInfo}`);
 });
 
 console.log(`✅ Stack initialized with ${evalStack.size()} evaluations`);
@@ -94,13 +129,18 @@ server.onConnect(async client => {
 
   // Pop the next evaluation from the stack
   const evaluation = evalStack.pop();
-  console.log(`📋 Assigning evaluation: "${evaluation.name}" (${evaluation.id})`);
+  const providerInfo = evaluation.model?.main_model?.provider ? ` using ${evaluation.model.main_model.provider}` : ' using default provider';
+  console.log(`📋 Assigning evaluation: "${evaluation.name}" (${evaluation.id})${providerInfo}`);
   console.log(`📊 Remaining evaluations in stack: ${evalStack.size()}`);
 
   try {
     console.log('🔄 Starting evaluation...');
+    if (evaluation.model?.main_model?.provider) {
+      console.log(`🔧 Using LLM provider: ${evaluation.model.main_model.provider} with model: ${evaluation.model.main_model.model}`);
+    }
+
     let response = await client.evaluate(evaluation);
-    
+
     console.log('✅ Evaluation completed!');
     console.log(`📊 Response for "${evaluation.name}":`, JSON.stringify(response, null, 2));
   } catch (error) {
diff --git a/eval-server/nodejs/examples/with-http-wrapper.js b/eval-server/nodejs/examples/with-http-wrapper.js
index ae017ac7599..2ec9d0f16c7 100644
--- a/eval-server/nodejs/examples/with-http-wrapper.js
+++ b/eval-server/nodejs/examples/with-http-wrapper.js
@@ -11,30 +11,30 @@ import { HTTPWrapper } from '../src/lib/HTTPWrapper.js';
 
 console.log('🔧 Creating EvalServer...');
 const evalServer = new EvalServer({
-  authKey: 'hello',
+  // No authKey - authentication disabled for automated mode
   host: '127.0.0.1',
-  port: 8080
+  port: 8082
 });
 
 console.log('🔧 Creating HTTP wrapper...');
 const httpWrapper = new HTTPWrapper(evalServer, {
-  port: 8081,
+  port: 8080,
   host: '127.0.0.1'
 });
 
 
 console.log('🔧 Starting EvalServer...');
 await evalServer.start();
-console.log('✅ EvalServer started on ws://127.0.0.1:8080');
+console.log('✅ EvalServer started on ws://127.0.0.1:8082');
 
 console.log('🔧 Starting HTTP wrapper...');
 await httpWrapper.start();
-console.log('✅ HTTP API started on http://127.0.0.1:8081');
+console.log('✅ HTTP API started on http://127.0.0.1:8080');
 
 console.log('⏳ Waiting for DevTools client to connect...');
-console.log('   WebSocket URL: ws://127.0.0.1:8080');
-console.log('   HTTP API URL: http://127.0.0.1:8081');
-console.log('   Auth Key: hello');
+console.log('   WebSocket URL: ws://127.0.0.1:8082');
+console.log('   HTTP API URL: http://127.0.0.1:8080');
+console.log('   Auth: Disabled (automated mode)');
 
 // Add periodic status check
 setInterval(() => {
diff --git a/eval-server/nodejs/package-lock.json b/eval-server/nodejs/package-lock.json
index 494fa5e41b8..99f3ff787ca 100644
--- a/eval-server/nodejs/package-lock.json
+++ b/eval-server/nodejs/package-lock.json
@@ -16,6 +16,9 @@
         "winston": "^3.11.0",
         "ws": "^8.16.0"
       },
+      "bin": {
+        "eval-server": "src/cli/index.js"
+      },
       "devDependencies": {
         "@types/ws": "^8.5.10"
       },
diff --git a/eval-server/nodejs/src/api-server.js b/eval-server/nodejs/src/api-server.js
index 7b0b6355cdd..2713da4858b 100644
--- a/eval-server/nodejs/src/api-server.js
+++ b/eval-server/nodejs/src/api-server.js
@@ -255,7 +255,7 @@ class APIServer {
   }
 
   /**
-   * Handle OpenAI Responses API compatible requests
+   * Handle OpenAI Responses API compatible requests with nested model format
    */
   async handleResponsesRequest(requestBody) {
     try {
@@ -264,12 +264,20 @@ class APIServer {
         throw new Error('Missing or invalid "input" field. Expected a string.');
       }
 
-      // Merge request parameters with config defaults
-      const modelConfig = this.mergeModelConfig(requestBody);
-      
+      // Handle nested model configuration directly
+      const nestedModelConfig = this.processNestedModelConfig(requestBody);
+
+      const redact = (mk) => ({
+        ...mk,
+        api_key: mk?.api_key ? `${String(mk.api_key).slice(0, 4)}...` : undefined
+      });
       logger.info('Processing responses request:', {
         input: requestBody.input,
-        modelConfig
+        modelConfig: {
+          main_model: redact(nestedModelConfig.main_model),
+          mini_model: redact(nestedModelConfig.mini_model),
+          nano_model: redact(nestedModelConfig.nano_model),
+        }
       });
 
       // Find a connected and ready client
@@ -279,7 +287,7 @@ class APIServer {
       }
 
       // Create a dynamic evaluation for this request
-      const evaluation = this.createDynamicEvaluation(requestBody.input, modelConfig);
+      const evaluation = this.createDynamicEvaluationNested(requestBody.input, nestedModelConfig);
 
       // Execute the evaluation on the DevTools client
       logger.info('Executing evaluation on DevTools client', {
@@ -288,10 +296,10 @@ class APIServer {
       });
 
       const result = await this.evaluationServer.executeEvaluation(readyClient, evaluation);
-      
+
       // Debug: log the result structure
       logger.debug('executeEvaluation result:', result);
-      
+
       // Extract the response text from the result
       const responseText = this.extractResponseText(result);
 
@@ -305,19 +313,51 @@ class APIServer {
   }
 
   /**
-   * Merge request model parameters with config.yaml defaults
+   * Process nested model configuration from request body
+   * @param {Object} requestBody - Request body containing optional model configuration
+   * @returns {import('./types/model-config').ModelConfig} Nested model configuration
    */
-  mergeModelConfig(requestBody) {
+  processNestedModelConfig(requestBody) {
     const defaults = this.configDefaults?.model || {};
-    
+
+    // If nested format is provided, use it directly with fallbacks
+    if (requestBody.model) {
+      return {
+        main_model: requestBody.model.main_model || this.createDefaultModelConfig('main', defaults),
+        mini_model: requestBody.model.mini_model || this.createDefaultModelConfig('mini', defaults),
+        nano_model: requestBody.model.nano_model || this.createDefaultModelConfig('nano', defaults)
+      };
+    }
+
+    // No model config provided, use defaults
     return {
-      main_model: requestBody.main_model || defaults.main_model || 'gpt-4.1',
-      mini_model: requestBody.mini_model || defaults.mini_model || 'gpt-4.1-mini',
-      nano_model: requestBody.nano_model || defaults.nano_model || 'gpt-4.1-nano',
-      provider: requestBody.provider || defaults.provider || 'openai'
+      main_model: this.createDefaultModelConfig('main', defaults),
+      mini_model: this.createDefaultModelConfig('mini', defaults),
+      nano_model: this.createDefaultModelConfig('nano', defaults)
     };
   }
 
+  /**
+   * Create default model configuration for a tier
+   * @param {'main' | 'mini' | 'nano'} tier - Model tier
+   * @param {Object} defaults - Default configuration from config.yaml
+   * @returns {import('./types/model-config').ModelTierConfig} Model tier configuration
+   */
+  createDefaultModelConfig(tier, defaults) {
+    const defaultModels = {
+      main: defaults.main_model || 'gpt-4',
+      mini: defaults.mini_model || 'gpt-4-mini',
+      nano: defaults.nano_model || 'gpt-3.5-turbo'
+    };
+
+    return {
+      provider: defaults.provider || 'openai',
+      model: defaultModels[tier],
+      api_key: process.env.OPENAI_API_KEY
+    };
+  }
+
+
   /**
    * Find a connected and ready client
    */
@@ -331,34 +371,37 @@ class APIServer {
   }
 
   /**
-   * Create a dynamic evaluation object for the API request
+   * Create a dynamic evaluation object with nested model configuration
+   * @param {string} input - Input message for the evaluation
+   * @param {import('./types/model-config').ModelConfig} nestedModelConfig - Model configuration
+   * @returns {import('./types/model-config').EvaluationRequest} Evaluation request object
    */
-  createDynamicEvaluation(input, modelConfig) {
+  createDynamicEvaluationNested(input, nestedModelConfig) {
     const evaluationId = `api-eval-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
-    
+
     return {
       id: evaluationId,
-      name: 'OpenAI API Request',
-      description: 'Dynamic evaluation created from OpenAI Responses API request',
+      name: 'API Request',
+      description: 'Dynamic evaluation created from API request',
       enabled: true,
       tool: 'chat',
       timeout: 1500000, // 25 minutes
       input: {
-        message: input,
-        reasoning: 'API request processing'
+        message: input
       },
-      model: modelConfig,
+      model: nestedModelConfig,
       validation: {
         type: 'none' // No validation needed for API responses
       },
       metadata: {
-        tags: ['api', 'dynamic', 'openai-responses'],
+        tags: ['api', 'dynamic'],
         priority: 'high',
-        source: 'openai-api'
+        source: 'api'
       }
     };
   }
 
+
   /**
    * Extract response text from evaluation result
    */
diff --git a/eval-server/nodejs/src/client-manager.js b/eval-server/nodejs/src/client-manager.js
index c2dfdbdbbd3..d21b88dbd11 100644
--- a/eval-server/nodejs/src/client-manager.js
+++ b/eval-server/nodejs/src/client-manager.js
@@ -51,27 +51,26 @@ class ClientManager {
    * Precedence logic:
    * 1. API calls OR individual test YAML models (highest priority - either overrides everything)
    * 2. config.yaml defaults (fallback only when neither API nor test YAML specify models)
+   * @param {Object} evaluation - Evaluation object with optional model configuration
+   * @param {import('../types/model-config').ModelConfig} apiModelOverride - Optional API model override
+   * @returns {import('../types/model-config').ModelConfig} Final model configuration
    */
   applyModelPrecedence(evaluation, apiModelOverride = null) {
     // Check if API override is provided
     if (apiModelOverride) {
       // API model override takes precedence over everything
-      return {
-        ...(this.configDefaults?.model || {}), // Use config as base
-        ...apiModelOverride // API overrides everything
-      };
+      // Ensure nested format is used
+      return apiModelOverride;
     }
-    
+
     // Check if evaluation has its own model config from YAML
     const testModel = evaluation.model;
     if (testModel && Object.keys(testModel).length > 0) {
-      // Test YAML model takes precedence, use config.yaml only for missing fields
-      return {
-        ...(this.configDefaults?.model || {}), // Config as fallback base
-        ...testModel // Test YAML overrides config
-      };
+      // Test YAML model takes precedence
+      // Ensure nested format is returned
+      return testModel;
     }
-    
+
     // Neither API nor test YAML specified models, use config.yaml defaults only
     return this.configDefaults?.model || {};
   }
diff --git a/eval-server/nodejs/src/config.js b/eval-server/nodejs/src/config.js
index 632d0de167a..4bde4e52d27 100644
--- a/eval-server/nodejs/src/config.js
+++ b/eval-server/nodejs/src/config.js
@@ -7,21 +7,58 @@ export const CONFIG = {
     port: parseInt(process.env.PORT) || 8080,
     host: process.env.HOST || 'localhost'
   },
-  
+
   llm: {
     apiKey: process.env.OPENAI_API_KEY,
     model: process.env.JUDGE_MODEL || 'gpt-4',
     temperature: parseFloat(process.env.JUDGE_TEMPERATURE) || 0.1
   },
-  
+
+  // LLM Provider Configuration for configure_llm API
+  providers: {
+    openai: {
+      apiKey: process.env.OPENAI_API_KEY
+    },
+    litellm: {
+      endpoint: process.env.LITELLM_ENDPOINT,
+      apiKey: process.env.LITELLM_API_KEY
+    },
+    groq: {
+      apiKey: process.env.GROQ_API_KEY
+    },
+    openrouter: {
+      apiKey: process.env.OPENROUTER_API_KEY
+    }
+  },
+
+  // Default model configuration
+  defaults: {
+    provider: process.env.DEFAULT_PROVIDER || 'openai',
+    mainModel: process.env.DEFAULT_MAIN_MODEL || 'gpt-4',
+    miniModel: process.env.DEFAULT_MINI_MODEL || 'gpt-4-mini',
+    nanoModel: process.env.DEFAULT_NANO_MODEL || 'gpt-3.5-turbo'
+  },
+
   logging: {
     level: process.env.LOG_LEVEL || 'info',
     dir: process.env.LOG_DIR || './logs'
   },
-  
+
   rpc: {
     timeout: parseInt(process.env.RPC_TIMEOUT) || 1500000, // 25 minutes default
     maxConcurrentEvaluations: parseInt(process.env.MAX_CONCURRENT_EVALUATIONS) || 10
+  },
+
+  security: {
+    authSecretKey: process.env.AUTH_SECRET_KEY
+  },
+
+  clients: {
+    dir: process.env.CLIENTS_DIR || './clients'
+  },
+
+  evals: {
+    dir: process.env.EVALS_DIR || './evals'
   }
 };
 
diff --git a/eval-server/nodejs/src/lib/EvalServer.js b/eval-server/nodejs/src/lib/EvalServer.js
index 5471720bc57..d174c7a947e 100644
--- a/eval-server/nodejs/src/lib/EvalServer.js
+++ b/eval-server/nodejs/src/lib/EvalServer.js
@@ -196,6 +196,13 @@ export class EvalServer extends EventEmitter {
     return this.evaluationLoader.getAllEvaluations();
   }
 
+  /**
+   * Get the client manager instance
+   */
+  getClientManager() {
+    return this.clientManager;
+  }
+
   /**
    * Handle new WebSocket connections
    */
@@ -265,6 +272,12 @@ export class EvalServer extends EventEmitter {
         return;
       }
 
+      // Handle RPC requests from client to server
+      if (data.jsonrpc === '2.0' && data.method && data.id) {
+        await this.handleRpcRequest(connection, data);
+        return;
+      }
+
       // Handle other message types
       switch (data.type) {
         case 'register':
@@ -313,6 +326,147 @@ export class EvalServer extends EventEmitter {
     }
   }
 
+  /**
+   * Handle RPC requests from client to server
+   */
+  async handleRpcRequest(connection, request) {
+    try {
+      const { method, params, id } = request;
+
+      logger.info('Received RPC request', {
+        connectionId: connection.id,
+        clientId: connection.clientId,
+        method,
+        requestId: id
+      });
+
+      let result = null;
+
+      switch (method) {
+        case 'configure_llm':
+          result = await this.handleConfigureLLM(connection, params);
+          break;
+        default:
+          // JSON-RPC: Method not found
+          this.sendMessage(connection.ws, {
+            jsonrpc: '2.0',
+            error: {
+              code: -32601,
+              message: `Method not found: ${method}`
+            },
+            id
+          });
+          return;
+      }
+
+      // Send success response
+      this.sendMessage(connection.ws, {
+        jsonrpc: '2.0',
+        result,
+        id
+      });
+
+    } catch (error) {
+      logger.error('RPC request failed', {
+        connectionId: connection.id,
+        clientId: connection.clientId,
+        method: request.method,
+        requestId: request.id,
+        error: error.message
+      });
+
+      // Send error response
+      this.sendMessage(connection.ws, {
+        jsonrpc: '2.0',
+        error: {
+          code: -32603, // Internal error
+          message: error.message
+        },
+        id: request.id
+      });
+    }
+  }
+
+  /**
+   * Handle configure_llm RPC method
+   */
+  async handleConfigureLLM(connection, params) {
+    if (!connection.registered) {
+      throw new Error('Client must be registered before configuring LLM');
+    }
+
+    const { provider, apiKey, endpoint, models, partial = false } = params;
+
+    // Validate inputs
+    const supportedProviders = ['openai', 'litellm', 'groq', 'openrouter'];
+    if (partial) {
+      // For partial updates, validate only provided fields
+      if (provider && !supportedProviders.includes(provider)) {
+        throw new Error(`Unsupported provider: ${provider}. Supported providers: ${supportedProviders.join(', ')}`);
+      }
+      if (models && models.main === '') {
+        throw new Error('Main model cannot be empty');
+      }
+    } else {
+      // For full updates, require provider and main model
+      if (!provider || !supportedProviders.includes(provider)) {
+        throw new Error(`Unsupported or missing provider: ${provider ?? '(none)'}. Supported providers: ${supportedProviders.join(', ')}`);
+      }
+      if (!models || !models.main) {
+        throw new Error('Main model is required');
+      }
+    }
+
+    // Store configuration for this client connection
+    if (!connection.llmConfig) {
+      connection.llmConfig = {};
+    }
+
+    // Apply configuration (full or partial update)
+    if (partial && connection.llmConfig) {
+      // Partial update - merge with existing config
+      connection.llmConfig = {
+        ...connection.llmConfig,
+        provider: provider || connection.llmConfig.provider,
+        apiKey: apiKey || connection.llmConfig.apiKey,
+        endpoint: endpoint || connection.llmConfig.endpoint,
+        models: {
+          ...connection.llmConfig.models,
+          ...models
+        }
+      };
+    } else {
+      // Full update - replace entire config
+      connection.llmConfig = {
+        provider,
+        apiKey: apiKey || CONFIG.providers[provider]?.apiKey,
+        endpoint: endpoint || CONFIG.providers[provider]?.endpoint,
+        models: {
+          main: models.main,
+          mini: models.mini || models.main,
+          nano: models.nano || models.mini || models.main
+        }
+      };
+    }
+
+    logger.info('LLM configuration updated', {
+      clientId: connection.clientId,
+      provider: connection.llmConfig.provider,
+      models: connection.llmConfig.models,
+      hasApiKey: !!connection.llmConfig.apiKey,
+      hasEndpoint: !!connection.llmConfig.endpoint
+    });
+
+    return {
+      status: 'success',
+      message: 'LLM configuration updated successfully',
+      appliedConfig: {
+        provider: connection.llmConfig.provider,
+        models: connection.llmConfig.models
+      }
+    };
+  }
+
   /**
    * Handle client registration
    */
@@ -562,6 +716,35 @@ export class EvalServer extends EventEmitter {
         'running'
       );
 
+      // Prepare model configuration - use client config if available, otherwise evaluation config, otherwise defaults
+      let modelConfig = evaluation.model || {};
+
+      if (connection.llmConfig) {
+        // New nested format: separate config objects for each model tier
+        modelConfig = {
+          main_model: {
+            provider: connection.llmConfig.provider,
+            model: connection.llmConfig.models.main,
+            api_key: connection.llmConfig.apiKey,
+            endpoint: connection.llmConfig.endpoint
+          },
+          mini_model: {
+            provider: connection.llmConfig.provider,
+            model: connection.llmConfig.models.mini,
+            api_key: connection.llmConfig.apiKey,
+            endpoint: connection.llmConfig.endpoint
+          },
+          nano_model: {
+            provider: connection.llmConfig.provider,
+            model: connection.llmConfig.models.nano,
+            api_key: connection.llmConfig.apiKey,
+            endpoint: connection.llmConfig.endpoint
+          },
+          // Include any evaluation-specific overrides
+          ...modelConfig
+        };
+      }
+
       // Prepare RPC request
       const rpcRequest = {
         jsonrpc: '2.0',
@@ -572,7 +755,7 @@ export class EvalServer extends EventEmitter {
           url: evaluation.target?.url || evaluation.url,
           tool: evaluation.tool,
           input: evaluation.input,
-          model: evaluation.model,
+          model: modelConfig,
           timeout: evaluation.timeout || 30000,
           metadata: {
             tags: evaluation.metadata?.tags || [],
diff --git a/front_end/panels/ai_chat/BUILD.gn b/front_end/panels/ai_chat/BUILD.gn
index 1f733392a0e..464fde0fa98 100644
--- a/front_end/panels/ai_chat/BUILD.gn
+++ b/front_end/panels/ai_chat/BUILD.gn
@@ -58,6 +58,7 @@ devtools_module("ai_chat") {
     "core/PageInfoManager.ts",
     "core/AgentNodes.ts",
     "core/GraphHelpers.ts",
+    "core/LLMConfigurationManager.ts",
     "core/ToolNameMap.ts",
     "core/ToolSurfaceProvider.ts",
     "core/StateGraph.ts",
@@ -214,6 +215,7 @@ _ai_chat_sources = [
     "core/PageInfoManager.ts",
     "core/AgentNodes.ts",
     "core/GraphHelpers.ts",
+    "core/LLMConfigurationManager.ts",
     "core/ToolNameMap.ts",
     "core/ToolSurfaceProvider.ts",
     "core/StateGraph.ts",
diff --git a/front_end/panels/ai_chat/LLM/OpenAIProvider.ts b/front_end/panels/ai_chat/LLM/OpenAIProvider.ts
index b8c30ad3239..dbfc03b3b30 100644
--- a/front_end/panels/ai_chat/LLM/OpenAIProvider.ts
+++ b/front_end/panels/ai_chat/LLM/OpenAIProvider.ts
@@ -7,6 +7,7 @@ import { LLMBaseProvider } from './LLMProvider.js';
 import { LLMRetryManager } from './LLMErrorHandler.js';
 import { LLMResponseParser } from './LLMResponseParser.js';
 import { createLogger } from '../core/Logger.js';
+import { BUILD_CONFIG } from '../core/BuildConfig.js';
 
 const logger = createLogger('OpenAIProvider');
 
@@ -522,9 +523,17 @@ export class OpenAIProvider extends LLMBaseProvider {
    * Validate that required credentials are available for OpenAI
    */
   validateCredentials(): {isValid: boolean, message: string, missingItems?: string[]} {
+    // In AUTOMATED_MODE, skip credential validation since API keys are provided dynamically
+    if (BUILD_CONFIG.AUTOMATED_MODE) {
+      return {
+        isValid: true,
+        message: 'OpenAI credentials validation skipped in AUTOMATED_MODE (API keys provided dynamically).'
+      };
+    }
+
     const storageKeys = this.getCredentialStorageKeys();
     const apiKey = localStorage.getItem(storageKeys.apiKey!);
-    
+
     if (!apiKey) {
       return {
         isValid: false,
@@ -532,7 +541,7 @@ export class OpenAIProvider extends LLMBaseProvider {
         missingItems: ['API Key']
       };
     }
-    
+
     return {
       isValid: true,
       message: 'OpenAI credentials are configured correctly.'
diff --git a/front_end/panels/ai_chat/common/EvaluationConfig.ts b/front_end/panels/ai_chat/common/EvaluationConfig.ts
index be5169fe6cb..cda09b94b0e 100644
--- a/front_end/panels/ai_chat/common/EvaluationConfig.ts
+++ b/front_end/panels/ai_chat/common/EvaluationConfig.ts
@@ -30,7 +30,7 @@ class EvaluationConfigStore {
   private static instance: EvaluationConfigStore;
   private config: EvaluationConfiguration = {
     enabled: false,
-    endpoint: 'ws://localhost:8080',
+    endpoint: 'ws://localhost:8082',
     secretKey: '',
     clientId: ''
   };
@@ -50,16 +50,23 @@ class EvaluationConfigStore {
 
   private loadFromLocalStorage(): void {
     try {
-      // In automated mode, set default to enabled if not already set
-      if (BUILD_CONFIG.AUTOMATED_MODE && 
-          localStorage.getItem('ai_chat_evaluation_enabled') === null) {
-        localStorage.setItem('ai_chat_evaluation_enabled', 'true');
-        logger.info('Automated mode: defaulted evaluation to enabled');
+      // In automated mode, set defaults if not already set
+      if (BUILD_CONFIG.AUTOMATED_MODE) {
+        if (localStorage.getItem('ai_chat_evaluation_enabled') === null) {
+          localStorage.setItem('ai_chat_evaluation_enabled', 'true');
+          logger.info('Automated mode: defaulted evaluation to enabled');
+        }
+        if (localStorage.getItem('ai_chat_evaluation_endpoint') === null) {
+          localStorage.setItem('ai_chat_evaluation_endpoint', 'ws://localhost:8082');
+          logger.info('Automated mode: defaulted endpoint to ws://localhost:8082');
+        }
+        // No secret key needed in automated mode
       }
 
       const enabled = localStorage.getItem('ai_chat_evaluation_enabled') === 'true';
-      const endpoint = localStorage.getItem('ai_chat_evaluation_endpoint') || 'ws://localhost:8080';
-      const secretKey = localStorage.getItem('ai_chat_evaluation_secret_key') || '';
+      const endpoint = localStorage.getItem('ai_chat_evaluation_endpoint') || 'ws://localhost:8082';
+      // Don't use secretKey in automated mode
+      const secretKey = BUILD_CONFIG.AUTOMATED_MODE ? undefined : (localStorage.getItem('ai_chat_evaluation_secret_key') || '');
       const clientId = localStorage.getItem('ai_chat_evaluation_client_id') || '';
 
       this.config = {
@@ -98,7 +105,10 @@ class EvaluationConfigStore {
     try {
       localStorage.setItem('ai_chat_evaluation_enabled', String(this.config.enabled));
       localStorage.setItem('ai_chat_evaluation_endpoint', this.config.endpoint);
-      localStorage.setItem('ai_chat_evaluation_secret_key', this.config.secretKey || '');
+      // Don't save secret key in automated mode
+      if (!BUILD_CONFIG.AUTOMATED_MODE) {
+        localStorage.setItem('ai_chat_evaluation_secret_key', this.config.secretKey || '');
+      }
       localStorage.setItem('ai_chat_evaluation_client_id', this.config.clientId || '');
     } catch (error) {
       logger.warn('Failed to save evaluation config to localStorage:', error);
diff --git a/front_end/panels/ai_chat/core/AgentNodes.ts b/front_end/panels/ai_chat/core/AgentNodes.ts
index b2a476b6f34..cbfe05c52b8 100644
--- a/front_end/panels/ai_chat/core/AgentNodes.ts
+++ b/front_end/panels/ai_chat/core/AgentNodes.ts
@@ -15,6 +15,7 @@ import { ToolSurfaceProvider } from './ToolSurfaceProvider.js';
 import { createLogger } from './Logger.js';
 import type { AgentState } from './State.js';
 import type { Runnable } from './Types.js';
+import { LLMConfigurationManager } from './LLMConfigurationManager.js';
 import { AgentErrorHandler } from './AgentErrorHandler.js';
 import { createTracingProvider, withTracingContext } from '../tracing/TracingConfig.js';
 import * as ToolNameMap from './ToolNameMap.js';
@@ -812,13 +813,17 @@ export function createToolExecutorNode(state: AgentState, provider: LLMProvider,
 
         const result = await withTracingContext(executionContext, async () => {
           logger.debug(`Inside withTracingContext for tool: ${toolName}`);
-          const apiKeyFromState = (state.context as any)?.apiKey;
+
+          // Get configuration from manager (supports overrides)
+          const configManager = LLMConfigurationManager.getInstance();
+          const config = configManager.getConfiguration();
+
           return await selectedTool.execute(toolArgs as any, {
-            apiKey: apiKeyFromState,
-            provider: this.provider,
-            model: this.modelName,
-            miniModel: this.miniModel,
-            nanoModel: this.nanoModel,
+            apiKey: config.apiKey,
+            provider: config.provider,
+            model: config.mainModel,
+            miniModel: config.miniModel,
+            nanoModel: config.nanoModel,
             abortSignal: signal || state.context.abortSignal,
             ...(configurableDescriptor ? { agentDescriptor: configurableDescriptor } : {})
           });
diff --git a/front_end/panels/ai_chat/core/AgentService.ts b/front_end/panels/ai_chat/core/AgentService.ts
index c7258b68a98..b244be45583 100644
--- a/front_end/panels/ai_chat/core/AgentService.ts
+++ b/front_end/panels/ai_chat/core/AgentService.ts
@@ -1,6 +1,7 @@
 // Copyright 2025 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
+// Cache break: 2025-09-17T22:47:00Z - Add AUTOMATED_MODE bypass for createAgentGraph API key validation
 
 import * as Common from '../../../core/common/common.js';
 import * as i18n from '../../../core/i18n/i18n.js';
@@ -14,13 +15,16 @@ import { AgentDescriptorRegistry } from './AgentDescriptorRegistry.js';
 import {type AgentState, createInitialState, createUserMessage} from './State.js';
 import type {CompiledGraph} from './Types.js';
 import { LLMClient } from '../LLM/LLMClient.js';
+import { LLMConfigurationManager } from './LLMConfigurationManager.js';
 import { createTracingProvider, getCurrentTracingContext } from '../tracing/TracingConfig.js';
 import type { TracingProvider, TracingContext } from '../tracing/TracingProvider.js';
 import { AgentRunnerEventBus } from '../agent_framework/AgentRunnerEventBus.js';
 import { AgentRunner } from '../agent_framework/AgentRunner.js';
 import type { AgentSession, AgentMessage } from '../agent_framework/AgentSessionTypes.js';
 import type { LLMProvider } from '../LLM/LLMTypes.js';
+import { BUILD_CONFIG } from './BuildConfig.js';
 
+// Cache break: 2025-09-17T17:54:00Z - Force rebuild with AUTOMATED_MODE bypass
 const logger = createLogger('AgentService');
 
 /**
@@ -58,6 +62,7 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
   #tracingProvider!: TracingProvider;
   #sessionId: string;
   #activeAgentSessions = new Map<string, AgentSession>();
+  #configManager: LLMConfigurationManager;
 
   // Global registry for all active executions
   private static activeExecutions = new Map<string, AbortController>();
@@ -96,7 +101,10 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
 
   constructor() {
     super();
-    
+
+    // Initialize configuration manager
+    this.#configManager = LLMConfigurationManager.getInstance();
+
     // Initialize tracing
     this.#sessionId = this.generateSessionId();
     this.#initializeTracing();
@@ -115,6 +123,9 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
     
     // Subscribe to AgentRunner events
     AgentRunnerEventBus.getInstance().addEventListener('agent-progress', this.#handleAgentProgress.bind(this));
+
+    // Subscribe to configuration changes
+    this.#configManager.addChangeListener(this.#handleConfigurationChange.bind(this));
   }
 
   /**
@@ -139,59 +150,63 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
    */
   async #initializeLLMClient(): Promise<void> {
     const llm = LLMClient.getInstance();
-    
-    // Get configuration from localStorage
-    const provider = localStorage.getItem('ai_chat_provider') || 'openai';
-    const openaiKey = localStorage.getItem('ai_chat_api_key') || '';
-    const litellmKey = localStorage.getItem('ai_chat_litellm_api_key') || '';
-    const litellmEndpoint = localStorage.getItem('ai_chat_litellm_endpoint') || '';
-    const groqKey = localStorage.getItem('ai_chat_groq_api_key') || '';
-    const openrouterKey = localStorage.getItem('ai_chat_openrouter_api_key') || '';
-    
+
+    // Get configuration from manager (with override support)
+    const config = this.#configManager.getConfiguration();
+    const provider = config.provider;
+    const apiKey = config.apiKey;
+    const endpoint = config.endpoint;
+
     const providers = [];
-    
-    // Only add the selected provider
-    if (provider === 'openai' && openaiKey) {
-      providers.push({ 
-        provider: 'openai' as const, 
-        apiKey: openaiKey 
-      });
-    }
-      
-    if (provider === 'litellm' && litellmEndpoint) {
-      providers.push({ 
-        provider: 'litellm' as const, 
-        apiKey: litellmKey, // Can be empty for some LiteLLM endpoints
-        providerURL: litellmEndpoint 
-      });
-    }
-      
-    if (provider === 'groq' && groqKey) {
-      providers.push({ 
-        provider: 'groq' as const, 
-        apiKey: groqKey 
-      });
+
+    // Validate and add the selected provider
+    // Skip credential checks in AUTOMATED_MODE where API keys come from request body
+    const validation = this.#configManager.validateConfiguration(BUILD_CONFIG.AUTOMATED_MODE);
+    if (!validation.isValid) {
+      throw new Error(`Configuration validation failed: ${validation.errors.join(', ')}`);
     }
-      
-    if (provider === 'openrouter' && openrouterKey) {
-      providers.push({ 
-        provider: 'openrouter' as const, 
-        apiKey: openrouterKey 
-      });
+
+    // Only add the selected provider if it has valid configuration
+    switch (provider) {
+      case 'openai':
+        if (apiKey) {
+          providers.push({
+            provider: 'openai' as const,
+            apiKey
+          });
+        }
+        break;
+      case 'litellm':
+        if (endpoint) {
+          providers.push({
+            provider: 'litellm' as const,
+            apiKey: apiKey || '', // Can be empty for some LiteLLM endpoints
+            providerURL: endpoint
+          });
+        }
+        break;
+      case 'groq':
+        if (apiKey) {
+          providers.push({
+            provider: 'groq' as const,
+            apiKey
+          });
+        }
+        break;
+      case 'openrouter':
+        if (apiKey) {
+          providers.push({
+            provider: 'openrouter' as const,
+            apiKey
+          });
+        }
+        break;
     }
-    
+
     if (providers.length === 0) {
-      let errorMessage = 'OpenAI API key is required for this configuration';
-      if (provider === 'litellm') {
-        errorMessage = 'LiteLLM endpoint is required for this configuration';
-      } else if (provider === 'groq') {
-        errorMessage = 'Groq API key is required for this configuration';
-      } else if (provider === 'openrouter') {
-        errorMessage = 'OpenRouter API key is required for this configuration';
-      }
-      throw new Error(errorMessage);
+      throw new Error(`No valid configuration found for provider ${provider}`);
     }
-    
+
     await llm.initialize({ providers });
     logger.info('LLM client initialized successfully', {
       selectedProvider: provider,
@@ -213,9 +228,9 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
       // Check if the configuration requires an API key
       const requiresApiKey = this.#doesCurrentConfigRequireApiKey();
       
-      // If API key is required but not provided, throw error
-      if (requiresApiKey && !apiKey) {
-        const provider = localStorage.getItem('ai_chat_provider') || 'openai';
+      // If API key is required but not provided, throw error (unless in AUTOMATED_MODE)
+      if (requiresApiKey && !apiKey && !BUILD_CONFIG.AUTOMATED_MODE) {
+        const provider = this.#configManager.getProvider();
         let providerName = 'OpenAI';
         if (provider === 'litellm') {
           providerName = 'LiteLLM';
@@ -227,13 +242,13 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
         throw new Error(`${providerName} API key is required for this configuration`);
       }
 
-      // Determine selected provider for primary graph execution
-      const selectedProvider = (localStorage.getItem('ai_chat_provider') || 'openai') as LLMProvider;
+      // Get provider from configuration manager
+      const config = this.#configManager.getConfiguration();
 
       // Mini and nano models are injected by caller (validated upstream)
 
       // Will throw error if model/provider configuration is invalid
-      this.#graph = createAgentGraph(apiKey, modelName, selectedProvider, miniModel, nanoModel);
+      this.#graph = createAgentGraph(apiKey, modelName, config.provider, miniModel, nanoModel);
 
       // Stash apiKey in state context for downstream tools that need it
       if (!this.#state.context) { (this.#state as any).context = {}; }
@@ -313,14 +328,51 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
     await this.#initializeTracing();
   }
 
+  /**
+   * Handle configuration changes from LLMConfigurationManager
+   */
+  #handleConfigurationChange(): void {
+    logger.info('LLM configuration changed, reinitializing if needed');
+
+    // If we're initialized, we need to reinitialize with new configuration
+    if (this.#isInitialized) {
+      // Mark as uninitialized to force reinit on next use
+      this.#isInitialized = false;
+      this.#graph = undefined;
+
+      logger.info('Marked agent service for reinitialization due to config change');
+    }
+  }
+
+  /**
+   * Public method to refresh credentials and agent service
+   * Can be called from settings dialog or other components
+   */
+  async refreshCredentials(): Promise<void> {
+    logger.info('Refreshing credentials and reinitializing agent service');
+
+    this.#isInitialized = false;
+    this.#graph = undefined;
+
+    // Force reinitialization on next use
+    try {
+      const config = this.#configManager.getConfiguration();
+      await this.initialize(config.apiKey || null, config.mainModel, config.miniModel || '', config.nanoModel || '');
+      logger.info('Agent service reinitialized successfully');
+    } catch (error) {
+      logger.error('Failed to reinitialize agent service:', error);
+      throw error;
+    }
+  }
+
   /**
    * Sends a message to the AI agent
    */
   async sendMessage(text: string, imageInput?: ImageInputData, selectedAgentType?: string | null): Promise<ChatMessage> {
     // Check if the current configuration requires an API key
     const requiresApiKey = this.#doesCurrentConfigRequireApiKey();
-    
-    if (requiresApiKey && !this.#apiKey) {
+
+    if (requiresApiKey && !this.#apiKey && !BUILD_CONFIG.AUTOMATED_MODE) {
       throw new Error('API key not set. Please set the API key in settings.');
     }
 
@@ -328,6 +380,13 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
       throw new Error('Empty message. Please enter some text.');
     }
 
+    // In AUTOMATED_MODE, ensure the graph is initialized even without API key
+    if (BUILD_CONFIG.AUTOMATED_MODE && !this.#graph) {
+      const config = this.#configManager.getConfiguration();
+      // Initialize with empty API key in AUTOMATED_MODE - will be overridden by request
+      await this.initialize('', config.mainModel, config.miniModel || '', config.nanoModel || '');
+    }
+
     // Create a user message
     const userMessage = createUserMessage(text, imageInput);
 
@@ -712,7 +771,7 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
   #doesCurrentConfigRequireApiKey(): boolean {
     try {
       // Check the selected provider
-      const selectedProvider = localStorage.getItem('ai_chat_provider') || 'openai';
+      const selectedProvider = this.#configManager.getProvider();
       
       // OpenAI provider always requires an API key
       if (selectedProvider === 'openai') {
diff --git a/front_end/panels/ai_chat/core/LLMConfigurationManager.ts b/front_end/panels/ai_chat/core/LLMConfigurationManager.ts
new file mode 100644
index 00000000000..2ce3aa2160f
--- /dev/null
+++ b/front_end/panels/ai_chat/core/LLMConfigurationManager.ts
@@ -0,0 +1,428 @@
+// Copyright 2025 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+// Cache break: 2025-09-18T19:00:00Z - Add skipCredentialChecks + preserve credentials + secure logging
+
+import { createLogger } from './Logger.js';
+import type { LLMProvider } from '../LLM/LLMTypes.js';
+
+const logger = createLogger('LLMConfigurationManager');
+
+/**
+ * Configuration interface for LLM settings
+ */
+export interface LLMConfig {
+  provider: LLMProvider;
+  apiKey?: string;
+  endpoint?: string; // For LiteLLM
+  mainModel: string;
+  miniModel?: string;
+  nanoModel?: string;
+}
+
+/**
+ * Local storage keys for LLM configuration
+ */
+const STORAGE_KEYS = {
+  PROVIDER: 'ai_chat_provider',
+  MODEL_SELECTION: 'ai_chat_model_selection',
+  MINI_MODEL: 'ai_chat_mini_model',
+  NANO_MODEL: 'ai_chat_nano_model',
+  OPENAI_API_KEY: 'ai_chat_api_key',
+  LITELLM_ENDPOINT: 'ai_chat_litellm_endpoint',
+  LITELLM_API_KEY: 'ai_chat_litellm_api_key',
+  GROQ_API_KEY: 'ai_chat_groq_api_key',
+  OPENROUTER_API_KEY: 'ai_chat_openrouter_api_key',
+} as const;
+
+/**
+ * Centralized LLM configuration manager with override capabilities.
+ * Supports both manual mode (localStorage-based) and automated mode (override-based).
+ */
+export class LLMConfigurationManager {
+  private static instance: LLMConfigurationManager;
+  private overrideConfig?: Partial<LLMConfig>; // Override for automated mode
+  private changeListeners: Array<() => void> = [];
+
+  private constructor() {
+    // Listen for localStorage changes from other tabs (manual mode)
+    window.addEventListener('storage', this.handleStorageChange.bind(this));
+  }
+
+  /**
+   * Get the singleton instance
+   */
+  static getInstance(): LLMConfigurationManager {
+    if (!LLMConfigurationManager.instance) {
+      LLMConfigurationManager.instance = new LLMConfigurationManager();
+    }
+    return LLMConfigurationManager.instance;
+  }
+
+  /**
+   * Get the current provider with override fallback
+   */
+  getProvider(): LLMProvider {
+    if (this.overrideConfig?.provider) {
+      return this.overrideConfig.provider;
+    }
+    const stored = localStorage.getItem(STORAGE_KEYS.PROVIDER);
+    return (stored as LLMProvider) || 'openai';
+  }
+
+  /**
+   * Get the main model with override fallback
+   */
+  getMainModel(): string {
+    if (this.overrideConfig?.mainModel) {
+      return this.overrideConfig.mainModel;
+    }
+    return localStorage.getItem(STORAGE_KEYS.MODEL_SELECTION) || '';
+  }
+
+  /**
+   * Get the mini model with override fallback
+   */
+  getMiniModel(): string {
+    if (this.overrideConfig?.miniModel) {
+      return this.overrideConfig.miniModel;
+    }
+    return localStorage.getItem(STORAGE_KEYS.MINI_MODEL) || '';
+  }
+
+  /**
+   * Get the nano model with override fallback
+   */
+  getNanoModel(): string {
+    if (this.overrideConfig?.nanoModel) {
+      return this.overrideConfig.nanoModel;
+    }
+    return localStorage.getItem(STORAGE_KEYS.NANO_MODEL) || '';
+  }
+
+  /**
+   * Get the API key for the current provider with override fallback
+   */
+  getApiKey(): string {
+    if (this.overrideConfig?.apiKey) {
+      return this.overrideConfig.apiKey;
+    }
+
+    const provider = this.getProvider();
+    switch (provider) {
+      case 'openai':
+        return localStorage.getItem(STORAGE_KEYS.OPENAI_API_KEY) || '';
+      case 'litellm':
+        return localStorage.getItem(STORAGE_KEYS.LITELLM_API_KEY) || '';
+      case 'groq':
+        return localStorage.getItem(STORAGE_KEYS.GROQ_API_KEY) || '';
+      case 'openrouter':
+        return localStorage.getItem(STORAGE_KEYS.OPENROUTER_API_KEY) || '';
+      default:
+        return '';
+    }
+  }
+
+  /**
+   * Get the endpoint (primarily for LiteLLM) with override fallback
+   */
+  getEndpoint(): string | undefined {
+    if (this.overrideConfig?.endpoint) {
+      return this.overrideConfig.endpoint;
+    }
+
+    const provider = this.getProvider();
+    if (provider === 'litellm') {
+      return localStorage.getItem(STORAGE_KEYS.LITELLM_ENDPOINT) || undefined;
+    }
+    return undefined;
+  }
+
+  /**
+   * Get the complete current configuration
+   */
+  getConfiguration(): LLMConfig {
+    return {
+      provider: this.getProvider(),
+      apiKey: this.getApiKey(),
+      endpoint: this.getEndpoint(),
+      mainModel: this.getMainModel(),
+      miniModel: this.getMiniModel(),
+      nanoModel: this.getNanoModel(),
+    };
+  }
+
+  /**
+   * Set override configuration (for automated mode per-request overrides)
+   */
+  setOverride(config: Partial<LLMConfig>): void {
+    logger.info('Setting configuration override', {
+      provider: config.provider,
+      mainModel: config.mainModel,
+      hasApiKey: !!config.apiKey,
+      hasEndpoint: !!config.endpoint
+    });
+
+    this.overrideConfig = { ...config };
+    this.notifyListeners();
+  }
+
+  /**
+   * Clear override configuration
+   */
+  clearOverride(): void {
+    if (this.overrideConfig) {
+      logger.info('Clearing configuration override');
+      this.overrideConfig = undefined;
+      this.notifyListeners();
+    }
+  }
+
+  /**
+   * Check if override is currently active
+   */
+  hasOverride(): boolean {
+    return !!this.overrideConfig;
+  }
+
+  /**
+   * Save configuration to localStorage (for manual mode and persistent automated mode)
+   */
+  saveConfiguration(config: LLMConfig): void {
+    logger.info('Saving configuration to localStorage', {
+      provider: config.provider,
+      mainModel: config.mainModel,
+      hasApiKey: !!config.apiKey,
+      hasEndpoint: !!config.endpoint
+    });
+
+    // Save provider
+    localStorage.setItem(STORAGE_KEYS.PROVIDER, config.provider);
+
+    // Save models
+    localStorage.setItem(STORAGE_KEYS.MODEL_SELECTION, config.mainModel);
+    if (config.miniModel) {
+      localStorage.setItem(STORAGE_KEYS.MINI_MODEL, config.miniModel);
+    } else {
+      localStorage.removeItem(STORAGE_KEYS.MINI_MODEL);
+    }
+    if (config.nanoModel) {
+      localStorage.setItem(STORAGE_KEYS.NANO_MODEL, config.nanoModel);
+    } else {
+      localStorage.removeItem(STORAGE_KEYS.NANO_MODEL);
+    }
+
+    // Save provider-specific settings
+    this.saveProviderSpecificSettings(config);
+
+    // Notify listeners of configuration change
+    this.notifyListeners();
+  }
+
+  /**
+   * Apply partial configuration updates (merges with existing configuration)
+   */
+  applyPartialConfiguration(partial: Partial<LLMConfig>): void {
+    const current = this.loadConfiguration();
+
+    // Merge configurations, preserving existing values where partial doesn't provide them
+    const merged: LLMConfig = {
+      provider: partial.provider ?? current.provider,
+      mainModel: partial.mainModel ?? current.mainModel,
+      miniModel: partial.miniModel ?? current.miniModel,
+      nanoModel: partial.nanoModel ?? current.nanoModel,
+      apiKey: partial.apiKey ?? current.apiKey,
+      endpoint: partial.endpoint ?? current.endpoint,
+    };
+
+    logger.info('Applying partial configuration update', {
+      current: {
+        provider: current.provider,
+        mainModel: current.mainModel,
+        hasApiKey: !!current.apiKey
+      },
+      partial: {
+        provider: partial.provider,
+        mainModel: partial.mainModel,
+        hasApiKey: !!partial.apiKey
+      },
+      merged: {
+        provider: merged.provider,
+        mainModel: merged.mainModel,
+        hasApiKey: !!merged.apiKey
+      }
+    });
+
+    // Save the merged configuration
+    this.saveConfiguration(merged);
+  }
+
+  /**
+   * Load configuration from localStorage
+   */
+  loadConfiguration(): LLMConfig {
+    return {
+      provider: this.getProvider(),
+      apiKey: this.getApiKey(),
+      endpoint: this.getEndpoint(),
+      mainModel: this.getMainModel(),
+      miniModel: this.getMiniModel(),
+      nanoModel: this.getNanoModel(),
+    };
+  }
+
+  /**
+   * Add a listener for configuration changes
+   */
+  addChangeListener(listener: () => void): void {
+    this.changeListeners.push(listener);
+  }
+
+  /**
+   * Remove a configuration change listener
+   */
+  removeChangeListener(listener: () => void): void {
+    const index = this.changeListeners.indexOf(listener);
+    if (index !== -1) {
+      this.changeListeners.splice(index, 1);
+    }
+  }
+
+  /**
+   * Validate the current configuration
+   * @param skipCredentialChecks When true, bypasses API key/endpoint validation for AUTOMATED_MODE
+   */
+  validateConfiguration(skipCredentialChecks = false): { isValid: boolean; errors: string[] } {
+    const config = this.getConfiguration();
+    const errors: string[] = [];
+
+    // Check provider
+    if (!config.provider) {
+      errors.push('Provider is required');
+    }
+
+    // Check main model
+    if (!config.mainModel) {
+      errors.push('Main model is required');
+    }
+
+    // Provider-specific validation - skip credential checks in AUTOMATED_MODE
+    if (!skipCredentialChecks) {
+      switch (config.provider) {
+        case 'openai':
+        case 'groq':
+        case 'openrouter':
+          if (!config.apiKey) {
+            errors.push(`API key is required for ${config.provider}`);
+          }
+          break;
+        case 'litellm':
+          if (!config.endpoint) {
+            errors.push('Endpoint is required for LiteLLM');
+          }
+          break;
+      }
+    }
+
+    return {
+      isValid: errors.length === 0,
+      errors
+    };
+  }
+
+  /**
+   * Save provider-specific settings to localStorage
+   * Only modifies settings for the active provider, preserving other providers' credentials
+   */
+  private saveProviderSpecificSettings(config: LLMConfig): void {
+    // Save current provider's settings only (do not clear others)
+    switch (config.provider) {
+      case 'openai':
+        if (config.apiKey) {
+          localStorage.setItem(STORAGE_KEYS.OPENAI_API_KEY, config.apiKey);
+        } else {
+          localStorage.removeItem(STORAGE_KEYS.OPENAI_API_KEY);
+        }
+        break;
+      case 'litellm':
+        if (config.endpoint) {
+          localStorage.setItem(STORAGE_KEYS.LITELLM_ENDPOINT, config.endpoint);
+        } else {
+          localStorage.removeItem(STORAGE_KEYS.LITELLM_ENDPOINT);
+        }
+        if (config.apiKey) {
+          localStorage.setItem(STORAGE_KEYS.LITELLM_API_KEY, config.apiKey);
+        } else {
+          localStorage.removeItem(STORAGE_KEYS.LITELLM_API_KEY);
+        }
+        break;
+      case 'groq':
+        if (config.apiKey) {
+          localStorage.setItem(STORAGE_KEYS.GROQ_API_KEY, config.apiKey);
+        } else {
+          localStorage.removeItem(STORAGE_KEYS.GROQ_API_KEY);
+        }
+        break;
+      case 'openrouter':
+        if (config.apiKey) {
+          localStorage.setItem(STORAGE_KEYS.OPENROUTER_API_KEY, config.apiKey);
+        } else {
+          localStorage.removeItem(STORAGE_KEYS.OPENROUTER_API_KEY);
+        }
+        break;
+    }
+  }
+
+  /**
+   * Handle localStorage changes from other tabs
+   */
+  private handleStorageChange(event: StorageEvent): void {
+    if (event.key && Object.values(STORAGE_KEYS).includes(event.key as any)) {
+      const sensitiveKeys = new Set([
+        STORAGE_KEYS.OPENAI_API_KEY,
+        STORAGE_KEYS.LITELLM_API_KEY,
+        STORAGE_KEYS.GROQ_API_KEY,
+        STORAGE_KEYS.OPENROUTER_API_KEY,
+      ]);
+      const redacted =
+        sensitiveKeys.has(event.key as any) ? '(redacted)' :
+        (event.newValue ? `${event.newValue.slice(0, 8)}…` : null);
+      logger.debug('Configuration changed in another tab', {
+        key: event.key,
+        newValue: redacted
+      });
+      this.notifyListeners();
+    }
+  }
+
+  /**
+   * Notify all listeners of configuration changes
+   */
+  private notifyListeners(): void {
+    this.changeListeners.forEach(listener => {
+      try {
+        listener();
+      } catch (error) {
+        logger.error('Error in configuration change listener:', error);
+      }
+    });
+  }
+
+  /**
+   * Get debug information about current configuration state
+   */
+  getDebugInfo(): Record<string, any> {
+    const redact = (cfg?: Partial<LLMConfig>) => cfg ? {
+      ...cfg,
+      apiKey: cfg.apiKey ? '(redacted)' : undefined
+    } : undefined;
+
+    return {
+      hasOverride: this.hasOverride(),
+      overrideConfig: redact(this.overrideConfig),
+      currentConfig: redact(this.getConfiguration()),
+      validation: this.validateConfiguration(),
+      listenerCount: this.changeListeners.length,
+    };
+  }
+}
\ No newline at end of file
diff --git a/front_end/panels/ai_chat/evaluation/EvaluationAgent.ts b/front_end/panels/ai_chat/evaluation/EvaluationAgent.ts
index 7564b17927c..b152fa2f4b5 100644
--- a/front_end/panels/ai_chat/evaluation/EvaluationAgent.ts
+++ b/front_end/panels/ai_chat/evaluation/EvaluationAgent.ts
@@ -4,6 +4,7 @@
 
 import { WebSocketRPCClient } from '../common/WebSocketRPCClient.js';
 import { getEvaluationConfig, getEvaluationClientId } from '../common/EvaluationConfig.js';
+import { BUILD_CONFIG } from '../core/BuildConfig.js';
 import { ToolRegistry } from '../agent_framework/ConfigurableAgentTool.js';
 import { AgentService } from '../core/AgentService.js';
 import { createLogger } from '../core/Logger.js';
@@ -252,6 +253,14 @@ export class EvaluationAgent {
   }
 
   private async handleAuthRequest(message: RegistrationAckMessage): Promise<void> {
+    // In automated mode, skip authentication entirely
+    if (BUILD_CONFIG.AUTOMATED_MODE) {
+      logger.info('Automated mode: Skipping authentication verification');
+      const authMessage = createAuthVerifyMessage(message.clientId, true);
+      this.client?.send(authMessage);
+      return;
+    }
+
     if (!message.serverSecretKey) {
       logger.error('Server did not provide secret key for verification');
       this.disconnect();
@@ -265,10 +274,10 @@ export class EvaluationAgent {
     // Verify if the server's secret key matches the client's configured key
     const verified = clientSecretKey === message.serverSecretKey;
 
-    logger.info('Verifying secret key', { 
+    logger.info('Verifying secret key', {
       hasClientKey: !!clientSecretKey,
       hasServerKey: !!message.serverSecretKey,
-      verified 
+      verified
     });
 
     // Send verification response
diff --git a/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts b/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts
index 2bd5255275e..992a44681ec 100644
--- a/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts
+++ b/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts
@@ -1,11 +1,15 @@
 // Copyright 2025 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
+// Cache break: 2025-09-17T17:38:00Z - Fixed API key validation
 
+import { BUILD_CONFIG } from '../../core/BuildConfig.js';
 import { WebSocketRPCClient } from '../../common/WebSocketRPCClient.js';
+import { LLMConfigurationManager, type LLMConfig } from '../../core/LLMConfigurationManager.js';
 import { getEvaluationConfig, getEvaluationClientId } from '../../common/EvaluationConfig.js';
 import { ToolRegistry, ConfigurableAgentTool } from '../../agent_framework/ConfigurableAgentTool.js';
 import { AgentService } from '../../core/AgentService.js';
+import { AIChatPanel } from '../../ui/AIChatPanel.js';
 import { createLogger } from '../../core/Logger.js';
 import { createTracingProvider, withTracingContext, isTracingEnabled, getTracingConfig } from '../../tracing/TracingConfig.js';
 import { AgentDescriptorRegistry, type AgentDescriptor } from '../../core/AgentDescriptorRegistry.js';
@@ -22,17 +26,21 @@ import {
   EvaluationRequest,
   EvaluationSuccessResponse,
   EvaluationErrorResponse,
+  LLMConfigurationRequest,
+  LLMConfigurationResponse,
   ErrorCodes,
   isWelcomeMessage,
   isRegistrationAckMessage,
   isEvaluationRequest,
+  isLLMConfigurationRequest,
   isPongMessage,
   createRegisterMessage,
   createReadyMessage,
   createAuthVerifyMessage,
   createStatusMessage,
   createSuccessResponse,
-  createErrorResponse
+  createErrorResponse,
+  createLLMConfigurationResponse
 } from './EvaluationProtocol.js';
 
 const logger = createLogger('EvaluationAgent');
@@ -186,6 +194,9 @@ export class EvaluationAgent {
       else if (isEvaluationRequest(message)) {
         await this.handleEvaluationRequest(message);
       }
+      else if (isLLMConfigurationRequest(message)) {
+        await this.handleLLMConfigurationRequest(message);
+      }
       else if (isPongMessage(message)) {
         logger.debug('Received pong');
       }
@@ -268,6 +279,14 @@ export class EvaluationAgent {
   }
 
   private async handleAuthRequest(message: RegistrationAckMessage): Promise<void> {
+    // In automated mode, skip authentication entirely
+    if (BUILD_CONFIG.AUTOMATED_MODE) {
+      logger.info('Automated mode: Skipping authentication verification');
+      const authMessage = createAuthVerifyMessage(message.clientId, true);
+      this.client?.send(authMessage);
+      return;
+    }
+
     if (!message.serverSecretKey) {
       logger.error('Server did not provide secret key for verification');
       this.disconnect();
@@ -315,6 +334,7 @@ export class EvaluationAgent {
   private async handleEvaluationRequest(request: EvaluationRequest): Promise<void> {
     const { params, id } = request;
     const startTime = Date.now();
+    let hasSetEarlyOverride = false;
 
     logger.info('Received evaluation request', {
       evaluationId: params.evaluationId,
@@ -325,6 +345,76 @@ export class EvaluationAgent {
       modelConfig: params.model
     });
 
+    // CRITICAL FIX: Set configuration override early for any model configuration provided
+    // This allows API keys from request body to be available for UI-level validation
+    logger.info('DEBUG: Checking for model configuration override', {
+      evaluationId: params.evaluationId,
+      hasModel: !!params.model,
+      model: params.model
+    });
+
+    if (params.model && (params.model.main_model || params.model.provider || params.model.api_key)) {
+      const configManager = LLMConfigurationManager.getInstance();
+
+      // Extract configuration from nested model structure - handle both flat and nested formats
+      const mainModel = params.model.main_model as any;
+
+      // For nested format: main_model: { provider: "openai", model: "gpt-4", api_key: "key" }
+      // For flat format: { provider: "openai", main_model: "gpt-4", api_key: "key" }
+      const provider = mainModel?.provider || params.model.provider || 'openai';
+      const apiKey = mainModel?.api_key || params.model.api_key;
+      const modelName = mainModel?.model || mainModel || params.model.main_model;
+      const miniModel = (params.model.mini_model as any)?.model || params.model.mini_model;
+      const nanoModel = (params.model.nano_model as any)?.model || params.model.nano_model;
+
+      logger.info('DEBUG: Extracted model configuration', {
+        evaluationId: params.evaluationId,
+        mainModel,
+        provider,
+        hasApiKey: !!apiKey,
+        apiKeyLength: apiKey?.length,
+        modelName,
+        miniModel,
+        nanoModel
+      });
+
+      if (apiKey) {
+        logger.info('Setting early configuration override for evaluation', {
+          evaluationId: params.evaluationId,
+          provider,
+          hasApiKey: !!apiKey,
+          modelName
+        });
+
+        configManager.setOverride({
+          provider,
+          apiKey,
+          mainModel: modelName,
+          miniModel: miniModel || modelName,
+          nanoModel: nanoModel || miniModel || modelName
+        });
+        hasSetEarlyOverride = true;
+
+        logger.info('DEBUG: Early override set successfully', {
+          evaluationId: params.evaluationId
+        });
+      } else {
+        logger.warn('DEBUG: No API key found in model configuration', {
+          evaluationId: params.evaluationId,
+          mainModel,
+          provider
+        });
+      }
+    } else {
+      logger.warn('DEBUG: No model configuration found for override', {
+        evaluationId: params.evaluationId,
+        hasModel: !!params.model,
+        hasMainModel: !!(params.model?.main_model),
+        hasProvider: !!(params.model?.provider),
+        hasApiKey: !!(params.model?.api_key)
+      });
+    }
+
     // Track active evaluation
     this.activeEvaluations.set(params.evaluationId, {
       startTime,
@@ -455,15 +545,41 @@ export class EvaluationAgent {
         this.sendStatus(params.evaluationId, 'running', 0.5, 'Processing chat request...');
         
         // Merge model configuration - prefer params.model over params.input model fields
+        // Also check for early override configuration from LLMConfigurationManager
+        const configManager = LLMConfigurationManager.getInstance();
+        const hasOverride = configManager.hasOverride?.() || false;
+        const overrideConfig = hasOverride ? configManager.getConfiguration() : null;
+
         const mergedInput = {
           ...params.input,
           ...(params.model && {
-            main_model: params.model.main_model,
-            mini_model: params.model.mini_model,
-            nano_model: params.model.nano_model,
-            provider: params.model.provider
+            // Extract nested model configuration properly
+            main_model: (params.model.main_model as any)?.model || params.model.main_model,
+            mini_model: (params.model.mini_model as any)?.model || params.model.mini_model,
+            nano_model: (params.model.nano_model as any)?.model || params.model.nano_model,
+            provider: (params.model.main_model as any)?.provider || params.model.provider,
+            api_key: (params.model.main_model as any)?.api_key || (params.model as any).api_key
+          }),
+          // Apply early override configuration if available
+          ...(overrideConfig && {
+            provider: overrideConfig.provider || ((params.model?.main_model as any)?.provider || params.model?.provider),
+            api_key: overrideConfig.apiKey || ((params.model?.main_model as any)?.api_key || (params.model as any)?.api_key),
+            main_model: overrideConfig.mainModel || ((params.model?.main_model as any)?.model || params.model?.main_model),
+            mini_model: overrideConfig.miniModel || ((params.model?.mini_model as any)?.model || params.model?.mini_model),
+            nano_model: overrideConfig.nanoModel || ((params.model?.nano_model as any)?.model || params.model?.nano_model),
+            endpoint: overrideConfig.endpoint || ((params.model as any)?.endpoint || (params.model?.main_model as any)?.endpoint)
           })
         };
+
+        logger.info('DEBUG: Created merged input for chat evaluation', {
+          evaluationId: params.evaluationId,
+          hasOverrideConfig: !!overrideConfig,
+          overrideConfig,
+          mergedInput: {
+            ...mergedInput,
+            api_key: mergedInput.api_key ? `${mergedInput.api_key.substring(0, 10)}...` : undefined
+          }
+        });
         
         toolResult = await this.executeChatEvaluation(
           mergedInput,
@@ -580,6 +696,17 @@ export class EvaluationAgent {
 
     } finally {
       this.activeEvaluations.delete(params.evaluationId);
+
+      // Clean up early override if we set one
+      if (hasSetEarlyOverride) {
+        try {
+          const configManager = LLMConfigurationManager.getInstance();
+          configManager.clearOverride();
+          logger.info('Cleared early configuration override', { evaluationId: params.evaluationId });
+        } catch (clearError) {
+          logger.warn('Failed to clear early configuration override:', clearError);
+        }
+      }
     }
   }
 
@@ -753,25 +880,50 @@ export class EvaluationAgent {
       }, timeout);
 
       let chatObservationId: string | undefined;
+      // Get configuration manager for override support (defined outside try-catch for finally block)
+      const configManager = LLMConfigurationManager.getInstance();
       const orchestratorDescriptor = await this.orchestratorDescriptorPromise;
 
       try {
+
         // Get or create AgentService instance
         const agentService = AgentService.getInstance();
-        
-        // Use explicit models from constructor
-        const modelName = this.judgeModel;
-        const miniModel = this.miniModel;
-        const nanoModel = this.nanoModel;
-        
-        logger.info('Initializing AgentService for chat evaluation', {
-          modelName,
-          hasApiKey: !!agentService.getApiKey(),
+
+        // Get current configuration as baseline
+        const config = configManager.getConfiguration();
+
+        // Consolidate configuration with proper fallbacks
+        // Priority: input values -> existing config -> defaults
+        const provider = input.provider || config.provider || 'openai';
+        const mainModel = input.main_model || config.mainModel || this.judgeModel;
+        const miniModel = input.mini_model ?? config.miniModel ?? this.miniModel;
+        const nanoModel = input.nano_model ?? config.nanoModel ?? this.nanoModel;
+        const apiKey = input.api_key ?? config.apiKey ?? agentService.getApiKey();
+        const endpoint = input.endpoint ?? config.endpoint;
+
+        logger.info('Setting consolidated configuration override for chat evaluation', {
+          provider: provider,
+          mainModel: mainModel,
+          hasApiKey: !!apiKey,
+          hasEndpoint: !!endpoint,
           isInitialized: agentService.isInitialized()
         });
-        
-        // Always reinitialize with the current model and explicit mini/nano
-        await agentService.initialize(agentService.getApiKey(), modelName, miniModel, nanoModel);
+
+        // Set single consolidated override with all fallback values
+        configManager.setOverride({
+          provider: provider,
+          mainModel: mainModel,
+          miniModel: miniModel,
+          nanoModel: nanoModel,
+          apiKey: apiKey || undefined,
+          endpoint: endpoint || undefined
+        });
+
+        // Send the message using AgentService directly but with configuration override
+        // The configuration override ensures it uses the API key from the request
+        const finalMessage: ChatMessage = tracingContext
+          ? await withTracingContext(tracingContext, () => agentService.sendMessage(input.message))
+          : await agentService.sendMessage(input.message);
         
         // Create a child observation for the chat execution
         if (tracingContext) {
@@ -782,7 +934,7 @@ export class EvaluationAgent {
               name: 'Chat Execution',
               type: 'span',
               startTime: new Date(),
-              input: { message: input.message, model: modelName },
+              input: { message: input.message, model: mainModel },
               metadata: {
                 evaluationType: 'chat',
                 ...(orchestratorDescriptor ? {
@@ -798,11 +950,6 @@ export class EvaluationAgent {
           }
         }
         
-        // Send the message with the evaluation tracing context
-        const finalMessage: ChatMessage = tracingContext 
-          ? await withTracingContext(tracingContext, () => agentService.sendMessage(input.message))
-          : await agentService.sendMessage(input.message);
-        
         clearTimeout(timer);
         
         // Extract the response text from the final message
@@ -840,23 +987,23 @@ export class EvaluationAgent {
         const result = {
           response: responseText,
           messages: agentService.getMessages(),
-          modelUsed: modelName,
+          modelUsed: mainModel,
           timestamp: new Date().toISOString(),
           evaluationMetadata: {
             evaluationType: 'chat',
-            actualModelUsed: modelName
+            actualModelUsed: mainModel
           }
         };
         
         logger.info('Chat evaluation completed successfully', {
           responseLength: responseText.length,
           messageCount: result.messages.length,
-          modelUsed: modelName,
+          modelUsed: mainModel,
           evaluationId: tracingContext?.traceId
         });
         
         resolve(result);
-        
+
       } catch (error) {
         clearTimeout(timer);
         
@@ -882,10 +1029,133 @@ export class EvaluationAgent {
             logger.warn('Failed to update chat execution observation with error:', updateError);
           }
         }
-        
+
         logger.error('Chat evaluation failed:', error);
         reject(error);
+      } finally {
+        // Clear configuration override after evaluation
+        configManager.clearOverride();
       }
     });
   }
+
+  /**
+   * Handle LLM configuration requests for persistent configuration
+   */
+  private async handleLLMConfigurationRequest(request: LLMConfigurationRequest): Promise<void> {
+    const { params, id } = request;
+
+    logger.info('Received LLM configuration request', {
+      provider: params.provider,
+      hasApiKey: !!params.apiKey,
+      models: params.models,
+      partial: params.partial
+    });
+
+    try {
+      // Get configuration manager
+      const configManager = LLMConfigurationManager.getInstance();
+
+      // Store current config for potential rollback
+      const currentConfig = configManager.loadConfiguration();
+
+      // Handle configuration update based on partial flag
+      if (params.partial) {
+        // Use the new partial configuration method
+        const partialConfig: Partial<LLMConfig> = {};
+        if (params.provider !== undefined) partialConfig.provider = params.provider;
+        if (params.apiKey !== undefined) partialConfig.apiKey = params.apiKey;
+        if (params.endpoint !== undefined) partialConfig.endpoint = params.endpoint;
+        if (params.models?.main !== undefined) partialConfig.mainModel = params.models.main;
+        if (params.models?.mini !== undefined) partialConfig.miniModel = params.models.mini;
+        if (params.models?.nano !== undefined) partialConfig.nanoModel = params.models.nano;
+
+        configManager.applyPartialConfiguration(partialConfig);
+      } else {
+        // Full configuration update
+        const fullConfig: LLMConfig = {
+          provider: params.provider,
+          apiKey: params.apiKey,
+          endpoint: params.endpoint,
+          mainModel: params.models.main,
+          miniModel: params.models?.mini,
+          nanoModel: params.models?.nano
+        };
+        configManager.saveConfiguration(fullConfig);
+      }
+
+      // Validate the saved configuration
+      const postSaveValidation = configManager.validateConfiguration();
+
+      if (!postSaveValidation.isValid) {
+        // Restore the original config if validation fails
+        configManager.saveConfiguration(currentConfig);
+
+        // Send error response with validation errors
+        const errorResponse = createErrorResponse(
+          id,
+          ErrorCodes.INVALID_PARAMS,
+          'Invalid configuration',
+          { errors: postSaveValidation.errors }
+        );
+
+        if (this.client) {
+          this.client.send(errorResponse);
+        }
+
+        logger.error('Configuration validation failed', {
+          errors: postSaveValidation.errors
+        });
+
+        return;
+      }
+
+      // Reinitialize AgentService with new configuration
+      const agentService = AgentService.getInstance();
+      await agentService.refreshCredentials();
+
+      // Get the applied configuration
+      const appliedConfiguration = configManager.loadConfiguration();
+
+      // Prepare response with applied configuration
+      const appliedConfig = {
+        provider: appliedConfiguration.provider,
+        models: {
+          main: appliedConfiguration.mainModel,
+          mini: appliedConfiguration.miniModel || '',
+          nano: appliedConfiguration.nanoModel || ''
+        }
+      };
+
+      // Send success response
+      const response = createLLMConfigurationResponse(id, appliedConfig);
+
+      if (this.client) {
+        this.client.send(response);
+      }
+
+      logger.info('LLM configuration applied successfully', {
+        provider: appliedConfiguration.provider,
+        mainModel: appliedConfiguration.mainModel
+      });
+
+    } catch (error) {
+      logger.error('Failed to apply LLM configuration:', error);
+
+      // Send error response
+      const errorResponse = createErrorResponse(
+        id,
+        ErrorCodes.INTERNAL_ERROR,
+        'Failed to apply LLM configuration',
+        {
+          error: error instanceof Error ? error.message : String(error),
+          timestamp: new Date().toISOString()
+        }
+      );
+
+      if (this.client) {
+        this.client.send(errorResponse);
+      }
+    }
+  }
 }
diff --git a/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts b/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts
index c9d4c8e9acb..f0efc557b9c 100644
--- a/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts
+++ b/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts
@@ -90,6 +90,8 @@ export interface EvaluationParams {
     mini_model?: string;
     nano_model?: string;
     provider?: string;
+    api_key?: string;    // New: per-request API key
+    endpoint?: string;   // New: per-request endpoint (LiteLLM)
   };
   timeout: number;
   metadata: {
@@ -250,4 +252,83 @@ export function createErrorResponse(
     },
     id
   };
+}
+
+// LLM Configuration JSON-RPC Messages
+
+export interface LLMConfigurationRequest {
+  jsonrpc: '2.0';
+  method: 'configure_llm';
+  params: LLMConfigurationParams;
+  id: string;
+}
+
+export interface LLMConfigurationParams {
+  provider: 'openai' | 'litellm' | 'groq' | 'openrouter';
+  apiKey?: string;
+  endpoint?: string; // For LiteLLM
+  models: {
+    main: string;
+    mini?: string;
+    nano?: string;
+  };
+  // Optional: only update specific fields
+  partial?: boolean;
+}
+
+export interface LLMConfigurationResponse {
+  jsonrpc: '2.0';
+  result: {
+    status: 'success';
+    message: string;
+    appliedConfig: {
+      provider: string;
+      models: {
+        main: string;
+        mini: string;
+        nano: string;
+      };
+    };
+  };
+  id: string;
+}
+
+// Type guard for LLM configuration
+export function isLLMConfigurationRequest(msg: any): msg is LLMConfigurationRequest {
+  return msg?.jsonrpc === '2.0' && msg?.method === 'configure_llm';
+}
+
+// Helper function for LLM configuration
+export function createLLMConfigurationRequest(
+  id: string,
+  params: LLMConfigurationParams
+): LLMConfigurationRequest {
+  return {
+    jsonrpc: '2.0',
+    method: 'configure_llm',
+    params,
+    id
+  };
+}
+
+export function createLLMConfigurationResponse(
+  id: string,
+  appliedConfig: {
+    provider: string;
+    models: {
+      main: string;
+      mini: string;
+      nano: string;
+    };
+  }
+): LLMConfigurationResponse {
+  return {
+    jsonrpc: '2.0',
+    result: {
+      status: 'success',
+      message: 'LLM configuration updated successfully',
+      appliedConfig
+    },
+    id
+  };
 }
\ No newline at end of file
diff --git a/front_end/panels/ai_chat/ui/AIChatPanel.ts b/front_end/panels/ai_chat/ui/AIChatPanel.ts
index 662c0f25c78..b38228a4293 100644
--- a/front_end/panels/ai_chat/ui/AIChatPanel.ts
+++ b/front_end/panels/ai_chat/ui/AIChatPanel.ts
@@ -12,6 +12,7 @@ import * as Lit from '../../../ui/lit/lit.js';
 import * as VisualLogging from '../../../ui/visual_logging/visual_logging.js';
 import {AgentService, Events as AgentEvents} from '../core/AgentService.js';
 import { LLMClient } from '../LLM/LLMClient.js';
+import { LLMConfigurationManager } from '../core/LLMConfigurationManager.js';
 import { LLMProviderRegistry } from '../LLM/LLMProviderRegistry.js';
 import { OpenAIProvider } from '../LLM/OpenAIProvider.js';
 import { LiteLLMProvider } from '../LLM/LiteLLMProvider.js';
@@ -332,27 +333,28 @@ export class AIChatPanel extends UI.Panel.Panel {
   }
 
   static getMiniModel(): string {
-    const instance = AIChatPanel.instance();
-    
-    // Validate the model selection before returning
-    instance.#validateAndFixModelSelections();
-    
-    return instance.#miniModel || instance.#selectedModel;
+    const configManager = LLMConfigurationManager.getInstance();
+    const miniModel = configManager.getMiniModel();
+
+    // Fallback to main model if mini model not set
+    return miniModel || configManager.getMainModel();
   }
 
   static getNanoModel(): string {
-    const instance = AIChatPanel.instance();
-    
-    // Validate the model selection before returning
-    instance.#validateAndFixModelSelections();
-    
-    return instance.#nanoModel || instance.#miniModel || instance.#selectedModel;
+    const configManager = LLMConfigurationManager.getInstance();
+    const nanoModel = configManager.getNanoModel();
+    const miniModel = configManager.getMiniModel();
+    const mainModel = configManager.getMainModel();
+
+    // Fallback hierarchy: nano -> mini -> main
+    return nanoModel || miniModel || mainModel;
   }
 
   static getNanoModelWithProvider(): { model: string, provider: 'openai' | 'litellm' | 'groq' | 'openrouter' } {
+    const configManager = LLMConfigurationManager.getInstance();
     const modelName = AIChatPanel.getNanoModel();
-    const provider = AIChatPanel.getProviderForModel(modelName);
-    
+    const provider = configManager.getProvider();
+
     return {
       model: modelName,
       provider: provider
@@ -360,9 +362,10 @@ export class AIChatPanel extends UI.Panel.Panel {
   }
 
   static getMiniModelWithProvider(): { model: string, provider: 'openai' | 'litellm' | 'groq' | 'openrouter' } {
+    const configManager = LLMConfigurationManager.getInstance();
     const modelName = AIChatPanel.getMiniModel();
-    const provider = AIChatPanel.getProviderForModel(modelName);
-    
+    const provider = configManager.getProvider();
+
     return {
       model: modelName,
       provider: provider
@@ -734,6 +737,7 @@ export class AIChatPanel extends UI.Panel.Panel {
   #apiKey: string | null = null; // Regular API key
   #evaluationAgent: EvaluationAgent | null = null; // Evaluation agent for this tab
   #mcpUnsubscribe: (() => void) | null = null;
+  #configManager: LLMConfigurationManager;
 
   // Store bound event listeners to properly add/remove without duplications
   #boundOnMessagesChanged?: (e: Common.EventTarget.EventTargetEvent<ChatMessage[]>) => void;
@@ -746,6 +750,9 @@ export class AIChatPanel extends UI.Panel.Panel {
   constructor() {
     super(AIChatPanel.panelName);
 
+    // Initialize configuration manager
+    this.#configManager = LLMConfigurationManager.getInstance();
+
     // Initialize storage monitoring for debugging
     StorageMonitor.getInstance();
     
@@ -1112,6 +1119,23 @@ export class AIChatPanel extends UI.Panel.Panel {
     return this.#selectedModel;
   }
 
+  /**
+   * Set LLM configuration programmatically (for manual mode and persistent automated mode)
+   */
+  setLLMConfiguration(config: import('../core/LLMConfigurationManager.js').LLMConfig): void {
+    logger.info('Setting LLM configuration programmatically', {
+      provider: config.provider,
+      mainModel: config.mainModel,
+      hasApiKey: !!config.apiKey
+    });
+
+    // Save configuration to localStorage
+    this.#configManager.saveConfiguration(config);
+
+    // Refresh the agent service with new configuration
+    this.refreshCredentials();
+  }
+
   /**
    * Public method to refresh credential validation and agent service
    * Can be called from settings dialog or other components