Prewarm LLM provider connections

.changeset/prewarm-llm-connections.md

@@ -0,0 +1,9 @@
+---
+'@livekit/agents': patch
+'@livekit/agents-plugin-anthropic': patch
+'@livekit/agents-plugin-google': patch
+'@livekit/agents-plugin-mistralai': patch
+'@livekit/agents-plugin-openai': patch
+---
+
+Prewarm LLM provider connections before the first inference request.

agents/src/inference/llm.test.ts

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: 2026 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { beforeAll, describe, expect, it } from 'vitest';
+import { beforeAll, describe, expect, it, vi } from 'vitest';
 import * as agents from '../index.js';
 import { ChatContext } from '../llm/index.js';
 import { initializeLogger } from '../log.js';
@@ -15,6 +15,44 @@ beforeAll(() => {
 type CapturedHeaders = Record<string, string>;
 type CompletionChunk = Record<string, unknown>;
 
+describe('inference.LLM prewarm', () => {
+  it('refreshes the access token before listing models and forwards cancellation', async () => {
+    const llm = new LLM({
+      model: 'openai/gpt-4o-mini',
+      apiKey: 'test-key',
+      apiSecret: 'test-secret',
+      baseURL: 'https://example.livekit.cloud',
+    });
+    const internal = llm as unknown as {
+      client: {
+        apiKey: string;
+        models: {
+          list: (options: { signal?: AbortSignal }) => Promise<void>;
+        };
+      };
+    };
+    let apiKeyAtList = '';
+    let prewarmSignal: AbortSignal | undefined;
+    const modelsList = vi.fn(async (options: { signal?: AbortSignal }) => {
+      apiKeyAtList = internal.client.apiKey;
+      prewarmSignal = options.signal;
+    });
+    internal.client.models.list = modelsList;
+
+    llm.prewarm();
+    await new Promise<void>((resolve) => setImmediate(resolve));
+
+    expect(modelsList).toHaveBeenCalledTimes(1);
+    expect(apiKeyAtList).not.toBe('placeholder');
+    expect(apiKeyAtList.split('.')).toHaveLength(3);
+    expect(prewarmSignal).toBeInstanceOf(AbortSignal);
+    expect(prewarmSignal?.aborted).toBe(false);
+
+    await llm.aclose();
+    expect(prewarmSignal?.aborted).toBe(true);
+  });
+});
+
 /**
  * Build an LLM, stub its OpenAI client's chat.completions.create, start a chat
  * stream with the given per-call value, drain the stream, and return the headers

agents/src/inference/llm.ts

@@ -261,6 +261,11 @@ export class LLM extends llm.LLM {
     return 'livekit';
   }
 
+  protected override async _prewarmImpl(signal: AbortSignal): Promise<void> {
+    this.client.apiKey = await createAccessToken(this.opts.apiKey, this.opts.apiSecret);
+    await this.client.models.list({ signal });
+  }
+
   static fromModelString(modelString: string): LLM {
     return new LLM({ model: modelString });
   }

agents/src/llm/llm.test.ts

@@ -1,10 +1,10 @@
 // SPDX-FileCopyrightText: 2026 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { beforeAll, describe, expect, it } from 'vitest';
+import { beforeAll, describe, expect, it, vi } from 'vitest';
 import { initializeLogger } from '../log.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
-import { delay } from '../utils.js';
+import { Future, Task, delay } from '../utils.js';
 import { ChatContext, FunctionCall } from './chat_context.js';
 import { type ChatChunk, LLM, LLMStream } from './llm.js';
 import type { ToolChoice, ToolCtxInput } from './tool_context.js';
@@ -59,6 +59,110 @@ class MockLLM extends LLM {
   }
 }
 
+class PrewarmLLM extends MockLLM {
+  constructor(private readonly prewarmImpl: (signal: AbortSignal) => Promise<void>) {
+    super([]);
+  }
+
+  protected override _prewarmImpl(signal: AbortSignal): Promise<void> {
+    return this.prewarmImpl(signal);
+  }
+}
+
+const waitForTasks = () => new Promise<void>((resolve) => setImmediate(resolve));
+
+describe('LLM prewarm lifecycle', () => {
+  it('is a no-op when the provider does not override _prewarmImpl', async () => {
+    const taskFrom = vi.spyOn(Task, 'from');
+    const llm = new MockLLM([]);
+
+    try {
+      llm.prewarm();
+      await llm.aclose();
+
+      expect(taskFrom).not.toHaveBeenCalled();
+    } finally {
+      taskFrom.mockRestore();
+    }
+  });
+
+  it('schedules fire-and-forget work without surfacing provider rejection', async () => {
+    const unhandledRejection = vi.fn();
+    process.on('unhandledRejection', unhandledRejection);
+    const llm = new PrewarmLLM(async () => {
+      throw new Error('provider unavailable');
+    });
+
+    try {
+      expect(() => llm.prewarm()).not.toThrow();
+      await waitForTasks();
+
+      expect(unhandledRejection).not.toHaveBeenCalled();
+    } finally {
+      process.off('unhandledRejection', unhandledRejection);
+    }
+  });
+
+  it('invokes the provider hook once across repeated calls, including after failure', async () => {
+    const prewarmImpl = vi.fn(async () => {
+      throw new Error('provider unavailable');
+    });
+    const llm = new PrewarmLLM(prewarmImpl);
+
+    llm.prewarm();
+    llm.prewarm();
+    await waitForTasks();
+    llm.prewarm();
+    await waitForTasks();
+
+    expect(prewarmImpl).toHaveBeenCalledTimes(1);
+  });
+
+  it('aborts an in-flight prewarm and waits for provider cleanup during close', async () => {
+    const started = new Future<void>();
+    let signal: AbortSignal | undefined;
+    let cleanedUp = false;
+    const llm = new PrewarmLLM(
+      (prewarmSignal) =>
+        new Promise<void>((resolve) => {
+          signal = prewarmSignal;
+          started.resolve();
+          prewarmSignal.addEventListener(
+            'abort',
+            () => {
+              setImmediate(() => {
+                cleanedUp = true;
+                resolve();
+              });
+            },
+            { once: true },
+          );
+        }),
+    );
+
+    llm.prewarm();
+    await started.await;
+    const closing = llm.aclose();
+
+    expect(signal?.aborted).toBe(true);
+    expect(cleanedUp).toBe(false);
+
+    await closing;
+    expect(cleanedUp).toBe(true);
+  });
+
+  it('does not start prewarm work after close wins the lifecycle race', async () => {
+    const prewarmImpl = vi.fn(async () => {});
+    const llm = new PrewarmLLM(prewarmImpl);
+
+    await llm.aclose();
+    llm.prewarm();
+    await waitForTasks();
+
+    expect(prewarmImpl).not.toHaveBeenCalled();
+  });
+});
+
 describe('LLMStream.collect', () => {
   beforeAll(() => {
     initializeLogger({ pretty: false });

agents/src/llm/llm.ts

@@ -9,7 +9,7 @@ import { log } from '../log.js';
 import type { LLMMetrics } from '../metrics/base.js';
 import { recordException, traceTypes, tracer } from '../telemetry/index.js';
 import { type APIConnectOptions, intervalForRetry } from '../types.js';
-import { AsyncIterableQueue, delay, startSoon, toError } from '../utils.js';
+import { AsyncIterableQueue, Task, delay, startSoon, toError } from '../utils.js';
 import { type ChatContext, type ChatRole, type FunctionCall } from './chat_context.js';
 import {
   type ToolChoice,
@@ -65,6 +65,9 @@ export type LLMCallbacks = {
 };
 
 export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCallbacks>) {
+  #prewarmTask?: Task<void>;
+  #closed = false;
+
   constructor() {
     super();
   }
@@ -120,14 +123,45 @@ export abstract class LLM extends (EventEmitter as new () => TypedEmitter<LLMCal
   }): LLMStream;
 
   /**
-   * Pre-warm connection to the LLM service
+   * Pre-warm connection to the LLM service.
+   *
+   * Establishes DNS resolution and the TLS connection to the provider before the first inference
+   * request, reducing time-to-first-token on the initial reply. Non-blocking (fire-and-forget) and
+   * idempotent; calls made after {@link aclose} are ignored. Providers enable it by overriding
+   * {@link _prewarmImpl}.
    */
   prewarm(): void {
-    // Default implementation - subclasses can override
+    if (this.#closed || this._prewarmImpl === LLM.prototype._prewarmImpl) {
+      return;
+    }
+
+    if (this.#prewarmTask) {
+      return;
+    }
+
+    this.#prewarmTask = Task.from(async (controller) => {
+      try {
+        await this._prewarmImpl(controller.signal);
+      } catch {
+        // Prewarm is best-effort and must not affect session startup.
+      }
+    });
   }
 
+  /**
+   * Performs a provider-specific, token-free request that initializes DNS, TLS, authentication,
+   * and keep-alive state.
+   *
+   * @remarks Exceptions are swallowed by {@link prewarm}. Implementations must honor `signal` so
+   * {@link aclose} can cancel and await in-flight work.
+   */
+  protected async _prewarmImpl(_signal: AbortSignal): Promise<void> {}
+
   async aclose(): Promise<void> {
-    // Default implementation - subclasses can override
+    this.#closed = true;
+    if (this.#prewarmTask) {
+      await this.#prewarmTask.cancelAndWait();
+    }
   }
 }
 

agents/src/voice/agent_activity.ts

@@ -478,6 +478,10 @@ export class AgentActivity implements RecognitionHooks {
   async start(options?: { reuseResources?: ReusableResources }): Promise<void> {
     const unlock = await this.lock.lock();
     try {
+      if (this.llm instanceof LLM) {
+        this.llm.prewarm();
+      }
+
       await this._startSession({
         spanName: 'start_agent_activity',
         runOnEnter: true,

agents/src/voice/agent_activity_prewarm.test.ts

@@ -0,0 +1,60 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { Agent } from './agent.js';
+import { AgentSession } from './agent_session.js';
+import { FakeLLM, type FakeLLMResponse } from './testing/fake_llm.js';
+
+class TrackingLLM extends FakeLLM {
+  prewarmCalls = 0;
+
+  constructor(
+    responses: FakeLLMResponse[] = [],
+    private readonly events: string[] = [],
+  ) {
+    super(responses);
+  }
+
+  override chat(options: Parameters<FakeLLM['chat']>[0]) {
+    this.events.push('chat');
+    return super.chat(options);
+  }
+
+  protected override async _prewarmImpl(_signal: AbortSignal): Promise<void> {
+    this.prewarmCalls += 1;
+    this.events.push('prewarm');
+  }
+}
+
+describe('AgentActivity LLM prewarm', () => {
+  it('prewarms an agent-level LLM exactly once before its first inference', async () => {
+    const sessionLLM = new TrackingLLM();
+    const agentEvents: string[] = [];
+    const agentLLM = new TrackingLLM([{ input: 'hello', content: 'hi' }], agentEvents);
+    const session = new AgentSession({
+      llm: sessionLLM,
+      vad: null,
+      turnHandling: { turnDetection: null },
+    });
+
+    expect(sessionLLM.prewarmCalls).toBe(1);
+    expect(agentLLM.prewarmCalls).toBe(0);
+
+    try {
+      await session.start({
+        agent: new Agent({ instructions: '', llm: agentLLM }),
+      });
+
+      expect(agentLLM.prewarmCalls).toBe(1);
+      expect(sessionLLM.prewarmCalls).toBe(1);
+
+      await session.run({ userInput: 'hello' }).wait();
+
+      expect(agentEvents).toEqual(['prewarm', 'chat']);
+      expect(agentLLM.prewarmCalls).toBe(1);
+    } finally {
+      await session.close();
+    }
+  });
+});

agents/src/voice/agent_session.ts

@@ -42,6 +42,7 @@ import type {
   ToolContextLike,
 } from '../llm/index.js';
 import { ToolContext, toToolContext } from '../llm/index.js';
+import { LLM as BaseLLM } from '../llm/llm.js';
 import type { LLMError } from '../llm/llm.js';
 import { log } from '../log.js';
 import { type ModelUsage, ModelUsageCollector, filterZeroValues } from '../metrics/model_usage.js';
@@ -560,6 +561,11 @@ export class AgentSession<
       this.llm = llm;
     }
 
+    // Eagerly establish DNS/TLS to the LLM provider so the first inference request is faster.
+    if (this.llm instanceof BaseLLM) {
+      this.llm.prewarm();
+    }
+
     if (typeof tts === 'string') {
       this.tts = InferenceTTS.fromModelString(tts);
     } else {

plugins/anthropic/src/llm.test.ts

@@ -3,10 +3,35 @@
 // SPDX-License-Identifier: Apache-2.0
 import type Anthropic from '@anthropic-ai/sdk';
 import { llm } from '@livekit/agents';
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
 import { z } from 'zod';
 import { LLM } from './llm.js';
 
+describe('Anthropic LLM prewarm', () => {
+  it('lists one model with the prewarm cancellation signal', async () => {
+    let prewarmSignal: AbortSignal | undefined;
+    const modelsList = vi.fn(
+      async (_params: { limit: number }, options: { signal?: AbortSignal }) => {
+        prewarmSignal = options.signal;
+      },
+    );
+    const client = {
+      baseURL: 'https://api.anthropic.test',
+      models: { list: modelsList },
+    } as unknown as Anthropic;
+    const llm = new LLM({ model: 'claude-sonnet-4-6', client });
+
+    llm.prewarm();
+    await new Promise<void>((resolve) => setImmediate(resolve));
+
+    expect(modelsList).toHaveBeenCalledWith({ limit: 1 }, { signal: expect.any(AbortSignal) });
+    expect(prewarmSignal?.aborted).toBe(false);
+
+    await llm.aclose();
+    expect(prewarmSignal?.aborted).toBe(true);
+  });
+});
+
 function messageStartEvent(): Anthropic.MessageStreamEvent {
   return {
     type: 'message_start',