diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb6e3ce..3b6bee2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -28,6 +28,7 @@ jobs: - name: Run E2E tests env: + MOCK: true OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} OPENAI_MODEL: ${{ secrets.OPENAI_MODEL }} diff --git a/packages/agent/__tests__/e2e/react-loop.e2e.test.ts b/packages/agent/__tests__/e2e/react-loop.e2e.test.ts index 6c61bd9..dbdb356 100644 --- a/packages/agent/__tests__/e2e/react-loop.e2e.test.ts +++ b/packages/agent/__tests__/e2e/react-loop.e2e.test.ts @@ -5,7 +5,8 @@ * receives a question, invokes a tool, receives the tool result, and * produces a final text answer. * - * Skipped when OPENAI_API_KEY is not set (uses OpenAI-compatible adapter). + * Skipped when OPENAI_API_KEY is not set (uses OpenAI-compatible adapter), + * unless MOCK=true. * Falls back to ANTHROPIC_API_KEY if OPENAI_API_KEY is unavailable. * * Environment variables (OpenAI-compatible): @@ -17,9 +18,11 @@ * ANTHROPIC_API_KEY — API key * ANTHROPIC_BASE_URL — Base URL * ANTHROPIC_MODEL — Model name + * + * MOCK — Set to "true" to use a mock server (no real API needed) */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { registerProvider, createModel, @@ -29,28 +32,50 @@ import { } from "@openlinkos/ai"; import { createAgent } from "../../src/index.js"; import type { ToolDefinition } from "../../src/types.js"; +import { createMockOpenAI } from "../helpers/mock-openai-server.js"; +import type { MockOpenAI } from "../helpers/mock-openai-server.js"; const OPENAI_KEY = process.env.OPENAI_API_KEY; const ANTHROPIC_KEY = process.env.ANTHROPIC_API_KEY; -const HAS_PROVIDER = !!(OPENAI_KEY || ANTHROPIC_KEY); +const USE_MOCK = process.env.MOCK === "true"; +const HAS_PROVIDER = !!(OPENAI_KEY || ANTHROPIC_KEY || USE_MOCK); + +describe.skipIf(!HAS_PROVIDER)("Agent ReAct loop E2E", () => { + let mockServer: MockOpenAI | null = null; + + beforeAll(async () => { + if (USE_MOCK) { + mockServer = await createMockOpenAI(); + } + }, 30_000); -function setupModel() { - clearProviders(); - if (OPENAI_KEY) { - registerProvider(createOpenAIProvider()); - const modelName = process.env.OPENAI_MODEL ?? "gpt-4o-mini"; - return createModel(`openai:${modelName}`, { - ...(process.env.OPENAI_BASE_URL ? { baseURL: process.env.OPENAI_BASE_URL } : {}), + afterAll(() => { + mockServer?.close(); + }); + + function setupModel() { + clearProviders(); + if (USE_MOCK && mockServer) { + registerProvider(createOpenAIProvider()); + return createModel("openai:mock-model", { + baseURL: mockServer.url, + apiKey: "mock-key", + }); + } + if (OPENAI_KEY) { + registerProvider(createOpenAIProvider()); + const modelName = process.env.OPENAI_MODEL ?? "gpt-4o-mini"; + return createModel(`openai:${modelName}`, { + ...(process.env.OPENAI_BASE_URL ? { baseURL: process.env.OPENAI_BASE_URL } : {}), + }); + } + registerProvider(createAnthropicProvider()); + const modelName = process.env.ANTHROPIC_MODEL ?? "claude-sonnet-4-20250514"; + return createModel(`anthropic:${modelName}`, { + ...(process.env.ANTHROPIC_BASE_URL ? { baseURL: process.env.ANTHROPIC_BASE_URL } : {}), }); } - registerProvider(createAnthropicProvider()); - const modelName = process.env.ANTHROPIC_MODEL ?? "claude-sonnet-4-20250514"; - return createModel(`anthropic:${modelName}`, { - ...(process.env.ANTHROPIC_BASE_URL ? { baseURL: process.env.ANTHROPIC_BASE_URL } : {}), - }); -} -describe.skipIf(!HAS_PROVIDER)("Agent ReAct loop E2E", () => { it("completes a full ReAct cycle: question → tool call → answer", async () => { const model = setupModel(); diff --git a/packages/agent/__tests__/e2e/tool-calling.e2e.test.ts b/packages/agent/__tests__/e2e/tool-calling.e2e.test.ts index a3fd6a0..758c035 100644 --- a/packages/agent/__tests__/e2e/tool-calling.e2e.test.ts +++ b/packages/agent/__tests__/e2e/tool-calling.e2e.test.ts @@ -4,7 +4,8 @@ * Verifies that the agent correctly invokes tools with the right parameters, * receives results, and incorporates them into the final response. * - * Skipped when OPENAI_API_KEY is not set (uses OpenAI-compatible adapter). + * Skipped when OPENAI_API_KEY is not set (uses OpenAI-compatible adapter), + * unless MOCK=true. * Falls back to ANTHROPIC_API_KEY if OPENAI_API_KEY is unavailable. * * Environment variables (OpenAI-compatible): @@ -16,9 +17,11 @@ * ANTHROPIC_API_KEY — API key * ANTHROPIC_BASE_URL — Base URL * ANTHROPIC_MODEL — Model name + * + * MOCK — Set to "true" to use a mock server (no real API needed) */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { registerProvider, createModel, @@ -28,28 +31,50 @@ import { } from "@openlinkos/ai"; import { createAgent } from "../../src/index.js"; import type { ToolDefinition } from "../../src/types.js"; +import { createMockOpenAI } from "../helpers/mock-openai-server.js"; +import type { MockOpenAI } from "../helpers/mock-openai-server.js"; const OPENAI_KEY = process.env.OPENAI_API_KEY; const ANTHROPIC_KEY = process.env.ANTHROPIC_API_KEY; -const HAS_PROVIDER = !!(OPENAI_KEY || ANTHROPIC_KEY); - -function setupModel() { - clearProviders(); - if (OPENAI_KEY) { - registerProvider(createOpenAIProvider()); - const modelName = process.env.OPENAI_MODEL ?? "gpt-4o-mini"; - return createModel(`openai:${modelName}`, { - ...(process.env.OPENAI_BASE_URL ? { baseURL: process.env.OPENAI_BASE_URL } : {}), +const USE_MOCK = process.env.MOCK === "true"; +const HAS_PROVIDER = !!(OPENAI_KEY || ANTHROPIC_KEY || USE_MOCK); + +describe.skipIf(!HAS_PROVIDER)("Agent tool-calling E2E", () => { + let mockServer: MockOpenAI | null = null; + + beforeAll(async () => { + if (USE_MOCK) { + mockServer = await createMockOpenAI(); + } + }, 30_000); + + afterAll(() => { + mockServer?.close(); + }); + + function setupModel() { + clearProviders(); + if (USE_MOCK && mockServer) { + registerProvider(createOpenAIProvider()); + return createModel("openai:mock-model", { + baseURL: mockServer.url, + apiKey: "mock-key", + }); + } + if (OPENAI_KEY) { + registerProvider(createOpenAIProvider()); + const modelName = process.env.OPENAI_MODEL ?? "gpt-4o-mini"; + return createModel(`openai:${modelName}`, { + ...(process.env.OPENAI_BASE_URL ? { baseURL: process.env.OPENAI_BASE_URL } : {}), + }); + } + registerProvider(createAnthropicProvider()); + const modelName = process.env.ANTHROPIC_MODEL ?? "claude-sonnet-4-20250514"; + return createModel(`anthropic:${modelName}`, { + ...(process.env.ANTHROPIC_BASE_URL ? { baseURL: process.env.ANTHROPIC_BASE_URL } : {}), }); } - registerProvider(createAnthropicProvider()); - const modelName = process.env.ANTHROPIC_MODEL ?? "claude-sonnet-4-20250514"; - return createModel(`anthropic:${modelName}`, { - ...(process.env.ANTHROPIC_BASE_URL ? { baseURL: process.env.ANTHROPIC_BASE_URL } : {}), - }); -} -describe.skipIf(!HAS_PROVIDER)("Agent tool-calling E2E", () => { it("passes correct parameters to tool and uses result", async () => { const model = setupModel(); let receivedParams: Record | undefined; diff --git a/packages/agent/__tests__/helpers/mock-openai-server.ts b/packages/agent/__tests__/helpers/mock-openai-server.ts new file mode 100644 index 0000000..55f127a --- /dev/null +++ b/packages/agent/__tests__/helpers/mock-openai-server.ts @@ -0,0 +1,362 @@ +/** + * Mock OpenAI-compatible server for E2E tests (agent package copy). + * + * Duplicated from packages/ai/__tests__/helpers/mock-openai-server.ts + * because vitest cannot resolve test helpers across package boundaries. + */ + +import { createServer, type IncomingMessage, type ServerResponse } from "http"; + +export interface MockOpenAI { + url: string; + close: () => void; +} + +export function createMockOpenAI(): Promise { + const server = createServer(handler); + + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + resolve({ + url: `http://127.0.0.1:${port}/v1`, + close: () => server.close(), + }); + }); + }); +} + +// --------------------------------------------------------------------------- +// Request handler +// --------------------------------------------------------------------------- + +interface MockTool { + type: string; + function: { + name: string; + description?: string; + parameters?: Record; + }; +} + +interface MockMessage { + role: string; + content?: string | null; + tool_call_id?: string; +} + +interface MockAssistantMessage extends MockMessage { + tool_calls?: Array<{ id: string; type: string; function: { name: string; arguments: string } }>; +} + +function handler(req: IncomingMessage, res: ServerResponse) { + if (req.method !== "POST") { + res.writeHead(405).end(); + return; + } + + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf-8"); + let body: Record; + try { + body = JSON.parse(raw) as Record; + } catch { + res.writeHead(400, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "Invalid JSON" })); + return; + } + + const path = req.url ?? ""; + if (path.endsWith("/chat/completions")) { + handleChatCompletions(body, res); + } else { + res.writeHead(404, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "Not found" })); + } + }); +} + +// --------------------------------------------------------------------------- +// Chat completions +// --------------------------------------------------------------------------- + +function handleChatCompletions(body: Record, res: ServerResponse) { + const isStream = body.stream === true; + const tools = (body.tools as MockTool[] | undefined) ?? []; + const messages = (body.messages as (MockMessage | MockAssistantMessage)[] | undefined) ?? []; + + res.writeHead(200, { "Content-Type": "application/json" }); + + const hasTools = Array.isArray(tools) && tools.length > 0; + const lastUserMsg = findLastUserMessage(messages); + const lastMessage = messages.length > 0 ? messages[messages.length - 1] : null; + const isToolResult = lastMessage?.role === "tool"; + + if (isToolResult) { + const text = generateToolResultSummary(messages); + if (isStream) { + writeStreamedText(res, text); + } else { + const response = buildChatResponse({ + content: text, + toolCalls: undefined, + finishReason: "stop", + }); + res.end(JSON.stringify(response)); + } + return; + } + + const isJsonRequest = !!lastUserMsg && /JSON|json object|respond with/i.test(lastUserMsg); + const isToolRequest = hasTools && lastUserMsg && !isToolResult && + (/tool|use|调用|weather|calculate|population|what is|how many/i.test(lastUserMsg) || + /multiply|add|subtract|divide/i.test(lastUserMsg)); + + if (isToolRequest && hasTools) { + const toolName = tools[0].function.name; + const multiArgs = generateMultiToolArgs(toolName, lastUserMsg); + + if (multiArgs && multiArgs.length > 1) { + const toolCalls = multiArgs.map((args, i) => ({ + id: `call_mock_00${i + 1}`, + name: toolName, + arguments: JSON.stringify(args), + })); + + if (isStream) { + writeStreamedMultiToolCall(res, toolName, multiArgs); + } else { + const response = buildChatResponse({ + content: null, + toolCalls, + finishReason: "tool_calls", + }); + res.end(JSON.stringify(response)); + } + } else { + const toolArgs = generateMockToolArgs(toolName, lastUserMsg); + if (isStream) { + writeStreamedToolCall(res, toolName, toolArgs); + } else { + const response = buildChatResponse({ + content: null, + toolCalls: [{ id: "call_mock_001", name: toolName, arguments: JSON.stringify(toolArgs) }], + finishReason: "tool_calls", + }); + res.end(JSON.stringify(response)); + } + } + } else if (isJsonRequest) { + const jsonContent = generateMockJsonResponse(lastUserMsg); + if (isStream) { + writeStreamedText(res, jsonContent); + } else { + const response = buildChatResponse({ + content: jsonContent, + toolCalls: undefined, + finishReason: "stop", + }); + res.end(JSON.stringify(response)); + } + } else { + const text = "Hello! I am a mock response from the test server."; + if (isStream) { + writeStreamedText(res, text); + } else { + const response = buildChatResponse({ + content: text, + toolCalls: undefined, + finishReason: "stop", + }); + res.end(JSON.stringify(response)); + } + } +} + +// --------------------------------------------------------------------------- +// Response builders +// --------------------------------------------------------------------------- + +function buildChatResponse(opts: { + content: string | null; + toolCalls?: Array<{ id: string; name: string; arguments: string }>; + finishReason: string; +}) { + const message: Record = { + role: "assistant", + content: opts.content, + }; + if (opts.toolCalls?.length) { + message.tool_calls = opts.toolCalls.map((tc) => ({ + id: tc.id, + type: "function", + function: { name: tc.name, arguments: tc.arguments }, + })); + } + return { + id: "chatcmpl-mock-001", + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ message, finish_reason: opts.finishReason, index: 0 }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }; +} + +// --------------------------------------------------------------------------- +// Streaming helpers +// --------------------------------------------------------------------------- + +function writeStreamedText(res: ServerResponse, text: string) { + const chunk1 = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: { role: "assistant" }, finish_reason: null, index: 0 }] }; + const chunk2 = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: { content: text }, finish_reason: null, index: 0 }] }; + const chunk3 = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: {}, finish_reason: "stop", index: 0 }], usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 } }; + res.write(`data: ${JSON.stringify(chunk1)}\n\n`); + res.write(`data: ${JSON.stringify(chunk2)}\n\n`); + res.write(`data: ${JSON.stringify(chunk3)}\n\n`); + res.write("data: [DONE]\n\n"); + res.end(); +} + +function writeStreamedToolCall(res: ServerResponse, toolName: string, toolArgs: Record) { + const chunk1 = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: { role: "assistant" }, finish_reason: null, index: 0 }] }; + const chunk2 = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: { tool_calls: [{ index: 0, id: "call_mock_001", function: { name: toolName, arguments: JSON.stringify(toolArgs) } }] }, finish_reason: null, index: 0 }] }; + const chunk3 = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: {}, finish_reason: "tool_calls", index: 0 }], usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 } }; + res.write(`data: ${JSON.stringify(chunk1)}\n\n`); + res.write(`data: ${JSON.stringify(chunk2)}\n\n`); + res.write(`data: ${JSON.stringify(chunk3)}\n\n`); + res.write("data: [DONE]\n\n"); + res.end(); +} + +function writeStreamedMultiToolCall(res: ServerResponse, toolName: string, argsList: Array>) { + const chunk1 = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: { role: "assistant" }, finish_reason: null, index: 0 }] }; + res.write(`data: ${JSON.stringify(chunk1)}\n\n`); + for (let i = 0; i < argsList.length; i++) { + const tc = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: { tool_calls: [{ index: i, id: `call_mock_00${i + 1}`, function: { name: toolName, arguments: JSON.stringify(argsList[i]) } }] }, finish_reason: null, index: 0 }] }; + res.write(`data: ${JSON.stringify(tc)}\n\n`); + } + const chunkFinal = { id: "chatcmpl-mock-stream", object: "chat.completion.chunk", created: Math.floor(Date.now() / 1000), model: "mock-model", choices: [{ delta: {}, finish_reason: "tool_calls", index: 0 }], usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 } }; + res.write(`data: ${JSON.stringify(chunkFinal)}\n\n`); + res.write("data: [DONE]\n\n"); + res.end(); +} + +// --------------------------------------------------------------------------- +// Mock data generators +// --------------------------------------------------------------------------- + +function findLastUserMessage(messages: MockMessage[]): string | null { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === "user" && messages[i].content) { + return messages[i].content; + } + } + return null; +} + +function generateMockToolArgs(toolName: string, userMsg: string | null): Record { + const msg = userMsg ?? ""; + const numbers = msg.match(/\d+/g)?.map(Number) ?? []; + switch (toolName) { + case "get_weather": { + const cityMatch = msg.match(/(?:in|for|of)\s+(\w+)/i); + return { city: cityMatch ? cityMatch[1] : "Paris" }; + } + case "calculate": + return { a: numbers[0] ?? 7, b: numbers[1] ?? 6, operator: msg.includes("multipl") ? "multiply" : msg.includes("add") ? "add" : "multiply" }; + case "get_population": { + const cities = msg.match(/[A-Z][a-z]+/g) ?? ["Tokyo"]; + return { city: cities[0] }; + } + default: + return {}; + } +} + +function generateMultiToolArgs(toolName: string, userMsg: string | null): Array> | null { + if (!userMsg) return null; + if (toolName === "get_population") { + const cityPatterns = [ + { regex: /\bTokyo\b/i, city: "Tokyo" }, + { regex: /\bLondon\b/i, city: "London" }, + { regex: /\bParis\b/i, city: "Paris" }, + { regex: /\bNew\s*York\b/i, city: "New York" }, + ]; + const found = cityPatterns.filter(p => p.regex.test(userMsg)); + if (found.length >= 2) { + return found.map(f => ({ city: f.city })); + } + } + return null; +} + +function generateMockJsonResponse(userMsg: string | null): string { + const msg = userMsg ?? ""; + if (/name.*Alice/.test(msg)) { + return JSON.stringify({ name: "Alice", age: 30 }); + } + return JSON.stringify({ result: "mock" }); +} + +function generateToolResultSummary(messages: (MockMessage | MockAssistantMessage)[]): string { + const toolResults: Array<{ name: string; args: string; result: string }> = []; + + for (let i = 0; i < messages.length; i++) { + const msg = messages[i]; + if (msg.role === "assistant" && (msg as MockAssistantMessage).tool_calls) { + const tcs = (msg as MockAssistantMessage).tool_calls!; + for (const tc of tcs) { + const toolResultMsg = messages.find( + (m, j) => j > i && m.role === "tool" && m.content && m.tool_call_id === tc.id, + ); + toolResults.push({ name: tc.function.name, args: tc.function.arguments, result: toolResultMsg?.content ?? "" }); + } + } + } + + if (toolResults.length === 0) { + return "Based on the tool results, I have the information you requested."; + } + + const allSameTool = toolResults.every(r => r.name === toolResults[0].name); + + if (allSameTool && toolResults[0].name === "get_population") { + const parts: string[] = []; + for (const tr of toolResults) { + try { + const parsed = JSON.parse(tr.result); + parts.push(`${parsed.city}: ${parsed.population.toLocaleString()}`); + } catch { + parts.push(tr.result); + } + } + return `Here are the populations: ${parts.join(", ")}.`; + } + + const summaries: string[] = []; + for (const tr of toolResults) { + try { + const parsed = JSON.parse(tr.result); + switch (tr.name) { + case "get_weather": + summaries.push(`The weather in ${parsed.city} is ${parsed.condition} with ${parsed.temp_c}°C.`); + break; + case "calculate": + summaries.push(`The calculation result is ${parsed.result}.`); + break; + case "get_population": + summaries.push(`${parsed.city} has a population of ${parsed.population.toLocaleString()}.`); + break; + default: + summaries.push(`Result: ${JSON.stringify(parsed)}`); + } + } catch { + summaries.push(`Result: ${tr.result}`); + } + } + return summaries.join(" "); +} diff --git a/packages/ai/__tests__/e2e/openai-compat.e2e.test.ts b/packages/ai/__tests__/e2e/openai-compat.e2e.test.ts index e7f9c4b..65d06e2 100644 --- a/packages/ai/__tests__/e2e/openai-compat.e2e.test.ts +++ b/packages/ai/__tests__/e2e/openai-compat.e2e.test.ts @@ -2,45 +2,77 @@ * OpenAI-compatible adapter E2E tests — generate, stream, tools, structured output. * * Tests the OpenAI Chat Completions API protocol adapter with any compatible - * endpoint. Skipped when OPENAI_API_KEY is not set. + * endpoint. Skipped when OPENAI_API_KEY is not set (unless MOCK=true). * * Environment variables: * OPENAI_API_KEY — API key for the OpenAI-compatible endpoint * OPENAI_BASE_URL — Base URL (default: https://api.openai.com/v1) * OPENAI_MODEL — Model name (default: gpt-4o-mini) + * MOCK — Set to "true" to use a mock server (no real API needed) */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { OpenAIProvider } from "../../src/providers/openai.js"; import { collectText } from "../../src/stream.js"; import type { ToolDefinition } from "../../src/types.js"; +import { createMockOpenAI } from "../helpers/mock-openai-server.js"; +import type { MockOpenAI } from "../helpers/mock-openai-server.js"; const API_KEY = process.env.OPENAI_API_KEY; const BASE_URL = process.env.OPENAI_BASE_URL; const MODEL = process.env.OPENAI_MODEL ?? "gpt-4o-mini"; +const USE_MOCK = process.env.MOCK === "true"; -describe.skipIf(!API_KEY)("OpenAI-compatible adapter E2E", () => { - const provider = new OpenAIProvider(); - const messages = [{ role: "user" as const, content: "Say hello in one word." }]; - const options = { - modelName: MODEL, - ...(BASE_URL ? { baseURL: BASE_URL } : {}), - }; +const HAS_PROVIDER = !!(API_KEY || USE_MOCK); + +describe.skipIf(!HAS_PROVIDER)("OpenAI-compatible adapter E2E", () => { + let mockServer: MockOpenAI | null = null; + let baseURL = BASE_URL; + let apiKey = API_KEY ?? ""; + let model = MODEL; + + beforeAll(async () => { + if (USE_MOCK) { + mockServer = await createMockOpenAI(); + baseURL = mockServer.url; + apiKey = "mock-key"; + model = "mock-model"; + } + }, 30_000); + + afterAll(() => { + mockServer?.close(); + }); + + const getProvider = () => new OpenAIProvider(); + const getOptions = () => ({ + modelName: model, + ...(baseURL ? { baseURL } : {}), + }); it("generate returns text", async () => { - const response = await provider.generate(messages, options); + const provider = getProvider(); + const response = await provider.generate( + [{ role: "user" as const, content: "Say hello in one word." }], + { ...getOptions(), apiKey }, + ); expect(response.text).toBeTruthy(); expect(response.finishReason).toBe("stop"); expect(response.usage.totalTokens).toBeGreaterThan(0); }, 30_000); it("stream returns text deltas", async () => { - const stream = await provider.stream(messages, options); + const provider = getProvider(); + const stream = await provider.stream( + [{ role: "user" as const, content: "Say hello in one word." }], + { ...getOptions(), apiKey }, + ); const text = await collectText(stream); expect(text.length).toBeGreaterThan(0); }, 30_000); it("generateWithTools triggers tool call", async () => { + const provider = getProvider(); const tools: ToolDefinition[] = [ { name: "get_weather", @@ -56,7 +88,7 @@ describe.skipIf(!API_KEY)("OpenAI-compatible adapter E2E", () => { const response = await provider.generateWithTools( [{ role: "user", content: "What is the weather in Paris? Use the tool." }], tools, - options, + { ...getOptions(), apiKey }, ); expect(response.toolCalls.length).toBeGreaterThan(0); @@ -64,6 +96,7 @@ describe.skipIf(!API_KEY)("OpenAI-compatible adapter E2E", () => { }, 30_000); it("structured output via responseFormat", async () => { + const provider = getProvider(); const response = await provider.generate( [ { @@ -73,7 +106,8 @@ describe.skipIf(!API_KEY)("OpenAI-compatible adapter E2E", () => { }, ], { - ...options, + ...getOptions(), + apiKey, responseFormat: { type: "json", schema: { diff --git a/packages/ai/__tests__/helpers/mock-openai-server.ts b/packages/ai/__tests__/helpers/mock-openai-server.ts new file mode 100644 index 0000000..5c62bf1 --- /dev/null +++ b/packages/ai/__tests__/helpers/mock-openai-server.ts @@ -0,0 +1,538 @@ +/** + * Mock OpenAI-compatible server for E2E tests. + * + * Creates a local HTTP server that responds to OpenAI Chat Completions API + * requests with predictable mock responses. Supports non-streaming, streaming, + * tool calls, and JSON mode responses. + */ + +import { createServer, type Server, type IncomingMessage, type ServerResponse } from "http"; + +export interface MockOpenAI { + /** Base URL including /v1 path, e.g. http://127.0.0.1:54321/v1 */ + url: string; + /** Shut down the server. */ + close: () => void; +} + +/** + * Start a mock OpenAI-compatible server on a random port. + * + * Routes handled: + * POST /v1/chat/completions — non-streaming & streaming chat completions + */ +export function createMockOpenAI(): Promise { + const server = createServer(handler); + + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + resolve({ + url: `http://127.0.0.1:${port}/v1`, + close: () => server.close(), + }); + }); + }); +} + +// --------------------------------------------------------------------------- +// Request handler +// --------------------------------------------------------------------------- + +function handler(req: IncomingMessage, res: ServerResponse) { + // Only handle POST + if (req.method !== "POST") { + res.writeHead(405).end(); + return; + } + + // Read body + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf-8"); + let body: Record; + try { + body = JSON.parse(raw) as Record; + } catch { + res.writeHead(400, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "Invalid JSON" })); + return; + } + + const path = req.url ?? ""; + + if (path.endsWith("/chat/completions")) { + handleChatCompletions(body, res); + } else { + res.writeHead(404, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "Not found" })); + } + }); +} + +// --------------------------------------------------------------------------- +// Chat completions +// --------------------------------------------------------------------------- + +interface MockTool { + type: string; + function: { + name: string; + description?: string; + parameters?: Record; + }; +} + +interface MockMessage { + role: string; + content?: string | null; + tool_call_id?: string; +} + +interface MockAssistantMessage extends MockMessage { + tool_calls?: Array<{ id: string; type: string; function: { name: string; arguments: string } }>; +} + +function handleChatCompletions(body: Record, res: ServerResponse) { + const isStream = body.stream === true; + const tools = (body.tools as MockTool[] | undefined) ?? []; + const messages = (body.messages as (MockMessage | MockAssistantMessage)[] | undefined) ?? []; + const responseFormat = body.response_format as Record | undefined; + + res.writeHead(200, { "Content-Type": "application/json" }); + + // Determine what kind of response to return + const hasTools = Array.isArray(tools) && tools.length > 0; + const lastUserMsg = findLastUserMessage(messages); + const lastMessage = messages.length > 0 ? messages[messages.length - 1] : null; + const isToolResult = lastMessage?.role === "tool"; + + // If the last message is a tool result, generate a text summary + if (isToolResult) { + const text = generateToolResultSummary(messages); + if (isStream) { + writeStreamedText(res, text); + } else { + const response = buildChatResponse({ + content: text, + toolCalls: undefined, + finishReason: "stop", + }); + res.end(JSON.stringify(response)); + } + return; + } + + // Check if user is requesting JSON/structured output (the adapter doesn't send + // response_format to the server, so we detect it from the prompt content) + const isJsonRequest = !!lastUserMsg && /JSON|json object|respond with/i.test(lastUserMsg); + + // Check if the last user message is a tool-related request + const isToolRequest = hasTools && lastUserMsg && !isToolResult && + (/tool|use|调用|weather|calculate|population|what is|how many/i.test(lastUserMsg) || + /multiply|add|subtract|divide/i.test(lastUserMsg)); + + if (isToolRequest && hasTools) { + // Check if multiple tool calls are needed + const toolName = tools[0].function.name; + const multiArgs = generateMultiToolArgs(toolName, lastUserMsg); + + if (multiArgs && multiArgs.length > 1) { + // Return multiple tool calls + const toolCalls = multiArgs.map((args, i) => ({ + id: `call_mock_00${i + 1}`, + name: toolName, + arguments: JSON.stringify(args), + })); + + if (isStream) { + writeStreamedMultiToolCall(res, toolName, multiArgs); + } else { + const response = buildChatResponse({ + content: null, + toolCalls, + finishReason: "tool_calls", + }); + res.end(JSON.stringify(response)); + } + } else { + // Single tool call + const toolArgs = generateMockToolArgs(toolName, lastUserMsg); + + if (isStream) { + writeStreamedToolCall(res, toolName, toolArgs); + } else { + const response = buildChatResponse({ + content: null, + toolCalls: [{ id: "call_mock_001", name: toolName, arguments: JSON.stringify(toolArgs) }], + finishReason: "tool_calls", + }); + res.end(JSON.stringify(response)); + } + } + } else if (isJsonRequest) { + // Return JSON formatted text + const jsonContent = generateMockJsonResponse(lastUserMsg); + if (isStream) { + writeStreamedText(res, jsonContent); + } else { + const response = buildChatResponse({ + content: jsonContent, + toolCalls: undefined, + finishReason: "stop", + }); + res.end(JSON.stringify(response)); + } + } else { + // Regular text response + const text = "Hello! I am a mock response from the test server."; + if (isStream) { + writeStreamedText(res, text); + } else { + const response = buildChatResponse({ + content: text, + toolCalls: undefined, + finishReason: "stop", + }); + res.end(JSON.stringify(response)); + } + } +} + +// --------------------------------------------------------------------------- +// Response builders +// --------------------------------------------------------------------------- + +function buildChatResponse(opts: { + content: string | null; + toolCalls?: Array<{ id: string; name: string; arguments: string }>; + finishReason: string; +}) { + const message: Record = { + role: "assistant", + content: opts.content, + }; + if (opts.toolCalls?.length) { + message.tool_calls = opts.toolCalls.map((tc) => ({ + id: tc.id, + type: "function", + function: { + name: tc.name, + arguments: tc.arguments, + }, + })); + } + return { + id: "chatcmpl-mock-001", + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [ + { + message, + finish_reason: opts.finishReason, + index: 0, + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + }, + }; +} + +// --------------------------------------------------------------------------- +// Streaming helpers +// --------------------------------------------------------------------------- + +function writeStreamedText(res: ServerResponse, text: string) { + // First chunk: role + const chunk1 = { + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ delta: { role: "assistant" }, finish_reason: null, index: 0 }], + }; + + // Content chunk + const chunk2 = { + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ delta: { content: text }, finish_reason: null, index: 0 }], + }; + + // Final chunk with usage + const chunk3 = { + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ delta: {}, finish_reason: "stop", index: 0 }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }; + + res.write(`data: ${JSON.stringify(chunk1)}\n\n`); + res.write(`data: ${JSON.stringify(chunk2)}\n\n`); + res.write(`data: ${JSON.stringify(chunk3)}\n\n`); + res.write("data: [DONE]\n\n"); + res.end(); +} + +function writeStreamedToolCall(res: ServerResponse, toolName: string, toolArgs: Record) { + // Role chunk + const chunk1 = { + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ delta: { role: "assistant" }, finish_reason: null, index: 0 }], + }; + + // Tool call chunk + const chunk2 = { + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ + delta: { + tool_calls: [{ + index: 0, + id: "call_mock_001", + function: { name: toolName, arguments: JSON.stringify(toolArgs) }, + }], + }, + finish_reason: null, + index: 0, + }], + }; + + // Final chunk + const chunk3 = { + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ delta: {}, finish_reason: "tool_calls", index: 0 }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }; + + res.write(`data: ${JSON.stringify(chunk1)}\n\n`); + res.write(`data: ${JSON.stringify(chunk2)}\n\n`); + res.write(`data: ${JSON.stringify(chunk3)}\n\n`); + res.write("data: [DONE]\n\n"); + res.end(); +} + +function writeStreamedMultiToolCall(res: ServerResponse, toolName: string, argsList: Array>) { + // Role chunk + const chunk1 = { + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ delta: { role: "assistant" }, finish_reason: null, index: 0 }], + }; + + // Multiple tool call chunks, one per tool + const toolCallChunks = argsList.map((args, i) => ({ + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ + delta: { + tool_calls: [{ + index: i, + id: `call_mock_00${i + 1}`, + function: { name: toolName, arguments: JSON.stringify(args) }, + }], + }, + finish_reason: null, + index: 0, + }], + })); + + // Final chunk + const chunkFinal = { + id: "chatcmpl-mock-stream", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "mock-model", + choices: [{ delta: {}, finish_reason: "tool_calls", index: 0 }], + usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 }, + }; + + res.write(`data: ${JSON.stringify(chunk1)}\n\n`); + for (const tc of toolCallChunks) { + res.write(`data: ${JSON.stringify(tc)}\n\n`); + } + res.write(`data: ${JSON.stringify(chunkFinal)}\n\n`); + res.write("data: [DONE]\n\n"); + res.end(); +} + +// --------------------------------------------------------------------------- +// Mock data generators +// --------------------------------------------------------------------------- + +function findLastUserMessage(messages: MockMessage[]): string | null { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === "user" && messages[i].content) { + return messages[i].content; + } + } + return null; +} + +/** + * Generate a text summary response after receiving tool results. + * This simulates the LLM processing tool call results and producing a final answer. + */ +function generateToolResultSummary(messages: (MockMessage | MockAssistantMessage)[]): string { + // Collect all tool results and their associated tool calls + const toolResults: Array<{ name: string; args: string; result: string }> = []; + + // Find all assistant messages with tool calls and their corresponding tool results + for (let i = 0; i < messages.length; i++) { + const msg = messages[i]; + if (msg.role === "assistant" && (msg as MockAssistantMessage).tool_calls) { + const tcs = (msg as MockAssistantMessage).tool_calls!; + for (const tc of tcs) { + // Find the corresponding tool result message + const toolResultMsg = messages.find( + (m, j) => j > i && m.role === "tool" && m.content && (m as unknown as { tool_call_id?: string }).tool_call_id === tc.id, + ); + toolResults.push({ + name: tc.function.name, + args: tc.function.arguments, + result: toolResultMsg?.content ?? "", + }); + } + } + } + + // If no structured results found, just collect recent tool result messages + if (toolResults.length === 0) { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === "tool" && messages[i].content) { + toolResults.push({ name: "unknown", args: "", result: messages[i].content! }); + } + } + } + + // Generate text based on tool results + if (toolResults.length === 0) { + return "Based on the tool results, I have the information you requested."; + } + + // If all results are from the same tool type, generate a combined summary + const allSameTool = toolResults.every(r => r.name === toolResults[0].name); + + if (allSameTool && toolResults[0].name === "get_population") { + const parts: string[] = []; + for (const tr of toolResults) { + try { + const parsed = JSON.parse(tr.result); + parts.push(`${parsed.city}: ${parsed.population.toLocaleString()}`); + } catch { + parts.push(tr.result); + } + } + return `Here are the populations: ${parts.join(", ")}.`; + } + + // Default: generate individual summaries + const summaries: string[] = []; + for (const tr of toolResults) { + try { + const parsed = JSON.parse(tr.result); + switch (tr.name) { + case "get_weather": + summaries.push(`The weather in ${parsed.city} is ${parsed.condition} with ${parsed.temp_c}°C.`); + break; + case "calculate": + summaries.push(`The calculation result is ${parsed.result}.`); + break; + case "get_population": + summaries.push(`${parsed.city} has a population of ${parsed.population.toLocaleString()}.`); + break; + default: + summaries.push(`Result: ${JSON.stringify(parsed)}`); + } + } catch { + summaries.push(`Result: ${tr.result}`); + } + } + return summaries.join(" "); +} + +function generateMockToolArgs(toolName: string, userMsg: string | null): Record { + const msg = userMsg ?? ""; + + // Extract numbers from the user message for calculation tools + const numbers = msg.match(/\d+/g)?.map(Number) ?? []; + + switch (toolName) { + case "get_weather": + // Try to extract city name + const cityMatch = msg.match(/(?:in|for|of)\s+(\w+)/i); + return { city: cityMatch ? cityMatch[1] : "Paris" }; + + case "calculate": + return { + a: numbers[0] ?? 7, + b: numbers[1] ?? 6, + operator: msg.includes("multipl") ? "multiply" : msg.includes("add") ? "add" : "multiply", + }; + + case "get_population": + // Extract city from message + const cities = msg.match(/[A-Z][a-z]+/g) ?? ["Tokyo"]; + return { city: cities[0] }; + + default: + return {}; + } +} + +/** + * Check if the user message requests multiple tool calls (e.g., "Tokyo and London"). + * Returns an array of tool arguments, one per call. + */ +function generateMultiToolArgs(toolName: string, userMsg: string | null): Array> | null { + if (!userMsg) return null; + + if (toolName === "get_population") { + // Look for multiple city names in the message + const cityPatterns = [ + { regex: /\bTokyo\b/i, city: "Tokyo" }, + { regex: /\bLondon\b/i, city: "London" }, + { regex: /\bParis\b/i, city: "Paris" }, + { regex: /\bNew\s*York\b/i, city: "New York" }, + ]; + const found = cityPatterns.filter(p => p.regex.test(userMsg)); + if (found.length >= 2) { + return found.map(f => ({ city: f.city })); + } + } + + return null; +} + +function generateMockJsonResponse(userMsg: string | null): string { + const msg = userMsg ?? ""; + + // Try to match requested fields from the message + if (/name.*Alice/.test(msg)) { + return JSON.stringify({ name: "Alice", age: 30 }); + } + + return JSON.stringify({ result: "mock" }); +}