|
1 | 1 | import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; |
| 2 | +import { z } from 'zod'; |
2 | 3 |
|
3 | 4 | const hoisted = vi.hoisted(() => ({ |
4 | 5 | generateText: vi.fn(), |
@@ -228,3 +229,154 @@ describe('agent', () => { |
228 | 229 | expect(callArgs?.maxTokens).toBeUndefined(); |
229 | 230 | }); |
230 | 231 | }); |
| 232 | + |
| 233 | +describe('agent session.send: structured output (responseSchema)', () => { |
| 234 | + beforeEach(() => { |
| 235 | + hoisted.generateText.mockReset(); |
| 236 | + hoisted.streamText.mockReset(); |
| 237 | + hoisted.getRecordedAgentOSUsage.mockReset(); |
| 238 | + }); |
| 239 | + |
| 240 | + afterEach(() => { |
| 241 | + vi.restoreAllMocks(); |
| 242 | + }); |
| 243 | + |
| 244 | + const Decision = z.object({ |
| 245 | + verdict: z.enum(['yes', 'no']), |
| 246 | + confidence: z.number().min(0).max(1), |
| 247 | + }); |
| 248 | + |
| 249 | + it('returns plain GenerateTextResult when responseSchema is omitted (regression guard)', async () => { |
| 250 | + hoisted.generateText.mockResolvedValueOnce({ |
| 251 | + provider: 'openai', |
| 252 | + model: 'gpt-4.1-mini', |
| 253 | + text: 'plain reply', |
| 254 | + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, |
| 255 | + toolCalls: [], |
| 256 | + finishReason: 'stop', |
| 257 | + }); |
| 258 | + const assistant = agent({ model: 'openai:gpt-4.1-mini' }); |
| 259 | + const session = assistant.session('demo'); |
| 260 | + const r = await session.send('hi'); |
| 261 | + expect(r.text).toBe('plain reply'); |
| 262 | + expect('object' in r).toBe(false); |
| 263 | + }); |
| 264 | + |
| 265 | + it('returns typed object alongside text when responseSchema is set', async () => { |
| 266 | + hoisted.generateText.mockResolvedValueOnce({ |
| 267 | + provider: 'openai', |
| 268 | + model: 'gpt-4.1-mini', |
| 269 | + text: '{"verdict":"yes","confidence":0.92}', |
| 270 | + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, |
| 271 | + toolCalls: [], |
| 272 | + finishReason: 'stop', |
| 273 | + }); |
| 274 | + const assistant = agent({ model: 'openai:gpt-4.1-mini' }); |
| 275 | + const session = assistant.session('demo'); |
| 276 | + const r = await session.send('decide', { responseSchema: Decision }); |
| 277 | + expect(r.object).toEqual({ verdict: 'yes', confidence: 0.92 }); |
| 278 | + expect(r.text).toBe('{"verdict":"yes","confidence":0.92}'); |
| 279 | + }); |
| 280 | + |
| 281 | + it('forwards _responseFormat to generateText when responseSchema is set (openai → json_schema)', async () => { |
| 282 | + hoisted.generateText.mockResolvedValueOnce({ |
| 283 | + provider: 'openai', |
| 284 | + model: 'gpt-4.1-mini', |
| 285 | + text: '{"verdict":"yes","confidence":0.5}', |
| 286 | + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, |
| 287 | + toolCalls: [], |
| 288 | + finishReason: 'stop', |
| 289 | + }); |
| 290 | + const assistant = agent({ model: 'openai:gpt-4.1-mini' }); |
| 291 | + await assistant.session('demo').send('decide', { |
| 292 | + responseSchema: Decision, |
| 293 | + schemaName: 'Decision', |
| 294 | + }); |
| 295 | + const callArgs = hoisted.generateText.mock.calls.at(-1)?.[0]; |
| 296 | + expect(callArgs?._responseFormat).toMatchObject({ |
| 297 | + type: 'json_schema', |
| 298 | + json_schema: { name: 'Decision', strict: true }, |
| 299 | + }); |
| 300 | + }); |
| 301 | + |
| 302 | + it('strips caller-provided tools when responseSchema is set and warns once', async () => { |
| 303 | + hoisted.generateText.mockResolvedValueOnce({ |
| 304 | + provider: 'openai', |
| 305 | + model: 'gpt-4.1-mini', |
| 306 | + text: '{"verdict":"no","confidence":0.1}', |
| 307 | + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, |
| 308 | + toolCalls: [], |
| 309 | + finishReason: 'stop', |
| 310 | + }); |
| 311 | + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); |
| 312 | + const fakeTool = { |
| 313 | + stub: { |
| 314 | + description: 'stub', |
| 315 | + inputSchema: { type: 'object' as const, properties: {} }, |
| 316 | + execute: async () => ({ success: true, output: {} }), |
| 317 | + }, |
| 318 | + }; |
| 319 | + const assistant = agent({ model: 'openai:gpt-4.1-mini', tools: fakeTool as any }); |
| 320 | + await assistant.session('demo').send('decide', { responseSchema: Decision }); |
| 321 | + const callArgs = hoisted.generateText.mock.calls.at(-1)?.[0]; |
| 322 | + expect(callArgs?.tools).toBeUndefined(); |
| 323 | + expect(warn).toHaveBeenCalledWith( |
| 324 | + expect.stringContaining('tools are ignored when responseSchema is set'), |
| 325 | + ); |
| 326 | + }); |
| 327 | + |
| 328 | + it('throws ObjectGenerationError when provider returns non-JSON text', async () => { |
| 329 | + hoisted.generateText.mockResolvedValueOnce({ |
| 330 | + provider: 'openai', |
| 331 | + model: 'gpt-4.1-mini', |
| 332 | + text: 'not json at all', |
| 333 | + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, |
| 334 | + toolCalls: [], |
| 335 | + finishReason: 'stop', |
| 336 | + }); |
| 337 | + const assistant = agent({ model: 'openai:gpt-4.1-mini' }); |
| 338 | + await expect( |
| 339 | + assistant.session('demo').send('decide', { responseSchema: Decision }), |
| 340 | + ).rejects.toThrow(/not valid JSON/); |
| 341 | + }); |
| 342 | + |
| 343 | + it('throws ObjectGenerationError when JSON fails Zod validation', async () => { |
| 344 | + hoisted.generateText.mockResolvedValueOnce({ |
| 345 | + provider: 'openai', |
| 346 | + model: 'gpt-4.1-mini', |
| 347 | + text: '{"verdict":"maybe","confidence":2}', |
| 348 | + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, |
| 349 | + toolCalls: [], |
| 350 | + finishReason: 'stop', |
| 351 | + }); |
| 352 | + const assistant = agent({ model: 'openai:gpt-4.1-mini' }); |
| 353 | + await expect( |
| 354 | + assistant.session('demo').send('decide', { responseSchema: Decision }), |
| 355 | + ).rejects.toThrow(/Zod validation/); |
| 356 | + }); |
| 357 | + |
| 358 | + it('preserves session memory across schema-aware sends', async () => { |
| 359 | + hoisted.generateText.mockResolvedValue({ |
| 360 | + provider: 'openai', |
| 361 | + model: 'gpt-4.1-mini', |
| 362 | + text: '{"verdict":"yes","confidence":0.8}', |
| 363 | + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, |
| 364 | + toolCalls: [], |
| 365 | + finishReason: 'stop', |
| 366 | + }); |
| 367 | + const assistant = agent({ model: 'openai:gpt-4.1-mini' }); |
| 368 | + const session = assistant.session('demo'); |
| 369 | + await session.send('first', { responseSchema: Decision }); |
| 370 | + await session.send('second', { responseSchema: Decision }); |
| 371 | + expect(hoisted.generateText).toHaveBeenNthCalledWith( |
| 372 | + 2, |
| 373 | + expect.objectContaining({ |
| 374 | + messages: [ |
| 375 | + { role: 'user', content: 'first' }, |
| 376 | + { role: 'assistant', content: '{"verdict":"yes","confidence":0.8}' }, |
| 377 | + { role: 'user', content: 'second' }, |
| 378 | + ], |
| 379 | + }), |
| 380 | + ); |
| 381 | + }); |
| 382 | +}); |
0 commit comments