diff --git a/.changeset/smart-wasps-develop.md b/.changeset/smart-wasps-develop.md new file mode 100644 index 000000000000..dc2bc0de112b --- /dev/null +++ b/.changeset/smart-wasps-develop.md @@ -0,0 +1,5 @@ +--- +"@ai-sdk/openai": patch +--- + +feat(openai): add opt-in pass-through for unsupported file media types diff --git a/content/providers/01-ai-sdk-providers/03-openai.mdx b/content/providers/01-ai-sdk-providers/03-openai.mdx index fd02fb28c8a5..c19ec234686f 100644 --- a/content/providers/01-ai-sdk-providers/03-openai.mdx +++ b/content/providers/01-ai-sdk-providers/03-openai.mdx @@ -165,6 +165,10 @@ The following provider options are available: Whether to store the generation. Defaults to `true`. +- **passThroughUnsupportedFiles** _boolean_ + + Whether to pass through non-image file types as generic input files. Defaults to `false`, which restricts inline file inputs to images and PDFs. Enable this when the target OpenAI Responses model supports additional file media types. + - **maxToolCalls** _integer_ The maximum number of total calls to built-in tools that can be processed in a response. This maximum number applies across all built-in tool calls, not per individual tool. diff --git a/examples/ai-functions/src/generate-text/openai/responses-unsupported-file-type.ts b/examples/ai-functions/src/generate-text/openai/responses-unsupported-file-type.ts new file mode 100644 index 000000000000..69c02f633003 --- /dev/null +++ b/examples/ai-functions/src/generate-text/openai/responses-unsupported-file-type.ts @@ -0,0 +1,36 @@ +import { + openai, + type OpenAILanguageModelResponsesOptions, +} from '@ai-sdk/openai'; +import { generateText } from 'ai'; +import { run } from '../../lib/run'; + +run(async () => { + const result = await generateText({ + model: openai.responses('gpt-4.1-nano'), + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'What names appear in this CSV? Reply with just the names.', + }, + { + type: 'file', + filename: 'names.csv', + mediaType: 'text/csv', + data: Buffer.from('name,role\nAda,engineer\nGrace,scientist\n'), + }, + ], + }, + ], + providerOptions: { + openai: { + passThroughUnsupportedFiles: true, + } satisfies OpenAILanguageModelResponsesOptions, + }, + }); + + console.log(result.text); +}); diff --git a/packages/openai/src/responses/convert-to-openai-responses-input.test.ts b/packages/openai/src/responses/convert-to-openai-responses-input.test.ts index 837d05c39c25..ee69be9c31fa 100644 --- a/packages/openai/src/responses/convert-to-openai-responses-input.test.ts +++ b/packages/openai/src/responses/convert-to-openai-responses-input.test.ts @@ -444,6 +444,44 @@ describe('convertToOpenAIResponsesInput', () => { ).rejects.toThrow('file part media type text/plain'); }); + it('should pass through unsupported file types when enabled', async () => { + const base64Data = 'bmFtZSxyb2xlCkFkYSxlbmdpbmVlcgo='; + + const result = await convertToOpenAIResponsesInput({ + prompt: [ + { + role: 'user', + content: [ + { + type: 'file', + mediaType: 'text/csv', + data: base64Data, + filename: 'names.csv', + }, + ], + }, + ], + toolNameMapping: testToolNameMapping, + systemMessageMode: 'system', + providerOptionsName: 'openai', + passThroughUnsupportedFiles: true, + store: true, + }); + + expect(result.input).toEqual([ + { + role: 'user', + content: [ + { + type: 'input_file', + filename: 'names.csv', + file_data: `data:text/csv;base64,${base64Data}`, + }, + ], + }, + ]); + }); + it('should convert PDF file parts with URL to input_file with file_url', async () => { const result = await convertToOpenAIResponsesInput({ prompt: [ diff --git a/packages/openai/src/responses/convert-to-openai-responses-input.ts b/packages/openai/src/responses/convert-to-openai-responses-input.ts index c24e4dfe2526..64acd87bb682 100644 --- a/packages/openai/src/responses/convert-to-openai-responses-input.ts +++ b/packages/openai/src/responses/convert-to-openai-responses-input.ts @@ -52,6 +52,7 @@ export async function convertToOpenAIResponsesInput({ systemMessageMode, providerOptionsName, fileIdPrefixes, + passThroughUnsupportedFiles = false, store, hasConversation = false, hasLocalShellTool = false, @@ -64,6 +65,7 @@ export async function convertToOpenAIResponsesInput({ systemMessageMode: 'system' | 'developer' | 'remove'; providerOptionsName: string; fileIdPrefixes?: readonly string[]; + passThroughUnsupportedFiles?: boolean; store: boolean; hasConversation?: boolean; // when true, skip assistant messages that already have item IDs hasLocalShellTool?: boolean; @@ -116,12 +118,10 @@ export async function convertToOpenAIResponsesInput({ return { type: 'input_text', text: part.text }; } case 'file': { - if (part.mediaType.startsWith('image/')) { - const mediaType = - part.mediaType === 'image/*' - ? 'image/jpeg' - : part.mediaType; + const mediaType = + part.mediaType === 'image/*' ? 'image/jpeg' : part.mediaType; + if (mediaType.startsWith('image/')) { return { type: 'input_image', ...(part.data instanceof URL @@ -135,28 +135,38 @@ export async function convertToOpenAIResponsesInput({ detail: part.providerOptions?.[providerOptionsName]?.imageDetail, }; - } else if (part.mediaType === 'application/pdf') { - if (part.data instanceof URL) { - return { - type: 'input_file', - file_url: part.data.toString(), - }; - } + } + + if (part.data instanceof URL) { return { type: 'input_file', - ...(typeof part.data === 'string' && - isFileId(part.data, fileIdPrefixes) - ? { file_id: part.data } - : { - filename: part.filename ?? `part-${index}.pdf`, - file_data: `data:application/pdf;base64,${convertToBase64(part.data)}`, - }), + file_url: part.data.toString(), }; - } else { + } + + if ( + mediaType !== 'application/pdf' && + !passThroughUnsupportedFiles + ) { throw new UnsupportedFunctionalityError({ - functionality: `file part media type ${part.mediaType}`, + functionality: `file part media type ${mediaType}`, }); } + + return { + type: 'input_file', + ...(typeof part.data === 'string' && + isFileId(part.data, fileIdPrefixes) + ? { file_id: part.data } + : { + filename: + part.filename ?? + (mediaType === 'application/pdf' + ? `part-${index}.pdf` + : `part-${index}`), + file_data: `data:${mediaType};base64,${convertToBase64(part.data)}`, + }), + }; } } }), diff --git a/packages/openai/src/responses/openai-responses-language-model.ts b/packages/openai/src/responses/openai-responses-language-model.ts index 5085b3757b1a..b48c133184e8 100644 --- a/packages/openai/src/responses/openai-responses-language-model.ts +++ b/packages/openai/src/responses/openai-responses-language-model.ts @@ -229,6 +229,8 @@ export class OpenAIResponsesLanguageModel implements LanguageModelV3 { : modelCapabilities.systemMessageMode), providerOptionsName, fileIdPrefixes: this.config.fileIdPrefixes, + passThroughUnsupportedFiles: + openaiOptions?.passThroughUnsupportedFiles ?? false, store: openaiOptions?.store ?? true, hasConversation: openaiOptions?.conversation != null, hasLocalShellTool: hasOpenAITool('openai.local_shell'), diff --git a/packages/openai/src/responses/openai-responses-options.ts b/packages/openai/src/responses/openai-responses-options.ts index 9b412795b6d6..751d2b93c71b 100644 --- a/packages/openai/src/responses/openai-responses-options.ts +++ b/packages/openai/src/responses/openai-responses-options.ts @@ -269,6 +269,15 @@ export const openaiLanguageModelResponsesOptionsSchema = lazySchema(() => */ store: z.boolean().nullish(), + /** + * Whether to pass through non-image file types as generic input files. + * + * By default, inline file inputs are restricted to images and PDFs. + * Enable this when the target OpenAI Responses model supports additional + * file media types, such as text/csv. + */ + passThroughUnsupportedFiles: z.boolean().optional(), + /** * Whether to use strict JSON schema validation. * Defaults to `true`.