From f0a2d8d55e8b68dd9870cb29bb3cb7d5468b0fad Mon Sep 17 00:00:00 2001 From: Stainless Bot <107565488+stainless-bot@users.noreply.github.com> Date: Thu, 9 May 2024 16:19:10 -0400 Subject: [PATCH] feat(api): add message image content (#834) --- .stats.yml | 2 +- api.md | 6 + src/lib/AssistantStream.ts | 11 +- src/resources/beta/threads/index.ts | 6 + src/resources/beta/threads/messages.ts | 110 +++++++++++++++++- src/resources/beta/threads/runs/runs.ts | 17 +-- src/resources/beta/threads/threads.ts | 22 ++-- src/resources/files.ts | 16 ++- .../beta/threads/messages.test.ts | 7 +- .../beta/threads/runs/runs.test.ts | 6 +- .../beta/threads/threads.test.ts | 12 +- 11 files changed, 171 insertions(+), 44 deletions(-) diff --git a/.stats.yml b/.stats.yml index 50c6b293..52e87d1b 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,2 +1,2 @@ configured_endpoints: 64 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-2e14236d4015bf3b956290ea8b656224a0c7b206a356c6af2a7ae43fdbceb04c.yml +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-084b8f68408c6b689a55200a78bcf233769bfcd8e999d9fadaeb399152b05bcd.yml diff --git a/api.md b/api.md index 18cdd9e6..17a3f963 100644 --- a/api.md +++ b/api.md @@ -349,14 +349,20 @@ Types: - ImageFileContentBlock - ImageFileDelta - ImageFileDeltaBlock +- ImageURL +- ImageURLContentBlock +- ImageURLDelta +- ImageURLDeltaBlock - Message - MessageContent - MessageContentDelta +- MessageContentPartParam - MessageDeleted - MessageDelta - MessageDeltaEvent - Text - TextContentBlock +- TextContentBlockParam - TextDelta - TextDeltaBlock diff --git a/src/lib/AssistantStream.ts b/src/lib/AssistantStream.ts index a2974826..de7511b5 100644 --- a/src/lib/AssistantStream.ts +++ b/src/lib/AssistantStream.ts @@ -7,6 +7,7 @@ import { ImageFile, TextDelta, Messages, + MessageContent, } from 'openai/resources/beta/threads/messages'; import * as Core from 'openai/core'; import { RequestOptions } from 'openai/core'; @@ -87,7 +88,7 @@ export class AssistantStream #messageSnapshot: Message | undefined; #finalRun: Run | undefined; #currentContentIndex: number | undefined; - #currentContent: TextContentBlock | ImageFileContentBlock | undefined; + #currentContent: MessageContent | undefined; #currentToolCallIndex: number | undefined; #currentToolCall: ToolCall | undefined; @@ -624,10 +625,8 @@ export class AssistantStream currentContent, ); } else { - snapshot.content[contentElement.index] = contentElement as - | TextContentBlock - | ImageFileContentBlock; - //This is a new element + snapshot.content[contentElement.index] = contentElement as MessageContent; + // This is a new element newContent.push(contentElement); } } @@ -650,7 +649,7 @@ export class AssistantStream #accumulateContent( contentElement: MessageContentDelta, - currentContent: TextContentBlock | ImageFileContentBlock | undefined, + currentContent: MessageContent | undefined, ): TextContentBlock | ImageFileContentBlock { return AssistantStream.accumulateDelta(currentContent as unknown as Record, contentElement) as | TextContentBlock diff --git a/src/resources/beta/threads/index.ts b/src/resources/beta/threads/index.ts index d0ebb179..b55f67ed 100644 --- a/src/resources/beta/threads/index.ts +++ b/src/resources/beta/threads/index.ts @@ -11,14 +11,20 @@ export { ImageFileContentBlock, ImageFileDelta, ImageFileDeltaBlock, + ImageURL, + ImageURLContentBlock, + ImageURLDelta, + ImageURLDeltaBlock, Message, MessageContent, MessageContentDelta, + MessageContentPartParam, MessageDeleted, MessageDelta, MessageDeltaEvent, Text, TextContentBlock, + TextContentBlockParam, TextDelta, TextDeltaBlock, MessageCreateParams, diff --git a/src/resources/beta/threads/messages.ts b/src/resources/beta/threads/messages.ts index 8ce714f5..a1f90e1e 100644 --- a/src/resources/beta/threads/messages.ts +++ b/src/resources/beta/threads/messages.ts @@ -249,9 +249,16 @@ export namespace FilePathDeltaAnnotation { export interface ImageFile { /** * The [File](https://platform.openai.com/docs/api-reference/files) ID of the image - * in the message content. + * in the message content. Set `purpose="vision"` when uploading the File if you + * need to later display the file content. */ file_id: string; + + /** + * Specifies the detail level of the image if specified by the user. `low` uses + * fewer tokens, you can opt in to high resolution using `high`. + */ + detail?: 'auto' | 'low' | 'high'; } /** @@ -268,9 +275,16 @@ export interface ImageFileContentBlock { } export interface ImageFileDelta { + /** + * Specifies the detail level of the image if specified by the user. `low` uses + * fewer tokens, you can opt in to high resolution using `high`. + */ + detail?: 'auto' | 'low' | 'high'; + /** * The [File](https://platform.openai.com/docs/api-reference/files) ID of the image - * in the message content. + * in the message content. Set `purpose="vision"` when uploading the File if you + * need to later display the file content. */ file_id?: string; } @@ -293,6 +307,63 @@ export interface ImageFileDeltaBlock { image_file?: ImageFileDelta; } +export interface ImageURL { + /** + * The external URL of the image, must be a supported image types: jpeg, jpg, png, + * gif, webp. + */ + url: string; + + /** + * Specifies the detail level of the image. `low` uses fewer tokens, you can opt in + * to high resolution using `high`. Default value is `auto` + */ + detail?: 'auto' | 'low' | 'high'; +} + +/** + * References an image URL in the content of a message. + */ +export interface ImageURLContentBlock { + image_url: ImageURL; + + /** + * The type of the content part. + */ + type: 'image_url'; +} + +export interface ImageURLDelta { + /** + * Specifies the detail level of the image. `low` uses fewer tokens, you can opt in + * to high resolution using `high`. + */ + detail?: 'auto' | 'low' | 'high'; + + /** + * The URL of the image, must be a supported image types: jpeg, jpg, png, gif, + * webp. + */ + url?: string; +} + +/** + * References an image URL in the content of a message. + */ +export interface ImageURLDeltaBlock { + /** + * The index of the content part in the message. + */ + index: number; + + /** + * Always `image_url`. + */ + type: 'image_url'; + + image_url?: ImageURLDelta; +} + /** * Represents a message within a * [thread](https://platform.openai.com/docs/api-reference/threads). @@ -406,13 +477,19 @@ export namespace Message { * References an image [File](https://platform.openai.com/docs/api-reference/files) * in the content of a message. */ -export type MessageContent = ImageFileContentBlock | TextContentBlock; +export type MessageContent = ImageFileContentBlock | ImageURLContentBlock | TextContentBlock; + +/** + * References an image [File](https://platform.openai.com/docs/api-reference/files) + * in the content of a message. + */ +export type MessageContentDelta = ImageFileDeltaBlock | TextDeltaBlock | ImageURLDeltaBlock; /** * References an image [File](https://platform.openai.com/docs/api-reference/files) * in the content of a message. */ -export type MessageContentDelta = ImageFileDeltaBlock | TextDeltaBlock; +export type MessageContentPartParam = ImageFileContentBlock | ImageURLContentBlock | TextContentBlockParam; export interface MessageDeleted { id: string; @@ -479,6 +556,21 @@ export interface TextContentBlock { type: 'text'; } +/** + * The text content that is part of a message. + */ +export interface TextContentBlockParam { + /** + * Text content to be sent to the model + */ + text: string; + + /** + * Always `text`. + */ + type: 'text'; +} + export interface TextDelta { annotations?: Array; @@ -507,9 +599,9 @@ export interface TextDeltaBlock { export interface MessageCreateParams { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: @@ -591,14 +683,20 @@ export namespace Messages { export import ImageFileContentBlock = MessagesAPI.ImageFileContentBlock; export import ImageFileDelta = MessagesAPI.ImageFileDelta; export import ImageFileDeltaBlock = MessagesAPI.ImageFileDeltaBlock; + export import ImageURL = MessagesAPI.ImageURL; + export import ImageURLContentBlock = MessagesAPI.ImageURLContentBlock; + export import ImageURLDelta = MessagesAPI.ImageURLDelta; + export import ImageURLDeltaBlock = MessagesAPI.ImageURLDeltaBlock; export import Message = MessagesAPI.Message; export import MessageContent = MessagesAPI.MessageContent; export import MessageContentDelta = MessagesAPI.MessageContentDelta; + export import MessageContentPartParam = MessagesAPI.MessageContentPartParam; export import MessageDeleted = MessagesAPI.MessageDeleted; export import MessageDelta = MessagesAPI.MessageDelta; export import MessageDeltaEvent = MessagesAPI.MessageDeltaEvent; export import Text = MessagesAPI.Text; export import TextContentBlock = MessagesAPI.TextContentBlock; + export import TextContentBlockParam = MessagesAPI.TextContentBlockParam; export import TextDelta = MessagesAPI.TextDelta; export import TextDeltaBlock = MessagesAPI.TextDeltaBlock; export import MessagesPage = MessagesAPI.MessagesPage; diff --git a/src/resources/beta/threads/runs/runs.ts b/src/resources/beta/threads/runs/runs.ts index 18095886..d188edb2 100644 --- a/src/resources/beta/threads/runs/runs.ts +++ b/src/resources/beta/threads/runs/runs.ts @@ -9,6 +9,7 @@ import { sleep } from 'openai/core'; import { RunSubmitToolOutputsParamsStream } from 'openai/lib/AssistantStream'; import * as RunsAPI from 'openai/resources/beta/threads/runs/runs'; import * as AssistantsAPI from 'openai/resources/beta/assistants'; +import * as MessagesAPI from 'openai/resources/beta/threads/messages'; import * as ThreadsAPI from 'openai/resources/beta/threads/threads'; import * as StepsAPI from 'openai/resources/beta/threads/runs/steps'; import { CursorPage, type CursorPageParams } from 'openai/pagination'; @@ -747,9 +748,9 @@ export interface RunCreateParamsBase { export namespace RunCreateParams { export interface AdditionalMessage { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: @@ -999,9 +1000,9 @@ export interface RunCreateAndPollParams { export namespace RunCreateAndPollParams { export interface AdditionalMessage { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: @@ -1204,9 +1205,9 @@ export interface RunCreateAndStreamParams { export namespace RunCreateAndStreamParams { export interface AdditionalMessage { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: @@ -1409,9 +1410,9 @@ export interface RunStreamParams { export namespace RunStreamParams { export interface AdditionalMessage { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: diff --git a/src/resources/beta/threads/threads.ts b/src/resources/beta/threads/threads.ts index b8b3ff2b..7bd86fa5 100644 --- a/src/resources/beta/threads/threads.ts +++ b/src/resources/beta/threads/threads.ts @@ -284,9 +284,9 @@ export interface ThreadCreateParams { export namespace ThreadCreateParams { export interface Message { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: @@ -623,9 +623,9 @@ export namespace ThreadCreateAndRunParams { export namespace Thread { export interface Message { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: @@ -973,9 +973,9 @@ export namespace ThreadCreateAndRunPollParams { export namespace Thread { export interface Message { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: @@ -1302,9 +1302,9 @@ export namespace ThreadCreateAndRunStreamParams { export namespace Thread { export interface Message { /** - * The content of the message. + * The text contents of the message. */ - content: string; + content: string | Array; /** * The role of the entity that is creating the message. Allowed values include: @@ -1503,14 +1503,20 @@ export namespace Threads { export import ImageFileContentBlock = MessagesAPI.ImageFileContentBlock; export import ImageFileDelta = MessagesAPI.ImageFileDelta; export import ImageFileDeltaBlock = MessagesAPI.ImageFileDeltaBlock; + export import ImageURL = MessagesAPI.ImageURL; + export import ImageURLContentBlock = MessagesAPI.ImageURLContentBlock; + export import ImageURLDelta = MessagesAPI.ImageURLDelta; + export import ImageURLDeltaBlock = MessagesAPI.ImageURLDeltaBlock; export import Message = MessagesAPI.Message; export import MessageContent = MessagesAPI.MessageContent; export import MessageContentDelta = MessagesAPI.MessageContentDelta; + export import MessageContentPartParam = MessagesAPI.MessageContentPartParam; export import MessageDeleted = MessagesAPI.MessageDeleted; export import MessageDelta = MessagesAPI.MessageDelta; export import MessageDeltaEvent = MessagesAPI.MessageDeltaEvent; export import Text = MessagesAPI.Text; export import TextContentBlock = MessagesAPI.TextContentBlock; + export import TextContentBlockParam = MessagesAPI.TextContentBlockParam; export import TextDelta = MessagesAPI.TextDelta; export import TextDeltaBlock = MessagesAPI.TextDeltaBlock; export import MessagesPage = MessagesAPI.MessagesPage; diff --git a/src/resources/files.ts b/src/resources/files.ts index 63dff5bd..5d284a07 100644 --- a/src/resources/files.ts +++ b/src/resources/files.ts @@ -149,10 +149,17 @@ export interface FileObject { /** * The intended purpose of the file. Supported values are `assistants`, - * `assistants_output`, `batch`, `batch_output`, `fine-tune`, and - * `fine-tune-results`. + * `assistants_output`, `batch`, `batch_output`, `fine-tune`, `fine-tune-results` + * and `vision`. */ - purpose: 'assistants' | 'assistants_output' | 'batch' | 'batch_output' | 'fine-tune' | 'fine-tune-results'; + purpose: + | 'assistants' + | 'assistants_output' + | 'batch' + | 'batch_output' + | 'fine-tune' + | 'fine-tune-results' + | 'vision'; /** * @deprecated: Deprecated. The current status of the file, which can be either @@ -178,7 +185,8 @@ export interface FileCreateParams { * * Use "assistants" for * [Assistants](https://platform.openai.com/docs/api-reference/assistants) and - * [Messages](https://platform.openai.com/docs/api-reference/messages), "batch" for + * [Message](https://platform.openai.com/docs/api-reference/messages) files, + * "vision" for Assistants image file inputs, "batch" for * [Batch API](https://platform.openai.com/docs/guides/batch), and "fine-tune" for * [Fine-tuning](https://platform.openai.com/docs/api-reference/fine-tuning). */ diff --git a/tests/api-resources/beta/threads/messages.test.ts b/tests/api-resources/beta/threads/messages.test.ts index 262ff178..01268586 100644 --- a/tests/api-resources/beta/threads/messages.test.ts +++ b/tests/api-resources/beta/threads/messages.test.ts @@ -10,7 +10,10 @@ const openai = new OpenAI({ describe('resource messages', () => { test('create: only required params', async () => { - const responsePromise = openai.beta.threads.messages.create('string', { content: 'x', role: 'user' }); + const responsePromise = openai.beta.threads.messages.create('string', { + content: 'string', + role: 'user', + }); const rawResponse = await responsePromise.asResponse(); expect(rawResponse).toBeInstanceOf(Response); const response = await responsePromise; @@ -22,7 +25,7 @@ describe('resource messages', () => { test('create: required and optional params', async () => { const response = await openai.beta.threads.messages.create('string', { - content: 'x', + content: 'string', role: 'user', attachments: [ { diff --git a/tests/api-resources/beta/threads/runs/runs.test.ts b/tests/api-resources/beta/threads/runs/runs.test.ts index 85d97c34..3ee6ecb4 100644 --- a/tests/api-resources/beta/threads/runs/runs.test.ts +++ b/tests/api-resources/beta/threads/runs/runs.test.ts @@ -27,7 +27,7 @@ describe('resource runs', () => { additional_messages: [ { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string', @@ -58,7 +58,7 @@ describe('resource runs', () => { }, { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string', @@ -89,7 +89,7 @@ describe('resource runs', () => { }, { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string', diff --git a/tests/api-resources/beta/threads/threads.test.ts b/tests/api-resources/beta/threads/threads.test.ts index f2521cd5..4c425625 100644 --- a/tests/api-resources/beta/threads/threads.test.ts +++ b/tests/api-resources/beta/threads/threads.test.ts @@ -35,7 +35,7 @@ describe('resource threads', () => { messages: [ { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string', @@ -66,7 +66,7 @@ describe('resource threads', () => { }, { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string', @@ -97,7 +97,7 @@ describe('resource threads', () => { }, { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string', @@ -214,7 +214,7 @@ describe('resource threads', () => { messages: [ { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string', @@ -245,7 +245,7 @@ describe('resource threads', () => { }, { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string', @@ -276,7 +276,7 @@ describe('resource threads', () => { }, { role: 'user', - content: 'x', + content: 'string', attachments: [ { file_id: 'string',