From f0a2d8d55e8b68dd9870cb29bb3cb7d5468b0fad Mon Sep 17 00:00:00 2001
From: Stainless Bot <107565488+stainless-bot@users.noreply.github.com>
Date: Thu, 9 May 2024 16:19:10 -0400
Subject: [PATCH] feat(api): add message image content (#834)
---
.stats.yml | 2 +-
api.md | 6 +
src/lib/AssistantStream.ts | 11 +-
src/resources/beta/threads/index.ts | 6 +
src/resources/beta/threads/messages.ts | 110 +++++++++++++++++-
src/resources/beta/threads/runs/runs.ts | 17 +--
src/resources/beta/threads/threads.ts | 22 ++--
src/resources/files.ts | 16 ++-
.../beta/threads/messages.test.ts | 7 +-
.../beta/threads/runs/runs.test.ts | 6 +-
.../beta/threads/threads.test.ts | 12 +-
11 files changed, 171 insertions(+), 44 deletions(-)
diff --git a/.stats.yml b/.stats.yml
index 50c6b293..52e87d1b 100644
--- a/.stats.yml
+++ b/.stats.yml
@@ -1,2 +1,2 @@
configured_endpoints: 64
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-2e14236d4015bf3b956290ea8b656224a0c7b206a356c6af2a7ae43fdbceb04c.yml
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-084b8f68408c6b689a55200a78bcf233769bfcd8e999d9fadaeb399152b05bcd.yml
diff --git a/api.md b/api.md
index 18cdd9e6..17a3f963 100644
--- a/api.md
+++ b/api.md
@@ -349,14 +349,20 @@ Types:
- ImageFileContentBlock
- ImageFileDelta
- ImageFileDeltaBlock
+- ImageURL
+- ImageURLContentBlock
+- ImageURLDelta
+- ImageURLDeltaBlock
- Message
- MessageContent
- MessageContentDelta
+- MessageContentPartParam
- MessageDeleted
- MessageDelta
- MessageDeltaEvent
- Text
- TextContentBlock
+- TextContentBlockParam
- TextDelta
- TextDeltaBlock
diff --git a/src/lib/AssistantStream.ts b/src/lib/AssistantStream.ts
index a2974826..de7511b5 100644
--- a/src/lib/AssistantStream.ts
+++ b/src/lib/AssistantStream.ts
@@ -7,6 +7,7 @@ import {
ImageFile,
TextDelta,
Messages,
+ MessageContent,
} from 'openai/resources/beta/threads/messages';
import * as Core from 'openai/core';
import { RequestOptions } from 'openai/core';
@@ -87,7 +88,7 @@ export class AssistantStream
#messageSnapshot: Message | undefined;
#finalRun: Run | undefined;
#currentContentIndex: number | undefined;
- #currentContent: TextContentBlock | ImageFileContentBlock | undefined;
+ #currentContent: MessageContent | undefined;
#currentToolCallIndex: number | undefined;
#currentToolCall: ToolCall | undefined;
@@ -624,10 +625,8 @@ export class AssistantStream
currentContent,
);
} else {
- snapshot.content[contentElement.index] = contentElement as
- | TextContentBlock
- | ImageFileContentBlock;
- //This is a new element
+ snapshot.content[contentElement.index] = contentElement as MessageContent;
+ // This is a new element
newContent.push(contentElement);
}
}
@@ -650,7 +649,7 @@ export class AssistantStream
#accumulateContent(
contentElement: MessageContentDelta,
- currentContent: TextContentBlock | ImageFileContentBlock | undefined,
+ currentContent: MessageContent | undefined,
): TextContentBlock | ImageFileContentBlock {
return AssistantStream.accumulateDelta(currentContent as unknown as Record, contentElement) as
| TextContentBlock
diff --git a/src/resources/beta/threads/index.ts b/src/resources/beta/threads/index.ts
index d0ebb179..b55f67ed 100644
--- a/src/resources/beta/threads/index.ts
+++ b/src/resources/beta/threads/index.ts
@@ -11,14 +11,20 @@ export {
ImageFileContentBlock,
ImageFileDelta,
ImageFileDeltaBlock,
+ ImageURL,
+ ImageURLContentBlock,
+ ImageURLDelta,
+ ImageURLDeltaBlock,
Message,
MessageContent,
MessageContentDelta,
+ MessageContentPartParam,
MessageDeleted,
MessageDelta,
MessageDeltaEvent,
Text,
TextContentBlock,
+ TextContentBlockParam,
TextDelta,
TextDeltaBlock,
MessageCreateParams,
diff --git a/src/resources/beta/threads/messages.ts b/src/resources/beta/threads/messages.ts
index 8ce714f5..a1f90e1e 100644
--- a/src/resources/beta/threads/messages.ts
+++ b/src/resources/beta/threads/messages.ts
@@ -249,9 +249,16 @@ export namespace FilePathDeltaAnnotation {
export interface ImageFile {
/**
* The [File](https://platform.openai.com/docs/api-reference/files) ID of the image
- * in the message content.
+ * in the message content. Set `purpose="vision"` when uploading the File if you
+ * need to later display the file content.
*/
file_id: string;
+
+ /**
+ * Specifies the detail level of the image if specified by the user. `low` uses
+ * fewer tokens, you can opt in to high resolution using `high`.
+ */
+ detail?: 'auto' | 'low' | 'high';
}
/**
@@ -268,9 +275,16 @@ export interface ImageFileContentBlock {
}
export interface ImageFileDelta {
+ /**
+ * Specifies the detail level of the image if specified by the user. `low` uses
+ * fewer tokens, you can opt in to high resolution using `high`.
+ */
+ detail?: 'auto' | 'low' | 'high';
+
/**
* The [File](https://platform.openai.com/docs/api-reference/files) ID of the image
- * in the message content.
+ * in the message content. Set `purpose="vision"` when uploading the File if you
+ * need to later display the file content.
*/
file_id?: string;
}
@@ -293,6 +307,63 @@ export interface ImageFileDeltaBlock {
image_file?: ImageFileDelta;
}
+export interface ImageURL {
+ /**
+ * The external URL of the image, must be a supported image types: jpeg, jpg, png,
+ * gif, webp.
+ */
+ url: string;
+
+ /**
+ * Specifies the detail level of the image. `low` uses fewer tokens, you can opt in
+ * to high resolution using `high`. Default value is `auto`
+ */
+ detail?: 'auto' | 'low' | 'high';
+}
+
+/**
+ * References an image URL in the content of a message.
+ */
+export interface ImageURLContentBlock {
+ image_url: ImageURL;
+
+ /**
+ * The type of the content part.
+ */
+ type: 'image_url';
+}
+
+export interface ImageURLDelta {
+ /**
+ * Specifies the detail level of the image. `low` uses fewer tokens, you can opt in
+ * to high resolution using `high`.
+ */
+ detail?: 'auto' | 'low' | 'high';
+
+ /**
+ * The URL of the image, must be a supported image types: jpeg, jpg, png, gif,
+ * webp.
+ */
+ url?: string;
+}
+
+/**
+ * References an image URL in the content of a message.
+ */
+export interface ImageURLDeltaBlock {
+ /**
+ * The index of the content part in the message.
+ */
+ index: number;
+
+ /**
+ * Always `image_url`.
+ */
+ type: 'image_url';
+
+ image_url?: ImageURLDelta;
+}
+
/**
* Represents a message within a
* [thread](https://platform.openai.com/docs/api-reference/threads).
@@ -406,13 +477,19 @@ export namespace Message {
* References an image [File](https://platform.openai.com/docs/api-reference/files)
* in the content of a message.
*/
-export type MessageContent = ImageFileContentBlock | TextContentBlock;
+export type MessageContent = ImageFileContentBlock | ImageURLContentBlock | TextContentBlock;
+
+/**
+ * References an image [File](https://platform.openai.com/docs/api-reference/files)
+ * in the content of a message.
+ */
+export type MessageContentDelta = ImageFileDeltaBlock | TextDeltaBlock | ImageURLDeltaBlock;
/**
* References an image [File](https://platform.openai.com/docs/api-reference/files)
* in the content of a message.
*/
-export type MessageContentDelta = ImageFileDeltaBlock | TextDeltaBlock;
+export type MessageContentPartParam = ImageFileContentBlock | ImageURLContentBlock | TextContentBlockParam;
export interface MessageDeleted {
id: string;
@@ -479,6 +556,21 @@ export interface TextContentBlock {
type: 'text';
}
+/**
+ * The text content that is part of a message.
+ */
+export interface TextContentBlockParam {
+ /**
+ * Text content to be sent to the model
+ */
+ text: string;
+
+ /**
+ * Always `text`.
+ */
+ type: 'text';
+}
+
export interface TextDelta {
annotations?: Array;
@@ -507,9 +599,9 @@ export interface TextDeltaBlock {
export interface MessageCreateParams {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
@@ -591,14 +683,20 @@ export namespace Messages {
export import ImageFileContentBlock = MessagesAPI.ImageFileContentBlock;
export import ImageFileDelta = MessagesAPI.ImageFileDelta;
export import ImageFileDeltaBlock = MessagesAPI.ImageFileDeltaBlock;
+ export import ImageURL = MessagesAPI.ImageURL;
+ export import ImageURLContentBlock = MessagesAPI.ImageURLContentBlock;
+ export import ImageURLDelta = MessagesAPI.ImageURLDelta;
+ export import ImageURLDeltaBlock = MessagesAPI.ImageURLDeltaBlock;
export import Message = MessagesAPI.Message;
export import MessageContent = MessagesAPI.MessageContent;
export import MessageContentDelta = MessagesAPI.MessageContentDelta;
+ export import MessageContentPartParam = MessagesAPI.MessageContentPartParam;
export import MessageDeleted = MessagesAPI.MessageDeleted;
export import MessageDelta = MessagesAPI.MessageDelta;
export import MessageDeltaEvent = MessagesAPI.MessageDeltaEvent;
export import Text = MessagesAPI.Text;
export import TextContentBlock = MessagesAPI.TextContentBlock;
+ export import TextContentBlockParam = MessagesAPI.TextContentBlockParam;
export import TextDelta = MessagesAPI.TextDelta;
export import TextDeltaBlock = MessagesAPI.TextDeltaBlock;
export import MessagesPage = MessagesAPI.MessagesPage;
diff --git a/src/resources/beta/threads/runs/runs.ts b/src/resources/beta/threads/runs/runs.ts
index 18095886..d188edb2 100644
--- a/src/resources/beta/threads/runs/runs.ts
+++ b/src/resources/beta/threads/runs/runs.ts
@@ -9,6 +9,7 @@ import { sleep } from 'openai/core';
import { RunSubmitToolOutputsParamsStream } from 'openai/lib/AssistantStream';
import * as RunsAPI from 'openai/resources/beta/threads/runs/runs';
import * as AssistantsAPI from 'openai/resources/beta/assistants';
+import * as MessagesAPI from 'openai/resources/beta/threads/messages';
import * as ThreadsAPI from 'openai/resources/beta/threads/threads';
import * as StepsAPI from 'openai/resources/beta/threads/runs/steps';
import { CursorPage, type CursorPageParams } from 'openai/pagination';
@@ -747,9 +748,9 @@ export interface RunCreateParamsBase {
export namespace RunCreateParams {
export interface AdditionalMessage {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
@@ -999,9 +1000,9 @@ export interface RunCreateAndPollParams {
export namespace RunCreateAndPollParams {
export interface AdditionalMessage {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
@@ -1204,9 +1205,9 @@ export interface RunCreateAndStreamParams {
export namespace RunCreateAndStreamParams {
export interface AdditionalMessage {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
@@ -1409,9 +1410,9 @@ export interface RunStreamParams {
export namespace RunStreamParams {
export interface AdditionalMessage {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
diff --git a/src/resources/beta/threads/threads.ts b/src/resources/beta/threads/threads.ts
index b8b3ff2b..7bd86fa5 100644
--- a/src/resources/beta/threads/threads.ts
+++ b/src/resources/beta/threads/threads.ts
@@ -284,9 +284,9 @@ export interface ThreadCreateParams {
export namespace ThreadCreateParams {
export interface Message {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
@@ -623,9 +623,9 @@ export namespace ThreadCreateAndRunParams {
export namespace Thread {
export interface Message {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
@@ -973,9 +973,9 @@ export namespace ThreadCreateAndRunPollParams {
export namespace Thread {
export interface Message {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
@@ -1302,9 +1302,9 @@ export namespace ThreadCreateAndRunStreamParams {
export namespace Thread {
export interface Message {
/**
- * The content of the message.
+ * The text contents of the message.
*/
- content: string;
+ content: string | Array;
/**
* The role of the entity that is creating the message. Allowed values include:
@@ -1503,14 +1503,20 @@ export namespace Threads {
export import ImageFileContentBlock = MessagesAPI.ImageFileContentBlock;
export import ImageFileDelta = MessagesAPI.ImageFileDelta;
export import ImageFileDeltaBlock = MessagesAPI.ImageFileDeltaBlock;
+ export import ImageURL = MessagesAPI.ImageURL;
+ export import ImageURLContentBlock = MessagesAPI.ImageURLContentBlock;
+ export import ImageURLDelta = MessagesAPI.ImageURLDelta;
+ export import ImageURLDeltaBlock = MessagesAPI.ImageURLDeltaBlock;
export import Message = MessagesAPI.Message;
export import MessageContent = MessagesAPI.MessageContent;
export import MessageContentDelta = MessagesAPI.MessageContentDelta;
+ export import MessageContentPartParam = MessagesAPI.MessageContentPartParam;
export import MessageDeleted = MessagesAPI.MessageDeleted;
export import MessageDelta = MessagesAPI.MessageDelta;
export import MessageDeltaEvent = MessagesAPI.MessageDeltaEvent;
export import Text = MessagesAPI.Text;
export import TextContentBlock = MessagesAPI.TextContentBlock;
+ export import TextContentBlockParam = MessagesAPI.TextContentBlockParam;
export import TextDelta = MessagesAPI.TextDelta;
export import TextDeltaBlock = MessagesAPI.TextDeltaBlock;
export import MessagesPage = MessagesAPI.MessagesPage;
diff --git a/src/resources/files.ts b/src/resources/files.ts
index 63dff5bd..5d284a07 100644
--- a/src/resources/files.ts
+++ b/src/resources/files.ts
@@ -149,10 +149,17 @@ export interface FileObject {
/**
* The intended purpose of the file. Supported values are `assistants`,
- * `assistants_output`, `batch`, `batch_output`, `fine-tune`, and
- * `fine-tune-results`.
+ * `assistants_output`, `batch`, `batch_output`, `fine-tune`, `fine-tune-results`
+ * and `vision`.
*/
- purpose: 'assistants' | 'assistants_output' | 'batch' | 'batch_output' | 'fine-tune' | 'fine-tune-results';
+ purpose:
+ | 'assistants'
+ | 'assistants_output'
+ | 'batch'
+ | 'batch_output'
+ | 'fine-tune'
+ | 'fine-tune-results'
+ | 'vision';
/**
* @deprecated: Deprecated. The current status of the file, which can be either
@@ -178,7 +185,8 @@ export interface FileCreateParams {
*
* Use "assistants" for
* [Assistants](https://platform.openai.com/docs/api-reference/assistants) and
- * [Messages](https://platform.openai.com/docs/api-reference/messages), "batch" for
+ * [Message](https://platform.openai.com/docs/api-reference/messages) files,
+ * "vision" for Assistants image file inputs, "batch" for
* [Batch API](https://platform.openai.com/docs/guides/batch), and "fine-tune" for
* [Fine-tuning](https://platform.openai.com/docs/api-reference/fine-tuning).
*/
diff --git a/tests/api-resources/beta/threads/messages.test.ts b/tests/api-resources/beta/threads/messages.test.ts
index 262ff178..01268586 100644
--- a/tests/api-resources/beta/threads/messages.test.ts
+++ b/tests/api-resources/beta/threads/messages.test.ts
@@ -10,7 +10,10 @@ const openai = new OpenAI({
describe('resource messages', () => {
test('create: only required params', async () => {
- const responsePromise = openai.beta.threads.messages.create('string', { content: 'x', role: 'user' });
+ const responsePromise = openai.beta.threads.messages.create('string', {
+ content: 'string',
+ role: 'user',
+ });
const rawResponse = await responsePromise.asResponse();
expect(rawResponse).toBeInstanceOf(Response);
const response = await responsePromise;
@@ -22,7 +25,7 @@ describe('resource messages', () => {
test('create: required and optional params', async () => {
const response = await openai.beta.threads.messages.create('string', {
- content: 'x',
+ content: 'string',
role: 'user',
attachments: [
{
diff --git a/tests/api-resources/beta/threads/runs/runs.test.ts b/tests/api-resources/beta/threads/runs/runs.test.ts
index 85d97c34..3ee6ecb4 100644
--- a/tests/api-resources/beta/threads/runs/runs.test.ts
+++ b/tests/api-resources/beta/threads/runs/runs.test.ts
@@ -27,7 +27,7 @@ describe('resource runs', () => {
additional_messages: [
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',
@@ -58,7 +58,7 @@ describe('resource runs', () => {
},
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',
@@ -89,7 +89,7 @@ describe('resource runs', () => {
},
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',
diff --git a/tests/api-resources/beta/threads/threads.test.ts b/tests/api-resources/beta/threads/threads.test.ts
index f2521cd5..4c425625 100644
--- a/tests/api-resources/beta/threads/threads.test.ts
+++ b/tests/api-resources/beta/threads/threads.test.ts
@@ -35,7 +35,7 @@ describe('resource threads', () => {
messages: [
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',
@@ -66,7 +66,7 @@ describe('resource threads', () => {
},
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',
@@ -97,7 +97,7 @@ describe('resource threads', () => {
},
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',
@@ -214,7 +214,7 @@ describe('resource threads', () => {
messages: [
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',
@@ -245,7 +245,7 @@ describe('resource threads', () => {
},
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',
@@ -276,7 +276,7 @@ describe('resource threads', () => {
},
{
role: 'user',
- content: 'x',
+ content: 'string',
attachments: [
{
file_id: 'string',