feat(openai_dart)!: Add multi-modal support (#218)

davidmigloz · Nov 15, 2023 · 14c8e7e · 14c8e7e
1 parent 73f3791
commit 14c8e7e
Show file tree

Hide file tree

Showing 13 changed files with 3,647 additions and 719 deletions.
diff --git a/packages/openai_dart/README.md b/packages/openai_dart/README.md
@@ -59,13 +59,11 @@ final res = await client.createChatCompletion(
   request: CreateChatCompletionRequest(
     model: ChatCompletionModel.modelId('gpt-4'),
     messages: [
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.system,
+      ChatCompletionMessage.system(
         content: 'You are a helpful assistant.',
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.user,
-        content: 'Hello!',
+      ChatCompletionMessage.user(
+        content: ChatCompletionUserMessageContent.string('Hello!'),
       ),
     ],
     temperature: 0,
@@ -79,29 +77,72 @@ print(res.choices.first.message.content);
 - `ChatCompletionModel.modelId('model-id')`: the model ID as string (e.g. `'gpt-4'` or your fine-tuned model ID).
 - `ChatCompletionModel.model(ChatCompletionModels.gpt4)`: a value from `ChatCompletionModels` enum which lists all of the available models.
 
+`ChatCompletionMessage` is a sealed class that supports the following message types:
+- `ChatCompletionMessage.system()`: a system message.
+- `ChatCompletionMessage.user()`: a user message.
+- `ChatCompletionMessage.assistant()`: an assistant message.
+- `ChatCompletionMessage.tool()`: a tool message.
+- `ChatCompletionMessage.function()`: a function message.
+
 **Stream chat completion:**
 
 ```dart
 final stream = client.createChatCompletionStream(
   request: CreateChatCompletionRequest(
     model: ChatCompletionModel.modelId('gpt-3.5-turbo'),
     messages: [
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.system,
+      ChatCompletionMessage.system(
         content:
             'You are a helpful assistant that replies only with numbers '
             'in order without any spaces or commas',
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.user,
-        content: 'List the numbers from 1 to 9',
+      ChatCompletionMessage.user(
+        content: ChatCompletionUserMessageContent.string(
+          'List the numbers from 1 to 9',
+        ),
       ),
     ],
   ),
 );
 await for (final res in stream) {
   print(res.choices.first.delta.content);
 }
+// 123
+// 456
+// 789
+```
+
+**Multi-modal prompt:**
+
+```dart
+final res = await client.createChatCompletion(
+  request: CreateChatCompletionRequest(
+    model: ChatCompletionModel.model(
+      ChatCompletionModels.gpt4VisionPreview,
+    ),
+    messages: [
+      ChatCompletionMessage.system(
+        content: 'You are a helpful assistant.',
+      ),
+      ChatCompletionMessage.user(
+        content: ChatCompletionUserMessageContent.parts(
+          [
+            ChatCompletionMessageContentPart.text(
+              text: 'What fruit is this?',
+            ),
+            ChatCompletionMessageContentPart.image(
+              imageUrl: ChatCompletionMessageImageUrl(
+                url: 'https://upload.wikimedia.org/wikipedia/commons/9/92/95apple.jpeg',
+              ),
+            ),
+          ],
+        ),
+      ),
+    ],
+  ),
+);
+print(res.choices.first.message.content);
+// The fruit in the image is an apple.
 ```
 
 **JSON mode:**
@@ -113,15 +154,15 @@ final res = await client.createChatCompletion(
       ChatCompletionModels.gpt41106Preview,
     ),
     messages: [
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.system,
+      ChatCompletionMessage.system(
         content:
           'You are a helpful assistant. That extracts names from text '
           'and returns them in a JSON array.',
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.user,
-        content: 'John, Mary, and Peter.',
+      ChatCompletionMessage.user(
+        content: ChatCompletionUserMessageContent.string(
+          'John, Mary, and Peter.',
+        ),
       ),
     ],
     temperature: 0,
@@ -166,13 +207,13 @@ final res1 = await client.createChatCompletion(
       ChatCompletionModels.gpt35Turbo,
     ),
     messages: [
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.system,
+      ChatCompletionMessage.system(
         content: 'You are a helpful assistant.',
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.user,
-        content: 'What’s the weather like in Boston right now?',
+      ChatCompletionMessage.user(
+        content: ChatCompletionUserMessageContent.string(
+          'What’s the weather like in Boston right now?',
+        ),
       ),
     ],
     tools: [tool],
@@ -194,16 +235,15 @@ final res2 = await client.createChatCompletion(
   request: CreateChatCompletionRequest(
     model: ChatCompletionModel.modelId('gpt-3.5-turbo'),
     messages: [
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.system,
+      ChatCompletionMessage.system(
         content: 'You are a helpful assistant.',
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.user,
-        content: 'What’s the weather like in Boston right now?',
+      ChatCompletionMessage.user(
+        content: ChatCompletionUserMessageContent.string(
+          'What’s the weather like in Boston right now?',
+        ),
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.tool,
+      ChatCompletionMessage.tool(
         toolCallId: toolCall.id,
         content: json.encode(functionResult),
       ),
@@ -242,13 +282,13 @@ final res1 = await client.createChatCompletion(
   request: CreateChatCompletionRequest(
     model: ChatCompletionModel.modelId('gpt-3.5-turbo'),
     messages: [
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.system,
+      ChatCompletionMessage.system(
         content: 'You are a helpful assistant.',
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.user,
-        content: 'What’s the weather like in Boston right now?',
+      ChatCompletionMessage.user(
+        content: ChatCompletionUserMessageContent.string(
+          'What’s the weather like in Boston right now?',
+        ),
       ),
     ],
     functions: [function],
@@ -264,16 +304,15 @@ final res2 = await client.createChatCompletion(
   request: CreateChatCompletionRequest(
     model: ChatCompletionModel.modelId('gpt-3.5-turbo'),
     messages: [
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.system,
+      ChatCompletionMessage.system(
         content: 'You are a helpful assistant.',
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.user,
-        content: 'What’s the weather like in Boston right now?',
+      ChatCompletionMessage.user(
+        content: ChatCompletionUserMessageContent.string(
+          'What’s the weather like in Boston right now?',
+        ),
       ),
-      ChatCompletionMessage(
-        role: ChatCompletionMessageRole.function,
+      ChatCompletionMessage.function(
         name: function.name,
         content: json.encode(functionResult),
       ),

diff --git a/packages/openai_dart/example/openai_dart_example.dart b/packages/openai_dart/example/openai_dart_example.dart
@@ -23,13 +23,11 @@ Future<void> _chatCompletions(final OpenAIClient client) async {
     request: const CreateChatCompletionRequest(
       model: ChatCompletionModel.modelId('gpt-4'),
       messages: [
-        ChatCompletionMessage(
-          role: ChatCompletionMessageRole.system,
+        ChatCompletionMessage.system(
           content: 'You are a helpful assistant.',
         ),
-        ChatCompletionMessage(
-          role: ChatCompletionMessageRole.user,
-          content: 'Hello!',
+        ChatCompletionMessage.user(
+          content: ChatCompletionUserMessageContent.string('Hello!'),
         ),
       ],
       temperature: 0,