From 5608c93361dfa78859c844b9f3206876aef293d9 Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Wed, 13 Mar 2024 09:49:00 -0400 Subject: [PATCH 1/8] add generateImage function --- src/lib/ai.ts | 30 ++++++++++++++++++++++++++++++ src/lib/commands/DALLE3Command.ts | 29 +++++++++++++++++++++++++++++ src/lib/commands/index.ts | 2 ++ 3 files changed, 61 insertions(+) create mode 100644 src/lib/commands/DALLE3Command.ts diff --git a/src/lib/ai.ts b/src/lib/ai.ts index 5bed37ff..627022a8 100644 --- a/src/lib/ai.ts +++ b/src/lib/ai.ts @@ -396,3 +396,33 @@ export const textToSpeech = async (message: string): Promise => { return objectUrl; }; + +type dalle3ImageSize = "1024x1024" | "1792x1024" | "1024x1792"; + +export const generateImage = async ( + prompt: string, + n: number = 1, + size: dalle3ImageSize = "1024x1024" +): Promise => { + const { currentProvider } = getSettings(); + if (!currentProvider?.apiKey) { + throw new Error("Missing OpenAI API Key"); + } + + const { openai } = createClient(currentProvider?.apiKey, currentProvider?.apiUrl); + + try { + const response = await openai.images.generate({ + model: "dall-e-3", + prompt: prompt, + n: n, + size: size, + }); + + // Assuming the response structure has an array of generated images + const imageUrls = response.data.map((img: any) => img.url); + return imageUrls; + } catch (error: any) { + throw new Error(`Failed to generate image: ${error.message}`); + } +}; diff --git a/src/lib/commands/DALLE3Command.ts b/src/lib/commands/DALLE3Command.ts new file mode 100644 index 00000000..777e044d --- /dev/null +++ b/src/lib/commands/DALLE3Command.ts @@ -0,0 +1,29 @@ +import { ChatCraftCommand } from "../ChatCraftCommand"; +import { ChatCraftChat } from "../ChatCraftChat"; +import { ChatCraftHumanMessage } from "../ChatCraftMessage"; +import { generateImage } from "../../lib/ai"; + +export class DALLE3Command extends ChatCraftCommand { + constructor() { + super("DALLE3"); + } + + async execute(chat: ChatCraftChat, user: User | undefined, args?: string[]) { + // return chat.resetMessages(); + + if (!(args && args[0])) { + throw new Error("must include a prompt"); + } + const prompt = args.join(" "); + let imageUrls: string[] = []; + const text = `(DALL·E 3 result of the prompt: ${prompt})`; + + try { + imageUrls = await generateImage(prompt); + } catch (error: any) { + console.error("Failed to generate image:", error); + throw new Error("Failed to generate image ", error); + } + return chat.addMessage(new ChatCraftHumanMessage({ user, text, imageUrls })); + } +} diff --git a/src/lib/commands/index.ts b/src/lib/commands/index.ts index 562c3725..c5d7e649 100644 --- a/src/lib/commands/index.ts +++ b/src/lib/commands/index.ts @@ -7,6 +7,7 @@ import { SummaryCommand } from "./SummaryCommand"; import { HelpCommand } from "./HelpCommand"; import { ImportCommand } from "./ImportCommand"; import { CommandsHelpCommand } from "./CommandsHelpCommand"; +import { DALLE3Command } from "./DALLE3Command"; // Register all our commands ChatCraftCommandRegistry.registerCommand(new NewCommand()); @@ -15,3 +16,4 @@ ChatCraftCommandRegistry.registerCommand(new SummaryCommand()); ChatCraftCommandRegistry.registerCommand(new HelpCommand()); ChatCraftCommandRegistry.registerCommand(new CommandsHelpCommand()); ChatCraftCommandRegistry.registerCommand(new ImportCommand()); +ChatCraftCommandRegistry.registerCommand(new DALLE3Command()); From 4d77d2693a8a503e210ffbd1b46e4bb41790c9d9 Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:11:47 -0400 Subject: [PATCH 2/8] remove debug comment --- src/lib/ai.ts | 1 - src/lib/commands/DALLE3Command.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/lib/ai.ts b/src/lib/ai.ts index 627022a8..d485a70d 100644 --- a/src/lib/ai.ts +++ b/src/lib/ai.ts @@ -419,7 +419,6 @@ export const generateImage = async ( size: size, }); - // Assuming the response structure has an array of generated images const imageUrls = response.data.map((img: any) => img.url); return imageUrls; } catch (error: any) { diff --git a/src/lib/commands/DALLE3Command.ts b/src/lib/commands/DALLE3Command.ts index 777e044d..514b8c7a 100644 --- a/src/lib/commands/DALLE3Command.ts +++ b/src/lib/commands/DALLE3Command.ts @@ -9,8 +9,6 @@ export class DALLE3Command extends ChatCraftCommand { } async execute(chat: ChatCraftChat, user: User | undefined, args?: string[]) { - // return chat.resetMessages(); - if (!(args && args[0])) { throw new Error("must include a prompt"); } From 11118202310ab3607f112795c298b3a3fb662cd7 Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:36:43 -0400 Subject: [PATCH 3/8] change command to image --- src/lib/commands/{DALLE3Command.ts => ImageCommand.ts} | 4 ++-- src/lib/commands/index.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/lib/commands/{DALLE3Command.ts => ImageCommand.ts} (91%) diff --git a/src/lib/commands/DALLE3Command.ts b/src/lib/commands/ImageCommand.ts similarity index 91% rename from src/lib/commands/DALLE3Command.ts rename to src/lib/commands/ImageCommand.ts index 514b8c7a..896f8fe3 100644 --- a/src/lib/commands/DALLE3Command.ts +++ b/src/lib/commands/ImageCommand.ts @@ -3,9 +3,9 @@ import { ChatCraftChat } from "../ChatCraftChat"; import { ChatCraftHumanMessage } from "../ChatCraftMessage"; import { generateImage } from "../../lib/ai"; -export class DALLE3Command extends ChatCraftCommand { +export class ImageCommand extends ChatCraftCommand { constructor() { - super("DALLE3"); + super("image"); } async execute(chat: ChatCraftChat, user: User | undefined, args?: string[]) { diff --git a/src/lib/commands/index.ts b/src/lib/commands/index.ts index c5d7e649..37349b20 100644 --- a/src/lib/commands/index.ts +++ b/src/lib/commands/index.ts @@ -7,7 +7,7 @@ import { SummaryCommand } from "./SummaryCommand"; import { HelpCommand } from "./HelpCommand"; import { ImportCommand } from "./ImportCommand"; import { CommandsHelpCommand } from "./CommandsHelpCommand"; -import { DALLE3Command } from "./DALLE3Command"; +import { ImageCommand } from "./ImageCommand"; // Register all our commands ChatCraftCommandRegistry.registerCommand(new NewCommand()); @@ -16,4 +16,4 @@ ChatCraftCommandRegistry.registerCommand(new SummaryCommand()); ChatCraftCommandRegistry.registerCommand(new HelpCommand()); ChatCraftCommandRegistry.registerCommand(new CommandsHelpCommand()); ChatCraftCommandRegistry.registerCommand(new ImportCommand()); -ChatCraftCommandRegistry.registerCommand(new DALLE3Command()); +ChatCraftCommandRegistry.registerCommand(new ImageCommand()); From 55bb53563878f5b3f3f4c757a8f6124a12ab38ad Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:02:23 -0400 Subject: [PATCH 4/8] add image command instructions --- src/components/Message/AppMessage/Help.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/Message/AppMessage/Help.tsx b/src/components/Message/AppMessage/Help.tsx index 3850f7af..b6cb4cab 100644 --- a/src/components/Message/AppMessage/Help.tsx +++ b/src/components/Message/AppMessage/Help.tsx @@ -17,7 +17,9 @@ Some commands accept arguments as well. | /new | Creates a new chat. | | /clear | Erases all messages in the current chat. | | /summary [max-length] | Uses ChatGPT to create a summary of the current chat. Optionally takes a maximum word length (defaults to 500). | -| /import  | Loads the provided URL and imports the text. Where possible, ChatCraft will try to get raw text vs. HTML from sites like GitHub. NOTE: to prevent abuse, you must be logged into use the import command. |`; +| /import  | Loads the provided URL and imports the text. Where possible, ChatCraft will try to get raw text vs. HTML from sites like GitHub. NOTE: to prevent abuse, you must be logged into use the import command. | +| /image  | Creates an image using the provided prompt.| +`; const helpText = `## ChatCraft.org Help From 0e082f7decc105bd18be900e635455dea64a86dc Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Thu, 14 Mar 2024 21:19:35 -0400 Subject: [PATCH 5/8] add n parameter instructions --- src/lib/ai.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/ai.ts b/src/lib/ai.ts index d485a70d..e79d7028 100644 --- a/src/lib/ai.ts +++ b/src/lib/ai.ts @@ -401,6 +401,8 @@ type dalle3ImageSize = "1024x1024" | "1792x1024" | "1024x1792"; export const generateImage = async ( prompt: string, + //You can request 1 image at a time with DALL·E 3 (request more by making parallel requests) or up to 10 images at a time using DALL·E 2 with the n parameter. + //https://platform.openai.com/docs/guides/images/generations n: number = 1, size: dalle3ImageSize = "1024x1024" ): Promise => { From 1b1d5b132a1794ef1fdbb984a5ab55065759a2d0 Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Fri, 15 Mar 2024 01:19:39 -0400 Subject: [PATCH 6/8] use the currentProvider.createClient method --- src/lib/ai.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/ai.ts b/src/lib/ai.ts index e79d7028..c33dab48 100644 --- a/src/lib/ai.ts +++ b/src/lib/ai.ts @@ -411,7 +411,7 @@ export const generateImage = async ( throw new Error("Missing OpenAI API Key"); } - const { openai } = createClient(currentProvider?.apiKey, currentProvider?.apiUrl); + const { openai } = currentProvider.createClient(currentProvider.apiKey); try { const response = await openai.images.generate({ From e61b83ab02f21bfca8b1114116fab0983a06b365 Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:14:41 -0400 Subject: [PATCH 7/8] display no supported models error --- src/lib/ai.ts | 21 +++++++++++++++++++-- src/lib/commands/ImageCommand.ts | 9 ++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/lib/ai.ts b/src/lib/ai.ts index c33dab48..c8c589f3 100644 --- a/src/lib/ai.ts +++ b/src/lib/ai.ts @@ -397,6 +397,23 @@ export const textToSpeech = async (message: string): Promise => { return objectUrl; }; +/** + * Only meant to be used outside components or hooks + * where useModels cannot be used. + */ +export async function isGenerateImageSupported() { + const { currentProvider } = getSettings(); + if (!currentProvider.apiKey) { + throw new Error("Missing API Key"); + } + + return ( + (await currentProvider.queryModels(currentProvider.apiKey)).filter((model: string) => + model.includes("dall-e-3") + )?.length > 0 + ); +} + type dalle3ImageSize = "1024x1024" | "1792x1024" | "1024x1792"; export const generateImage = async ( @@ -407,7 +424,7 @@ export const generateImage = async ( size: dalle3ImageSize = "1024x1024" ): Promise => { const { currentProvider } = getSettings(); - if (!currentProvider?.apiKey) { + if (!currentProvider.apiKey) { throw new Error("Missing OpenAI API Key"); } @@ -424,6 +441,6 @@ export const generateImage = async ( const imageUrls = response.data.map((img: any) => img.url); return imageUrls; } catch (error: any) { - throw new Error(`Failed to generate image: ${error.message}`); + throw new Error(error); } }; diff --git a/src/lib/commands/ImageCommand.ts b/src/lib/commands/ImageCommand.ts index 896f8fe3..3404c52d 100644 --- a/src/lib/commands/ImageCommand.ts +++ b/src/lib/commands/ImageCommand.ts @@ -1,7 +1,7 @@ import { ChatCraftCommand } from "../ChatCraftCommand"; import { ChatCraftChat } from "../ChatCraftChat"; import { ChatCraftHumanMessage } from "../ChatCraftMessage"; -import { generateImage } from "../../lib/ai"; +import { generateImage, isGenerateImageSupported } from "../../lib/ai"; export class ImageCommand extends ChatCraftCommand { constructor() { @@ -9,6 +9,9 @@ export class ImageCommand extends ChatCraftCommand { } async execute(chat: ChatCraftChat, user: User | undefined, args?: string[]) { + if (!(await isGenerateImageSupported())) { + throw new Error("Failed to generate image, no image generation models available"); + } if (!(args && args[0])) { throw new Error("must include a prompt"); } @@ -19,8 +22,8 @@ export class ImageCommand extends ChatCraftCommand { try { imageUrls = await generateImage(prompt); } catch (error: any) { - console.error("Failed to generate image:", error); - throw new Error("Failed to generate image ", error); + console.error(`Failed to generate image: ${error.message}`); + throw new Error(`Failed to generate image: ${error.message}`); } return chat.addMessage(new ChatCraftHumanMessage({ user, text, imageUrls })); } From be3d7e932688ff7f25fb5e44be817170bff81867 Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Fri, 15 Mar 2024 10:22:08 -0400 Subject: [PATCH 8/8] refactor for same key value names --- src/lib/ai.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/ai.ts b/src/lib/ai.ts index c8c589f3..96d05d56 100644 --- a/src/lib/ai.ts +++ b/src/lib/ai.ts @@ -433,9 +433,9 @@ export const generateImage = async ( try { const response = await openai.images.generate({ model: "dall-e-3", - prompt: prompt, - n: n, - size: size, + prompt, + n, + size, }); const imageUrls = response.data.map((img: any) => img.url);