feat(core): ai images

packages/backend/server/src/data/migrations/utils/prompts.ts

@@ -284,4 +284,43 @@ export const prompts: Prompt[] = [
       },
     ],
   },
+  {
+    name: 'Make it real',
+    action: 'Make it real',
+    model: 'gpt-4-vision-preview',
+    messages: [
+      {
+        role: 'system',
+        content: `You are an expert web developer who specializes in building working website prototypes from low-fidelity wireframes.
+  Your job is to accept low-fidelity wireframes, then create a working prototype using HTML, CSS, and JavaScript, and finally send back the results.
+  The results should be a single HTML file.
+  Use tailwind to style the website.
+  Put any additional CSS styles in a style tag and any JavaScript in a script tag.
+  Use unpkg or skypack to import any required dependencies.
+  Use Google fonts to pull in any open source fonts you require.
+  If you have any images, load them from Unsplash or use solid colored rectangles.
+
+  The wireframes may include flow charts, diagrams, labels, arrows, sticky notes, and other features that should inform your work.
+  If there are screenshots or images, use them to inform the colors, fonts, and layout of your website.
+  Use your best judgement to determine whether what you see should be part of the user interface, or else is just an annotation.
+
+  Use what you know about applications and user experience to fill in any implicit business logic in the wireframes. Flesh it out, make it real!
+
+  The user may also provide you with the html of a previous design that they want you to iterate from.
+  In the wireframe, the previous design's html will appear as a white rectangle.
+  Use their notes, together with the previous design, to inform your next result.
+
+  Sometimes it's hard for you to read the writing in the wireframes.
+  For this reason, all text from the wireframes will be provided to you as a list of strings, separated by newlines.
+  Use the provided list of text from the wireframes as a reference if any text is hard to read.
+
+  You love your designers and want them to be happy. Incorporating their feedback and notes and producing working websites makes them happy.
+
+  When sent new wireframes, respond ONLY with the contents of the html file.
+
+  {{image}}
+  `,
+      },
+    ],
+  },
 ];

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -75,7 +75,7 @@ export class OpenAIProvider
             .filter(url => SIMPLE_IMAGE_URL_REGEX.test(url))
             .map(url => ({
               type: 'image_url',
-              image_url: { url, detail: 'low' },
+              image_url: { url, detail: 'high' },
             })),
         ];
         return {

packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts

@@ -1,7 +1,7 @@
 import { assertExists } from '@blocksuite/global/utils';
 import { AIProvider } from '@blocksuite/presets';
 
-import { textToTextStream } from './request';
+import { imageToTextStream, textToTextStream } from './request';
 
 export function setupAIProvider() {
   AIProvider.provideAction('chat', options => {
@@ -31,7 +31,10 @@ export function setupAIProvider() {
 
   AIProvider.provideAction('translate', options => {
     assertExists(options.stream);
-    const prompt = `Translate the following content to ${options.lang}: ${options.input}`;
+    const prompt = `Please translate the following content into ${options.lang} and return it to us, adhering to the original format of the content：
+
+    ${options.input}
+    `;
     return textToTextStream({
       docId: options.docId,
       workspaceId: options.workspaceId,
@@ -111,11 +114,174 @@ export function setupAIProvider() {
 
   AIProvider.provideAction('checkCodeErrors', options => {
     assertExists(options.stream);
-    const prompt = `Check the code errors in the following content: ${options.input}`;
+    const prompt = `Check the code errors in the following content and provide the corrected version:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('explainCode', options => {
+    assertExists(options.stream);
+    const prompt = `Explain the code in the following content, focusing on the logic, functions, and expected outcomes:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeArticle', options => {
+    assertExists(options.stream);
+    const prompt = `Write an article based on the following content, focusing on the main ideas, structure, and flow:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeTwitterPost', options => {
+    assertExists(options.stream);
+    const prompt = `Write a Twitter post based on the following content, keeping it concise and engaging:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writePoem', options => {
+    assertExists(options.stream);
+    const prompt = `Write a poem based on the following content, focusing on the emotions, imagery, and rhythm:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeOutline', options => {
+    assertExists(options.stream);
+    const prompt = `Write an outline from the following content in Markdown: ${options.input}`;
+
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeBlogPost', options => {
+    assertExists(options.stream);
+    const prompt = `Write a blog post based on the following content, focusing on the insights, analysis, and personal perspective:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('brainstorm', options => {
+    assertExists(options.stream);
+    const prompt = `Brainstorm ideas based on the following content, exploring different angles, perspectives, and approaches:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('findActions', options => {
+    assertExists(options.stream);
+    const prompt = `Find actions related to the following content and return content in markdown: ${options.input}`;
+
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('writeOutline', options => {
+    assertExists(options.stream);
+    const prompt = `Write an outline based on the following content, organizing the main points, subtopics, and structure:
+
+    ${options.input}
+    `;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('brainstormMindmap', options => {
+    assertExists(options.stream);
+    const prompt = `Use the nested unordered list syntax without other extra text style in Markdown to create a structure similar to a mind map without any unnecessary plain text description. Analyze the following questions or topics: ${options.input}`;
     return textToTextStream({
       docId: options.docId,
       workspaceId: options.workspaceId,
       prompt,
     });
   });
+
+  AIProvider.provideAction('explain', options => {
+    assertExists(options.stream);
+    const prompt = `Explain the following content in Markdown: ${options.input}`;
+
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+    });
+  });
+
+  AIProvider.provideAction('explainImage', options => {
+    assertExists(options.stream);
+    const prompt = `Describe the scene captured in this image, focusing on the details, colors, emotions, and any interactions between subjects or objects present.`;
+    return textToTextStream({
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      prompt,
+      attachments: options.attachments,
+    });
+  });
+
+  AIProvider.provideAction('makeItReal', options => {
+    assertExists(options.stream);
+    const promptName = 'Make it real';
+    return imageToTextStream({
+      promptName,
+      docId: options.docId,
+      workspaceId: options.workspaceId,
+      params: options.params,
+      attachments: options.attachments,
+      content:
+        options.content ||
+        'Here are the latest wireframes. Could you make a new website based on these wireframes and notes and send back just the html file?',
+    });
+  });
 }

packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts

@@ -1,26 +1,106 @@
 import { getBaseUrl } from '@affine/graphql';
 import { CopilotClient, toTextStream } from '@blocksuite/presets';
 
-const TIMEOUT = 5000;
+const TIMEOUT = 500000;
 
 export function textToTextStream({
   docId,
   workspaceId,
   prompt,
+  attachments,
+  params,
 }: {
   docId: string;
   workspaceId: string;
   prompt: string;
+  attachments?: string[];
+  params?: string;
 }): BlockSuitePresets.TextStream {
   const client = new CopilotClient(getBaseUrl());
   return {
     [Symbol.asyncIterator]: async function* () {
+      const hasAttachments = attachments && attachments.length > 0;
       const session = await client.createSession({
         workspaceId,
         docId,
-        promptName: 'Summary', // placeholder
+        promptName: hasAttachments ? 'debug:action:vision4' : 'Summary',
       });
-      const eventSource = client.textToTextStream(prompt, session);
+      if (hasAttachments) {
+        const messageId = await client.createMessage({
+          sessionId: session,
+          content: prompt,
+          attachments,
+          params,
+        });
+        const eventSource = client.textStream(messageId, session);
+        yield* toTextStream(eventSource, { timeout: TIMEOUT });
+      } else {
+        const eventSource = client.textToTextStream(prompt, session);
+        yield* toTextStream(eventSource, { timeout: TIMEOUT });
+      }
+    },
+  };
+}
+
+// Image to text(html)
+export function imageToTextStream({
+  docId,
+  workspaceId,
+  promptName,
+  ...options
+}: {
+  docId: string;
+  workspaceId: string;
+  promptName: string;
+  params?: string;
+  content: string;
+  attachments?: string[];
+}) {
+  const client = new CopilotClient(getBaseUrl());
+  return {
+    [Symbol.asyncIterator]: async function* () {
+      const sessionId = await client.createSession({
+        workspaceId,
+        docId,
+        promptName,
+      });
+      const messageId = await client.createMessage({
+        sessionId,
+        ...options,
+      });
+      const eventSource = client.textStream(messageId, sessionId);
+      yield* toTextStream(eventSource, { timeout: TIMEOUT });
+    },
+  };
+}
+
+// Image to images
+export function imageToImagesStream({
+  docId,
+  workspaceId,
+  promptName,
+  ...options
+}: {
+  docId: string;
+  workspaceId: string;
+  promptName: string;
+  content: string;
+  params?: string;
+  attachments?: string[];
+}) {
+  const client = new CopilotClient(getBaseUrl());
+  return {
+    [Symbol.asyncIterator]: async function* () {
+      const sessionId = await client.createSession({
+        workspaceId,
+        docId,
+        promptName,
+      });
+      const messageId = await client.createMessage({
+        sessionId,
+        ...options,
+      });
+      const eventSource = client.imagesStream(messageId, sessionId);
       yield* toTextStream(eventSource, { timeout: TIMEOUT });
     },
   };