diff --git a/packages/backend/server/src/data/migrations/utils/prompts.ts b/packages/backend/server/src/data/migrations/utils/prompts.ts index 645e54225791d..1060ec80b3ce7 100644 --- a/packages/backend/server/src/data/migrations/utils/prompts.ts +++ b/packages/backend/server/src/data/migrations/utils/prompts.ts @@ -286,7 +286,7 @@ export const prompts: Prompt[] = [ }, { name: 'Make it real', - action: 'image', + action: 'Make it real', model: 'gpt-4-vision-preview', messages: [ { diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts index f7be9b379fd78..b361a8d60ed24 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts @@ -31,7 +31,10 @@ export function setupAIProvider() { AIProvider.provideAction('translate', options => { assertExists(options.stream); - const prompt = `Translate the following content to ${options.lang}: ${options.input}`; + const prompt = `Please translate the following content into ${options.lang} and return it to us, adhering to the original format of the content: + + ${options.input} + `; return textToTextStream({ docId: options.docId, workspaceId: options.workspaceId, @@ -111,7 +114,144 @@ export function setupAIProvider() { AIProvider.provideAction('checkCodeErrors', options => { assertExists(options.stream); - const prompt = `Check the code errors in the following content: ${options.input}`; + const prompt = `Check the code errors in the following content and provide the corrected version: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('explainCode', options => { + assertExists(options.stream); + const prompt = `Explain the code in the following content, focusing on the logic, functions, and expected outcomes: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeArticle', options => { + assertExists(options.stream); + const prompt = `Write an article based on the following content, focusing on the main ideas, structure, and flow: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeTwitterPost', options => { + assertExists(options.stream); + const prompt = `Write a Twitter post based on the following content, keeping it concise and engaging: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writePoem', options => { + assertExists(options.stream); + const prompt = `Write a poem based on the following content, focusing on the emotions, imagery, and rhythm: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeOutline', options => { + assertExists(options.stream); + const prompt = `Write an outline from the following content in Markdown: ${options.input}`; + + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeBlogPost', options => { + assertExists(options.stream); + const prompt = `Write a blog post based on the following content, focusing on the insights, analysis, and personal perspective: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('brainstorm', options => { + assertExists(options.stream); + const prompt = `Brainstorm ideas based on the following content, exploring different angles, perspectives, and approaches: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('findActions', options => { + assertExists(options.stream); + const prompt = `Find actions related to the following content and return content in markdown: ${options.input}`; + + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeOutline', options => { + assertExists(options.stream); + const prompt = `Write an outline based on the following content, organizing the main points, subtopics, and structure: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('brainstormMindmap', options => { + assertExists(options.stream); + const prompt = `Use the nested unordered list syntax without other extra text style in Markdown to create a structure similar to a mind map without any unnecessary plain text description. Analyze the following questions or topics: ${options.input}`; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('explain', options => { + assertExists(options.stream); + const prompt = `Explain the following content in Markdown: ${options.input}`; + return textToTextStream({ docId: options.docId, workspaceId: options.workspaceId, @@ -119,11 +259,34 @@ export function setupAIProvider() { }); }); + AIProvider.provideAction('explainImage', options => { + assertExists(options.stream); + const prompt = `Describe the scene captured in this image, focusing on the details, colors, emotions, and any interactions between subjects or objects present.`; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + attachments: options.attachments, + }); + }); + AIProvider.provideAction('makeItReal', options => { assertExists(options.stream); const prompt = 'Make it real'; - return imageToTextStream({ + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, prompt, + attachments: options.attachments, + params: options.params, + }); + }); + + AIProvider.provideAction('makeItReal', options => { + assertExists(options.stream); + const prompt = 'Make it real'; + return imageToTextStream({ + promptName: prompt, docId: options.docId, workspaceId: options.workspaceId, params: options.params, diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts index 7223289643f60..81a81250ef612 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts @@ -1,27 +1,43 @@ import { getBaseUrl } from '@affine/graphql'; import { CopilotClient, toTextStream } from '@blocksuite/presets'; -const TIMEOUT = 5000; +const TIMEOUT = 500000; export function textToTextStream({ docId, workspaceId, prompt, + attachments, + params, }: { docId: string; workspaceId: string; prompt: string; + attachments?: string[]; + params?: string; }): BlockSuitePresets.TextStream { const client = new CopilotClient(getBaseUrl()); return { [Symbol.asyncIterator]: async function* () { + const hasAttachments = attachments && attachments.length > 0; const session = await client.createSession({ workspaceId, docId, - promptName: 'Summary', // placeholder + promptName: hasAttachments ? 'debug:action:vision4' : 'Summary', }); - const eventSource = client.textToTextStream(prompt, session); - yield* toTextStream(eventSource, { timeout: TIMEOUT }); + if (hasAttachments) { + const messageId = await client.createMessage({ + sessionId: session, + content: prompt, + attachments, + params, + }); + const eventSource = client.textStream(messageId, session); + yield* toTextStream(eventSource, { timeout: TIMEOUT }); + } else { + const eventSource = client.textToTextStream(prompt, session); + yield* toTextStream(eventSource, { timeout: TIMEOUT }); + } }, }; } @@ -30,13 +46,14 @@ export function textToTextStream({ export function imageToTextStream({ docId, workspaceId, - prompt: promptName, + promptName: promptName, ...options }: { docId: string; workspaceId: string; - prompt: string; + promptName: string; params?: string; + content: string; attachments?: string[]; }) { const client = new CopilotClient(getBaseUrl());