diff --git a/packages/backend/server/src/data/migrations/utils/prompts.ts b/packages/backend/server/src/data/migrations/utils/prompts.ts index 6818b0728c02..1060ec80b3ce 100644 --- a/packages/backend/server/src/data/migrations/utils/prompts.ts +++ b/packages/backend/server/src/data/migrations/utils/prompts.ts @@ -284,4 +284,43 @@ export const prompts: Prompt[] = [ }, ], }, + { + name: 'Make it real', + action: 'Make it real', + model: 'gpt-4-vision-preview', + messages: [ + { + role: 'system', + content: `You are an expert web developer who specializes in building working website prototypes from low-fidelity wireframes. + Your job is to accept low-fidelity wireframes, then create a working prototype using HTML, CSS, and JavaScript, and finally send back the results. + The results should be a single HTML file. + Use tailwind to style the website. + Put any additional CSS styles in a style tag and any JavaScript in a script tag. + Use unpkg or skypack to import any required dependencies. + Use Google fonts to pull in any open source fonts you require. + If you have any images, load them from Unsplash or use solid colored rectangles. + + The wireframes may include flow charts, diagrams, labels, arrows, sticky notes, and other features that should inform your work. + If there are screenshots or images, use them to inform the colors, fonts, and layout of your website. + Use your best judgement to determine whether what you see should be part of the user interface, or else is just an annotation. + + Use what you know about applications and user experience to fill in any implicit business logic in the wireframes. Flesh it out, make it real! + + The user may also provide you with the html of a previous design that they want you to iterate from. + In the wireframe, the previous design's html will appear as a white rectangle. + Use their notes, together with the previous design, to inform your next result. + + Sometimes it's hard for you to read the writing in the wireframes. + For this reason, all text from the wireframes will be provided to you as a list of strings, separated by newlines. + Use the provided list of text from the wireframes as a reference if any text is hard to read. + + You love your designers and want them to be happy. Incorporating their feedback and notes and producing working websites makes them happy. + + When sent new wireframes, respond ONLY with the contents of the html file. + + {{image}} + `, + }, + ], + }, ]; diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index 2084d6a5cb59..bea0f35d385c 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -75,7 +75,7 @@ export class OpenAIProvider .filter(url => SIMPLE_IMAGE_URL_REGEX.test(url)) .map(url => ({ type: 'image_url', - image_url: { url, detail: 'low' }, + image_url: { url, detail: 'high' }, })), ]; return { diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts index ed1d3a7af8c3..5131ef6c8644 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts @@ -1,7 +1,7 @@ import { assertExists } from '@blocksuite/global/utils'; import { AIProvider } from '@blocksuite/presets'; -import { textToTextStream } from './request'; +import { imageToTextStream, textToTextStream } from './request'; export function setupAIProvider() { AIProvider.provideAction('chat', options => { @@ -31,7 +31,10 @@ export function setupAIProvider() { AIProvider.provideAction('translate', options => { assertExists(options.stream); - const prompt = `Translate the following content to ${options.lang}: ${options.input}`; + const prompt = `Please translate the following content into ${options.lang} and return it to us, adhering to the original format of the content： + + ${options.input} + `; return textToTextStream({ docId: options.docId, workspaceId: options.workspaceId, @@ -111,11 +114,174 @@ export function setupAIProvider() { AIProvider.provideAction('checkCodeErrors', options => { assertExists(options.stream); - const prompt = `Check the code errors in the following content: ${options.input}`; + const prompt = `Check the code errors in the following content and provide the corrected version: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('explainCode', options => { + assertExists(options.stream); + const prompt = `Explain the code in the following content, focusing on the logic, functions, and expected outcomes: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeArticle', options => { + assertExists(options.stream); + const prompt = `Write an article based on the following content, focusing on the main ideas, structure, and flow: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeTwitterPost', options => { + assertExists(options.stream); + const prompt = `Write a Twitter post based on the following content, keeping it concise and engaging: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writePoem', options => { + assertExists(options.stream); + const prompt = `Write a poem based on the following content, focusing on the emotions, imagery, and rhythm: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeOutline', options => { + assertExists(options.stream); + const prompt = `Write an outline from the following content in Markdown: ${options.input}`; + + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeBlogPost', options => { + assertExists(options.stream); + const prompt = `Write a blog post based on the following content, focusing on the insights, analysis, and personal perspective: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('brainstorm', options => { + assertExists(options.stream); + const prompt = `Brainstorm ideas based on the following content, exploring different angles, perspectives, and approaches: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('findActions', options => { + assertExists(options.stream); + const prompt = `Find actions related to the following content and return content in markdown: ${options.input}`; + + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('writeOutline', options => { + assertExists(options.stream); + const prompt = `Write an outline based on the following content, organizing the main points, subtopics, and structure: + + ${options.input} + `; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('brainstormMindmap', options => { + assertExists(options.stream); + const prompt = `Use the nested unordered list syntax without other extra text style in Markdown to create a structure similar to a mind map without any unnecessary plain text description. Analyze the following questions or topics: ${options.input}`; return textToTextStream({ docId: options.docId, workspaceId: options.workspaceId, prompt, }); }); + + AIProvider.provideAction('explain', options => { + assertExists(options.stream); + const prompt = `Explain the following content in Markdown: ${options.input}`; + + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + }); + }); + + AIProvider.provideAction('explainImage', options => { + assertExists(options.stream); + const prompt = `Describe the scene captured in this image, focusing on the details, colors, emotions, and any interactions between subjects or objects present.`; + return textToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + prompt, + attachments: options.attachments, + }); + }); + + AIProvider.provideAction('makeItReal', options => { + assertExists(options.stream); + const promptName = 'Make it real'; + return imageToTextStream({ + promptName, + docId: options.docId, + workspaceId: options.workspaceId, + params: options.params, + attachments: options.attachments, + content: + options.content || + 'Here are the latest wireframes. Could you make a new website based on these wireframes and notes and send back just the html file?', + }); + }); } diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts index a484f8831fb4..ea11c88d3315 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts @@ -1,26 +1,106 @@ import { getBaseUrl } from '@affine/graphql'; import { CopilotClient, toTextStream } from '@blocksuite/presets'; -const TIMEOUT = 5000; +const TIMEOUT = 500000; export function textToTextStream({ docId, workspaceId, prompt, + attachments, + params, }: { docId: string; workspaceId: string; prompt: string; + attachments?: string[]; + params?: string; }): BlockSuitePresets.TextStream { const client = new CopilotClient(getBaseUrl()); return { [Symbol.asyncIterator]: async function* () { + const hasAttachments = attachments && attachments.length > 0; const session = await client.createSession({ workspaceId, docId, - promptName: 'Summary', // placeholder + promptName: hasAttachments ? 'debug:action:vision4' : 'Summary', }); - const eventSource = client.textToTextStream(prompt, session); + if (hasAttachments) { + const messageId = await client.createMessage({ + sessionId: session, + content: prompt, + attachments, + params, + }); + const eventSource = client.textStream(messageId, session); + yield* toTextStream(eventSource, { timeout: TIMEOUT }); + } else { + const eventSource = client.textToTextStream(prompt, session); + yield* toTextStream(eventSource, { timeout: TIMEOUT }); + } + }, + }; +} + +// Image to text(html) +export function imageToTextStream({ + docId, + workspaceId, + promptName, + ...options +}: { + docId: string; + workspaceId: string; + promptName: string; + params?: string; + content: string; + attachments?: string[]; +}) { + const client = new CopilotClient(getBaseUrl()); + return { + [Symbol.asyncIterator]: async function* () { + const sessionId = await client.createSession({ + workspaceId, + docId, + promptName, + }); + const messageId = await client.createMessage({ + sessionId, + ...options, + }); + const eventSource = client.textStream(messageId, sessionId); + yield* toTextStream(eventSource, { timeout: TIMEOUT }); + }, + }; +} + +// Image to images +export function imageToImagesStream({ + docId, + workspaceId, + promptName, + ...options +}: { + docId: string; + workspaceId: string; + promptName: string; + content: string; + params?: string; + attachments?: string[]; +}) { + const client = new CopilotClient(getBaseUrl()); + return { + [Symbol.asyncIterator]: async function* () { + const sessionId = await client.createSession({ + workspaceId, + docId, + promptName, + }); + const messageId = await client.createMessage({ + sessionId, + ...options, + }); + const eventSource = client.imagesStream(messageId, sessionId); yield* toTextStream(eventSource, { timeout: TIMEOUT }); }, };