diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts index ed1d3a7af8c36..9a7824462f5a3 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/provider.ts @@ -1,7 +1,7 @@ import { assertExists } from '@blocksuite/global/utils'; import { AIProvider } from '@blocksuite/presets'; -import { textToTextStream } from './request'; +import { imageToTextStream, textToTextStream } from './request'; export function setupAIProvider() { AIProvider.provideAction('chat', options => { @@ -118,4 +118,41 @@ export function setupAIProvider() { prompt, }); }); + + AIProvider.provideAction('makeItReal', options => { + assertExists(options.stream); + const prompt = `You are an expert web developer who specializes in building working website prototypes from low-fidelity wireframes. +Your job is to accept low-fidelity wireframes, then create a working prototype using HTML, CSS, and JavaScript, and finally send back the results. +The results should be a single HTML file. +Use tailwind to style the website. +Put any additional CSS styles in a style tag and any JavaScript in a script tag. +Use unpkg or skypack to import any required dependencies. +Use Google fonts to pull in any open source fonts you require. +If you have any images, load them from Unsplash or use solid colored rectangles. + +The wireframes may include flow charts, diagrams, labels, arrows, sticky notes, and other features that should inform your work. +If there are screenshots or images, use them to inform the colors, fonts, and layout of your website. +Use your best judgement to determine whether what you see should be part of the user interface, or else is just an annotation. + +Use what you know about applications and user experience to fill in any implicit business logic in the wireframes. Flesh it out, make it real! + +The user may also provide you with the html of a previous design that they want you to iterate from. +In the wireframe, the previous design's html will appear as a white rectangle. +Use their notes, together with the previous design, to inform your next result. + +Sometimes it's hard for you to read the writing in the wireframes. +For this reason, all text from the wireframes will be provided to you as a list of strings, separated by newlines. +Use the provided list of text from the wireframes as a reference if any text is hard to read. + +You love your designers and want them to be happy. Incorporating their feedback and notes and producing working websites makes them happy. + +When sent new wireframes, respond ONLY with the contents of the html file.`; + return imageToTextStream({ + docId: options.docId, + workspaceId: options.workspaceId, + params: options.params, + attachments: options.atachments, + prompt, + }); + }); } diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts index a484f8831fb4f..5a534a46a5d18 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/request.ts @@ -25,3 +25,57 @@ export function textToTextStream({ }, }; } + +// Image to text(html) +export function imageToTextStream({ + docId, + workspaceId, + ...options +}: { + docId: string; + workspaceId: string; + prompt: string; + params?: string; + attachments?: string[]; +}) { + const client = new CopilotClient(getBaseUrl()); + return { + [Symbol.asyncIterator]: async function* () { + const session = await client.createSession({ + workspaceId, + docId, + promptName: 'Summary', // placeholder + }); + const message = await client.createMessage(options); + const eventSource = client.textStream(message, session); + yield* toTextStream(eventSource, { timeout: TIMEOUT }); + }, + }; +} + +// Image to images +export function imageToImagesStream({ + docId, + workspaceId, + ...options +}: { + docId: string; + workspaceId: string; + prompt: string; + params?: string; + attachments?: string[]; +}) { + const client = new CopilotClient(getBaseUrl()); + return { + [Symbol.asyncIterator]: async function* () { + const session = await client.createSession({ + workspaceId, + docId, + promptName: 'Summary', // placeholder + }); + const message = await client.createMessage(options); + const eventSource = client.imagesStream(message, session); + yield* toTextStream(eventSource, { timeout: TIMEOUT }); + }, + }; +}