From 7663e98253e8f671b11877184a4972006b0a8963 Mon Sep 17 00:00:00 2001 From: KKKK Date: Fri, 4 Sep 2026 22:13:51 +0800 Subject: [PATCH 1/3] feat: add MCP timeline MP4 rendering Render authoritative Editor timelines through MCP, invalidate stale render pointers after edits, and keep existing Canvas timeline cards synchronized through the shared Command Kernel. --- src/core/commands/canvas-commands.ts | 18 +- src/core/commands/command-kernel.test.ts | 72 +++ src/core/commands/editor-commands.ts | 37 ++ src/core/commands/executor.ts | 11 +- src/core/commands/persist.ts | 75 +++ src/core/editor/render-project-timeline.ts | 504 ++++++++++++++++++ src/mcp/server.runtime.test.ts | 26 +- src/mcp/server.ts | 65 +++ src/mcp/tools.ts | 1 + .../api/-trusted-mutations.contract.test.ts | 9 + .../api/app/projects/$projectId/timeline.ts | 34 +- 11 files changed, 838 insertions(+), 14 deletions(-) create mode 100644 src/core/editor/render-project-timeline.ts diff --git a/src/core/commands/canvas-commands.ts b/src/core/commands/canvas-commands.ts index 734477e..95fef6e 100644 --- a/src/core/commands/canvas-commands.ts +++ b/src/core/commands/canvas-commands.ts @@ -51,8 +51,8 @@ export function buildTimelineCanvasCard({ name, durationSec, clipCount, - lastRenderAssetId = null, - lastRenderUrl = null, + lastRenderAssetId, + lastRenderUrl, referenceCardIds = [], }: { existing?: CanvasCard; @@ -65,6 +65,14 @@ export function buildTimelineCanvasCard({ referenceCardIds?: string[]; }): CanvasAssetCard { const cardId = timelineCanvasCardId(timelineId); + const hasRenderUpdate = + lastRenderAssetId !== undefined || lastRenderUrl !== undefined; + const resolvedRenderAssetId = + hasRenderUpdate + ? (lastRenderAssetId ?? null) + : (existing?.lastRenderAssetId ?? existing?.assetId ?? null); + const resolvedRenderUrl = + hasRenderUpdate ? (lastRenderUrl ?? null) : (existing?.url ?? null); const mergedRefs = Array.from( new Set([ ...(existing?.referenceCardIds ?? []), @@ -73,11 +81,11 @@ export function buildTimelineCanvasCard({ ); return { id: cardId, - assetId: lastRenderAssetId ?? existing?.assetId ?? null, + assetId: resolvedRenderAssetId, kind: 'asset', type: 'timeline', name, - url: lastRenderUrl ?? existing?.url ?? null, + url: resolvedRenderUrl, prompt: existing?.prompt ?? '', referenceCardIds: mergedRefs, workflowTemplateId: existing?.workflowTemplateId ?? null, @@ -94,7 +102,7 @@ export function buildTimelineCanvasCard({ durationSec, timelineId, clipCount, - lastRenderAssetId: lastRenderAssetId ?? existing?.lastRenderAssetId ?? null, + lastRenderAssetId: resolvedRenderAssetId, }; } export type CanvasCommandApplication = { diff --git a/src/core/commands/command-kernel.test.ts b/src/core/commands/command-kernel.test.ts index 378c9fd..ffee0d7 100644 --- a/src/core/commands/command-kernel.test.ts +++ b/src/core/commands/command-kernel.test.ts @@ -270,6 +270,78 @@ test('timeline node upsert preserves existing canvas references', () => { assert.deepEqual(timelineCard?.referenceCardIds, [assetCard.id]); assert.equal(timelineCard?.lastRenderAssetId, 'render-1'); assert.equal(timelineCard?.durationSec, 8); + + const invalidated = applyCanvasOperations(updated.document, [ + { + type: 'upsert_timeline_node', + timelineId: 'timeline-1', + name: 'Timeline 1', + durationSec: 8, + clipCount: 2, + lastRenderAssetId: null, + }, + ]).document.cards.find((card) => card.id === 'timeline:timeline-1'); + assert.equal(invalidated?.assetId, null); + assert.equal(invalidated?.url, null); + assert.equal(invalidated?.lastRenderAssetId, null); +}); + +test('UI timeline replacement invalidates stale renders when edited content changes', () => { + const timeline = applyEditorOperations( + createTimelineDocument({ projectId: 'project-1', name: 'Timeline 1' }), + [ + { + type: 'set_render', + assetId: 'render-1', + publicUrl: '/render.mp4', + }, + ] + ).document; + const result = executeBeatDesignCommand({ + envelope: { + commandId: createCommandId(), + projectId: 'project-1', + origin: 'ui', + command: { + type: 'editor.replace_document', + document: { ...timeline, captionStyle: 'bold' }, + }, + }, + documents: { timeline }, + }); + assert.equal(result.ok, true); + if (result.ok) { + assert.equal(result.data.timeline?.lastRenderAssetId, null); + assert.equal(result.data.timeline?.lastRenderUrl, null); + } +}); + +test('UI timeline replacement can record a fresh render without invalidating it', () => { + const timeline = createTimelineDocument({ + projectId: 'project-1', + name: 'Timeline 1', + }); + const result = executeBeatDesignCommand({ + envelope: { + commandId: createCommandId(), + projectId: 'project-1', + origin: 'ui', + command: { + type: 'editor.replace_document', + document: { + ...timeline, + lastRenderAssetId: 'render-1', + lastRenderUrl: '/render.mp4', + }, + }, + }, + documents: { timeline }, + }); + assert.equal(result.ok, true); + if (result.ok) { + assert.equal(result.data.timeline?.lastRenderAssetId, 'render-1'); + assert.equal(result.data.timeline?.lastRenderUrl, '/render.mp4'); + } }); test('add_clip with a stable clip id is idempotent', () => { diff --git a/src/core/commands/editor-commands.ts b/src/core/commands/editor-commands.ts index b893d9a..fdc9a35 100644 --- a/src/core/commands/editor-commands.ts +++ b/src/core/commands/editor-commands.ts @@ -127,6 +127,36 @@ export type EditorOperation = const listClipIds = (document: TimelineDocument) => new Set(document.tracks.flatMap((track) => track.clips.map((clip) => clip.id))); +const timelineRenderSource = (document: TimelineDocument) => { + const { + updatedAt: _updatedAt, + lastRenderAssetId: _lastRenderAssetId, + lastRenderUrl: _lastRenderUrl, + ...source + } = document; + return source; +}; + +export function invalidateTimelineRenderIfSourceChanged({ + previous, + next, +}: { + previous: TimelineDocument; + next: TimelineDocument; +}) { + if ( + JSON.stringify(timelineRenderSource(previous)) === + JSON.stringify(timelineRenderSource(next)) + ) { + return next; + } + return { + ...next, + lastRenderAssetId: null, + lastRenderUrl: null, + }; +} + export function applyEditorOperations( source: TimelineDocument, operations: readonly EditorOperation[] @@ -272,6 +302,13 @@ export function applyEditorOperations( changedIds.add(document.id); } + if (next !== document && operation.type !== 'set_render') { + next = invalidateTimelineRenderIfSourceChanged({ + previous: document, + next, + }); + } + if (next === document) { throw new BeatDesignCommandError( 'INVALID_COMMAND', diff --git a/src/core/commands/executor.ts b/src/core/commands/executor.ts index 6486cdb..05532d7 100644 --- a/src/core/commands/executor.ts +++ b/src/core/commands/executor.ts @@ -11,6 +11,7 @@ import { } from './canvas-commands'; import { applyEditorOperations, + invalidateTimelineRenderIfSourceChanged, type EditorOperation, } from './editor-commands'; import { @@ -81,12 +82,18 @@ export function executeBeatDesignCommand({ } if (command.type === 'editor.replace_document') { + const document = documents.timeline + ? invalidateTimelineRenderIfSourceChanged({ + previous: documents.timeline, + next: command.document, + }) + : command.document; return createCommandSuccess({ commandId, projectId, origin, - changedIds: [command.document.id], - data: { timeline: command.document }, + changedIds: [document.id], + data: { timeline: document }, }); } diff --git a/src/core/commands/persist.ts b/src/core/commands/persist.ts index 95516d9..564bd75 100644 --- a/src/core/commands/persist.ts +++ b/src/core/commands/persist.ts @@ -8,6 +8,7 @@ import { } from '@/core/projects/projects'; import { normalizeCommandAssetReferences } from './asset-boundary'; +import { timelineCanvasCardId } from './canvas-commands'; import { BeatDesignCommandError, createCommandFailure, @@ -58,6 +59,67 @@ type PersistCommandInput = { command: BeatDesignCommand; }; +const timelineClipCount = (document: NonNullable) => + document.tracks.reduce((count, track) => count + track.clips.length, 0); + +async function syncExistingTimelineCanvasCard({ + projectId, + timeline, +}: { + projectId: string; + timeline: NonNullable; +}) { + const cardId = timelineCanvasCardId(timeline.id); + for (let attempt = 0; attempt < 3; attempt += 1) { + const state = await loadProjectWithLatestSnapshot({ projectId }); + if (!state || !state.snapshot.cards.some((card) => card.id === cardId)) return null; + const commandId = createCommandId(); + const envelope: BeatDesignCommandEnvelope = { + commandId, + projectId, + origin: 'system', + expectedRevision: state.snapshotVersion, + idempotencyKey: commandId, + command: { + type: 'canvas.apply', + operations: [ + { + type: 'upsert_timeline_node', + timelineId: timeline.id, + name: timeline.name, + durationSec: timeline.duration, + clipCount: timelineClipCount(timeline), + lastRenderAssetId: timeline.lastRenderAssetId, + lastRenderUrl: timeline.lastRenderUrl, + }, + ], + }, + }; + const executed = executeBeatDesignCommand({ + envelope, + documents: { canvas: state.snapshot }, + }); + if (!executed.ok || !executed.data.canvas) { + return executed.ok ? 'Canvas timeline node could not be synchronized.' : executed.message; + } + try { + await saveProjectSnapshot({ + projectId, + document: executed.data.canvas, + baseVersion: state.snapshotVersion, + }); + return null; + } catch (error) { + if (!isVersionConflict(error) || attempt === 2) { + return error instanceof Error + ? error.message + : 'Canvas timeline node could not be synchronized.'; + } + } + } + return 'Canvas timeline node could not be synchronized.'; +} + export function validateExternalCommandAssetReferences({ origin, command, @@ -213,6 +275,19 @@ async function persistBeatDesignCommandOnce({ timeline: saved.document, }, }; + const canvasSyncWarning = await syncExistingTimelineCanvasCard({ + projectId, + timeline: saved.document, + }); + if (canvasSyncWarning) { + result = { + ...result, + warnings: [ + ...result.warnings, + `Timeline saved, but its Canvas card is still synchronizing: ${canvasSyncWarning}`, + ], + }; + } } return storeCommandReceipt({ diff --git a/src/core/editor/render-project-timeline.ts b/src/core/editor/render-project-timeline.ts new file mode 100644 index 0000000..b3524fc --- /dev/null +++ b/src/core/editor/render-project-timeline.ts @@ -0,0 +1,504 @@ +import { spawn } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { getResolvedCaptionStyle } from './captions'; +import { + getTimelineClipSource, + type TimelineClip, + type TimelineDocument, + type TimelineTrackKind, +} from './timeline-document'; +import { diagnoseTimeline } from './timeline-diagnostics'; +import { + LOCAL_PROJECT_ASSET_BUCKET, + LOCAL_PROJECT_ASSET_PROVIDER, + persistLocalProjectAsset, + removePersistedLocalProjectAsset, + resolveLocalProjectAssetPath, +} from '@/core/projects/local-project-assets'; +import { + deleteUserAssetById, + getProjectAssetById, + linkProjectAsset, + recordUserAsset, +} from '@/core/workspace-lib/assets/user-assets'; + +type MediaProbe = { + width: number | null; + height: number | null; + hasVideo: boolean; + hasAudio: boolean; +}; + +type RenderInput = { + clip: TimelineClip; + trackKind: TimelineTrackKind; + inputIndex: number; + filePath: string; + source: ReturnType; + probe: MediaProbe; +}; + +const ffmpegBin = () => process.env.BEATDESIGN_FFMPEG?.trim() || 'ffmpeg'; +const ffprobeBin = () => process.env.BEATDESIGN_FFPROBE?.trim() || 'ffprobe'; + +const runProcess = ( + command: string, + args: string[], + { captureStdout = false }: { captureStdout?: boolean } = {} +) => + new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout?.on('data', (chunk) => { + if (captureStdout) stdout += chunk.toString('utf8'); + }); + child.stderr?.on('data', (chunk) => { + if (stderr.length < 24_000) stderr += chunk.toString('utf8'); + }); + child.on('error', (error) => { + reject( + new Error( + `Unable to start ${command}. Ensure ffmpeg and ffprobe are on PATH or set BEATDESIGN_FFMPEG and BEATDESIGN_FFPROBE. ${error.message}` + ) + ); + }); + child.on('close', (code) => { + if (code === 0) { + resolve(stdout); + return; + } + reject( + new Error( + stderr.trim() || `${command} failed with exit code ${code ?? 'unknown'}.` + ) + ); + }); + }); + +async function probeMedia(filePath: string): Promise { + const output = await runProcess( + ffprobeBin(), + [ + '-v', + 'error', + '-show_entries', + 'stream=codec_type,width,height', + '-of', + 'json', + filePath, + ], + { captureStdout: true } + ); + const parsed = JSON.parse(output) as { + streams?: Array<{ codec_type?: string; width?: number; height?: number }>; + }; + const streams = parsed.streams ?? []; + const visual = streams.find((stream) => stream.codec_type === 'video'); + return { + width: typeof visual?.width === 'number' ? visual.width : null, + height: typeof visual?.height === 'number' ? visual.height : null, + hasVideo: Boolean(visual), + hasAudio: streams.some((stream) => stream.codec_type === 'audio'), + }; +} + +const time = (value: number) => Math.max(0, value).toFixed(3); + +const escapeFilterPath = (value: string) => + value.replaceAll('\\', '\\\\').replaceAll(':', '\\:').replaceAll("'", "\\'"); + +export function wrapCaptionForFfmpeg({ + text, + maxCharacters, +}: { + text: string; + maxCharacters: number; +}) { + const width = Math.max(4, Math.floor(maxCharacters)); + return text + .split('\n') + .flatMap((paragraph) => { + const words = paragraph.trim().split(/\s+/).filter(Boolean); + if (words.length === 0) return ['']; + const lines: string[] = []; + let line = ''; + for (const word of words) { + const candidate = line ? `${line} ${word}` : word; + if (!line || candidate.length <= width) { + line = candidate; + continue; + } + lines.push(line); + line = word; + } + if (line) lines.push(line); + return lines; + }) + .join('\n'); +} + +async function resolveRenderInputs({ + projectId, + document, +}: { + projectId: string; + document: TimelineDocument; +}) { + const inputs: RenderInput[] = []; + for (const track of document.tracks) { + if (track.hidden || track.muted || track.kind === 'caption') continue; + for (const clip of track.clips) { + if (clip.muted) continue; + const source = getTimelineClipSource(clip); + const asset = await getProjectAssetById({ + projectId, + assetId: source.assetId, + }); + if (!asset) { + throw new Error(`Timeline asset ${source.assetId} is unavailable.`); + } + if (asset.bucket !== LOCAL_PROJECT_ASSET_BUCKET || !asset.objectKey) { + throw new Error('MCP timeline rendering requires project-owned local assets.'); + } + const filePath = resolveLocalProjectAssetPath({ objectKey: asset.objectKey }); + const probe = await probeMedia(filePath); + if (clip.sourceType === 'audio' && !probe.hasAudio) { + throw new Error(`Timeline audio is unreadable: ${source.name}`); + } + if (clip.sourceType !== 'audio' && !probe.hasVideo) { + throw new Error(`Timeline visual is unreadable: ${source.name}`); + } + inputs.push({ + clip, + trackKind: track.kind, + inputIndex: inputs.length + 1, + filePath, + source, + probe, + }); + } + } + return inputs; +} + +async function buildFfmpegArgs({ + document, + inputs, + directory, + outputPath, +}: { + document: TimelineDocument; + inputs: RenderInput[]; + directory: string; + outputPath: string; +}) { + const firstVisual = inputs.find( + (input) => input.trackKind === 'video' && input.probe.hasVideo + ); + if (!firstVisual) throw new Error('The timeline has no readable visual clip.'); + const portrait = + (firstVisual.probe.height ?? 0) > (firstVisual.probe.width ?? 0); + const width = portrait ? 1080 : 1920; + const height = portrait ? 1920 : 1080; + const args = [ + '-hide_banner', + '-loglevel', + 'error', + '-y', + '-f', + 'lavfi', + '-i', + `color=c=black:s=${width}x${height}:r=30:d=${time(document.duration)}`, + ]; + for (const input of inputs) { + if (input.clip.sourceType === 'image') { + args.push( + '-loop', + '1', + '-framerate', + '30', + '-t', + time(input.clip.duration), + '-i', + input.filePath + ); + } else { + args.push('-i', input.filePath); + } + } + + const filters: string[] = []; + let visualLabel = 'canvas0'; + filters.push(`[0:v]setpts=PTS-STARTPTS[${visualLabel}]`); + let visualIndex = 0; + for (const input of inputs.filter((item) => item.trackKind === 'video')) { + visualIndex += 1; + const sourceLabel = `visual${visualIndex}`; + const nextLabel = `canvas${visualIndex}`; + filters.push( + `[${input.inputIndex}:v]trim=start=${time(input.source.inPoint)}:end=${time(input.source.outPoint)},` + + `setpts=PTS-STARTPTS+${time(input.clip.startTime)}/TB,` + + `scale=${width}:${height}:force_original_aspect_ratio=decrease,` + + `pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:black[${sourceLabel}]` + ); + filters.push( + `[${visualLabel}][${sourceLabel}]overlay=0:0:eof_action=pass:` + + `enable='between(t,${time(input.clip.startTime)},${time(input.clip.startTime + input.clip.duration)})'[${nextLabel}]` + ); + visualLabel = nextLabel; + } + + let overlayIndex = 0; + for (const input of inputs.filter((item) => item.trackKind === 'overlay')) { + if (!input.clip.overlay) continue; + overlayIndex += 1; + const preparedLabel = `overlay${overlayIndex}`; + const nextLabel = `withoverlay${overlayIndex}`; + const overlay = input.clip.overlay; + const overlayWidth = Math.max(1, Math.round(width * overlay.width)); + const steps = [ + `trim=duration=${time(input.clip.duration)}`, + `setpts=PTS-STARTPTS+${time(input.clip.startTime)}/TB`, + `scale=${overlayWidth}:-1`, + 'format=rgba', + `colorchannelmixer=aa=${overlay.opacity.toFixed(3)}`, + ]; + if (input.clip.fadeIn > 0) { + steps.push(`fade=t=in:st=0:d=${time(input.clip.fadeIn)}:alpha=1`); + } + if (input.clip.fadeOut > 0) { + steps.push( + `fade=t=out:st=${time(input.clip.duration - input.clip.fadeOut)}:d=${time(input.clip.fadeOut)}:alpha=1` + ); + } + if (overlay.rotation !== 0) { + steps.push( + `rotate=${(overlay.rotation * Math.PI / 180).toFixed(6)}:ow=rotw(iw):oh=roth(ih):c=none` + ); + } + filters.push(`[${input.inputIndex}:v]${steps.join(',')}[${preparedLabel}]`); + filters.push( + `[${visualLabel}][${preparedLabel}]overlay=` + + `x=${Math.round(width * overlay.x)}-overlay_w/2:` + + `y=${Math.round(height * overlay.y)}-overlay_h/2:eof_action=pass:` + + `enable='between(t,${time(input.clip.startTime)},${time(input.clip.startTime + input.clip.duration)})'[${nextLabel}]` + ); + visualLabel = nextLabel; + } + + const captionTrack = document.tracks.find((track) => track.kind === 'caption'); + if (captionTrack && !captionTrack.hidden && !captionTrack.muted) { + let captionIndex = 0; + for (const clip of captionTrack.clips) { + if (clip.muted || !clip.text?.trim()) continue; + captionIndex += 1; + const style = getResolvedCaptionStyle(document, clip); + const fontSize = Math.max(24, Math.round(height * style.fontScale)); + const maxCharacters = (width * style.maxWidth) / (fontSize * 0.56); + const captionText = style.uppercase + ? clip.text.toLocaleUpperCase() + : clip.text; + const textPath = join(directory, `caption-${captionIndex}.txt`); + await writeFile( + textPath, + wrapCaptionForFfmpeg({ text: captionText, maxCharacters }), + 'utf8' + ); + const nextLabel = `withcaption${captionIndex}`; + const options = [ + `font=${style.fontWeight >= 700 ? 'Arial Bold' : 'Arial'}`, + `textfile='${escapeFilterPath(textPath)}'`, + 'expansion=none', + `fontsize=${fontSize}`, + `fontcolor=${style.fillStyle}`, + `line_spacing=${Math.round(fontSize * (style.lineHeight - 1))}`, + 'x=(w-text_w)/2', + `y=h-${Math.round(height * style.bottomOffset)}-text_h`, + `enable='between(t,${time(clip.startTime)},${time(clip.startTime + clip.duration)})'`, + ]; + if (style.strokeStyle) { + options.push( + `borderw=${Math.max(2, Math.round(fontSize * style.strokeScale))}`, + 'bordercolor=black@0.9' + ); + } + if (style.backgroundStyle) { + options.push( + 'box=1', + 'boxcolor=black@0.82', + `boxborderw=${Math.max(4, Math.round(fontSize * style.horizontalPaddingScale))}` + ); + } + filters.push(`[${visualLabel}]drawtext=${options.join(':')}[${nextLabel}]`); + visualLabel = nextLabel; + } + } + filters.push(`[${visualLabel}]format=yuv420p[vout]`); + + const audioLabels: string[] = []; + let audioIndex = 0; + for (const input of inputs.filter((item) => item.probe.hasAudio)) { + audioIndex += 1; + const label = `audio${audioIndex}`; + const steps = [ + `atrim=start=${time(input.source.inPoint)}:end=${time(input.source.outPoint)}`, + 'asetpts=PTS-STARTPTS', + `volume=${Math.max(0, input.clip.volume).toFixed(3)}`, + ]; + if (input.clip.fadeIn > 0) { + steps.push(`afade=t=in:st=0:d=${time(input.clip.fadeIn)}`); + } + if (input.clip.fadeOut > 0) { + steps.push( + `afade=t=out:st=${time(input.clip.duration - input.clip.fadeOut)}:d=${time(input.clip.fadeOut)}` + ); + } + steps.push(`adelay=${Math.round(input.clip.startTime * 1000)}:all=1`); + filters.push(`[${input.inputIndex}:a]${steps.join(',')}[${label}]`); + audioLabels.push(`[${label}]`); + } + if (audioLabels.length > 0) { + filters.push( + `${audioLabels.join('')}amix=inputs=${audioLabels.length}:duration=longest:dropout_transition=0,` + + `atrim=duration=${time(document.duration)},apad=whole_dur=${time(document.duration)}[aout]` + ); + } + + args.push('-filter_complex', filters.join(';'), '-map', '[vout]'); + if (audioLabels.length > 0) { + args.push('-map', '[aout]', '-c:a', 'aac', '-b:a', '192k'); + } else { + args.push('-an'); + } + args.push( + '-t', + time(document.duration), + '-r', + '30', + '-c:v', + 'libx264', + '-preset', + 'medium', + '-crf', + '18', + '-pix_fmt', + 'yuv420p', + '-movflags', + '+faststart', + outputPath + ); + return { args, width, height }; +} + +export async function renderProjectTimelineToAsset({ + projectId, + document, + timelineRevision, +}: { + projectId: string; + document: TimelineDocument; + timelineRevision: number; +}) { + const blocking = diagnoseTimeline(document).find( + (diagnostic) => diagnostic.severity === 'error' + ); + if (blocking) { + throw new Error(`Timeline diagnostics failed: ${blocking.code}`); + } + const inputs = await resolveRenderInputs({ projectId, document }); + const directory = await mkdtemp(join(tmpdir(), 'beatdesign-render-')); + const outputPath = join(directory, 'timeline.mp4'); + try { + const plan = await buildFfmpegArgs({ + document, + inputs, + directory, + outputPath, + }); + await runProcess(ffmpegBin(), plan.args); + const bytes = new Uint8Array(await readFile(outputPath)); + const persisted = await persistLocalProjectAsset({ + projectId, + filename: `${document.name}-timeline.mp4`, + mimeType: 'video/mp4', + bytes, + }); + try { + const clipCount = document.tracks.reduce( + (count, track) => count + track.clips.length, + 0 + ); + const id = await recordUserAsset({ + id: persisted.assetId, + type: 'video', + source: 'derived', + bucket: LOCAL_PROJECT_ASSET_BUCKET, + objectKey: persisted.objectKey, + publicUrl: persisted.publicUrl, + mimeType: 'video/mp4', + sizeBytes: persisted.sizeBytes, + sha256: persisted.sha256, + filename: persisted.filename, + storageProvider: LOCAL_PROJECT_ASSET_PROVIDER, + assetClass: 'derived', + originProjectId: projectId, + width: plan.width, + height: plan.height, + durationMs: Math.round(document.duration * 1000), + metadata: { + operation: 'timeline_render', + timelineId: document.id, + timelineRevision, + clipCount, + renderer: 'mcp-ffmpeg', + }, + }); + await linkProjectAsset({ + projectId, + assetId: id, + role: 'generated', + assetRole: 'timeline_render', + metadata: { timelineId: document.id, timelineRevision }, + }); + return { + id, + type: 'video' as const, + publicUrl: persisted.publicUrl, + filename: persisted.filename, + mimeType: 'video/mp4' as const, + sizeBytes: persisted.sizeBytes, + width: plan.width, + height: plan.height, + durationMs: Math.round(document.duration * 1000), + localPath: persisted.filePath, + }; + } catch (error) { + await deleteUserAssetById(persisted.assetId).catch(() => undefined); + await removePersistedLocalProjectAsset(persisted.filePath); + throw error; + } + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +export async function removeRenderedProjectTimelineAsset({ + projectId, + assetId, +}: { + projectId: string; + assetId: string; +}) { + const asset = await getProjectAssetById({ projectId, assetId }); + if (!asset?.objectKey || asset.bucket !== LOCAL_PROJECT_ASSET_BUCKET) return; + const filePath = resolveLocalProjectAssetPath({ objectKey: asset.objectKey }); + try { + await deleteUserAssetById(assetId); + } finally { + await removePersistedLocalProjectAsset(filePath); + } +} diff --git a/src/mcp/server.runtime.test.ts b/src/mcp/server.runtime.test.ts index fee5336..b541bcc 100644 --- a/src/mcp/server.runtime.test.ts +++ b/src/mcp/server.runtime.test.ts @@ -8,20 +8,42 @@ import { canvasOperationSchema, editorOperationSchema, } from '@/core/commands/schema'; +import { applyEditorOperations } from '@/core/commands/editor-commands'; import { createTimelineDocument } from '@/core/editor/timeline-document'; import { listGenerationModelDescriptors } from '@/core/generation-providers'; import { BEATDESIGN_MCP_TOOL_NAMES } from './tools'; -test('MCP tool catalog is exactly 26 named tools', () => { +test('MCP tool catalog is exactly 27 named tools', () => { const source = readFileSync(new URL('./server.ts', import.meta.url), 'utf8'); const registered = [...source.matchAll(/server\.registerTool\(\s*'([^']+)'/g)].map( (match) => match[1] ); - assert.equal(BEATDESIGN_MCP_TOOL_NAMES.length, 26); + assert.equal(BEATDESIGN_MCP_TOOL_NAMES.length, 27); assert.deepEqual(registered, [...BEATDESIGN_MCP_TOOL_NAMES]); }); +test('timeline edits invalidate an older render before MCP re-renders', () => { + const source = createTimelineDocument({ + projectId: 'project-render-stale', + name: 'Render stale guard', + }); + const rendered = applyEditorOperations(source, [ + { + type: 'set_render', + assetId: 'render-old', + publicUrl: '/assets/render-old.mp4', + }, + ]).document; + assert.equal(rendered.lastRenderAssetId, 'render-old'); + + const edited = applyEditorOperations(rendered, [ + { type: 'set_caption_style', preset: 'bold' }, + ]).document; + assert.equal(edited.lastRenderAssetId, null); + assert.equal(edited.lastRenderUrl, null); +}); + test('project targeting accepts an explicit workspace handoff destination', () => { const source = readFileSync(new URL('./server.ts', import.meta.url), 'utf8'); const targetRegistration = source.match( diff --git a/src/mcp/server.ts b/src/mcp/server.ts index ff158f6..fd2f398 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -22,6 +22,10 @@ import { MAX_SRT_FILE_BYTES, } from '@/core/editor/captions'; import { diagnoseTimeline } from '@/core/editor/timeline-diagnostics'; +import { + removeRenderedProjectTimelineAsset, + renderProjectTimelineToAsset, +} from '@/core/editor/render-project-timeline'; import { loadProjectTimeline } from '@/core/editor/timeline-state'; import { syncGeneration } from '@/core/effects/generation-sync'; import { listProjectGenerations } from '@/core/effects/project-generations'; @@ -182,12 +186,14 @@ async function executeExternalCommand({ expectedRevision, commandId = createCommandId(), idempotencyKey = commandId, + maxAttempts, }: { projectId: string; command: unknown; expectedRevision?: number | null; commandId?: string; idempotencyKey?: string; + maxAttempts?: number; }) { if (!(await getActiveProject({ projectId }))) { throw new Error('Project not found.'); @@ -202,6 +208,7 @@ async function executeExternalCommand({ expectedRevision, command: parsed, }, + maxAttempts, }); } @@ -837,6 +844,64 @@ export function createBeatDesignMcpServer() { }) ); + server.registerTool( + 'bdesign_editor_render', + { + description: + 'Render the authoritative Editor timeline to a project-owned MP4 Asset with overlays, captions, and mixed audio. Requires local ffmpeg and ffprobe.', + inputSchema: z.object({ + projectId: idSchema.optional(), + expectedRevision: z.number().int().min(0).nullable().optional(), + }), + annotations: { destructiveHint: false, idempotentHint: false }, + }, + withToolErrors(async ({ projectId, expectedRevision }) => { + const project = await resolveScopedProject(projectId); + const timeline = await loadProjectTimeline(project.id); + if (!timeline) throw new Error('Timeline not found.'); + if ( + typeof expectedRevision === 'number' && + expectedRevision !== timeline.version + ) { + throw new Error( + `Timeline revision conflict. Read the latest timeline and retry with expectedRevision ${timeline.version}.` + ); + } + const asset = await renderProjectTimelineToAsset({ + projectId: project.id, + document: timeline.document, + timelineRevision: timeline.version, + }); + const result = await executeExternalCommand({ + projectId: project.id, + expectedRevision: timeline.version, + maxAttempts: 1, + command: { + type: 'editor.apply', + operations: [ + { + type: 'set_render', + assetId: asset.id, + publicUrl: asset.publicUrl, + }, + ], + }, + }); + if (!result.ok) { + await removeRenderedProjectTimelineAsset({ + projectId: project.id, + assetId: asset.id, + }); + throw new Error(result.message); + } + return { + asset, + timelineRevision: result.revision, + commandId: result.commandId, + }; + }) + ); + server.registerTool( 'bdesign_editor_snapshot', { diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 1b30154..5bd0c2a 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -21,6 +21,7 @@ export const BEATDESIGN_MCP_TOOL_NAMES = [ 'bdesign_editor_get', 'bdesign_editor_edit', 'bdesign_editor_import_srt', + 'bdesign_editor_render', 'bdesign_editor_snapshot', 'bdesign_editor_diagnostics', 'bdesign_editor_view', diff --git a/src/routes/api/-trusted-mutations.contract.test.ts b/src/routes/api/-trusted-mutations.contract.test.ts index 4642f34..300bf43 100644 --- a/src/routes/api/-trusted-mutations.contract.test.ts +++ b/src/routes/api/-trusted-mutations.contract.test.ts @@ -66,3 +66,12 @@ test('workspace multipart mutation routes enforce the same-origin request contra assert.match(source, /multipart\/form-data/); } }); + +test('legacy timeline replacement still goes through the shared Command Kernel', () => { + const source = readFileSync( + new URL('./app/projects/$projectId/timeline.ts', import.meta.url), + 'utf8' + ); + assert.match(source, /persistBeatDesignCommand/); + assert.doesNotMatch(source, /saveProjectTimeline\s*\(/); +}); diff --git a/src/routes/api/app/projects/$projectId/timeline.ts b/src/routes/api/app/projects/$projectId/timeline.ts index 31c8884..fb6305a 100644 --- a/src/routes/api/app/projects/$projectId/timeline.ts +++ b/src/routes/api/app/projects/$projectId/timeline.ts @@ -8,7 +8,6 @@ import { } from '@/core/editor/timeline-document'; import { loadProjectTimeline, - saveProjectTimeline, } from '@/core/editor/timeline-state'; import { createCommandId } from '@/core/commands/contracts'; import { persistBeatDesignCommand } from '@/core/commands/persist'; @@ -59,13 +58,38 @@ async function PUT({ payload.document, params.projectId ); - const saved = await saveProjectTimeline({ + const commandId = createCommandId(); + const result = await persistBeatDesignCommand({ projectId: params.projectId, - document, - baseVersion: + origin: 'ui', + commandId, + idempotencyKey: commandId, + expectedRevision: typeof payload.baseVersion === 'number' ? payload.baseVersion : null, + command: { type: 'editor.replace_document', document }, + }); + if (!result.ok) { + return Response.json( + { error: result.message }, + { + status: + result.code === 'NOT_FOUND' + ? 404 + : result.code === 'REVISION_CONFLICT' + ? 409 + : 400, + } + ); + } + if (!result.data.timeline) { + return Response.json({ error: 'Timeline command returned no document' }, { status: 500 }); + } + return Response.json({ + timeline: { + document: result.data.timeline, + version: result.revision, + }, }); - return Response.json({ timeline: saved }); } catch (error) { if (error instanceof RequestBodyTooLargeError) { return Response.json({ error: 'Timeline is too large' }, { status: 413 }); From 033f6fd6442b04ae212abd5b6a0a82f0a98f1422 Mon Sep 17 00:00:00 2001 From: KKKK Date: Fri, 4 Sep 2026 22:14:22 +0800 Subject: [PATCH 2/3] docs: document MCP timeline rendering Align the English, Chinese, and Japanese product documentation plus Agent integration skills with the 27-tool MCP catalog and the new authoritative MP4 export workflow. --- CHANGELOG.md | 2 ++ README.ja.md | 2 +- README.md | 2 +- README.zh-CN.md | 2 +- docs/MCP.md | 25 +++++++++++++------ docs/PRODUCT_PLAN_AND_STATUS.md | 10 ++++---- .../skills/beatdesign-workspace/SKILL.md | 13 ++++++++++ integrations/codex/beatdesign/README.md | 2 +- .../skills/beatdesign-workspace/SKILL.md | 6 +++++ .../skills/beatdesign-workspace/SKILL.md | 6 +++++ 10 files changed, 54 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0642251..85bbb8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to BeatDesign are documented in this file. - Image overlays in the local Editor, with UI and MCP controls for placement, size, opacity, rotation, fades, and replacement with any project-owned image Asset. - Four caption style presets plus per-cue text, timing, size, width, and vertical-position controls shared by the UI and MCP command path. +- MCP `bdesign_editor_render` support for rendering the authoritative Timeline to a project-owned MP4 with visible clips, overlays, caption burn-in, and mixed audio. - Japanese localization across the application and public READMEs. ### Changed @@ -21,6 +22,7 @@ All notable changes to BeatDesign are documented in this file. - Prepared connected local and generated references through the shared upload bridge before remote generation so providers receive public HTTPS media URLs. - Made static video previews seek past common opening black frames. +- Invalidated stale Timeline renders after render-affecting UI or MCP edits and synchronized the current render state with an existing Canvas Timeline card. ## [0.2.2] - 2026-09-03 diff --git a/README.ja.md b/README.ja.md index 86c7257..24aba96 100644 --- a/README.ja.md +++ b/README.ja.md @@ -82,7 +82,7 @@ pnpm dev ## AgentからBeatDesignを使う -BeatDesignはProject、Asset、Canvas、生成、Editor操作をカバーする26個のローカルMCPツールを提供します。Agentによる変更は同じProjectサービスを通り、ブラウザーのワークスペースに表示されます。 +BeatDesignはProject、Asset、Canvas、生成、Editor操作をカバーする27個のローカルMCPツールを提供し、正式なTimelineからのMP4書き出しにも対応します。Agentによる変更は同じProjectサービスを通り、ブラウザーのワークスペースに表示されます。 MCP Hostへ接続した後は、次のように依頼できます。 diff --git a/README.md b/README.md index 0d1ab58..a578539 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ Add your own [BeatAPI API key](https://beatapi.io/dashboard/apikeys) only when y ## Use BeatDesign with an Agent -BeatDesign exposes 26 local MCP tools for Projects, Assets, Canvas, generation, and Editor operations. Agent changes use the same project services and become visible in the browser workspace. +BeatDesign exposes 27 local MCP tools for Projects, Assets, Canvas, generation, and Editor operations, including authoritative MP4 timeline rendering. Agent changes use the same project services and become visible in the browser workspace. After connecting your MCP host, you can ask: diff --git a/README.zh-CN.md b/README.zh-CN.md index bcfb400..67abea1 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -82,7 +82,7 @@ pnpm dev ## 让 Agent 操作 BeatDesign -BeatDesign 提供 26 个本地 MCP 工具,覆盖 Project、Asset、Canvas、生成和 Editor 操作。Agent 修改会经过同一套项目服务,并显示在浏览器工作空间中。 +BeatDesign 提供 27 个本地 MCP 工具,覆盖 Project、Asset、Canvas、生成和 Editor 操作,包括从权威时间线导出 MP4。Agent 修改会经过同一套项目服务,并显示在浏览器工作空间中。 连接 MCP Host 后,可以直接提出这样的要求。 diff --git a/docs/MCP.md b/docs/MCP.md index 147585d..5db26f8 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -93,7 +93,7 @@ installation shapes. ## Tool groups -There are **26** tools: +There are **27** tools: - Project (5): list, get, create, target the current MCP session, and open a workspace review surface. @@ -103,8 +103,8 @@ There are **26** tools: continue-from-tail-frame. - Generation (5): list model capabilities, read one model, submit an asset-first request, refresh status, and list history. -- Editor (7): get, incremental edit, SRT import, semantic snapshot, diagnostics, - deep-link view, and command history. +- Editor (8): get, incremental edit, SRT import, authoritative MP4 render, + semantic snapshot, diagnostics, deep-link view, and command history. MCP writes use `origin=mcp` assigned inside the server. `canvas.apply` and `editor.apply` accept stable IDs, revisions, and idempotency keys. The server @@ -157,6 +157,14 @@ Use `update_caption` to tune one caption cue's normalized font size, maximum width, and bottom position without changing later cues; `set_caption_style` continues to select the shared visual preset. +Use `bdesign_editor_render` to render the authoritative saved timeline to a +project-owned MP4 Asset. The render includes visible video and image clips, +image overlays, caption burn-in, and mixed audio. The tool requires `ffmpeg` +and `ffprobe` on `PATH`, or explicit `BEATDESIGN_FFMPEG` and +`BEATDESIGN_FFPROBE` paths. If the timeline changes while a render is running, +the revision-checked commit rejects the stale output and removes that attempt's +temporary Asset. + For a newly connected Canvas node, append a `place_card` operation after its `upsert_card`. By default it places the target once to the right of the frames listed in `sourceCardIds`, or to the right of the card's `referenceCardIds` when @@ -171,12 +179,15 @@ generation independently of prompt text; BeatDesign does not insert synthetic - `bdesign_editor_snapshot` resolves active clips and source times; it does not rasterize a pixel frame yet. - `bdesign_asset_extract_frame` and `bdesign_canvas_continue_from_tail` decode - the local video file. MCP/Node uses `ffmpeg` on PATH (or `BEATDESIGN_FFMPEG`); - this is not a required system install for the browser UI. + the local video file. MCP/Node frame extraction and timeline rendering use + `ffmpeg` on PATH (or `BEATDESIGN_FFMPEG`); timeline rendering also uses + `ffprobe` (or `BEATDESIGN_FFPROBE`). These are not required system installs + for the browser UI. - SRT import validates the whole subtitle document before replacing the current caption track. Malformed input leaves the saved timeline unchanged. -- Browser-only MP4 export is not exposed as a headless MCP tool yet. Caption - burn-in is included when the browser exports an MP4. +- Browser-native MP4 export remains available without system FFmpeg. + `bdesign_editor_render` provides the corresponding MCP/Node export path and + writes the result back as a project Asset. - `bdesign_asset_import` copies a local image, video, or audio file into the project Asset library from an absolute path. It does not place the Asset on Canvas or Editor; use `bdesign_canvas_apply` or `bdesign_editor_edit` after. diff --git a/docs/PRODUCT_PLAN_AND_STATUS.md b/docs/PRODUCT_PLAN_AND_STATUS.md index 4dee187..f908a89 100644 --- a/docs/PRODUCT_PLAN_AND_STATUS.md +++ b/docs/PRODUCT_PLAN_AND_STATUS.md @@ -129,7 +129,7 @@ Codex / Claude Code / Other Agent ### 媒体技术 - 使用 WebCodecs + Mediabunny 在浏览器完成媒体检查、编码和 MP4 封装。 -- 核心浏览器预览和 MP4 导出不要求系统 FFmpeg;可选的 MCP/Node 视频抽帧使用 `PATH` 中的 `ffmpeg` 或 `BEATDESIGN_FFMPEG`。 +- 核心浏览器预览和 MP4 导出不要求系统 FFmpeg;MCP/Node 视频抽帧与权威时间线 MP4 导出使用 `PATH` 中的 `ffmpeg`/`ffprobe`,也可通过 `BEATDESIGN_FFMPEG` 和 `BEATDESIGN_FFPROBE` 指定。 - OpenReel 只作为时间线术语、文档模型和非破坏式编辑行为的参考;未打包其完整 UI 和应用外壳。 - 媒体 metadata 采用限并发队列并带超时释放;单个损坏媒体不会永久阻塞后续卡片。 @@ -144,7 +144,7 @@ Codex / Claude Code / Other Agent - Generation 的 `AssetFirstGenerationRequest` 已成为服务端权威输入:适配器媒体参数由 Asset ID 和当前 generation intent 编译,旧的客户端 URL 字段不再决定引用事实。 - UI 命令入口不再接受客户端 `origin`;服务端固定写入 `ui`,MCP 入口在内核边界固定写入 `mcp`。 - Provider Contract 已将逻辑模型目录与 BeatAPI effectId、上传路径和上游模型名拆开;BeatAPI 是默认实现,Fork 可在源码扩展点注册其他 Provider。 -- 本地 stdio MCP Server 提供 26 个工具(Project / Asset / Canvas / Generation / Editor),模型和参数通过 capability discovery 暴露;Canvas / Editor 增量操作使用完整 JSON Schema,Agent 可直接发现操作类型和参数。MCP 生成直接调用当前 Provider,本地产品不重复实现 API Key、余额、计费或限流策略,只透传 Provider 的结果与错误。`bdesign_project_target` 绑定当前会话项目,Project/Canvas/Editor view 工具返回 Codex Browser handoff;`bdesign_asset_import` 把本地文件导入项目 Asset 库;`bdesign_asset_extract_frame` 与 `bdesign_canvas_continue_from_tail` 负责抽帧续写;Editor MCP 可导入 SRT、放置和替换任意项目图片 Overlay,并调整叠层与单条字幕参数。 +- 本地 stdio MCP Server 提供 27 个工具(Project / Asset / Canvas / Generation / Editor),模型和参数通过 capability discovery 暴露;Canvas / Editor 增量操作使用完整 JSON Schema,Agent 可直接发现操作类型和参数。MCP 生成直接调用当前 Provider,本地产品不重复实现 API Key、余额、计费或限流策略,只透传 Provider 的结果与错误。`bdesign_project_target` 绑定当前会话项目,Project/Canvas/Editor view 工具返回 Codex Browser handoff;`bdesign_asset_import` 把本地文件导入项目 Asset 库;`bdesign_asset_extract_frame` 与 `bdesign_canvas_continue_from_tail` 负责抽帧续写;Editor MCP 可导入 SRT、放置和替换任意项目图片 Overlay、调整叠层与单条字幕参数,并通过 `bdesign_editor_render` 将权威时间线导出为项目内 MP4 Asset。 - Codex、Claude Code 与 WorkBuddy 接入包内含 `beatdesign-workspace` Skill,负责项目选择、字幕/续写工具编排、付费生成停点和可视化复核;三者共用同一 MCP 与本地 Project 数据,其中 Claude Code 和 WorkBuddy 使用本机 HTTP MCP。 ## 6. v0.2 Phase 1 本地已实现 @@ -159,7 +159,7 @@ Codex / Claude Code / Other Agent - Canvas -> Timeline Node -> Editor 连续工作流。 - Editor 自动保存接入命令入口,并补齐冲突三方合并、重复操作保护和稳定播放头时间。 - 图片 Clip、时间线拖拽调整持续时间与图片/视频统一视觉轨。 -- 本地 MCP Server 提供 26 个 Project、Asset、Canvas、Generation、Editor 工具;支持会话项目绑定、Canvas/Editor 可视化交接、从绝对路径导入本地素材、抽取尾帧续写、导入和精调 SRT 字幕,以及放置、替换和调整图片 Overlay。 +- 本地 MCP Server 提供 27 个 Project、Asset、Canvas、Generation、Editor 工具;支持会话项目绑定、Canvas/Editor 可视化交接、从绝对路径导入本地素材、抽取尾帧续写、导入和精调 SRT 字幕、放置/替换/调整图片 Overlay,以及权威时间线 MP4 导出。 - MCP 增量 Canvas/Editor 命令在短暂 revision 竞争时会基于最新权威文档限次自动重放;持续冲突返回最新 revision 和明确重试提示。 - Canvas 与 Editor 每 2 秒并在页面重新聚焦时检查 MCP 写入的新 revision。 @@ -189,9 +189,9 @@ Codex / Claude Code / Other Agent - MCP Resources 和更完整的 schema versioning。 - Agent Activity、命令审计和实时 UI 事件桥。 - 外部市场正式审核与上架。仓库已提供 Codex 本地插件、可直接添加的 Claude Code 仓库插件市场,以及符合目录结构的 WorkBuddy MCP + Skill Connector;这些本地接入包不等于已通过第三方市场审核。 -- headless 预览/导出本地媒体 Worker。 +- 独立的 headless 像素预览与后台媒体 Worker;当前 MCP MP4 导出在本地 MCP Server 进程中完成。 -当前可以称为“已支持本地 MCP 基础版”,但不能称为完整 Agent 编辑环境:像素级 Snapshot、headless 导出、实时 UI 事件和永久审计仍未实现。本地文件导入桥已完成;UI 当前使用 2 秒 revision 轮询和聚焦检查,而不是实时事件推送。 +当前可以称为“已支持本地 MCP 基础版”,但不能称为完整 Agent 编辑环境:像素级 Snapshot、独立后台媒体 Worker、实时 UI 事件和永久审计仍未实现。本地文件导入桥和 MCP 权威时间线 MP4 导出已完成;UI 当前使用 2 秒 revision 轮询和聚焦检查,而不是实时事件推送。 Canvas 的拖拽、缩放、视口和完整布局仍使用 revision-checked Snapshot 自动保存;Canvas/Timeline 业务 operation 已有 Command 合同,Timeline Node 回写也已接入 `/commands`。后续做 MCP parity 时,应继续把可语义化的 Canvas UI 动作迁移为 `canvas.apply`,不能让外部 Agent 调用完整 Snapshot 覆盖。 diff --git a/integrations/claude-code/beatdesign/skills/beatdesign-workspace/SKILL.md b/integrations/claude-code/beatdesign/skills/beatdesign-workspace/SKILL.md index bcf05c1..a45090c 100644 --- a/integrations/claude-code/beatdesign/skills/beatdesign-workspace/SKILL.md +++ b/integrations/claude-code/beatdesign/skills/beatdesign-workspace/SKILL.md @@ -64,6 +64,19 @@ opened when the host cannot open it. `bdesign_canvas_apply`, then focus the same card with `bdesign_canvas_view`. +## Export the authoritative timeline + +- When the user authorizes an export, call `bdesign_editor_get`, then call + `bdesign_editor_render` with the returned revision. The result is a + project-owned MP4 Asset containing visible clips, image overlays, caption + burn-in, and mixed audio. +- If rendering reports that `ffmpeg` or `ffprobe` is unavailable, stop and + explain that the MCP process needs those binaries on `PATH`, or absolute + `BEATDESIGN_FFMPEG` and `BEATDESIGN_FFPROBE` paths. +- Render-affecting edits invalidate the previous current render without + deleting its historical Asset. If the timeline changes while rendering, + read the latest revision and ask before starting another render. + ## Verify the visible result After a write, read the returned revision and changed IDs. Canvas and Editor diff --git a/integrations/codex/beatdesign/README.md b/integrations/codex/beatdesign/README.md index bb245a3..7e9bc68 100644 --- a/integrations/codex/beatdesign/README.md +++ b/integrations/codex/beatdesign/README.md @@ -8,7 +8,7 @@ Canvas or Editor URL to Codex's in-app Browser for visible review. ```text Browser: pnpm dev → http://127.0.0.1:3020 (Canvas/Editor review surface) -Agent: MCP stdio → pnpm mcp (Agent calls 26 tools) +Agent: MCP stdio → pnpm mcp (Agent calls 27 tools) Both processes share the same local SQLite + project files. ``` diff --git a/integrations/codex/beatdesign/skills/beatdesign-workspace/SKILL.md b/integrations/codex/beatdesign/skills/beatdesign-workspace/SKILL.md index 3b2daab..5785a2a 100644 --- a/integrations/codex/beatdesign/skills/beatdesign-workspace/SKILL.md +++ b/integrations/codex/beatdesign/skills/beatdesign-workspace/SKILL.md @@ -39,6 +39,12 @@ For Canvas-specific review, call `bdesign_canvas_view` with `cardId` so the work - After submission, follow generation status until it succeeds, fails, or needs user action. A successful generation creates an Asset but does not place itself. Read the current continuation card, preserve its generation settings, update the returned `generationCardId` with that output through `bdesign_canvas_apply`, then call `bdesign_canvas_view` with the same card ID. - If continuation returns `ok=false`, preserve its structured conflict or rollback result. Follow a returned retry instruction at most once after reading current state; do not blindly re-extract frames. +## Export the authoritative timeline + +- When the user authorizes an export, call `bdesign_editor_get`, then call `bdesign_editor_render` with the returned revision. The result is a project-owned MP4 Asset that includes visible clips, image overlays, caption burn-in, and mixed audio. +- If rendering reports that `ffmpeg` or `ffprobe` is unavailable, stop and explain that the MCP host needs those binaries on `PATH`, or absolute `BEATDESIGN_FFMPEG` and `BEATDESIGN_FFPROBE` paths. +- A render-affecting edit invalidates the previous current render without deleting its historical Asset. If the timeline changes during rendering, read the latest revision and ask before starting another potentially expensive render. + ## Verify what the user can see After a write, read the returned revision and changed IDs, then inspect the already-open Canvas or Editor. Canvas and Editor currently refresh external revisions within about two seconds and on focus. Confirm the intended card, clip, caption, duration, or media is visible; a successful database revision alone is not completion. diff --git a/integrations/workbuddy/beatdesign/skills/beatdesign-workspace/SKILL.md b/integrations/workbuddy/beatdesign/skills/beatdesign-workspace/SKILL.md index fbf4bb4..fcc3343 100644 --- a/integrations/workbuddy/beatdesign/skills/beatdesign-workspace/SKILL.md +++ b/integrations/workbuddy/beatdesign/skills/beatdesign-workspace/SKILL.md @@ -44,6 +44,12 @@ BeatDesign MCP 是结构化控制层,浏览器中的 BeatDesign 是用户审 - 抽帧和放置节点是本地操作;返回的生成请求是单独的远程付费动作,只有用户明确授权后才能提交。 - 生成成功只会创建 Asset。用 `bdesign_canvas_apply` 把输出更新到返回的生成卡片,再调用 `bdesign_canvas_view` 聚焦它。 +## 导出权威时间线 + +- 用户明确授权导出后,先调用 `bdesign_editor_get`,再使用返回的 revision 调用 `bdesign_editor_render`。结果是项目内 MP4 Asset,包含可见 Clip、图片 Overlay、烧录字幕和混合音频。 +- 若提示缺少 `ffmpeg` 或 `ffprobe`,请用户把两者加入 MCP 进程的 `PATH`,或设置绝对路径 `BEATDESIGN_FFMPEG` 和 `BEATDESIGN_FFPROBE`;环境未改变前不要重复调用。 +- 会影响画面的时间线修改会让上一版当前导出失效,但不会删除历史 Asset。若导出期间时间线发生变化,先读取最新 revision,再询问用户是否重新导出。 + ## 完成标准 写入后核对返回的 revision 和变更 ID,再调用对应 view tool。Canvas 和 Editor 当前通常会在约两秒内或页面重新聚焦时读到 Agent 修改。最终提供准确的 `workspaceUrl` 供用户审核;仅有数据库 revision 不代表用户已经看到结果。 From a19e25382a910f6eb1a100581f686dd02f8aea38 Mon Sep 17 00:00:00 2001 From: KKKK Date: Fri, 4 Sep 2026 22:19:24 +0800 Subject: [PATCH 3/3] docs: clarify MCP media runtime requirements Keep project guidance and third-party notices explicit that browser export stays FFmpeg-free while optional MCP rendering uses user-provided ffmpeg and ffprobe. --- AGENTS.md | 2 +- docs/PRODUCT_PLAN_AND_STATUS.md | 2 +- third_party/README.md | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index cf011f7..ffe4f11 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ Do not embed a proprietary general chatbot. BeatDesign must be complete without - Canvas layout snapshots are a UI persistence exception for drag, resize, and viewport state; external Agents still use semantic Canvas operations. - Agent changes must become visible in the browser workspace and remain inspectable, reversible where supported, and verifiable. A database revision alone is not proof of a successful user-visible operation. - Studio, Canvas, Editor, Assets, and MCP share project, task, asset, and generation services rather than duplicating business logic. -- Preview and MP4 export use browser-native WebCodecs and Mediabunny. Do not make system FFmpeg a requirement for the core localhost UI. Node-side MCP frame extraction may use `ffmpeg` from `PATH` or `BEATDESIGN_FFMPEG` and must fail with a clear setup error when unavailable. +- Preview and browser-side MP4 export use browser-native WebCodecs and Mediabunny. Do not make system FFmpeg a requirement for the core localhost UI. Node-side MCP frame extraction and Timeline rendering may use `ffmpeg` from `PATH` or `BEATDESIGN_FFMPEG`; Timeline rendering also uses `ffprobe` from `PATH` or `BEATDESIGN_FFPROBE`. These tools must fail with a clear setup error when a required binary is unavailable. ## Provider and storage boundary diff --git a/docs/PRODUCT_PLAN_AND_STATUS.md b/docs/PRODUCT_PLAN_AND_STATUS.md index f908a89..af5f84d 100644 --- a/docs/PRODUCT_PLAN_AND_STATUS.md +++ b/docs/PRODUCT_PLAN_AND_STATUS.md @@ -200,7 +200,7 @@ Canvas 的拖拽、缩放、视口和完整布局仍使用 revision-checked Snap - Electron/Tauri 壳。 - 内置 Node runtime、MCP 和媒体 Worker。 - 原生文件选择、系统集成、签名、公证和自动更新。 -- 桌面封装阶段再评估是否内置 FFmpeg;当前开源 localhost 的核心 UI 不依赖它,但 MCP/Node 抽帧工具需要用户提供本地 `ffmpeg`。 +- 桌面封装阶段再评估是否内置 FFmpeg;当前开源 localhost 的核心 UI 不依赖它,但 MCP/Node 抽帧需要用户提供本地 `ffmpeg`,权威时间线导出还需要 `ffprobe`。 ## 8. 路线图 diff --git a/third_party/README.md b/third_party/README.md index 3855d9d..33cf034 100644 --- a/third_party/README.md +++ b/third_party/README.md @@ -27,5 +27,7 @@ machine-readable pin is in The preserved license is in [`mediabunny/LICENSE`](./mediabunny/LICENSE). Mediabunny is consumed as an unmodified npm dependency. BeatDesign does not -bundle a native FFmpeg executable and does not require a system FFmpeg -installation. +bundle a native FFmpeg executable, and browser-native editing and MP4 export do +not require a system FFmpeg installation. Optional MCP/Node frame extraction +and Timeline rendering require user-provided `ffmpeg`; Timeline rendering also +requires `ffprobe`.