From 4d086febb7d02d8b0f74f51f4fcf8a59bc27bff6 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Fri, 4 Sep 2026 08:24:50 +0000 Subject: [PATCH 1/2] feat: add script-to-beats compiler for continuous video episodes (#6225) Pure module that partitions a scene script into beat-level video clip specs: word-count-driven duration snapped to a target frame grid (uniform or the MiniMax H3 family's 17n+5 grid), byte-stable bible descriptor injection for cast/locations/style, explicit speaker-clause dialogue formatting, and a max-chain-length rule that forces a fresh re-establish cut every N clips. First slice of the continuous-video episode feature (#6217); the prompt linter (#6226) and orchestrator (#6227) build on this. --- server/lib/README.md | 1 + server/lib/index.js | 1 + server/lib/scriptVideoCompiler.js | 225 +++++++++++++++++++++++++ server/lib/scriptVideoCompiler.test.js | 175 +++++++++++++++++++ 4 files changed, 402 insertions(+) create mode 100644 server/lib/scriptVideoCompiler.js create mode 100644 server/lib/scriptVideoCompiler.test.js diff --git a/server/lib/README.md b/server/lib/README.md index 6f2c2cbdf2..a4a765d917 100644 --- a/server/lib/README.md +++ b/server/lib/README.md @@ -125,6 +125,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub | `catalogUniverseTags.js` | Pure transform that rewrites legacy machine universe tags (`from-universe`, `universe:`) on backfilled catalog ingredients into friendly universe-NAME tags, preserving user tags + the structured `catalog_ingredient_refs` link. Used by the boot-time repair and the bible→catalog backfill. | | `comicScriptParser.js` | Marvel/DC-format comic script parser. | | `composeStyledPrompt.js` | Compose user prompt + negative with an optional style preset. | +| `scriptVideoCompiler.js` | Compiles a scene script + bible into beat-level continuous-video clip specs (duration/frame-grid snapping, byte-stable descriptor injection, chain-length fresh-cut rule). | | `creativeDirectorPresets.js` | Locked-at-creation aspect ratio + quality presets for the Creative Director. | | `creativeLatitude.js` | The IP-latitude clause every creative LLM request carries (`withCreativeLatitude`, `CREATIVE_LATITUDE_TOKENS`), plus the one creative-vs-operational table both stamp keys are matched against (`isCreativeStage` for stage names, `isCreativeRunSource` for run `source` tags). | | `universeBibleCompleteness.js` | Is a universe bible entry actually described? The shared per-kind expand-field vocabulary (`BIBLE_EXPAND_FIELDS`, `BIBLE_CORE_FIELDS`) and the `core`/`full` completeness predicates the quota-burn describe job scans with. Pure. | diff --git a/server/lib/index.js b/server/lib/index.js index 1fdc42257a..1393b06ae7 100644 --- a/server/lib/index.js +++ b/server/lib/index.js @@ -78,6 +78,7 @@ export * as catalogUniverseTags from './catalogUniverseTags.js'; export * from './canonPrompt.js'; export * from './comicScriptParser.js'; export * from './composeStyledPrompt.js'; +export * from './scriptVideoCompiler.js'; export * from './creativeDirectorPresets.js'; export * from './creativeLatitude.js'; // Namespaced: the editorial-check registry (#1284) lives in the editorial/ diff --git a/server/lib/scriptVideoCompiler.js b/server/lib/scriptVideoCompiler.js new file mode 100644 index 0000000000..413222be24 --- /dev/null +++ b/server/lib/scriptVideoCompiler.js @@ -0,0 +1,225 @@ +/** + * Pure compiler that turns a scene script into beat-level video clip specs + * for continuous chained video generation (issue #6217's Script-to-Beats + * Compiler slice). Each beat becomes one clip request: a duration snapped to + * a target frame grid, and a prompt carrying byte-stable "Bible" descriptors + * for the cast/locations/style referenced in that beat plus formatted + * dialogue clauses. + * + * Byte-stability matters more than prose quality here: `server/services/ + * videoGen/chainedVideo.js` already handles low-level chunk stitching, but + * chained video models mutate character faces/clothing/environment when a + * descriptor is reworded between clips. This module never rewrites a + * descriptor — it looks the same bible entry up verbatim for every beat that + * references it. + * + * Deliberately does NOT decide clip framing/camera language or write the + * "Hard cut to :" opener continuing clips need — that is authored + * content (or an LLM step) which `videoPromptLinter.js` (#6226) then checks + * for. This module only decides WHERE a chain must break (`cutType`). + * + * No I/O, no video-backend awareness — `continuousVideo.js` (#6227) composes + * this with the linter and the backend submission. + */ + +export const BEAT_MAX_WORDS = 35; +export const BEAT_MAX_SPEAKERS = 2; +export const MAX_CHAIN_LENGTH = 6; +export const WORDS_PER_SECOND = 2.5; +export const AIR_SECONDS = 1.5; +export const DEFAULT_FPS = 24; + +// The MiniMax H3 family's VAE decodes only frame counts on a 17n+5 grid (see +// h3FrameGrid in mediaModels.js) — a beat targeting that runtime needs its +// duration snapped to the same grid rather than a plain per-frame ceiling. +export const H3_FRAME_STEP = 17; +export const H3_FRAME_OFFSET = 5; + +const countWords = (text) => { + const t = (text || '').trim(); + return t ? t.split(/\s+/).length : 0; +}; + +/** Total words across a beat's action + dialogue lines. */ +export const beatWordCount = (beat) => (beat?.lines || []).reduce((sum, l) => sum + countWords(l.text), 0); + +/** + * Spoken-pace duration estimate: word count / WORDS_PER_SECOND, plus a fixed + * AIR_SECONDS pad so a clip doesn't cut the instant the last word lands. + */ +export const estimateBeatSeconds = (beat) => (beatWordCount(beat) / WORDS_PER_SECOND) + AIR_SECONDS; + +/** + * Snap a duration to a target frame grid. + * + * `grid: 'uniform'` (default) just ceilings to the nearest whole frame at + * `fps`. `grid: '17n+5'` snaps UP to the nearest H3-family frame count, + * mirroring `h3FrameGrid` in mediaModels.js. + * + * Always rounds UP — a beat's dialogue must fully fit inside the rendered + * clip, so under-snapping (truncating speech) is never acceptable while a + * slightly longer clip is. + */ +export function snapFramesToGrid({ seconds, fps = DEFAULT_FPS, grid = 'uniform' } = {}) { + const s = Number.isFinite(seconds) && seconds > 0 ? seconds : 0; + const f = Number.isFinite(fps) && fps > 0 ? fps : DEFAULT_FPS; + const wantFrames = Math.max(1, Math.ceil(s * f)); + if (grid === '17n+5') { + const n = Math.max(0, Math.ceil((wantFrames - H3_FRAME_OFFSET) / H3_FRAME_STEP)); + const frames = (n * H3_FRAME_STEP) + H3_FRAME_OFFSET; + return { frames, seconds: frames / f, fps: f }; + } + return { frames: wantFrames, seconds: wantFrames / f, fps: f }; +} + +/** + * Explicit speaker clause: `S1 (Speaker, voice): "..."`. `index` is the + * clip-local speaker ordinal (1-based) — callers assign it per beat, not + * globally, so a two-speaker beat is always `S1`/`S2`. + */ +export const formatDialogueLine = ({ index, speaker, voice, text }) => ( + `S${index} (${speaker}${voice ? `, ${voice}` : ''}): "${(text || '').trim()}"` +); + +/** + * Partition script lines into beats of <= maxWords words and <= maxSpeakers + * distinct dialogue speakers. A beat never splits a single line — a line + * longer than maxWords on its own still becomes (and closes) its own beat, + * so no dialogue/action text is ever truncated. + * + * @param {Array<{type: 'action'|'dialogue', speaker?: string, voice?: string, text: string}>} lines + */ +export function partitionLinesIntoBeats(lines, { maxWords = BEAT_MAX_WORDS, maxSpeakers = BEAT_MAX_SPEAKERS } = {}) { + const beats = []; + let current = null; + + const openBeat = () => { + current = { lines: [], speakers: [] }; + beats.push(current); + }; + + for (const line of lines || []) { + const words = countWords(line.text); + const speaker = line.type === 'dialogue' ? line.speaker : null; + if (!current) openBeat(); + + const nextWordCount = beatWordCount(current) + words; + const nextSpeakers = new Set(current.speakers); + if (speaker) nextSpeakers.add(speaker); + + const wouldOverflow = current.lines.length > 0 + && (nextWordCount > maxWords || nextSpeakers.size > maxSpeakers); + if (wouldOverflow) { + openBeat(); + if (speaker) current.speakers.push(speaker); + } else if (speaker && !current.speakers.includes(speaker)) { + current.speakers.push(speaker); + } + current.lines.push(line); + } + + return beats; +} + +/** + * Look up a bible descriptor by kind ('cast' | 'locations') and id. Returns + * the SAME string every time for the same id — the caller must never + * paraphrase it — or `null` when the bible has no entry for it. + */ +export const resolveBibleDescriptor = (bible, kind, id) => bible?.[kind]?.[id]?.descriptor ?? null; + +/** + * Compose one beat's clip prompt: style descriptor, the scene's location + * descriptor, then the byte-stable cast descriptor for every speaker in the + * beat (deduped, first-seen order), followed by the beat's action text and + * formatted dialogue clauses. + */ +export function buildBeatPrompt({ beat, bible, locationId }) { + const fragments = []; + if (bible?.styleDescriptor) fragments.push(bible.styleDescriptor); + const locationDescriptor = locationId ? resolveBibleDescriptor(bible, 'locations', locationId) : null; + if (locationDescriptor) fragments.push(locationDescriptor); + for (const speaker of beat.speakers) { + const castDescriptor = resolveBibleDescriptor(bible, 'cast', speaker); + if (castDescriptor) fragments.push(castDescriptor); + } + + const body = []; + let speakerIndex = 0; + const speakerOrdinal = new Map(); + for (const line of beat.lines) { + if (line.type === 'dialogue') { + if (!speakerOrdinal.has(line.speaker)) { + speakerIndex += 1; + speakerOrdinal.set(line.speaker, speakerIndex); + } + body.push(formatDialogueLine({ + index: speakerOrdinal.get(line.speaker), + speaker: line.speaker, + voice: line.voice, + text: line.text, + })); + } else if (line.text) { + body.push(line.text.trim()); + } + } + + return [...fragments, ...body].filter(Boolean).join(' '); +} + +/** + * Compile a script (an array of scenes, each `{ sceneId, location, lines }`) + * against a bible into an ordered array of clip specs. + * + * Chain rule: the first beat of every scene is always `cutType: 'fresh'` + * (a scene boundary is a natural re-establish point). Within a scene, a + * chain of `continue` clips runs until `maxChainLength` clips have been + * emitted since the last fresh cut, at which point the next beat is forced + * back to `fresh` and the count restarts. + */ +export function compileScriptToClips({ + scenes, + bible, + maxWords = BEAT_MAX_WORDS, + maxSpeakers = BEAT_MAX_SPEAKERS, + maxChainLength = MAX_CHAIN_LENGTH, + fps = DEFAULT_FPS, + frameGrid = 'uniform', +} = {}) { + const clips = []; + + (scenes || []).forEach((scene, sceneIndex) => { + const beats = partitionLinesIntoBeats(scene.lines, { maxWords, maxSpeakers }); + let chainPosition = 0; + + beats.forEach((beat, beatIndex) => { + let cutType; + if (beatIndex === 0 || chainPosition >= maxChainLength) { + cutType = 'fresh'; + chainPosition = 1; + } else { + cutType = 'continue'; + chainPosition += 1; + } + + const seconds = estimateBeatSeconds(beat); + const { frames, seconds: snappedSeconds } = snapFramesToGrid({ seconds, fps, grid: frameGrid }); + const prompt = buildBeatPrompt({ beat, bible, locationId: scene.location }); + + clips.push({ + sceneIndex, + sceneId: scene.sceneId ?? null, + beatIndex, + cutType, + chainPosition, + speakers: beat.speakers, + fps, + frames, + durationSeconds: snappedSeconds, + prompt, + }); + }); + }); + + return clips; +} diff --git a/server/lib/scriptVideoCompiler.test.js b/server/lib/scriptVideoCompiler.test.js new file mode 100644 index 0000000000..541bda0e79 --- /dev/null +++ b/server/lib/scriptVideoCompiler.test.js @@ -0,0 +1,175 @@ +import { describe, it, expect } from 'vitest'; +import { + BEAT_MAX_WORDS, + MAX_CHAIN_LENGTH, + DEFAULT_FPS, + partitionLinesIntoBeats, + snapFramesToGrid, + formatDialogueLine, + buildBeatPrompt, + compileScriptToClips, +} from './scriptVideoCompiler.js'; + +const bible = { + styleDescriptor: 'Gritty noir animation, high-contrast chiaroscuro lighting.', + cast: { + KESSA: { descriptor: 'KESSA: lean build, silver undercut, mid-30s, teal scarf.' }, + GIANT: { descriptor: 'GIANT: hulking build, shaved head, late-40s, red gauntlet.' }, + }, + locations: { + VAULT: { descriptor: 'INT. VAULT — cavernous concrete chamber, blue emergency lighting.' }, + }, +}; + +describe('partitionLinesIntoBeats', () => { + it('keeps lines together under the word/speaker limits', () => { + const lines = [ + { type: 'action', text: 'Kessa creeps along the wall.' }, + { type: 'dialogue', speaker: 'KESSA', text: 'Quiet.' }, + ]; + const beats = partitionLinesIntoBeats(lines); + expect(beats).toHaveLength(1); + expect(beats[0].lines).toHaveLength(2); + expect(beats[0].speakers).toEqual(['KESSA']); + }); + + it('splits a beat when a third distinct speaker would join it', () => { + const lines = [ + { type: 'dialogue', speaker: 'KESSA', text: 'Move.' }, + { type: 'dialogue', speaker: 'GIANT', text: 'Where?' }, + { type: 'dialogue', speaker: 'NARRATOR', text: 'They ran.' }, + ]; + const beats = partitionLinesIntoBeats(lines, { maxSpeakers: 2 }); + expect(beats).toHaveLength(2); + expect(beats[0].speakers).toEqual(['KESSA', 'GIANT']); + expect(beats[1].speakers).toEqual(['NARRATOR']); + }); + + it('splits a beat once the word count would exceed maxWords', () => { + const lines = [ + { type: 'action', text: 'word '.repeat(20).trim() }, + { type: 'action', text: 'word '.repeat(20).trim() }, + ]; + const beats = partitionLinesIntoBeats(lines, { maxWords: BEAT_MAX_WORDS }); + expect(beats).toHaveLength(2); + }); + + it('never splits a single line, even one longer than maxWords', () => { + const lines = [{ type: 'action', text: 'word '.repeat(50).trim() }]; + const beats = partitionLinesIntoBeats(lines, { maxWords: 35 }); + expect(beats).toHaveLength(1); + expect(beats[0].lines).toHaveLength(1); + }); +}); + +describe('snapFramesToGrid', () => { + it('ceilings to a whole frame under the uniform grid', () => { + const { frames, fps } = snapFramesToGrid({ seconds: 2.01, fps: 24, grid: 'uniform' }); + expect(fps).toBe(24); + expect(frames).toBe(Math.ceil(2.01 * 24)); + }); + + it('defaults to DEFAULT_FPS when fps is omitted', () => { + const { fps } = snapFramesToGrid({ seconds: 3 }); + expect(fps).toBe(DEFAULT_FPS); + }); + + it('snaps up to the nearest 17n+5 frame count', () => { + // 107 = 17*6 + 5. A request for exactly 107 frames worth of seconds + // should land there, not on the next rung. + const { frames } = snapFramesToGrid({ seconds: 107 / 24, fps: 24, grid: '17n+5' }); + expect(frames).toBe(107); + expect((frames - 5) % 17).toBe(0); + }); + + it('rounds a mid-grid request UP to the next 17n+5 rung, never down', () => { + // 108 pixel frames sits strictly between 107 (17*6+5) and 124 (17*7+5). + const { frames } = snapFramesToGrid({ seconds: 108 / 24, fps: 24, grid: '17n+5' }); + expect(frames).toBe(124); + }); +}); + +describe('formatDialogueLine', () => { + it('formats a speaker clause with a voice tag', () => { + expect(formatDialogueLine({ index: 1, speaker: 'KESSA', voice: 'whispered', text: 'Quiet.' })) + .toBe('S1 (KESSA, whispered): "Quiet."'); + }); + + it('omits the voice tag when absent', () => { + expect(formatDialogueLine({ index: 2, speaker: 'GIANT', text: 'Where?' })) + .toBe('S2 (GIANT): "Where?"'); + }); +}); + +describe('buildBeatPrompt', () => { + it('injects the SAME bible descriptor string across two different beats', () => { + const beatA = { lines: [{ type: 'dialogue', speaker: 'KESSA', text: 'Move.' }], speakers: ['KESSA'] }; + const beatB = { lines: [{ type: 'dialogue', speaker: 'KESSA', text: 'Now.' }], speakers: ['KESSA'] }; + const promptA = buildBeatPrompt({ beat: beatA, bible, locationId: 'VAULT' }); + const promptB = buildBeatPrompt({ beat: beatB, bible, locationId: 'VAULT' }); + expect(promptA).toContain(bible.cast.KESSA.descriptor); + expect(promptB).toContain(bible.cast.KESSA.descriptor); + // Byte-identical descriptor substring, not just semantically similar. + const descInA = promptA.slice(promptA.indexOf('KESSA:'), promptA.indexOf('KESSA:') + bible.cast.KESSA.descriptor.length); + const descInB = promptB.slice(promptB.indexOf('KESSA:'), promptB.indexOf('KESSA:') + bible.cast.KESSA.descriptor.length); + expect(descInA).toBe(descInB); + }); + + it('formats action text and dialogue clauses into the prompt body', () => { + const beat = { + lines: [ + { type: 'action', text: 'Kessa freezes.' }, + { type: 'dialogue', speaker: 'KESSA', text: 'Did you hear that?' }, + ], + speakers: ['KESSA'], + }; + const prompt = buildBeatPrompt({ beat, bible, locationId: 'VAULT' }); + expect(prompt).toContain('Kessa freezes.'); + expect(prompt).toContain('S1 (KESSA): "Did you hear that?"'); + }); +}); + +describe('compileScriptToClips', () => { + it('marks the first beat of a scene fresh, then continues within it', () => { + const scenes = [{ + sceneId: 'S1', + location: 'VAULT', + lines: [ + { type: 'dialogue', speaker: 'KESSA', text: 'Move.' }, + { type: 'dialogue', speaker: 'GIANT', text: 'Where?' }, + { type: 'dialogue', speaker: 'KESSA', text: 'There.' }, + ], + }]; + const clips = compileScriptToClips({ scenes, bible, maxWords: 2, maxSpeakers: 1 }); + expect(clips.length).toBeGreaterThan(1); + expect(clips[0].cutType).toBe('fresh'); + expect(clips.slice(1).every((c) => c.cutType === 'continue')).toBe(true); + }); + + it('forces a fresh re-establish after maxChainLength continue clips', () => { + const lines = Array.from({ length: 10 }, (_, i) => ( + { type: 'dialogue', speaker: 'KESSA', text: `Line ${i}.` } + )); + const scenes = [{ sceneId: 'S1', location: 'VAULT', lines }]; + const clips = compileScriptToClips({ scenes, bible, maxWords: 2, maxSpeakers: 1, maxChainLength: 3 }); + // Beats: 0=fresh,1=continue,2=continue,3=fresh(chain hit 3),4=continue,... + const cutTypes = clips.map((c) => c.cutType); + expect(cutTypes[0]).toBe('fresh'); + expect(cutTypes[3]).toBe('fresh'); + expect(cutTypes.filter((t) => t === 'fresh').length).toBeGreaterThan(1); + }); + + it('always starts a new scene with a fresh cut, even after a continue chain', () => { + const scenes = [ + { sceneId: 'S1', location: 'VAULT', lines: [{ type: 'action', text: 'Opening beat.' }] }, + { sceneId: 'S2', location: 'VAULT', lines: [{ type: 'action', text: 'Second scene beat.' }] }, + ]; + const clips = compileScriptToClips({ scenes, bible }); + expect(clips).toHaveLength(2); + expect(clips.every((c) => c.cutType === 'fresh')).toBe(true); + }); + + it('respects the configured maxChainLength constant as a default', () => { + expect(MAX_CHAIN_LENGTH).toBe(6); + }); +}); From dbd85a2548f69615e0ad723fcefda01ded19d5ee Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Fri, 4 Sep 2026 08:26:33 +0000 Subject: [PATCH 2/2] fix: address opencode review findings on the script-video compiler Copy the beat speaker array before handing it out on a clip spec (the source array is shared with partitionLinesIntoBeats' internal beat object), and replace the O(n^2) per-line beatWordCount rescan with a running word counter. --- server/lib/scriptVideoCompiler.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/server/lib/scriptVideoCompiler.js b/server/lib/scriptVideoCompiler.js index 413222be24..582b62f775 100644 --- a/server/lib/scriptVideoCompiler.js +++ b/server/lib/scriptVideoCompiler.js @@ -32,6 +32,10 @@ export const DEFAULT_FPS = 24; // The MiniMax H3 family's VAE decodes only frame counts on a 17n+5 grid (see // h3FrameGrid in mediaModels.js) — a beat targeting that runtime needs its // duration snapped to the same grid rather than a plain per-frame ceiling. +// Duplicated rather than imported: mediaModels.js is a large registry module +// (2000+ lines, 15 imports) this compiler has no other reason to pull in for +// two constants — see the "widely-reached module" import-scoping rule in +// server/AGENTS.md. export const H3_FRAME_STEP = 17; export const H3_FRAME_OFFSET = 5; @@ -92,9 +96,11 @@ export const formatDialogueLine = ({ index, speaker, voice, text }) => ( export function partitionLinesIntoBeats(lines, { maxWords = BEAT_MAX_WORDS, maxSpeakers = BEAT_MAX_SPEAKERS } = {}) { const beats = []; let current = null; + let currentWordCount = 0; const openBeat = () => { current = { lines: [], speakers: [] }; + currentWordCount = 0; beats.push(current); }; @@ -103,12 +109,11 @@ export function partitionLinesIntoBeats(lines, { maxWords = BEAT_MAX_WORDS, maxS const speaker = line.type === 'dialogue' ? line.speaker : null; if (!current) openBeat(); - const nextWordCount = beatWordCount(current) + words; - const nextSpeakers = new Set(current.speakers); - if (speaker) nextSpeakers.add(speaker); + const nextWordCount = currentWordCount + words; + const nextSpeakerCount = current.speakers.length + (speaker && !current.speakers.includes(speaker) ? 1 : 0); const wouldOverflow = current.lines.length > 0 - && (nextWordCount > maxWords || nextSpeakers.size > maxSpeakers); + && (nextWordCount > maxWords || nextSpeakerCount > maxSpeakers); if (wouldOverflow) { openBeat(); if (speaker) current.speakers.push(speaker); @@ -116,6 +121,7 @@ export function partitionLinesIntoBeats(lines, { maxWords = BEAT_MAX_WORDS, maxS current.speakers.push(speaker); } current.lines.push(line); + currentWordCount += words; } return beats; @@ -212,7 +218,7 @@ export function compileScriptToClips({ beatIndex, cutType, chainPosition, - speakers: beat.speakers, + speakers: [...beat.speakers], fps, frames, durationSeconds: snappedSeconds,