From f5078106c057af48aa1bd7e5bb03071dfff75706 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 23 Apr 2026 03:22:28 +0000 Subject: [PATCH] perf(engine): webp extraction format + injector MIME routing --- .../src/services/extractionCache.test.ts | 4 +- .../engine/src/services/extractionCache.ts | 15 ++-- .../src/services/videoFrameExtractor.test.ts | 85 +++++++++++++++++++ .../src/services/videoFrameExtractor.ts | 23 ++++- .../src/services/videoFrameInjector.test.ts | 34 ++++++++ .../engine/src/services/videoFrameInjector.ts | 14 ++- .../src/services/renderOrchestrator.ts | 12 ++- 7 files changed, 175 insertions(+), 12 deletions(-) diff --git a/packages/engine/src/services/extractionCache.test.ts b/packages/engine/src/services/extractionCache.test.ts index de327e55..09800925 100644 --- a/packages/engine/src/services/extractionCache.test.ts +++ b/packages/engine/src/services/extractionCache.test.ts @@ -36,7 +36,9 @@ describe("computeExtractionCacheKey", () => { it("prefixes the key with the schema version so a future on-disk format change cannot collide", () => { const key = computeExtractionCacheKey(base); - expect(key.startsWith("v1-")).toBe(true); + // Shape: `v-` — we don't pin N here so schema bumps (e.g. v1→v2 + // for the WebP change) stay local to the extractionCache module. + expect(key).toMatch(/^v\d+-[0-9a-f]{32}$/); }); it.each([ diff --git a/packages/engine/src/services/extractionCache.ts b/packages/engine/src/services/extractionCache.ts index 55cf8c79..baed73cb 100644 --- a/packages/engine/src/services/extractionCache.ts +++ b/packages/engine/src/services/extractionCache.ts @@ -36,11 +36,14 @@ import { join } from "path"; /** * Schema version embedded in every cache key. Bump whenever the on-disk - * format of extracted frames changes in a way that breaks older entries - * (e.g. the upcoming WebP-unified-format PR changes the frame extension - * and MIME handling, so it will bump this to 2). + * format of extracted frames changes in a way that breaks older entries. + * + * - v1: jpg / png output from the pre-WebP extractor. + * - v2: adds webp as an extraction format. A v2 key for webp content can + * never collide with a v1 jpg/png key, so both generations can coexist + * under the same cache root during migration without cross-serving. */ -const CACHE_SCHEMA_VERSION = 1; +const CACHE_SCHEMA_VERSION = 2; /** * Sentinel filename written inside each completed cache entry directory. @@ -62,7 +65,7 @@ export interface ExtractionCacheKeyInputs { /** Target fps for extracted frames. */ fps: number; /** Extracted frame format ("jpg" or "png"). */ - format: "jpg" | "png"; + format: "jpg" | "png" | "webp"; } /** @@ -143,7 +146,7 @@ export interface CacheHit { export function lookupCacheEntry( cacheRoot: string, key: string, - format: "jpg" | "png", + format: "jpg" | "png" | "webp", ): CacheHit | null { const { dir, sentinel } = resolveCacheEntryPaths(cacheRoot, key); if (!existsSync(sentinel)) return null; diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index a23a1d53..9a8133aa 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -495,6 +495,91 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames with extraction cache", () = expect(result2.extracted[0]?.totalFrames).toBe(result1.extracted[0]?.totalFrames); }, 60_000); + it("writes webp frames and reuses them on a second call", async () => { + const cacheRoot = join(FIXTURE_DIR, "cache-webp"); + mkdirSync(cacheRoot, { recursive: true }); + + const video: VideoElement = { + id: "vid", + src: SOURCE, + start: 0, + end: 1, + mediaStart: 0, + hasAudio: false, + }; + + const out1 = join(FIXTURE_DIR, "webp-1"); + mkdirSync(out1, { recursive: true }); + const r1 = await extractAllVideoFrames( + [video], + FIXTURE_DIR, + { fps: 30, outputDir: out1, format: "webp" }, + undefined, + { extractCacheDir: cacheRoot }, + ); + expect(r1.errors).toEqual([]); + expect(r1.phaseBreakdown.cacheMisses).toBe(1); + // Cache hit on second call confirms the on-disk files are valid webp + // (lookupCacheEntry filters by extension); assert the file list directly + // too so a broken libwebp encode would surface as an empty dir. + const cacheDir = r1.extracted[0]?.outputDir; + expect(cacheDir).toBeDefined(); + const frames = readdirSync(cacheDir!).filter((f) => f.endsWith(".webp")); + expect(frames.length).toBeGreaterThan(0); + + const out2 = join(FIXTURE_DIR, "webp-2"); + mkdirSync(out2, { recursive: true }); + const r2 = await extractAllVideoFrames( + [video], + FIXTURE_DIR, + { fps: 30, outputDir: out2, format: "webp" }, + undefined, + { extractCacheDir: cacheRoot }, + ); + expect(r2.phaseBreakdown.cacheHits).toBe(1); + expect(r2.phaseBreakdown.cacheMisses).toBe(0); + expect(r2.extracted[0]?.totalFrames).toBe(r1.extracted[0]?.totalFrames); + }, 60_000); + + it("does not cross-serve a jpg cache entry as webp (format is part of the key)", async () => { + const cacheRoot = join(FIXTURE_DIR, "cache-cross-format"); + mkdirSync(cacheRoot, { recursive: true }); + + const video: VideoElement = { + id: "vid", + src: SOURCE, + start: 0, + end: 1, + mediaStart: 0, + hasAudio: false, + }; + + // Populate the cache with a jpg entry first. + const outJpg = join(FIXTURE_DIR, "cross-jpg"); + mkdirSync(outJpg, { recursive: true }); + const rJpg = await extractAllVideoFrames( + [video], + FIXTURE_DIR, + { fps: 30, outputDir: outJpg, format: "jpg" }, + undefined, + { extractCacheDir: cacheRoot }, + ); + expect(rJpg.phaseBreakdown.cacheMisses).toBe(1); + + // Same inputs but format=webp — must be a fresh miss (different key). + const outWebp = join(FIXTURE_DIR, "cross-webp"); + mkdirSync(outWebp, { recursive: true }); + const rWebp = await extractAllVideoFrames( + [video], + FIXTURE_DIR, + { fps: 30, outputDir: outWebp, format: "webp" }, + undefined, + { extractCacheDir: cacheRoot }, + ); + expect(rWebp.phaseBreakdown.cacheHits).toBe(0); + expect(rWebp.phaseBreakdown.cacheMisses).toBe(1); + }, 60_000); + it("misses again when fps changes (keyed on fps)", async () => { const cacheRoot = join(FIXTURE_DIR, "cache-fps"); mkdirSync(cacheRoot, { recursive: true }); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 2a26ef1d..be1226bf 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -53,7 +53,17 @@ export interface ExtractionOptions { fps: number; outputDir: string; quality?: number; - format?: "jpg" | "png"; + /** + * On-disk frame format. + * + * - `"webp"` (recommended) — smaller files than jpg at equivalent quality, + * handles alpha natively, decoded natively by Chrome. Default for the + * producer's SDR extraction path. + * - `"jpg"` (legacy default) — opaque only, smallest for no-alpha content. + * - `"png"` — lossless, retained for external callers and alpha paths + * that specifically want PNG semantics. + */ + format?: "jpg" | "png" | "webp"; } export interface ExtractionPhaseBreakdown { @@ -220,8 +230,15 @@ export async function extractVideoFramesRange( vfFilters.push(`fps=${fps}`); args.push("-vf", vfFilters.join(",")); - args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"); - if (format === "png") args.push("-compression_level", "6"); + if (format === "webp") { + // libwebp: `-quality` is 0-100, higher = better (inverse of JPEG's -q:v). + // Lossy mode by default — near-lossless at quality=95 but ~5-10x smaller + // than PNG and typically smaller than a visually-equivalent JPEG. + args.push("-c:v", "libwebp", "-quality", String(quality), "-lossless", "0"); + } else { + args.push("-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"); + if (format === "png") args.push("-compression_level", "6"); + } args.push("-y", outputPattern); return new Promise((resolve, reject) => { diff --git a/packages/engine/src/services/videoFrameInjector.test.ts b/packages/engine/src/services/videoFrameInjector.test.ts index 6b9dabf6..36e84ec0 100644 --- a/packages/engine/src/services/videoFrameInjector.test.ts +++ b/packages/engine/src/services/videoFrameInjector.test.ts @@ -95,3 +95,37 @@ describe("InjectorCacheStats via frame-dataURI LRU", () => { await expect(cache.get(FRAME_A)).resolves.toMatch(/^data:image\/jpeg;base64,/); }); }); + +// The injector's MIME detection drives what Chrome does with the data URI — +// a mis-tagged WebP frame decodes as a broken image. This block verifies the +// mapping through the LRU's public surface (the only way MIME-tagged URIs +// leave the module). +describe("frame path → MIME type tagging", () => { + const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-injector-mime-")); + const WEBP = join(FIXTURE_DIR, "frame_00001.webp"); + const PNG = join(FIXTURE_DIR, "frame_00001.png"); + const JPG = join(FIXTURE_DIR, "frame_00001.jpg"); + const UNKNOWN = join(FIXTURE_DIR, "frame_00001.xyz"); + + beforeAll(() => { + // Content isn't validated, only the extension drives MIME selection. + writeFileSync(WEBP, Buffer.from("WEBP")); + writeFileSync(PNG, Buffer.from([0x89, 0x50, 0x4e, 0x47])); + writeFileSync(JPG, Buffer.from([0xff, 0xd8, 0xff, 0xe0])); + writeFileSync(UNKNOWN, Buffer.from("xx")); + }); + + afterAll(() => { + rmSync(FIXTURE_DIR, { recursive: true, force: true }); + }); + + it.each([ + [".webp path", WEBP, /^data:image\/webp;base64,/], + [".png path", PNG, /^data:image\/png;base64,/], + [".jpg path", JPG, /^data:image\/jpeg;base64,/], + ["unknown extension falls back to jpeg", UNKNOWN, /^data:image\/jpeg;base64,/], + ])("tags %s correctly", async (_label, path, mimePattern) => { + const cache = createCache(32); + await expect(cache.get(path)).resolves.toMatch(mimePattern); + }); +}); diff --git a/packages/engine/src/services/videoFrameInjector.ts b/packages/engine/src/services/videoFrameInjector.ts index a23e913c..12e825b4 100644 --- a/packages/engine/src/services/videoFrameInjector.ts +++ b/packages/engine/src/services/videoFrameInjector.ts @@ -36,6 +36,18 @@ export function createEmptyInjectorCacheStats(): InjectorCacheStats { return { hits: 0, misses: 0, inFlightCoalesced: 0, peakEntries: 0 }; } +/** + * Map a frame file path to the MIME type that should accompany its data URI. + * Chrome decodes webp, png, and jpeg natively inside ``. + * Unknown extensions default to JPEG — the legacy pre-WebP behavior — so + * callers that produce non-standard filenames don't regress. + */ +function mimeTypeForFramePath(framePath: string): string { + if (framePath.endsWith(".webp")) return "image/webp"; + if (framePath.endsWith(".png")) return "image/png"; + return "image/jpeg"; +} + /** * Exported for unit tests only — not part of the package's public API. * Used to validate the LRU / stats behavior without spinning up a Chrome page. @@ -83,7 +95,7 @@ function createFrameDataUriCache(cacheLimit: number, stats?: InjectorCacheStats) const pending = fs .readFile(framePath) .then((frameData) => { - const mimeType = framePath.endsWith(".png") ? "image/png" : "image/jpeg"; + const mimeType = mimeTypeForFramePath(framePath); const dataUri = `data:${mimeType};base64,${frameData.toString("base64")}`; return remember(framePath, dataUri); }) diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 31401b3f..00017c9c 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1112,7 +1112,17 @@ export async function executeRenderJob( extractionResult = await extractAllVideoFrames( composition.videos, projectDir, - { fps: job.config.fps, outputDir: join(workDir, "video-frames") }, + { + fps: job.config.fps, + outputDir: join(workDir, "video-frames"), + // WebP gives smaller files than JPEG at equivalent visual quality + // AND handles alpha natively — used by the producer as the single + // extraction format so we don't have a jpg/png split. HDR pixel + // paths still produce PNG on their own out-of-band codepath (see + // the HDR pre-extract block below); this setting only affects + // SDR frames that flow through the `` injector. + format: "webp", + }, abortSignal, // Forward extractCacheDir (when configured) so repeat renders of // the same source+window+fps+format pair skip Phase 3 entirely.