feat(engine): add HDR two-pass compositing — DOM layer + native HLG video

.gitignore

@@ -29,8 +29,14 @@ packages/producer/tests/*/failures/
 
 # Rendered output (not test fixtures — those use git LFS)
 output/
+renders/
 !packages/producer/tests/*/output/
 
+# Composition source media (large binaries)
+compositions/**/*.mp4
+compositions/**/*.mov
+compositions/**/*.MOV
+
 # npm pack artifacts
 *.tgz
 

packages/engine/src/index.ts

@@ -77,6 +77,8 @@ export {
   injectVideoFramesBatch,
   syncVideoFrameVisibility,
   cdpSessionCache,
+  initTransparentBackground,
+  captureAlphaPng,
   type BeginFrameResult,
 } from "./services/screenshotService.js";
 
@@ -154,15 +156,25 @@ export {
 
 export { downloadToTemp, isHttpUrl } from "./utils/urlDownloader.js";
 
+export { decodePng, decodePngToRgb48le, blitRgba8OverRgb48le } from "./utils/alphaBlit.js";
+
 export {
   initHdrReadback,
   uploadAndReadbackHdrFrame,
-  convertHdrFrameToRgb48le,
   float16ToPqRgb,
   buildHdrChromeArgs,
   launchHdrBrowser,
 } from "./services/hdrCapture.js";
 
+export { captureScreenshotWithAlpha } from "./services/screenshotService.js";
+
+export {
+  hideVideoElements,
+  showVideoElements,
+  queryVideoElementBounds,
+  type VideoElementBounds,
+} from "./services/videoFrameInjector.js";
+
 export {
   isHdrColorSpace,
   detectTransfer,

packages/engine/src/services/screenshotService.ts

@@ -133,6 +133,78 @@ export async function pageScreenshotCapture(page: Page, options: CaptureOptions)
   return Buffer.from(result.data, "base64");
 }
 
+/**
+ * Capture a screenshot with transparent background (PNG + alpha channel).
+ *
+ * Used in the two-pass HDR compositing pipeline — captures DOM content
+ * (text, graphics, SDR overlays) with transparency where the background shows,
+ * so it can be overlaid on top of native HDR video frames in FFmpeg.
+ *
+ * Sets and restores the background color override on every call. For sessions
+ * that capture many frames, prefer calling initTransparentBackground() once
+ * at session init, then captureAlphaPng() per frame to avoid the 2× CDP
+ * round-trip overhead.
+ */
+export async function captureScreenshotWithAlpha(
+  page: Page,
+  width: number,
+  height: number,
+): Promise<Buffer> {
+  const client = await getCdpSession(page);
+  // Force transparent background so the screenshot has a real alpha channel
+  await client.send("Emulation.setDefaultBackgroundColorOverride", {
+    color: { r: 0, g: 0, b: 0, a: 0 },
+  });
+  const result = await client.send("Page.captureScreenshot", {
+    format: "png",
+    fromSurface: true,
+    captureBeyondViewport: false,
+    optimizeForSpeed: false, // must be false to preserve alpha
+    clip: { x: 0, y: 0, width, height, scale: 1 },
+  });
+  // Restore opaque background for subsequent captures
+  await client.send("Emulation.setDefaultBackgroundColorOverride", {});
+  return Buffer.from(result.data, "base64");
+}
+
+/**
+ * Set the page background to transparent once for a dedicated HDR DOM session.
+ *
+ * Call this once after session initialization. Then use captureAlphaPng() per
+ * frame instead of captureScreenshotWithAlpha() to skip the per-frame CDP
+ * background override round-trips.
+ *
+ * Only use on sessions that are exclusively dedicated to transparent capture
+ * (e.g., the HDR two-pass DOM layer session) — the background will stay
+ * transparent for the lifetime of the session.
+ */
+export async function initTransparentBackground(page: Page): Promise<void> {
+  const client = await getCdpSession(page);
+  await client.send("Emulation.setDefaultBackgroundColorOverride", {
+    color: { r: 0, g: 0, b: 0, a: 0 },
+  });
+}
+
+/**
+ * Capture a transparent-background PNG screenshot without setting the
+ * background color override. Requires initTransparentBackground() to have
+ * been called once on this session.
+ *
+ * Faster than captureScreenshotWithAlpha() for per-frame use in the HDR
+ * two-pass compositing loop.
+ */
+export async function captureAlphaPng(page: Page, width: number, height: number): Promise<Buffer> {
+  const client = await getCdpSession(page);
+  const result = await client.send("Page.captureScreenshot", {
+    format: "png",
+    fromSurface: true,
+    captureBeyondViewport: false,
+    optimizeForSpeed: false, // must be false to preserve alpha
+    clip: { x: 0, y: 0, width, height, scale: 1 },
+  });
+  return Buffer.from(result.data, "base64");
+}
+
 export async function injectVideoFramesBatch(
   page: Page,
   updates: Array<{ videoId: string; dataUri: string }>,

packages/engine/src/services/streamingEncoder.ts

@@ -121,7 +121,13 @@ function buildStreamingArgs(
   // Input args: pipe from stdin
   const args: string[] = [];
   if (options.rawInputFormat) {
-    // Raw pixel input (HDR PQ-encoded rgb48le)
+    // Raw pixel input (HLG/PQ-encoded rgb48le from FFmpeg extraction).
+    // Tag the input with the correct color space so FFmpeg uses the right
+    // YUV matrix when converting rgb48le → yuv420p10le for encoding.
+    // Without these tags FFmpeg assumes bt709 and applies the wrong matrix.
+    const hdrTransfer = options.hdr?.transfer;
+    const inputColorTrc =
+      hdrTransfer === "pq" ? "smpte2084" : hdrTransfer === "hlg" ? "arib-std-b67" : undefined;
     args.push(
       "-f",
       "rawvideo",
@@ -131,9 +137,18 @@ function buildStreamingArgs(
       `${options.width}x${options.height}`,
       "-framerate",
       String(fps),
-      "-i",
-      "-",
     );
+    if (inputColorTrc) {
+      args.push(
+        "-color_primaries",
+        "bt2020",
+        "-color_trc",
+        inputColorTrc,
+        "-colorspace",
+        "bt2020nc",
+      );
+    }
+    args.push("-i", "-");
   } else {
     const inputCodec = imageFormat === "png" ? "png" : "mjpeg";
     args.push("-f", "image2pipe", "-vcodec", inputCodec, "-framerate", String(fps), "-i", "-");

packages/engine/src/services/videoFrameInjector.ts

@@ -117,3 +117,111 @@ export function createVideoFrameInjector(
     }
   };
 }
+
+// ── HDR compositing utilities ─────────────────────────────────────────────────
+
+/**
+ * Bounds and transform of a video element, queried from Chrome each frame.
+ * Used by the two-pass HDR compositing pipeline to position native HDR frames.
+ */
+export interface VideoElementBounds {
+  videoId: string;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  opacity: number;
+  /** CSS transform matrix as a DOMMatrix-compatible string, e.g. "matrix(1,0,0,1,0,0)" */
+  transform: string;
+  zIndex: number;
+  visible: boolean;
+}
+
+/**
+ * Hide specific video elements by ID. Used in Pass 1 of the HDR pipeline so
+ * Chrome screenshots only contain DOM content (text, overlays) with transparent
+ * holes where the HDR videos go.
+ */
+export async function hideVideoElements(page: Page, videoIds: string[]): Promise<void> {
+  if (videoIds.length === 0) return;
+  await page.evaluate((ids: string[]) => {
+    for (const id of ids) {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (el) {
+        el.style.setProperty("visibility", "hidden", "important");
+        el.style.setProperty("opacity", "0", "important");
+        // Also hide the injected render frame image if present
+        const img = document.getElementById(`__render_frame_${id}__`);
+        if (img) img.style.setProperty("visibility", "hidden", "important");
+      }
+    }
+  }, videoIds);
+}
+
+/**
+ * Restore visibility of video elements after a DOM screenshot.
+ */
+export async function showVideoElements(page: Page, videoIds: string[]): Promise<void> {
+  if (videoIds.length === 0) return;
+  await page.evaluate((ids: string[]) => {
+    for (const id of ids) {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (el) {
+        el.style.removeProperty("visibility");
+        el.style.removeProperty("opacity");
+        const img = document.getElementById(`__render_frame_${id}__`);
+        if (img) img.style.removeProperty("visibility");
+      }
+    }
+  }, videoIds);
+}
+
+/**
+ * Query the current bounds, transform, and visibility of video elements.
+ * Called after seeking (so GSAP has moved things) but before the screenshot.
+ */
+export async function queryVideoElementBounds(
+  page: Page,
+  videoIds: string[],
+): Promise<VideoElementBounds[]> {
+  if (videoIds.length === 0) return [];
+  return page.evaluate((ids: string[]): VideoElementBounds[] => {
+    return ids.map((id) => {
+      const el = document.getElementById(id) as HTMLVideoElement | null;
+      if (!el) {
+        return {
+          videoId: id,
+          x: 0,
+          y: 0,
+          width: 0,
+          height: 0,
+          opacity: 0,
+          transform: "none",
+          zIndex: 0,
+          visible: false,
+        };
+      }
+      const rect = el.getBoundingClientRect();
+      const style = window.getComputedStyle(el);
+      const zIndex = parseInt(style.zIndex) || 0;
+      const opacity = parseFloat(style.opacity) || 1;
+      const transform = style.transform || "none";
+      const visible =
+        style.visibility !== "hidden" &&
+        style.display !== "none" &&
+        rect.width > 0 &&
+        rect.height > 0;
+      return {
+        videoId: id,
+        x: Math.round(rect.x),
+        y: Math.round(rect.y),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        opacity,
+        transform,
+        zIndex,
+        visible,
+      };
+    });
+  }, videoIds);
+}