From eb1087968860f62d083a057713cb0002ebf25ab7 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 16 Apr 2026 22:58:31 -0700 Subject: [PATCH] feat(hdr): add z-ordered multi-layer compositing with PQ support Per-frame z-order analysis groups elements into DOM and HDR layers, composited bottom-to-top. Adjacent DOM elements merge into single screenshots. PQ (HDR10/smpte2084) support via sRGB-to-PQ LUT with 203-nit SDR reference white. queryElementStacking walks DOM for effective z-index, groupIntoLayers splits on HDR/DOM boundaries. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/engine/src/index.ts | 12 +- .../engine/src/services/videoFrameInjector.ts | 76 +++++++ packages/engine/src/utils/alphaBlit.test.ts | 196 ++++++++++++------ packages/engine/src/utils/alphaBlit.ts | 184 +++++++++++----- .../engine/src/utils/layerCompositor.test.ts | 100 +++++++++ packages/engine/src/utils/layerCompositor.ts | 47 +++++ .../src/services/renderOrchestrator.ts | 131 +++++++----- 7 files changed, 577 insertions(+), 169 deletions(-) create mode 100644 packages/engine/src/utils/layerCompositor.test.ts create mode 100644 packages/engine/src/utils/layerCompositor.ts diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index cba8278a..6050ac5c 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -156,7 +156,15 @@ export { export { downloadToTemp, isHttpUrl } from "./utils/urlDownloader.js"; -export { decodePng, decodePngToRgb48le, blitRgba8OverRgb48le } from "./utils/alphaBlit.js"; +export { + decodePng, + decodePngToRgb48le, + blitRgba8OverRgb48le, + blitRgb48leRegion, + getSrgbToHdrLut, +} from "./utils/alphaBlit.js"; + +export { groupIntoLayers, type CompositeLayer } from "./utils/layerCompositor.js"; export { initHdrReadback, @@ -172,7 +180,9 @@ export { hideVideoElements, showVideoElements, queryVideoElementBounds, + queryElementStacking, type VideoElementBounds, + type ElementStackingInfo, } from "./services/videoFrameInjector.js"; export { diff --git a/packages/engine/src/services/videoFrameInjector.ts b/packages/engine/src/services/videoFrameInjector.ts index 668cee44..758a98f1 100644 --- a/packages/engine/src/services/videoFrameInjector.ts +++ b/packages/engine/src/services/videoFrameInjector.ts @@ -225,3 +225,79 @@ export async function queryVideoElementBounds( }); }, videoIds); } + +/** + * Stacking info for a single timed element, used by the z-ordered layer compositor. + */ +export interface ElementStackingInfo { + id: string; + zIndex: number; + x: number; + y: number; + width: number; + height: number; + opacity: number; + visible: boolean; + isHdr: boolean; +} + +/** + * Query Chrome for ALL timed elements' stacking context. + * Returns z-index, bounds, opacity, and whether each element is a native HDR video. + * + * Queries every element with `data-start` (not just videos) so the layer compositor + * can determine z-ordering between DOM content and HDR video elements. + */ +export async function queryElementStacking( + page: Page, + nativeHdrVideoIds: Set, +): Promise { + const hdrIds = Array.from(nativeHdrVideoIds); + return page.evaluate((hdrIdList: string[]): ElementStackingInfo[] => { + const hdrSet = new Set(hdrIdList); + const elements = document.querySelectorAll("[data-start]"); + const results: ElementStackingInfo[] = []; + + // Walk up the DOM to find the effective z-index from the nearest + // positioned ancestor with a z-index. CSS z-index only applies to + // positioned elements; video elements inside positioned wrappers + // inherit the wrapper's stacking context. + function getEffectiveZIndex(node: Element): number { + let current: Element | null = node; + while (current) { + const cs = window.getComputedStyle(current); + const pos = cs.position; + const z = parseInt(cs.zIndex); + if (!Number.isNaN(z) && pos !== "static") return z; + current = current.parentElement; + } + return 0; + } + + for (const el of elements) { + const id = el.id; + if (!id) continue; + const rect = el.getBoundingClientRect(); + const style = window.getComputedStyle(el); + const zIndex = getEffectiveZIndex(el); + const opacity = parseFloat(style.opacity) || 1; + const visible = + style.visibility !== "hidden" && + style.display !== "none" && + rect.width > 0 && + rect.height > 0; + results.push({ + id, + zIndex, + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.round(rect.width), + height: Math.round(rect.height), + opacity, + visible, + isHdr: hdrSet.has(id), + }); + } + return results; + }, hdrIds); +} diff --git a/packages/engine/src/utils/alphaBlit.test.ts b/packages/engine/src/utils/alphaBlit.test.ts index 78b7bcb0..0fb0fb6f 100644 --- a/packages/engine/src/utils/alphaBlit.test.ts +++ b/packages/engine/src/utils/alphaBlit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { deflateSync } from "zlib"; -import { decodePng, blitRgba8OverRgb48le } from "./alphaBlit.js"; +import { decodePng, blitRgba8OverRgb48le, blitRgb48leRegion } from "./alphaBlit.js"; // ── PNG construction helpers ───────────────────────────────────────────────── @@ -189,67 +189,68 @@ function makeDomRgba( } describe("blitRgba8OverRgb48le", () => { - it("fully transparent DOM: HDR pixel passes through unchanged", () => { - const hdr = makeHdrFrame(1, 1, 32000, 40000, 50000); + it("fully transparent DOM: canvas unchanged", () => { + const canvas = makeHdrFrame(1, 1, 32000, 40000, 50000); const dom = makeDomRgba(1, 1, 255, 0, 0, 0); // red but alpha=0 - const out = blitRgba8OverRgb48le(dom, hdr, 1, 1); + blitRgba8OverRgb48le(dom, canvas, 1, 1); - expect(out.readUInt16LE(0)).toBe(32000); - expect(out.readUInt16LE(2)).toBe(40000); - expect(out.readUInt16LE(4)).toBe(50000); + expect(canvas.readUInt16LE(0)).toBe(32000); + expect(canvas.readUInt16LE(2)).toBe(40000); + expect(canvas.readUInt16LE(4)).toBe(50000); }); - it("fully opaque DOM: sRGB→HLG converted values", () => { - const hdr = makeHdrFrame(1, 1, 10000, 20000, 30000); + it("fully opaque DOM: sRGB→HLG converted values overwrite canvas", () => { + const canvas = makeHdrFrame(1, 1, 10000, 20000, 30000); const dom = makeDomRgba(1, 1, 255, 128, 0, 255); // R=255, G=128, B=0, full opaque - const out = blitRgba8OverRgb48le(dom, hdr, 1, 1); + blitRgba8OverRgb48le(dom, canvas, 1, 1); // sRGB 255 → HLG 65535 (white maps to white) // sRGB 128 → HLG ~46484 (mid-gray maps higher due to HLG OETF) // sRGB 0 → HLG 0 - expect(out.readUInt16LE(0)).toBe(65535); - expect(out.readUInt16LE(2)).toBeGreaterThan(40000); // HLG mid-gray > sRGB mid-gray - expect(out.readUInt16LE(2)).toBeLessThan(50000); - expect(out.readUInt16LE(4)).toBe(0); + expect(canvas.readUInt16LE(0)).toBe(65535); + expect(canvas.readUInt16LE(2)).toBeGreaterThan(40000); // HLG mid-gray > sRGB mid-gray + expect(canvas.readUInt16LE(2)).toBeLessThan(50000); + expect(canvas.readUInt16LE(4)).toBe(0); }); it("sRGB→HLG: black stays black, white stays white", () => { - const hdr = makeHdrFrame(1, 1, 0, 0, 0); + const canvasBlack = makeHdrFrame(1, 1, 0, 0, 0); const domBlack = makeDomRgba(1, 1, 0, 0, 0, 255); - const outBlack = blitRgba8OverRgb48le(domBlack, hdr, 1, 1); - expect(outBlack.readUInt16LE(0)).toBe(0); + blitRgba8OverRgb48le(domBlack, canvasBlack, 1, 1); + expect(canvasBlack.readUInt16LE(0)).toBe(0); + const canvasWhite = makeHdrFrame(1, 1, 0, 0, 0); const domWhite = makeDomRgba(1, 1, 255, 255, 255, 255); - const outWhite = blitRgba8OverRgb48le(domWhite, hdr, 1, 1); - expect(outWhite.readUInt16LE(0)).toBe(65535); + blitRgba8OverRgb48le(domWhite, canvasWhite, 1, 1); + expect(canvasWhite.readUInt16LE(0)).toBe(65535); }); - it("50% alpha: HLG-converted DOM blended with HDR", () => { + it("50% alpha: HLG-converted DOM blended with canvas", () => { // DOM: white (255, 255, 255) at alpha=128 (~50%) - // HDR: black (0, 0, 0) - const hdr = makeHdrFrame(1, 1, 0, 0, 0); + // Canvas: black (0, 0, 0) + const canvas = makeHdrFrame(1, 1, 0, 0, 0); const dom = makeDomRgba(1, 1, 255, 255, 255, 128); - const out = blitRgba8OverRgb48le(dom, hdr, 1, 1); + blitRgba8OverRgb48le(dom, canvas, 1, 1); // sRGB 255 → HLG 65535, blended 50/50 with black const alpha = 128 / 255; const expectedR = Math.round(65535 * alpha); - expect(out.readUInt16LE(0)).toBeCloseTo(expectedR, -1); + expect(canvas.readUInt16LE(0)).toBeCloseTo(expectedR, -1); }); - it("50% alpha blends with non-zero HDR", () => { - // DOM: 8-bit red=200, HDR: 16-bit red=32000, alpha=128 - const hdr = makeHdrFrame(1, 1, 32000, 0, 0); + it("50% alpha blends with non-zero canvas", () => { + // DOM: 8-bit red=200, canvas: 16-bit red=32000, alpha=128 + const canvas = makeHdrFrame(1, 1, 32000, 0, 0); const dom = makeDomRgba(1, 1, 200, 0, 0, 128); - const out = blitRgba8OverRgb48le(dom, hdr, 1, 1); + blitRgba8OverRgb48le(dom, canvas, 1, 1); - // sRGB 200 → HLG value, blended ~50/50 with HDR red=32000 + // sRGB 200 → HLG value, blended ~50/50 with canvas red=32000 // Result should be higher than 32000 (pulled up by the HLG-converted DOM value) - expect(out.readUInt16LE(0)).toBeGreaterThan(32000); + expect(canvas.readUInt16LE(0)).toBeGreaterThan(32000); }); it("handles a 2x2 frame correctly pixel-by-pixel", () => { - const hdr = makeHdrFrame(2, 2, 0, 0, 0); + const canvas = makeHdrFrame(2, 2, 0, 0, 0); // First pixel: fully opaque white. Others: fully transparent. const dom = new Uint8Array(2 * 2 * 4); dom[0] = 255; @@ -258,31 +259,107 @@ describe("blitRgba8OverRgb48le", () => { dom[3] = 255; // pixel 0: opaque white // pixels 1-3: alpha=0 (transparent) - const out = blitRgba8OverRgb48le(dom, hdr, 2, 2); + blitRgba8OverRgb48le(dom, canvas, 2, 2); // Pixel 0: sRGB white → HLG white (65535) - expect(out.readUInt16LE(0)).toBe(65535); - expect(out.readUInt16LE(2)).toBe(65535); - expect(out.readUInt16LE(4)).toBe(65535); - - // Pixel 1: transparent DOM → HDR black (0, 0, 0) - expect(out.readUInt16LE(6)).toBe(0); - expect(out.readUInt16LE(8)).toBe(0); - expect(out.readUInt16LE(10)).toBe(0); + expect(canvas.readUInt16LE(0)).toBe(65535); + expect(canvas.readUInt16LE(2)).toBe(65535); + expect(canvas.readUInt16LE(4)).toBe(65535); + + // Pixel 1: transparent DOM → canvas black (0, 0, 0) unchanged + expect(canvas.readUInt16LE(6)).toBe(0); + expect(canvas.readUInt16LE(8)).toBe(0); + expect(canvas.readUInt16LE(10)).toBe(0); }); +}); + +describe("blitRgba8OverRgb48le with PQ transfer", () => { + it("PQ: black stays black, white maps to PQ white", () => { + const canvasBlack = makeHdrFrame(1, 1, 0, 0, 0); + const domBlack = makeDomRgba(1, 1, 0, 0, 0, 255); + blitRgba8OverRgb48le(domBlack, canvasBlack, 1, 1, "pq"); + expect(canvasBlack.readUInt16LE(0)).toBe(0); + + const canvasWhite = makeHdrFrame(1, 1, 0, 0, 0); + const domWhite = makeDomRgba(1, 1, 255, 255, 255, 255); + blitRgba8OverRgb48le(domWhite, canvasWhite, 1, 1, "pq"); + // PQ white at SDR 203 nits is NOT 65535 (that's 10000 nits) + // SDR white in PQ ≈ 58% signal → ~38000 + const pqWhite = canvasWhite.readUInt16LE(0); + expect(pqWhite).toBeGreaterThan(30000); + expect(pqWhite).toBeLessThan(45000); + }); + + it("PQ mid-gray differs from HLG mid-gray", () => { + const canvasHlg = makeHdrFrame(1, 1, 0, 0, 0); + const canvasPq = makeHdrFrame(1, 1, 0, 0, 0); + const dom = makeDomRgba(1, 1, 128, 128, 128, 255); + + blitRgba8OverRgb48le(dom, canvasHlg, 1, 1, "hlg"); + blitRgba8OverRgb48le(dom, canvasPq, 1, 1, "pq"); + + const hlgVal = canvasHlg.readUInt16LE(0); + const pqVal = canvasPq.readUInt16LE(0); + // PQ and HLG encode mid-gray differently + expect(hlgVal).not.toBe(pqVal); + // Both should be non-zero + expect(hlgVal).toBeGreaterThan(0); + expect(pqVal).toBeGreaterThan(0); + }); +}); - it("output buffer has correct size", () => { - const hdr = makeHdrFrame(4, 3, 0, 0, 0); - const dom = makeDomRgba(4, 3, 0, 0, 0, 0); - const out = blitRgba8OverRgb48le(dom, hdr, 4, 3); - expect(out.length).toBe(4 * 3 * 6); +// ── blitRgb48leRegion tests ────────────────────────────────────────────────── + +describe("blitRgb48leRegion", () => { + it("copies a region at position (0,0) — full overlap", () => { + const canvas = Buffer.alloc(4 * 4 * 6); // 4x4 black + const source = makeHdrFrame(2, 2, 10000, 20000, 30000); + blitRgb48leRegion(canvas, source, 0, 0, 2, 2, 4); + expect(canvas.readUInt16LE(0)).toBe(10000); + expect(canvas.readUInt16LE(2)).toBe(20000); + expect(canvas.readUInt16LE(4)).toBe(30000); + expect(canvas.readUInt16LE(2 * 6)).toBe(0); + }); + + it("copies a region at offset position", () => { + const canvas = Buffer.alloc(4 * 4 * 6); + const source = makeHdrFrame(2, 2, 50000, 40000, 30000); + blitRgb48leRegion(canvas, source, 1, 1, 2, 2, 4); + expect(canvas.readUInt16LE(0)).toBe(0); + const off = (1 * 4 + 1) * 6; + expect(canvas.readUInt16LE(off)).toBe(50000); + }); + + it("clips when region extends beyond canvas edge", () => { + const canvas = Buffer.alloc(4 * 4 * 6); + const source = makeHdrFrame(3, 3, 10000, 20000, 30000); + blitRgb48leRegion(canvas, source, 2, 2, 3, 3, 4); + const off = (2 * 4 + 2) * 6; + expect(canvas.readUInt16LE(off)).toBe(10000); + const off2 = (3 * 4 + 3) * 6; + expect(canvas.readUInt16LE(off2)).toBe(10000); + expect(canvas.length).toBe(4 * 4 * 6); + }); + + it("applies opacity when provided", () => { + const canvas = Buffer.alloc(1 * 1 * 6); + const source = makeHdrFrame(1, 1, 40000, 40000, 40000); + blitRgb48leRegion(canvas, source, 0, 0, 1, 1, 1, 0.5); + expect(canvas.readUInt16LE(0)).toBe(20000); + }); + + it("no-op for zero-size region", () => { + const canvas = Buffer.alloc(4 * 4 * 6); + const source = makeHdrFrame(2, 2, 10000, 20000, 30000); + blitRgb48leRegion(canvas, source, 0, 0, 0, 0, 4); + expect(canvas.readUInt16LE(0)).toBe(0); }); }); // ── Round-trip test: decodePng → blitRgba8OverRgb48le ──────────────────────── describe("decodePng + blitRgba8OverRgb48le integration", () => { - it("transparent PNG overlay leaves HDR frame untouched", () => { + it("transparent PNG overlay leaves canvas untouched", () => { const width = 2; const height = 2; @@ -291,20 +368,19 @@ describe("decodePng + blitRgba8OverRgb48le integration", () => { const png = makePng(width, height, pixels); const { data: domRgba } = decodePng(png); - // HDR frame with known values - const hdr = makeHdrFrame(width, height, 10000, 20000, 30000); - - const out = blitRgba8OverRgb48le(domRgba, hdr, width, height); + // Canvas pre-filled with known HDR values + const canvas = makeHdrFrame(width, height, 10000, 20000, 30000); + blitRgba8OverRgb48le(domRgba, canvas, width, height); - // All pixels should be unchanged HDR + // All pixels should be unchanged for (let i = 0; i < width * height; i++) { - expect(out.readUInt16LE(i * 6 + 0)).toBe(10000); - expect(out.readUInt16LE(i * 6 + 2)).toBe(20000); - expect(out.readUInt16LE(i * 6 + 4)).toBe(30000); + expect(canvas.readUInt16LE(i * 6 + 0)).toBe(10000); + expect(canvas.readUInt16LE(i * 6 + 2)).toBe(20000); + expect(canvas.readUInt16LE(i * 6 + 4)).toBe(30000); } }); - it("fully opaque PNG overlay covers all HDR pixels (sRGB→HLG)", () => { + it("fully opaque PNG overlay overwrites all canvas pixels (sRGB→HLG)", () => { const width = 2; const height = 2; @@ -315,14 +391,14 @@ describe("decodePng + blitRgba8OverRgb48le integration", () => { const png = makePng(width, height, pixels); const { data: domRgba } = decodePng(png); - const hdr = makeHdrFrame(width, height, 50000, 40000, 30000); - const out = blitRgba8OverRgb48le(domRgba, hdr, width, height); + const canvas = makeHdrFrame(width, height, 50000, 40000, 30000); + blitRgba8OverRgb48le(domRgba, canvas, width, height); // sRGB blue (0,0,255) → HLG (0, 0, 65535) — black/white map identically for (let i = 0; i < width * height; i++) { - expect(out.readUInt16LE(i * 6 + 0)).toBe(0); - expect(out.readUInt16LE(i * 6 + 2)).toBe(0); - expect(out.readUInt16LE(i * 6 + 4)).toBe(65535); + expect(canvas.readUInt16LE(i * 6 + 0)).toBe(0); + expect(canvas.readUInt16LE(i * 6 + 2)).toBe(0); + expect(canvas.readUInt16LE(i * 6 + 4)).toBe(65535); } }); }); diff --git a/packages/engine/src/utils/alphaBlit.ts b/packages/engine/src/utils/alphaBlit.ts index ff4b749e..64882df6 100644 --- a/packages/engine/src/utils/alphaBlit.ts +++ b/packages/engine/src/utils/alphaBlit.ts @@ -282,113 +282,183 @@ export function decodePngToRgb48le(buf: Buffer): { width: number; height: number return { width, height, data: output }; } -// ── sRGB → HLG color conversion ─────────────────────────────────────────────── +// ── sRGB → HDR color conversion ─────────────────────────────────────────────── /** - * 256-entry LUT: sRGB 8-bit value → HLG 16-bit signal value. + * Build a 256-entry LUT: sRGB 8-bit value → HDR 16-bit signal value. * - * Converts DOM overlay pixels (Chrome sRGB) to HLG signal space so they - * composite correctly into the HLG/BT.2020 output without color shift. + * Pipeline per channel: sRGB EOTF (decode gamma) → linear → HDR OETF → 16-bit. * - * Pipeline per channel: sRGB EOTF (decode gamma) → linear → HLG OETF → 16-bit. - * - * Note: this converts the transfer function (gamma) but not the color primaries - * (bt709 → bt2020). For neutral/near-neutral content (text, UI elements) the - * gamut difference is negligible. Saturated sRGB colors may shift slightly. + * Note: converts the transfer function but not the color primaries (bt709 → bt2020). + * For neutral/near-neutral content (text, UI) the gamut difference is negligible. */ -function buildSrgbToHlgLut(): Uint16Array { +function buildSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array { const lut = new Uint16Array(256); // HLG OETF constants (Rec. 2100) - const a = 0.17883277; - const b = 1 - 4 * a; - const c = 0.5 - a * Math.log(4 * a); + const hlgA = 0.17883277; + const hlgB = 1 - 4 * hlgA; + const hlgC = 0.5 - hlgA * Math.log(4 * hlgA); + + // PQ (SMPTE 2084) OETF constants + const pqM1 = 0.1593017578125; + const pqM2 = 78.84375; + const pqC1 = 0.8359375; + const pqC2 = 18.8515625; + const pqC3 = 18.6875; + const pqMaxNits = 10000.0; + const sdrNits = 203.0; for (let i = 0; i < 256; i++) { - // sRGB EOTF: signal → linear + // sRGB EOTF: signal → linear (range 0–1, relative to SDR white) const v = i / 255; const linear = v <= 0.04045 ? v / 12.92 : Math.pow((v + 0.055) / 1.055, 2.4); - // HLG OETF: linear → HLG signal - const hlg = linear <= 1 / 12 ? Math.sqrt(3 * linear) : a * Math.log(12 * linear - b) + c; + let signal: number; + if (transfer === "hlg") { + signal = + linear <= 1 / 12 ? Math.sqrt(3 * linear) : hlgA * Math.log(12 * linear - hlgB) + hlgC; + } else { + // PQ OETF: linear light (in SDR nits) → PQ signal + const Lp = Math.max(0, (linear * sdrNits) / pqMaxNits); + const Lm1 = Math.pow(Lp, pqM1); + signal = Math.pow((pqC1 + pqC2 * Lm1) / (1.0 + pqC3 * Lm1), pqM2); + } - lut[i] = Math.min(65535, Math.round(hlg * 65535)); + lut[i] = Math.min(65535, Math.round(signal * 65535)); } return lut; } -const SRGB_TO_HLG = buildSrgbToHlgLut(); +const SRGB_TO_HLG = buildSrgbToHdrLut("hlg"); +const SRGB_TO_PQ = buildSrgbToHdrLut("pq"); + +/** Select the correct sRGB→HDR LUT for the given transfer function. */ +export function getSrgbToHdrLut(transfer: "hlg" | "pq"): Uint16Array { + return transfer === "pq" ? SRGB_TO_PQ : SRGB_TO_HLG; +} // ── Alpha compositing ───────────────────────────────────────────────────────── /** - * Alpha-composite a DOM RGBA overlay (8-bit sRGB) onto an HDR frame - * (rgb48le, HLG-encoded) in memory. - * - * DOM pixels are converted from sRGB to HLG signal space before blending - * so the composited output is uniformly HLG-encoded. Without this conversion, - * sRGB content (text, SDR video rendered by Chrome) would have incorrect - * gamma and appear orange/washed in HDR playback. + * Alpha-composite a DOM RGBA overlay (8-bit sRGB) onto an HDR canvas + * (rgb48le) in-place. * - * For each pixel: - * - If DOM alpha == 0 → copy HDR pixel unchanged - * - If DOM alpha == 255 → use DOM pixel (sRGB→HLG converted) - * - Otherwise → blend converted DOM with HDR in HLG signal domain + * DOM pixels are converted from sRGB to the target HDR signal space (HLG or PQ) + * before blending so the composited output is uniformly encoded. Without this + * conversion, sRGB content appears orange/washed in HDR playback. * - * @param domRgba Raw RGBA pixel data from decodePng() — width*height*4 bytes - * @param hdrRgb48 HDR frame in rgb48le format — width*height*6 bytes - * @returns New rgb48le buffer with DOM composited on top (HLG-encoded) + * @param domRgba Raw RGBA pixel data from decodePng() — width*height*4 bytes + * @param canvas HDR canvas in rgb48le format — width*height*6 bytes, mutated in-place + * @param width Canvas width in pixels + * @param height Canvas height in pixels + * @param transfer HDR transfer function — selects the correct sRGB→HDR LUT */ export function blitRgba8OverRgb48le( domRgba: Uint8Array, - hdrRgb48: Buffer, + canvas: Buffer, width: number, height: number, -): Buffer { + transfer: "hlg" | "pq" = "hlg", +): void { const pixelCount = width * height; - const out = Buffer.allocUnsafe(pixelCount * 6); - const lut = SRGB_TO_HLG; + const lut = transfer === "pq" ? SRGB_TO_PQ : SRGB_TO_HLG; for (let i = 0; i < pixelCount; i++) { const da = domRgba[i * 4 + 3] ?? 0; if (da === 0) { - // Fully transparent DOM pixel — copy HDR unchanged - out[i * 6 + 0] = hdrRgb48[i * 6 + 0] ?? 0; - out[i * 6 + 1] = hdrRgb48[i * 6 + 1] ?? 0; - out[i * 6 + 2] = hdrRgb48[i * 6 + 2] ?? 0; - out[i * 6 + 3] = hdrRgb48[i * 6 + 3] ?? 0; - out[i * 6 + 4] = hdrRgb48[i * 6 + 4] ?? 0; - out[i * 6 + 5] = hdrRgb48[i * 6 + 5] ?? 0; + continue; } else if (da === 255) { - // Fully opaque DOM pixel — convert sRGB → HLG const r16 = lut[domRgba[i * 4 + 0] ?? 0] ?? 0; const g16 = lut[domRgba[i * 4 + 1] ?? 0] ?? 0; const b16 = lut[domRgba[i * 4 + 2] ?? 0] ?? 0; - out.writeUInt16LE(r16, i * 6); - out.writeUInt16LE(g16, i * 6 + 2); - out.writeUInt16LE(b16, i * 6 + 4); + canvas.writeUInt16LE(r16, i * 6); + canvas.writeUInt16LE(g16, i * 6 + 2); + canvas.writeUInt16LE(b16, i * 6 + 4); } else { - // Partial alpha — convert sRGB→HLG then blend in HLG signal domain const alpha = da / 255; const invAlpha = 1 - alpha; - // Read HDR pixel (little-endian uint16, already HLG-encoded) - const hdrR = (hdrRgb48[i * 6 + 0] ?? 0) | ((hdrRgb48[i * 6 + 1] ?? 0) << 8); - const hdrG = (hdrRgb48[i * 6 + 2] ?? 0) | ((hdrRgb48[i * 6 + 3] ?? 0) << 8); - const hdrB = (hdrRgb48[i * 6 + 4] ?? 0) | ((hdrRgb48[i * 6 + 5] ?? 0) << 8); + const hdrR = (canvas[i * 6 + 0] ?? 0) | ((canvas[i * 6 + 1] ?? 0) << 8); + const hdrG = (canvas[i * 6 + 2] ?? 0) | ((canvas[i * 6 + 3] ?? 0) << 8); + const hdrB = (canvas[i * 6 + 4] ?? 0) | ((canvas[i * 6 + 5] ?? 0) << 8); - // Convert DOM sRGB → HLG signal const domR = lut[domRgba[i * 4 + 0] ?? 0] ?? 0; const domG = lut[domRgba[i * 4 + 1] ?? 0] ?? 0; const domB = lut[domRgba[i * 4 + 2] ?? 0] ?? 0; - out.writeUInt16LE(Math.round(domR * alpha + hdrR * invAlpha), i * 6); - out.writeUInt16LE(Math.round(domG * alpha + hdrG * invAlpha), i * 6 + 2); - out.writeUInt16LE(Math.round(domB * alpha + hdrB * invAlpha), i * 6 + 4); + canvas.writeUInt16LE(Math.round(domR * alpha + hdrR * invAlpha), i * 6); + canvas.writeUInt16LE(Math.round(domG * alpha + hdrG * invAlpha), i * 6 + 2); + canvas.writeUInt16LE(Math.round(domB * alpha + hdrB * invAlpha), i * 6 + 4); } } +} + +// ── Positioned HDR region copy ──────────────────────────────────────────────── - return out; +/** + * Copy a rectangular region of an rgb48le source onto an rgb48le canvas + * at position (dx, dy). Clips to canvas bounds. Optional opacity blending + * (0.0–1.0) over existing canvas content. + * + * @param canvas Destination rgb48le buffer (canvasWidth * canvasHeight * 6 bytes) + * @param source Source rgb48le buffer (sw * sh * 6 bytes) + * @param dx Destination X offset on canvas + * @param dy Destination Y offset on canvas + * @param sw Source width in pixels + * @param sh Source height in pixels + * @param canvasWidth Canvas width in pixels (needed for stride calculation) + * @param opacity Optional opacity 0.0–1.0 (default 1.0 = fully opaque copy) + */ +export function blitRgb48leRegion( + canvas: Buffer, + source: Buffer, + dx: number, + dy: number, + sw: number, + sh: number, + canvasWidth: number, + opacity?: number, +): void { + if (sw <= 0 || sh <= 0) return; + + const op = opacity ?? 1.0; + const canvasHeight = canvas.length / (canvasWidth * 6); + + const x0 = Math.max(0, dx); + const y0 = Math.max(0, dy); + const x1 = Math.min(canvasWidth, dx + sw); + const y1 = Math.min(canvasHeight, dy + sh); + if (x0 >= x1 || y0 >= y1) return; + + const clippedW = x1 - x0; + const srcOffsetX = x0 - dx; + const srcOffsetY = y0 - dy; + + if (op >= 0.999) { + for (let y = 0; y < y1 - y0; y++) { + const srcRowOff = ((srcOffsetY + y) * sw + srcOffsetX) * 6; + const dstRowOff = ((y0 + y) * canvasWidth + x0) * 6; + source.copy(canvas, dstRowOff, srcRowOff, srcRowOff + clippedW * 6); + } + } else { + const invOp = 1 - op; + for (let y = 0; y < y1 - y0; y++) { + for (let x = 0; x < clippedW; x++) { + const srcOff = ((srcOffsetY + y) * sw + srcOffsetX + x) * 6; + const dstOff = ((y0 + y) * canvasWidth + x0 + x) * 6; + const sr = source.readUInt16LE(srcOff); + const sg = source.readUInt16LE(srcOff + 2); + const sb = source.readUInt16LE(srcOff + 4); + const dr = canvas.readUInt16LE(dstOff); + const dg = canvas.readUInt16LE(dstOff + 2); + const db = canvas.readUInt16LE(dstOff + 4); + canvas.writeUInt16LE(Math.round(sr * op + dr * invOp), dstOff); + canvas.writeUInt16LE(Math.round(sg * op + dg * invOp), dstOff + 2); + canvas.writeUInt16LE(Math.round(sb * op + db * invOp), dstOff + 4); + } + } + } } diff --git a/packages/engine/src/utils/layerCompositor.test.ts b/packages/engine/src/utils/layerCompositor.test.ts new file mode 100644 index 00000000..9e8bc5b4 --- /dev/null +++ b/packages/engine/src/utils/layerCompositor.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { groupIntoLayers } from "./layerCompositor.js"; +import type { ElementStackingInfo } from "../services/videoFrameInjector.js"; + +function makeEl(id: string, zIndex: number, isHdr: boolean): ElementStackingInfo { + return { id, zIndex, x: 0, y: 0, width: 1920, height: 1080, opacity: 1, visible: true, isHdr }; +} + +describe("groupIntoLayers", () => { + it("single DOM element → 1 DOM layer", () => { + const layers = groupIntoLayers([makeEl("text", 0, false)]); + expect(layers).toHaveLength(1); + expect(layers[0]!.type).toBe("dom"); + }); + + it("single HDR element → 1 HDR layer", () => { + const layers = groupIntoLayers([makeEl("v-hdr", 0, true)]); + expect(layers).toHaveLength(1); + expect(layers[0]!.type).toBe("hdr"); + }); + + it("merges adjacent DOM elements into one layer", () => { + const elements = [makeEl("bg", 0, false), makeEl("text", 1, false), makeEl("logo", 2, false)]; + const layers = groupIntoLayers(elements); + expect(layers).toHaveLength(1); + expect(layers[0]!.type).toBe("dom"); + if (layers[0]!.type === "dom") { + expect(layers[0]!.elementIds).toEqual(["bg", "text", "logo"]); + } + }); + + it("splits on HDR/DOM boundary: DOM → HDR → DOM = 3 layers", () => { + const elements = [makeEl("bg", 0, false), makeEl("v-hdr", 1, true), makeEl("title", 2, false)]; + const layers = groupIntoLayers(elements); + expect(layers).toHaveLength(3); + expect(layers[0]!.type).toBe("dom"); + expect(layers[1]!.type).toBe("hdr"); + expect(layers[2]!.type).toBe("dom"); + }); + + it("merges adjacent DOM around multiple HDR: DOM → HDR → HDR → DOM = 4 layers", () => { + const elements = [ + makeEl("bg", 0, false), + makeEl("v-hdr1", 1, true), + makeEl("v-hdr2", 2, true), + makeEl("title", 3, false), + ]; + const layers = groupIntoLayers(elements); + expect(layers).toHaveLength(4); + expect(layers[0]!.type).toBe("dom"); + expect(layers[1]!.type).toBe("hdr"); + expect(layers[2]!.type).toBe("hdr"); + expect(layers[3]!.type).toBe("dom"); + }); + + it("complex case: DOM DOM HDR DOM HDR DOM = 5 layers (2 DOM merges)", () => { + const elements = [ + makeEl("bg", 0, false), + makeEl("caption", 1, false), + makeEl("v-hdr1", 2, true), + makeEl("text", 3, false), + makeEl("v-hdr2", 4, true), + makeEl("logo", 5, false), + ]; + const layers = groupIntoLayers(elements); + expect(layers).toHaveLength(5); + expect(layers.map((l) => l.type)).toEqual(["dom", "hdr", "dom", "hdr", "dom"]); + if (layers[0]!.type === "dom") { + expect(layers[0]!.elementIds).toEqual(["bg", "caption"]); + } + }); + + it("sorts by zIndex before grouping", () => { + const elements = [makeEl("title", 5, false), makeEl("v-hdr", 2, true), makeEl("bg", 0, false)]; + const layers = groupIntoLayers(elements); + expect(layers).toHaveLength(3); + expect(layers[0]!.type).toBe("dom"); // bg (z=0) + expect(layers[1]!.type).toBe("hdr"); // v-hdr (z=2) + expect(layers[2]!.type).toBe("dom"); // title (z=5) + }); + + it("includes invisible elements in correct z-position", () => { + const elements = [ + makeEl("bg", 0, false), + { ...makeEl("hidden-sdr", 1, false), visible: false }, + { ...makeEl("hidden-hdr", 2, true), visible: false }, + makeEl("title", 3, false), + ]; + const layers = groupIntoLayers(elements); + // All elements included — invisible SDR videos need their injected + // replacements hidden from other layers' screenshots + expect(layers).toHaveLength(3); + expect(layers[0]!.type).toBe("dom"); // bg + hidden-sdr (merged) + expect(layers[1]!.type).toBe("hdr"); // hidden-hdr + expect(layers[2]!.type).toBe("dom"); // title + if (layers[0]!.type === "dom") { + expect(layers[0]!.elementIds).toEqual(["bg", "hidden-sdr"]); + } + }); +}); diff --git a/packages/engine/src/utils/layerCompositor.ts b/packages/engine/src/utils/layerCompositor.ts new file mode 100644 index 00000000..670e7e60 --- /dev/null +++ b/packages/engine/src/utils/layerCompositor.ts @@ -0,0 +1,47 @@ +/** + * Layer Compositor — z-order analysis for multi-layer HDR compositing. + * + * Groups timed elements into z-ordered layers (DOM or HDR) for the + * per-frame compositing loop. Adjacent DOM elements merge into a single + * layer to minimize Chrome screenshots. + */ + +import type { ElementStackingInfo } from "../services/videoFrameInjector.js"; + +export type { ElementStackingInfo }; + +export type CompositeLayer = + | { type: "dom"; elementIds: string[] } + | { type: "hdr"; element: ElementStackingInfo }; + +/** + * Group z-sorted elements into composite layers. Adjacent DOM elements + * merge into a single layer. Each HDR video is its own layer. + * + * Elements are sorted by zIndex ascending (back to front) before grouping. + * Invisible elements are filtered out. + */ +export function groupIntoLayers(elements: ElementStackingInfo[]): CompositeLayer[] { + // Include ALL elements regardless of visibility. Video elements are hidden by + // the frame injector (HEVC can't decode in headless Chrome) but their injected + // replacements ARE visible. We need them in the correct z-ordered layer + // so they get hidden from other layers' DOM screenshots. + const sorted = [...elements].sort((a, b) => a.zIndex - b.zIndex); + + const layers: CompositeLayer[] = []; + + for (const el of sorted) { + if (el.isHdr) { + layers.push({ type: "hdr", element: el }); + } else { + const last = layers[layers.length - 1]; + if (last && last.type === "dom") { + last.elementIds.push(el.id); + } else { + layers.push({ type: "dom", elementIds: [el.id] }); + } + } + } + + return layers; +} diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 9e77f3f9..32180543 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -62,9 +62,11 @@ import { decodePng, decodePngToRgb48le, blitRgba8OverRgb48le, + blitRgb48leRegion, hideVideoElements, showVideoElements, - queryVideoElementBounds, + queryElementStacking, + groupIntoLayers, } from "@hyperframes/engine"; import { join, dirname, resolve } from "path"; import { randomUUID } from "crypto"; @@ -861,16 +863,13 @@ export async function executeRenderJob( job.framesRendered = 0; - // ── HDR two-pass compositing path ──────────────────────────────────── - // Pass 1: Capture DOM layer with alpha (Chrome, video elements hidden) - // Pass 2: Extract native HLG frames from video sources (FFmpeg) - // Composite: overlay DOM on top of HDR video in FFmpeg per-frame - // - // This preserves HDR luminance from video sources while correctly - // compositing DOM content (text, graphics, SDR overlays) on top. - // GSAP-animated video position/opacity is applied via queried bounds. + // ── HDR z-ordered multi-layer compositing ────────────────────────────── + // Per frame: query all elements' z-order, group into layers (DOM or HDR), + // composite bottom-to-top in Node.js memory. HDR layers use native + // pre-extracted HLG pixels; DOM layers use Chrome alpha screenshots + // with sRGB→HLG conversion. Video position/opacity applied via queried bounds. if (hasHdrVideo) { - log.info("[Render] HDR two-pass: DOM layer + native HLG video compositing"); + log.info("[Render] HDR layered composite: z-ordered DOM + native HLG video layers"); // Use NATIVE HDR IDs (probed before SDR→HDR conversion) so only originally-HDR // videos are hidden + extracted natively. SDR videos stay in the DOM screenshot @@ -969,59 +968,89 @@ export async function executeRenderJob( if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t); }, time); - // Inject SDR video frames into the DOM (the hook handles all videos, - // but hideVideoElements below will hide the HDR ones before screenshot) + // Inject SDR video frames into the DOM if (beforeCaptureHook) { await beforeCaptureHook(domSession.page, time); } - // Query video element positions BEFORE hiding (so GSAP has already moved them) - const bounds = await queryVideoElementBounds(domSession.page, hdrVideoIds); - const activeBounds = bounds.filter((b) => b.visible); - - // Pass 1: Hide HDR videos (and their injected frames), capture DOM with alpha. - // SDR video frames remain visible in the screenshot. - await hideVideoElements(domSession.page, hdrVideoIds); - const domPng = await captureAlphaPng(domSession.page, width, height); - await showVideoElements(domSession.page, hdrVideoIds); - - // Pass 2: Read pre-extracted HDR frame and composite with DOM layer - const activeVideoId = activeBounds[0]?.videoId ?? hdrVideoIds[0]; - const video = composition.videos.find((v) => v.id === activeVideoId); - const frameDir = activeVideoId ? hdrFrameDirs.get(activeVideoId) : undefined; - - let composited: Buffer; - if (video && frameDir) { - // Frame index within the video (1-based for FFmpeg image2 output) - const videoFrameIndex = Math.round((time - video.start) * job.config.fps) + 1; - const framePath = join( - frameDir, - `frame_${String(videoFrameIndex).padStart(4, "0")}.png`, + // Query ALL timed elements for z-order analysis + const stackingInfo = await queryElementStacking(domSession.page, nativeHdrVideoIds); + + // Group into z-ordered layers + const layers = groupIntoLayers(stackingInfo); + + if (i % 30 === 0) { + const hdrEl = stackingInfo.find((e) => e.isHdr); + const hdrInLayers = layers.some((l) => l.type === "hdr"); + process.stderr.write( + `[DBG] f${i} t=${time.toFixed(2)} hdr_el=${hdrEl ? `z=${hdrEl.zIndex},vis=${hdrEl.visible},w=${hdrEl.width}` : "NONE"} hdr_layer=${hdrInLayers} layers=${layers.length}\n`, ); + } - let hdrRgb: Buffer; - if (existsSync(framePath)) { - try { - hdrRgb = decodePngToRgb48le(readFileSync(framePath)).data; - } catch { - hdrRgb = Buffer.alloc(width * height * 6); + // Start with a black canvas + const canvas = Buffer.alloc(width * height * 6); + + // Composite layers bottom-to-top + for (const layer of layers) { + if (layer.type === "hdr") { + const el = layer.element; + const frameDir = hdrFrameDirs.get(el.id); + const video = composition.videos.find((v) => v.id === el.id); + if (!frameDir || !video) continue; + + const videoFrameIndex = Math.round((time - video.start) * job.config.fps) + 1; + const framePath = join( + frameDir, + `frame_${String(videoFrameIndex).padStart(4, "0")}.png`, + ); + + if (existsSync(framePath)) { + try { + const hdrRgb = decodePngToRgb48le(readFileSync(framePath)).data; + blitRgb48leRegion( + canvas, + hdrRgb, + el.x, + el.y, + el.width, + el.height, + width, + el.opacity < 0.999 ? el.opacity : undefined, + ); + } catch { + // Skip this HDR layer if decode fails + } } } else { - hdrRgb = Buffer.alloc(width * height * 6); - } + // DOM layer: hide elements NOT in this layer + all HDR videos. + // All elements (including invisible SDR videos) are in the stacking + // info so their injected replacements get hidden from other layers. + const allElementIds = stackingInfo.map((e) => e.id); + const layerIds = new Set(layer.elementIds); + const hideIds = allElementIds.filter( + (id) => !layerIds.has(id) || nativeHdrVideoIds.has(id), + ); + + await hideVideoElements(domSession.page, hideIds); + const domPng = await captureAlphaPng(domSession.page, width, height); + await showVideoElements(domSession.page, hideIds); + + // Re-seek GSAP to restore animated properties (opacity, transforms) + // that showVideoElements clobbered via removeProperty. + await domSession.page.evaluate((t: number) => { + if (window.__hf && typeof window.__hf.seek === "function") window.__hf.seek(t); + }, time); - // In-memory alpha composite: DOM PNG over HDR rgb48le - try { - const { data: domRgba } = decodePng(domPng); - composited = blitRgba8OverRgb48le(domRgba, hdrRgb, width, height); - } catch { - composited = hdrRgb; + try { + const { data: domRgba } = decodePng(domPng); + blitRgba8OverRgb48le(domRgba, canvas, width, height, effectiveHdr!.transfer); + } catch { + // Skip this DOM layer if decode fails + } } - } else { - composited = Buffer.alloc(width * height * 6); } - hdrEncoder.writeFrame(composited); + hdrEncoder.writeFrame(canvas); job.framesRendered = i + 1; if ((i + 1) % 10 === 0 || i + 1 === job.totalFrames!) {