diff --git a/apps/playground/src/m5-worker-proof.ts b/apps/playground/src/m5-worker-proof.ts index d6e71e9..2de3d42 100644 --- a/apps/playground/src/m5-worker-proof.ts +++ b/apps/playground/src/m5-worker-proof.ts @@ -868,10 +868,7 @@ function assertNativeOutput( ); const color = frame.colorSpace; requireFixture( - color.fullRange !== true && - (color.matrix === null || color.matrix === "bt709") && - (color.primaries === null || color.primaries === "bt709") && - (color.transfer === null || color.transfer === "bt709"), + matchesDecodedBt709ColorSpace(color), `decoded color metadata contradicts BT.709 limited range at ordinal ${String( expected.ordinal )}` @@ -882,6 +879,18 @@ function assertNativeOutput( ); } +function matchesDecodedBt709ColorSpace(color: VideoColorSpace): boolean { + const limitedBt709 = color.fullRange !== true && + (color.matrix === null || color.matrix === "bt709") && + (color.primaries === null || color.primaries === "bt709") && + (color.transfer === null || color.transfer === "bt709"); + const webKitNormalizedBt709 = color.fullRange === true && + color.matrix === "bt709" && + color.primaries === "bt709" && + color.transfer === "iec61966-2-1"; + return limitedBt709 || webKitNormalizedBt709; +} + function updateCreditEvidence( credit: { maxOutstandingFrames: number; diff --git a/docs/superpowers/plans/2026-07-15-grass-rabbit-vercel-git-deployment.md b/docs/superpowers/plans/2026-07-15-grass-rabbit-vercel-git-deployment.md new file mode 100644 index 0000000..393fd37 --- /dev/null +++ b/docs/superpowers/plans/2026-07-15-grass-rabbit-vercel-git-deployment.md @@ -0,0 +1,48 @@ +# Grass Rabbit Vercel Git Deployment Implementation Plan + +> **For agentic workers:** Execute each task in order and verify the Git-trigger and proxy contract before handoff. + +**Goal:** Deploy the grass-rabbit demo from Git pushes while rebuilding it only when files under `examples/grass-rabbit` change, and keep it reachable at `pixelpoint.io/aval/`. + +**Architecture:** Keep `aval-demo` as its own Vercel project rooted at `examples/grass-rabbit`. Its Vercel build first compiles the local workspace packages needed by the demo, then runs the existing Vite `/aval/` build. A repository-scoped ignored-build command skips commits with no changes inside the demo directory. The PixelPoint website continues to reverse-proxy `/aval` to the demo project's stable production alias. + +**Tech Stack:** Vercel Git integration, npm workspaces, Vite, Vercel rewrites + +--- + +### Task 1: Make the demo build reproducible on Vercel + +**Files:** +- Modify: `examples/grass-rabbit/package.json` +- Modify: `examples/grass-rabbit/vercel.json` + +- [x] Add a Vercel build script that builds required local workspace packages before Vite. +- [x] Declare the build command and output directory in version-controlled Vercel configuration. +- [x] Add an ignored-build command that proceeds only when `examples/grass-rabbit` changed. +- [x] Validate the JSON and run the exact production build locally. + +### Task 2: Persist the public `/aval` proxy + +**Files:** +- Preserve/verify: `/Users/alex/Projects/pixelpoint-website/vercel.json` +- Preserve/verify: `/Users/alex/Projects/pixelpoint-website/.gitignore` + +- [x] Verify `/aval`, `/aval/`, and nested assets proxy to the `aval-demo` production alias. +- [x] Ensure the proxy and `.vercel` ignore rule remain as intentional website-repository changes. + +### Task 3: Connect the existing Vercel project to GitHub + +**Files:** +- Local-only: `examples/grass-rabbit/.vercel/project.json` + +- [x] Link the local demo directory to the existing `pixelpoint/aval-demo` project. +- [x] Connect that Vercel project to `pixel-point/aval` on GitHub. +- [x] Confirm the project root is `examples/grass-rabbit` and the production branch is `main`. + +### Task 4: Verify the complete deployment contract + +- [x] Confirm the built HTML and assets use the `/aval/` base path. +- [x] Verify the production build includes the current Safari runtime fix. +- [x] Test the built demo in WebKit through the `/aval/` route. +- [x] Confirm unrelated commits are skipped by the ignored-build command and demo-directory commits proceed. +- [x] Report the exact files that need to be committed and pushed in each repository. diff --git a/etc/api/player-web.api.md b/etc/api/player-web.api.md index 439f7ab..32ea3de 100644 --- a/etc/api/player-web.api.md +++ b/etc/api/player-web.api.md @@ -592,6 +592,8 @@ export class BrowserFrameBackend implements FrameRendererBackend_2 { setPresentationGeometry(geometry: Readonly): boolean; // (undocumented) upload(kind: FrameTextureKind, index: number, pixels: Uint8Array): void; + // (undocumented) + uploadFrame(kind: FrameTextureKind, index: number, frame: CopyableVideoFrame, layout: Readonly): void; } // @public (undocumented) @@ -632,6 +634,8 @@ export class BrowserOpaqueFrameBackend implements OpaqueFrameRendererBackend { readPixels(): Uint8Array; // (undocumented) upload(kind: FrameTextureKind, index: number, pixels: Uint8Array): void; + // (undocumented) + uploadFrame(kind: FrameTextureKind, index: number, frame: CopyableVideoFrame, layout: Readonly): void; } // @public @deprecated (undocumented) @@ -2220,6 +2224,18 @@ interface FrameRendererTimerHost { export { FrameRendererTimerHost } export { FrameRendererTimerHost as OpaqueFrameRendererTimerHost } +// @public (undocumented) +export interface FrameSourceLayout { + // (undocumented) + readonly height: number; + // (undocumented) + readonly width: number; + // (undocumented) + readonly x: number; + // (undocumented) + readonly y: number; +} + // @public (undocumented) type FrameTextureKind = "resident" | "stream"; export { FrameTextureKind } @@ -3210,6 +3226,8 @@ export interface OpaqueFrameRendererBackend { readPixels?(): Uint8Array; // (undocumented) upload(kind: FrameTextureKind, index: number, pixels: Uint8Array): void; + // (undocumented) + uploadFrame?(kind: FrameTextureKind, index: number, frame: CopyableVideoFrame, layout: Readonly): void; } // @public (undocumented) diff --git a/examples/grass-rabbit/.gitignore b/examples/grass-rabbit/.gitignore new file mode 100644 index 0000000..245259b --- /dev/null +++ b/examples/grass-rabbit/.gitignore @@ -0,0 +1,2 @@ +.vercel +.env* diff --git a/examples/grass-rabbit/package.json b/examples/grass-rabbit/package.json index 859b8b0..0eca72f 100644 --- a/examples/grass-rabbit/package.json +++ b/examples/grass-rabbit/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "build": "vite build --base=/aval/", + "build:vercel": "npm --prefix ../.. run build -w @pixel-point/aval-graph && npm --prefix ../.. run build -w @pixel-point/aval-format && npm --prefix ../.. run build -w @pixel-point/aval-player-web && npm --prefix ../.. run build -w @pixel-point/aval-element && npm --prefix ../.. run build -w @pixel-point/aval-grass-rabbit-example", "compile": "avl compile motion.json --out public/grass-rabbit.avl --force", "dev": "vite --host 127.0.0.1" }, diff --git a/examples/grass-rabbit/vercel.json b/examples/grass-rabbit/vercel.json index 69f3ee3..9544ce8 100644 --- a/examples/grass-rabbit/vercel.json +++ b/examples/grass-rabbit/vercel.json @@ -1,4 +1,10 @@ { + "$schema": "https://openapi.vercel.sh/vercel.json", + "framework": "vite", + "installCommand": "npm --prefix ../.. ci", + "buildCommand": "npm run build:vercel", + "outputDirectory": "dist", + "ignoreCommand": "git diff --quiet HEAD^ HEAD ./", "rewrites": [ { "source": "/aval/grass-rabbit.avl", diff --git a/packages/player-web/src/decoder-worker/core-validation.ts b/packages/player-web/src/decoder-worker/core-validation.ts index 68f41cc..8137664 100644 --- a/packages/player-web/src/decoder-worker/core-validation.ts +++ b/packages/player-web/src/decoder-worker/core-validation.ts @@ -262,11 +262,7 @@ export function validateDecodedFrame( true ); } - if ( - !isNonContradictoryBt709Limited(frame.colorSpace) || - expected.colorSpace !== null && - !matchesColorSpace(frame.colorSpace, expected.colorSpace) - ) { + if (!matchesDecodedBt709ColorSpace(frame.colorSpace, expected.colorSpace)) { throw new DecoderWorkerCoreError( "DECODER_OUTPUT_INVALID", "decoder output color space did not match the configured rendition", @@ -291,6 +287,34 @@ function isNonContradictoryBt709Limited(actual: VideoColorSpace): boolean { ); } +function matchesDecodedBt709ColorSpace( + actual: VideoColorSpace, + expected: DecoderWorkerColorSpaceExpectation | null +): boolean { + if (isNonContradictoryBt709Limited(actual)) { + return expected === null || matchesColorSpace(actual, expected); + } + return isWebKitNormalizedBt709(actual) && + (expected === null || isExactBt709Limited(expected)); +} + +/** WebKit exposes decoded BT.709 video through this complete normalized tuple. */ +function isWebKitNormalizedBt709(actual: VideoColorSpace): boolean { + return actual.fullRange === true && + actual.matrix === "bt709" && + actual.primaries === "bt709" && + actual.transfer === "iec61966-2-1"; +} + +function isExactBt709Limited( + expected: DecoderWorkerColorSpaceExpectation +): boolean { + return expected.fullRange === false && + expected.matrix === "bt709" && + expected.primaries === "bt709" && + expected.transfer === "bt709"; +} + export function normalizeCoreError( error: unknown, code: DecoderWorkerErrorCode, diff --git a/packages/player-web/src/decoder-worker/decoder-worker.test.ts b/packages/player-web/src/decoder-worker/decoder-worker.test.ts index 1055c3c..34f4c58 100644 --- a/packages/player-web/src/decoder-worker/decoder-worker.test.ts +++ b/packages/player-web/src/decoder-worker/decoder-worker.test.ts @@ -22,6 +22,7 @@ import { DECODER_WORKER_PROTOCOL_VERSION, type DecoderWorkerClientPort, type DecoderWorkerAvcConfig, + type DecoderWorkerColorSpaceExpectation, type DecoderWorkerLimits, type DecoderWorkerMessagePort, type DecoderWorkerSample @@ -794,6 +795,90 @@ describe("dedicated decoder worker boundary", () => { await fixture.dispose(); }); + it.each([ + ["unspecified", null], + ["exact BT.709 limited", { + fullRange: false, + matrix: "bt709", + primaries: "bt709", + transfer: "bt709" + }] + ] as const)( + "accepts WebKit-normalized BT.709 output with %s expectations", + async (_label, expectedColorSpace) => { + const fixture = createFixture( + { maxOutstandingFrames: 1 }, + { expectedColorSpace } + ); + await fixture.configure(); + await fixture.client.activateGeneration(1); + await fixture.client.submit(1, createUnitSamples(0, 0, 1)); + + const frame = fixture.decoder.emitNext({ + codedWidth: 2, + displayWidth: 2, + fullRange: true, + matrix: "bt709", + primaries: "bt709", + transfer: "iec61966-2-1" + }); + await fixture.client.waitForFrames(1, { timeoutMs: 100 }); + fixture.client.takeFrame()?.close(); + await drainMessages(); + + expect(frame.closeCalls).toBe(1); + expect(fixture.decoder.closeCalls).toBe(0); + await fixture.dispose(); + } + ); + + it.each([ + { + label: "a near-match output tuple", + expectedColorSpace: null, + outputTransfer: "bt709" + }, + { + label: "a noncanonical configured expectation", + expectedColorSpace: { + fullRange: false, + matrix: "bt709", + primaries: "bt709", + transfer: "iec61966-2-1" + }, + outputTransfer: "iec61966-2-1" + } + ] as const)( + "rejects WebKit color-space compatibility for $label", + async ({ expectedColorSpace, outputTransfer }) => { + const fixture = createFixture( + { maxOutstandingFrames: 1 }, + { expectedColorSpace } + ); + await fixture.configure(); + await fixture.client.activateGeneration(1); + await fixture.client.submit(1, createUnitSamples(0, 0, 1)); + const waiting = fixture.client.waitForFrames(1, { timeoutMs: 100 }); + + const frame = fixture.decoder.emitNext({ + codedWidth: 2, + displayWidth: 2, + fullRange: true, + matrix: "bt709", + primaries: "bt709", + transfer: outputTransfer + }); + await expect(waiting).rejects.toMatchObject({ + code: "DECODER_OUTPUT_INVALID", + fatal: true + }); + expect(frame.closeCalls).toBe(1); + expect(fixture.decoder.closeCalls).toBe(1); + + await fixture.dispose(); + } + ); + it("closes the decoder when decode() throws and settles the client failure", async () => { const fixture = createFixture({}, { decoderRejectTag: 0 }); await fixture.configure(); @@ -1004,6 +1089,7 @@ function createFixture( config: VideoDecoderConfig ) => VideoDecoderConfig; readonly codec?: DecoderWorkerAvcConfig["codec"]; + readonly expectedColorSpace?: DecoderWorkerColorSpaceExpectation | null; } = {} ): Fixture { const { clientPort, workerPort } = createPortPair(); @@ -1073,7 +1159,7 @@ function createFixture( displayWidth: 2, displayHeight: 2, visibleRect: { x: 0, y: 0, width: 2, height: 2 }, - colorSpace: null + colorSpace: options.expectedColorSpace ?? null }, limits }), @@ -1151,9 +1237,9 @@ class FakeVideoFrame { public readonly visibleRect = { x: 0, y: 0, width: 2, height: 2 }; public readonly colorSpace: { readonly fullRange: boolean | null; - readonly matrix: null; - readonly primaries: null; - readonly transfer: null; + readonly matrix: VideoMatrixCoefficients | null; + readonly primaries: VideoColorPrimaries | null; + readonly transfer: VideoTransferCharacteristics | null; }; public closeCalls = 0; @@ -1163,6 +1249,9 @@ class FakeVideoFrame { readonly codedWidth: number; readonly displayWidth: number; readonly fullRange?: boolean; + readonly matrix?: VideoMatrixCoefficients; + readonly primaries?: VideoColorPrimaries; + readonly transfer?: VideoTransferCharacteristics; } ) { this.timestamp = chunk.timestamp; @@ -1173,9 +1262,9 @@ class FakeVideoFrame { this.displayHeight = 2; this.colorSpace = { fullRange: geometry.fullRange ?? null, - matrix: null, - primaries: null, - transfer: null + matrix: geometry.matrix ?? null, + primaries: geometry.primaries ?? null, + transfer: geometry.transfer ?? null }; } @@ -1251,6 +1340,9 @@ class FakeVideoDecoder implements WorkerVideoDecoderAdapter { readonly codedWidth: number; readonly displayWidth: number; readonly fullRange?: boolean; + readonly matrix?: VideoMatrixCoefficients; + readonly primaries?: VideoColorPrimaries; + readonly transfer?: VideoTransferCharacteristics; } = { codedWidth: 2, displayWidth: 2 diff --git a/packages/player-web/src/index.ts b/packages/player-web/src/index.ts index 4040227..d0d44d8 100644 --- a/packages/player-web/src/index.ts +++ b/packages/player-web/src/index.ts @@ -346,6 +346,7 @@ export { type FrameRendererOptions, type FrameRendererSnapshot, type FrameRendererTimerHost, + type FrameSourceLayout, type FrameTextureKind } from "./runtime/frame-renderer.js"; export { diff --git a/packages/player-web/src/runtime/browser-avc-candidate-factories.ts b/packages/player-web/src/runtime/browser-avc-candidate-factories.ts index 2a76f29..1aaa788 100644 --- a/packages/player-web/src/runtime/browser-avc-candidate-factories.ts +++ b/packages/player-web/src/runtime/browser-avc-candidate-factories.ts @@ -28,7 +28,9 @@ import { } from "./frame-renderer-browser.js"; import { FrameRenderer, + type CopyableVideoFrame, type FrameRendererBackend, + type FrameSourceLayout, type FrameTextureLayout, type FrameTextureKind, type LegacyOpaqueFrameRendererBackend @@ -426,6 +428,12 @@ class TrackedBrowserBackend implements FrameRendererBackend { public readonly limits; public readonly readPixels?: () => Uint8Array; + public readonly uploadFrame?: ( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ) => void; public constructor(backend: FrameRendererBackend) { this.#backend = backend; @@ -434,6 +442,11 @@ class TrackedBrowserBackend implements FrameRendererBackend { if (readPixels !== undefined) { this.readPixels = () => readPixels.call(backend); } + const uploadFrame = backend.uploadFrame; + if (uploadFrame !== undefined) { + this.uploadFrame = (kind, index, frame, layout) => + uploadFrame.call(backend, kind, index, frame, layout); + } } public get alive(): boolean { @@ -470,6 +483,12 @@ class TrackedBrowserBackend implements FrameRendererBackend { class LegacyOpaqueBackendAdapter implements FrameRendererBackend { public readonly limits; public readonly readPixels?: () => Uint8Array; + public readonly uploadFrame?: ( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ) => void; readonly #legacy: LegacyOpaqueFrameRendererBackend; public constructor( @@ -481,6 +500,11 @@ class LegacyOpaqueBackendAdapter implements FrameRendererBackend { if (readPixels !== undefined) { this.readPixels = () => readPixels.call(legacy); } + const uploadFrame = legacy.uploadFrame; + if (uploadFrame !== undefined) { + this.uploadFrame = (kind, index, frame, layout) => + uploadFrame.call(legacy, kind, index, frame, layout); + } } public allocate(layout: FrameTextureLayout, slots: number): void { diff --git a/packages/player-web/src/runtime/browser-presentation-planes.test.ts b/packages/player-web/src/runtime/browser-presentation-planes.test.ts index 3d1ceb3..44c2b2e 100644 --- a/packages/player-web/src/runtime/browser-presentation-planes.test.ts +++ b/packages/player-web/src/runtime/browser-presentation-planes.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; import { BrowserPresentationPlanes } from "./browser-presentation-planes.js"; +import type { + CopyableVideoFrame, + FrameSourceLayout, + FrameTextureKind +} from "./frame-renderer.js"; import { FakePresentableBackend, fakeCanvas, @@ -38,4 +43,43 @@ describe("BrowserPresentationPlanes", () => { planes.dispose(); expect(animated.canvas).toMatchObject({ width: 0, height: 0 }); }); + + it("preserves optional native frame upload through the attached backend", () => { + class NativeUploadBackend extends FakePresentableBackend { + public readonly uploads: Array<{ + kind: FrameTextureKind; + index: number; + frame: CopyableVideoFrame; + layout: Readonly; + }> = []; + + public uploadFrame( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ): void { + this.uploads.push({ kind, index, frame, layout }); + } + } + + const animated = fakeCanvas(); + const backend = new NativeUploadBackend(); + const planes = new BrowserPresentationPlanes({ + animatedCanvas: animated.canvas, + canvas: logicalCanvas(), + maxBackingBytes: 8 * 1024 * 1024, + createBackend: () => backend + }); + const attached = planes.createFrameBackend(); + const frame = Object.freeze({}) as unknown as CopyableVideoFrame; + const layout = Object.freeze({ x: 1, y: 2, width: 3, height: 4 }); + + expect(attached.uploadFrame).toBeTypeOf("function"); + attached.uploadFrame?.("stream", 5, frame, layout); + expect(backend.uploads).toEqual([{ kind: "stream", index: 5, frame, layout }]); + + attached.dispose(); + planes.dispose(); + }); }); diff --git a/packages/player-web/src/runtime/browser-presentation-planes.ts b/packages/player-web/src/runtime/browser-presentation-planes.ts index 50b87e4..3ca1e6c 100644 --- a/packages/player-web/src/runtime/browser-presentation-planes.ts +++ b/packages/player-web/src/runtime/browser-presentation-planes.ts @@ -2,7 +2,9 @@ import type { CanvasV01 } from "@pixel-point/aval-format"; import { RendererUnavailableError, - type FrameRendererBackend + type CopyableVideoFrame, + type FrameRendererBackend, + type FrameSourceLayout } from "./frame-renderer.js"; import type { FrameTextureKind, FrameTextureLayout } from "./frame-renderer.js"; import { @@ -745,6 +747,7 @@ interface ValidatedPresentationBackend { readonly setPresentationGeometry: PresentableFrameBackend["setPresentationGeometry"]; readonly allocate: PresentableFrameBackend["allocate"]; readonly upload: PresentableFrameBackend["upload"]; + readonly uploadFrame: NonNullable | null; readonly draw: PresentableFrameBackend["draw"]; readonly readPixels: NonNullable | null; readonly dispose: () => unknown; @@ -783,6 +786,7 @@ function capturePresentationBackend( ) as unknown; const allocateImplementation = Reflect.get(value, "allocate") as unknown; const uploadImplementation = Reflect.get(value, "upload") as unknown; + const uploadFrameImplementation = Reflect.get(value, "uploadFrame") as unknown; const drawImplementation = Reflect.get(value, "draw") as unknown; const readPixelsImplementation = Reflect.get(value, "readPixels") as unknown; if ( @@ -790,6 +794,10 @@ function capturePresentationBackend( typeof allocateImplementation !== "function" || typeof uploadImplementation !== "function" || typeof drawImplementation !== "function" || + ( + uploadFrameImplementation !== undefined && + typeof uploadFrameImplementation !== "function" + ) || ( readPixelsImplementation !== undefined && typeof readPixelsImplementation !== "function" @@ -835,6 +843,21 @@ function capturePresentationBackend( ) => { Reflect.apply(uploadImplementation, value, [kind, index, pixels]); }, + uploadFrame: uploadFrameImplementation === undefined + ? null + : ( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ) => { + Reflect.apply(uploadFrameImplementation, value, [ + kind, + index, + frame, + layout + ]); + }, draw: (kind: FrameTextureKind, index: number) => { Reflect.apply(drawImplementation, value, [kind, index]); }, @@ -884,6 +907,12 @@ class AttachedPresentationBackend implements PresentableFrameBackend { public readonly limits; public readonly readPixels?: () => Uint8Array; + public readonly uploadFrame?: ( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ) => void; public constructor( backend: Readonly, @@ -901,6 +930,14 @@ class AttachedPresentationBackend implements PresentableFrameBackend { return pixels; }; } + const uploadFrame = backend.uploadFrame; + if (uploadFrame !== null) { + this.uploadFrame = (kind, index, frame, layout) => { + this.#assertActive(); + uploadFrame(kind, index, frame, layout); + this.#assertActive(); + }; + } } public setPresentationGeometry( diff --git a/packages/player-web/src/runtime/frame-renderer-browser.test.ts b/packages/player-web/src/runtime/frame-renderer-browser.test.ts index 8b28444..cf0748a 100644 --- a/packages/player-web/src/runtime/frame-renderer-browser.test.ts +++ b/packages/player-web/src/runtime/frame-renderer-browser.test.ts @@ -5,7 +5,10 @@ import { BrowserFrameBackend, FRAME_FRAGMENT_SHADER_SOURCE } from "./frame-renderer-browser.js"; -import type { FrameTextureLayout } from "./frame-renderer.js"; +import type { + CopyableVideoFrame, + FrameTextureLayout +} from "./frame-renderer.js"; import { deriveFrameSamplingLayout } from "./frame-renderer-validation.js"; @@ -206,6 +209,30 @@ describe("browser profile-neutral frame backend", () => { }); }); + it("uploads a native VideoFrame into the exact texture-array rectangle", () => { + const fixture = createRecordingCanvas(); + const backend = new BrowserFrameBackend(fixture.canvas); + const frame = {} as CopyableVideoFrame; + + backend.allocate(PACKED_LAYOUT, 3); + backend.uploadFrame("resident", 1, frame, { + x: 2, + y: 3, + width: 4, + height: 5 + }); + + expect(fixture.gl.nativeUploads).toEqual([{ + x: 2, + y: 3, + width: 4, + height: 5, + depth: 1, + layer: 1, + source: frame + }]); + }); + it("keeps opaque alpha exactly one and freezes premultiplied shader math", () => { const fixture = createRecordingCanvas(); const backend = new BrowserFrameBackend(fixture.canvas); @@ -750,6 +777,15 @@ class RecordingGl { readonly layer: number; readonly byteLength: number; }> = []; + public readonly nativeUploads: Array<{ + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + readonly depth: number; + readonly layer: number; + readonly source: TexImageSource; + }> = []; public readonly createdTextures: WebGLTexture[] = []; public readonly deletedTextures: WebGLTexture[] = []; public readonly createdShaders: WebGLShader[] = []; @@ -884,23 +920,35 @@ class RecordingGl { public texSubImage3D( _target: number, _level: number, - _x: number, - _y: number, + x: number, + y: number, layer: number, width: number, height: number, depth: number, _format: number, _type: number, - pixels: Uint8Array + source: Uint8Array | TexImageSource ): void { - this.uploads.push({ - width, - height, - depth, - layer, - byteLength: pixels.byteLength - }); + if (source instanceof Uint8Array) { + this.uploads.push({ + width, + height, + depth, + layer, + byteLength: source.byteLength + }); + } else { + this.nativeUploads.push({ + x, + y, + width, + height, + depth, + layer, + source + }); + } } public readPixels(): void {} diff --git a/packages/player-web/src/runtime/frame-renderer-browser.ts b/packages/player-web/src/runtime/frame-renderer-browser.ts index f95dcba..9dc34d8 100644 --- a/packages/player-web/src/runtime/frame-renderer-browser.ts +++ b/packages/player-web/src/runtime/frame-renderer-browser.ts @@ -1,8 +1,10 @@ import { RendererDisposedError, RendererUnavailableError, + type CopyableVideoFrame, type FrameRendererBackend, type FrameRendererBackendLimits, + type FrameSourceLayout, type FrameTextureLayout, type FrameTextureKind, type LegacyOpaqueFrameRendererBackend, @@ -221,6 +223,36 @@ export class BrowserFrameBackend implements FrameRendererBackend { this.#assertAllocated(); } + public uploadFrame( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ): void { + this.#assertAllocated(); + validateSourceLayout(layout, this.#codedWidth, this.#codedHeight); + const gl = this.#gl; + gl.bindTexture(gl.TEXTURE_2D_ARRAY, this.#textureFor(kind)); + gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1); + gl.texSubImage3D( + gl.TEXTURE_2D_ARRAY, + 0, + layout.x, + layout.y, + index, + layout.width, + layout.height, + 1, + gl.RGBA, + gl.UNSIGNED_BYTE, + frame as VideoFrame + ); + // A native-source overload can fail only through the GL error channel. + // Always consume it so FrameRenderer can retry with its RGBA copy fallback. + assertNoGlError(gl, `${kind} native frame upload`); + this.#assertAllocated(); + } + /** Resize/redraw only; decoder, textures, graph time, and generations survive. */ public setPresentationGeometry( geometry: Readonly @@ -657,6 +689,15 @@ implements LegacyOpaqueFrameRendererBackend { this.#backend.upload(kind, index, pixels); } + public uploadFrame( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ): void { + this.#backend.uploadFrame(kind, index, frame, layout); + } + public draw(kind: FrameTextureKind, index: number): void { this.#backend.draw(kind, index); } @@ -670,6 +711,29 @@ implements LegacyOpaqueFrameRendererBackend { } } +function validateSourceLayout( + layout: Readonly, + codedWidth: number, + codedHeight: number +): void { + if ( + layout === null || + typeof layout !== "object" || + !Number.isSafeInteger(layout.x) || + !Number.isSafeInteger(layout.y) || + !Number.isSafeInteger(layout.width) || + !Number.isSafeInteger(layout.height) || + layout.x < 0 || + layout.y < 0 || + layout.width < 1 || + layout.height < 1 || + layout.x + layout.width > codedWidth || + layout.y + layout.height > codedHeight + ) { + throw new RangeError("native frame upload layout is out of bounds"); + } +} + function requirePositiveGlLimit(value: unknown, label: string): number { if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) { throw new RendererUnavailableError(`${label} is invalid`); diff --git a/packages/player-web/src/runtime/frame-renderer.test.ts b/packages/player-web/src/runtime/frame-renderer.test.ts index 0921c5b..00c85d1 100644 --- a/packages/player-web/src/runtime/frame-renderer.test.ts +++ b/packages/player-web/src/runtime/frame-renderer.test.ts @@ -139,6 +139,64 @@ describe("profile-neutral frame renderer", () => { }); }); + it("uploads a validated native frame when the backend supports it", async () => { + const backend = new FakeNativeBackend(); + const renderer = new FrameRenderer(backend, LAYOUT); + const source = borrowedFrame(31, { + copyFailure: new TypeError("RGBA copy is unsupported") + }); + + await expect(renderer.uploadResident(1, source.source)).resolves.toEqual({ + kind: "resident", + layer: 1, + resourceGeneration: 1 + }); + expect(source.copyOptions()).toBeUndefined(); + expect(source.closeCalls()).toBe(1); + expect(backend.uploads).toHaveLength(0); + expect(backend.frameUploads).toEqual([{ + kind: "resident", + index: 1, + frame: source.source.frame, + layout: { x: 0, y: 0, width: 4, height: 12 } + }]); + }); + + it("falls back to the bounded RGBA copy when native upload fails", async () => { + const backend = new FakeNativeBackend(); + backend.frameUploadFailure = new TypeError("native upload is unsupported"); + const renderer = new FrameRenderer(backend, LAYOUT); + const source = borrowedFrame(19); + + await expect(renderer.uploadStreaming(0, 1, source.source)).resolves.toMatchObject({ + kind: "stream", + slot: 0, + pathGeneration: 1 + }); + expect(source.copyOptions()).toEqual({ + rect: { x: 0, y: 0, width: 4, height: 12 }, + format: "RGBA", + layout: [{ offset: 0, stride: 64 }] + }); + expect(source.closeCalls()).toBe(1); + expect(backend.uploads).toHaveLength(1); + }); + + it("does not commit a native upload that reenters renderer disposal", async () => { + const backend = new FakeNativeBackend(); + const renderer = new FrameRenderer(backend, LAYOUT); + const source = borrowedFrame(13); + backend.frameUploadAction = () => renderer.dispose(); + + await expect(renderer.uploadResident(0, source.source)).resolves.toBeNull(); + expect(source.closeCalls()).toBe(1); + expect(renderer.snapshot()).toMatchObject({ + state: "disposed", + uploadedResidentLayers: 0, + residentUploads: 0 + }); + }); + it("bounds browser-owned coded padding while copying only exact storage pixels", async () => { const backend = new FakeBackend(); const renderer = new FrameRenderer(backend, LAYOUT); @@ -606,6 +664,38 @@ class FakeBackend implements FrameRendererBackend { } } +class FakeNativeBackend extends FakeBackend { + public readonly frameUploads: Array<{ + readonly kind: FrameTextureKind; + readonly index: number; + readonly frame: CopyableVideoFrame; + readonly layout: Readonly<{ + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + }>; + }> = []; + public frameUploadFailure: Error | null = null; + public frameUploadAction: (() => void) | null = null; + + public uploadFrame( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly<{ + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + }> + ): void { + if (this.frameUploadFailure !== null) throw this.frameUploadFailure; + this.frameUploadAction?.(); + this.frameUploads.push({ kind, index, frame, layout }); + } +} + class FakeLegacyBackend implements LegacyOpaqueFrameRendererBackend { public readonly limits = Object.freeze({ maxTextureSize: 8_192, diff --git a/packages/player-web/src/runtime/frame-renderer.ts b/packages/player-web/src/runtime/frame-renderer.ts index 093811b..5ad7b36 100644 --- a/packages/player-web/src/runtime/frame-renderer.ts +++ b/packages/player-web/src/runtime/frame-renderer.ts @@ -61,11 +61,25 @@ export interface FrameRendererBackendLimits { export type FrameTextureKind = "resident" | "stream"; +export interface FrameSourceLayout { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + /** Small injectable boundary used for deterministic ownership tests. */ export interface FrameRendererBackend { readonly limits: Readonly; allocate(layout: FrameTextureLayout, streamingSlots: number): void; upload(kind: FrameTextureKind, index: number, pixels: Uint8Array): void; + /** Optional native-source path for browsers without VideoFrame RGBA copy. */ + uploadFrame?( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ): void; draw(kind: FrameTextureKind, index: number): void; readPixels?(): Uint8Array; dispose(): void; @@ -76,6 +90,12 @@ export interface LegacyOpaqueFrameRendererBackend { readonly limits: Readonly; allocate(layout: LegacyOpaqueFrameTextureLayout, streamingSlots: number): void; upload(kind: FrameTextureKind, index: number, pixels: Uint8Array): void; + uploadFrame?( + kind: FrameTextureKind, + index: number, + frame: CopyableVideoFrame, + layout: Readonly + ): void; draw(kind: FrameTextureKind, index: number): void; readPixels?(): Uint8Array; dispose(): void; @@ -146,9 +166,8 @@ const DEFAULT_RENDERER_TIMERS: Readonly = const LEGACY_VISIBLE_FRAME_OPTIONS = new WeakSet(); /** - * Owns one bounded staging buffer and serializes every async frame copy before - * passing packed RGBA bytes to an injected WebGL backend. Source frames are - * always closed exactly once by this class after ownership is transferred. + * Serializes native frame uploads and owns one bounded RGBA staging fallback. + * Source frames are always closed exactly once after ownership is transferred. */ export class FrameRenderer { readonly #layout: Readonly; @@ -268,9 +287,8 @@ export class FrameRenderer { return this.#queueUpload( source, resourceGeneration, - (pixels) => { - this.#requireActiveBackend().upload("resident", layer, pixels); - this.#assertUploadCurrent(resourceGeneration); + Object.freeze({ kind: "resident", index: layer }), + () => { this.#uploadedResidentLayers.add(layer); this.#residentUploads += 1; return Object.freeze({ @@ -294,9 +312,8 @@ export class FrameRenderer { return this.#queueUpload( source, resourceGeneration, - (pixels) => { - this.#requireActiveBackend().upload("stream", slot, pixels); - this.#assertUploadCurrent(resourceGeneration); + Object.freeze({ kind: "stream", index: slot }), + () => { this.#streamingUploads += 1; const uploadSerial = this.#nextStreamingUploadSerial; this.#nextStreamingUploadSerial += 1; @@ -513,7 +530,11 @@ export class FrameRenderer { #queueUpload( source: BorrowedVideoFrame, resourceGeneration: number, - upload: (pixels: Uint8Array) => T + destination: Readonly<{ + readonly kind: FrameTextureKind; + readonly index: number; + }>, + commit: () => T ): Promise { validateFrameGeneration(resourceGeneration, "resource generation"); validateFrameObject(source, "borrowed video frame"); @@ -538,25 +559,54 @@ export class FrameRenderer { this.#layout, this.#legacyVisibleFrameGeometry ); - const staging = this.#staging; - staging.fill(0); - const copy = this.#trackSourceCopy(source.frame.copyTo(staging, { - rect: copyLayout.rect, - format: "RGBA", - layout: [ - { - offset: copyLayout.offset, - stride: copyLayout.stride + const backend = this.#requireActiveBackend(); + let uploaded = false; + if (backend.uploadFrame !== undefined) { + try { + backend.uploadFrame( + destination.kind, + destination.index, + source.frame, + copyLayout.source + ); + uploaded = true; + } catch { + if ( + this.#state !== "active" || + resourceGeneration !== this.#resourceGeneration + ) { + throw new RendererUploadAbortedError( + "renderer changed during native frame upload" + ); } - ] - })); - const copiedPlanes = await awaitRendererCopy( - copy, - this.#uploadAbort.signal, - this.#copyTimeoutMs, - this.#timers - ); - validateCopiedPlaneLayout(copiedPlanes, copyLayout); + // Some engines expose the WebGL overload but reject a native + // VideoFrame at runtime. The bounded RGBA copy remains the fallback. + } + } + if (!uploaded) { + const staging = this.#staging; + staging.fill(0); + const copy = this.#trackSourceCopy(source.frame.copyTo(staging, { + rect: copyLayout.rect, + format: "RGBA", + layout: [ + { + offset: copyLayout.offset, + stride: copyLayout.stride + } + ] + })); + const copiedPlanes = await awaitRendererCopy( + copy, + this.#uploadAbort.signal, + this.#copyTimeoutMs, + this.#timers + ); + validateCopiedPlaneLayout(copiedPlanes, copyLayout); + this.#assertUploadCurrent(resourceGeneration); + backend.upload(destination.kind, destination.index, staging); + } + this.#assertUploadCurrent(resourceGeneration); if ( this.#state !== "active" || resourceGeneration !== this.#resourceGeneration @@ -564,7 +614,7 @@ export class FrameRenderer { this.#staleUploads += 1; return; } - result = upload(staging); + result = commit(); } catch (error) { if ( this.#state === "disposed" || @@ -785,7 +835,8 @@ function validateFrameGeometry( return Object.freeze({ rect: visible, offset: 0, - stride: width * 4 + stride: width * 4, + source: Object.freeze({ x: 0, y: 0, width, height }) }); } if ( @@ -808,7 +859,8 @@ function validateFrameGeometry( return Object.freeze({ rect: visible, offset: (y * geometry.codedWidth + x) * 4, - stride: geometry.codedWidth * 4 + stride: geometry.codedWidth * 4, + source: Object.freeze({ x, y, width, height }) }); } @@ -824,6 +876,7 @@ interface FrameCopyLayout { readonly rect: DOMRectReadOnly; readonly offset: number; readonly stride: number; + readonly source: Readonly; } function validateCopiedPlaneLayout( @@ -937,6 +990,12 @@ function adaptLegacyOpaqueBackend( limits: backend.limits, allocate, upload: (kind, index, pixels) => backend.upload(kind, index, pixels), + ...(backend.uploadFrame === undefined + ? {} + : { + uploadFrame: (kind, index, frame, layout) => + backend.uploadFrame!(kind, index, frame, layout) + }), draw: (kind, index) => backend.draw(kind, index), dispose: () => backend.dispose() }; diff --git a/tests/browser/m5-opaque-avc-worker.spec.ts b/tests/browser/m5-opaque-avc-worker.spec.ts index c08fae3..ab7a424 100644 --- a/tests/browser/m5-opaque-avc-worker.spec.ts +++ b/tests/browser/m5-opaque-avc-worker.spec.ts @@ -217,10 +217,15 @@ test("decodes the compiled opaque path through one dedicated worker", async ({ expect(report.colorSpaceVariants.length).toBeGreaterThan(0); for (const color of report.colorSpaceVariants) { - expect(color.fullRange).not.toBe(true); - expect([null, "bt709"]).toContain(color.matrix); - expect([null, "bt709"]).toContain(color.primaries); - expect([null, "bt709"]).toContain(color.transfer); + const limitedBt709 = color.fullRange !== true && + [null, "bt709"].includes(color.matrix) && + [null, "bt709"].includes(color.primaries) && + [null, "bt709"].includes(color.transfer); + const webKitNormalizedBt709 = color.fullRange === true && + color.matrix === "bt709" && + color.primaries === "bt709" && + color.transfer === "iec61966-2-1"; + expect(limitedBt709 || webKitNormalizedBt709).toBe(true); } expect(report.credit).toMatchObject({ diff --git a/tests/browser/m9-production-engine-capability.spec.ts b/tests/browser/m9-production-engine-capability.spec.ts index 27cb31d..90e13d4 100644 --- a/tests/browser/m9-production-engine-capability.spec.ts +++ b/tests/browser/m9-production-engine-capability.spec.ts @@ -7,6 +7,7 @@ test("built public element records an honest functional-engine capability outcom const element = document.createElement("aval-player") as unknown as HTMLElement & { src: string; prepare(): Promise; + pause(): void; dispose(): Promise; getDiagnostics(): { readiness: string; @@ -23,17 +24,96 @@ test("built public element records an honest functional-engine capability outcom element.src = `/__m8__/asset?session=${encodeURIComponent(session)}&fixture=user-states`; document.querySelector("[data-certification-stage]")!.append(element); await element.prepare(); + element.pause(); const ready = element.getDiagnostics(); + let renderedPixels: null | { + width: number; + height: number; + nonTransparent: number; + nonZeroColor: number; + visibleWidth: number; + visibleHeight: number; + } = null; + if (ready.readiness === "interactiveReady") { + const canvas = element.shadowRoot?.querySelector( + 'canvas[data-aval-layer="animated"]' + ); + if (canvas === null || canvas === undefined) { + throw new Error("animated presentation canvas is unavailable"); + } + await new Promise((resolve) => requestAnimationFrame(() => resolve())); + const bitmap = await createImageBitmap(canvas); + const snapshot = document.createElement("canvas"); + snapshot.width = bitmap.width; + snapshot.height = bitmap.height; + const context = snapshot.getContext("2d", { willReadFrequently: true }); + if (context === null) throw new Error("pixel evidence context is unavailable"); + context.drawImage(bitmap, 0, 0); + bitmap.close(); + const pixels = context.getImageData(0, 0, snapshot.width, snapshot.height).data; + let nonTransparent = 0; + let nonZeroColor = 0; + let minimumX = snapshot.width; + let minimumY = snapshot.height; + let maximumX = -1; + let maximumY = -1; + for (let offset = 0; offset < pixels.length; offset += 4) { + if (pixels[offset + 3] === 0) continue; + nonTransparent += 1; + if (pixels[offset] !== 0 || pixels[offset + 1] !== 0 || pixels[offset + 2] !== 0) { + nonZeroColor += 1; + } + const pixel = offset / 4; + const x = pixel % snapshot.width; + const y = Math.floor(pixel / snapshot.width); + minimumX = Math.min(minimumX, x); + minimumY = Math.min(minimumY, y); + maximumX = Math.max(maximumX, x); + maximumY = Math.max(maximumY, y); + } + renderedPixels = { + width: snapshot.width, + height: snapshot.height, + nonTransparent, + nonZeroColor, + visibleWidth: maximumX < minimumX ? 0 : maximumX - minimumX + 1, + visibleHeight: maximumY < minimumY ? 0 : maximumY - minimumY + 1 + }; + } element.remove(); await element.dispose(); - return { ready, terminal: element.getDiagnostics().outstanding }; + return { + ready, + renderedPixels, + terminal: element.getDiagnostics().outstanding + }; }, `m9-engine-${testInfo.project.name}`); const supported = result.ready.readiness === "interactiveReady"; + const animationRequired = /(?:chromium|webkit)-engine-production-probe$/u.test( + testInfo.project.name + ); testInfo.annotations.push({ type: "functional-engine-capability", description: supported ? "production-animation-supported" : "production-animation-unsupported" }); + if (animationRequired) { + expect(result.ready).toMatchObject({ + readiness: "interactiveReady", + mode: "animated", + staticReason: null, + lastFailure: null + }); + expect(result.renderedPixels?.width).toBeGreaterThan(0); + expect(result.renderedPixels?.height).toBeGreaterThan(0); + expect( + (result.renderedPixels?.width ?? 0) / (result.renderedPixels?.height ?? 1) + ).toBeCloseTo(16 / 9, 6); + expect(result.renderedPixels?.nonTransparent).toBeGreaterThan(100); + expect(result.renderedPixels?.nonZeroColor).toBeGreaterThan(100); + expect(result.renderedPixels?.visibleWidth).toBeGreaterThan(8); + expect(result.renderedPixels?.visibleHeight).toBeGreaterThan(8); + } if (supported) { expect(result.ready).toMatchObject({ mode: "animated", staticReason: null, lastFailure: null }); expect(result.ready.runtime.selectedRendition).not.toBeNull();