diff --git a/icons/icons/png/1024x1024.png b/icons/icons/png/1024x1024.png index c194c3fff..7cc41c028 100644 Binary files a/icons/icons/png/1024x1024.png and b/icons/icons/png/1024x1024.png differ diff --git a/icons/icons/png/128x128.png b/icons/icons/png/128x128.png index 8dc2b50fd..82b456e9d 100644 Binary files a/icons/icons/png/128x128.png and b/icons/icons/png/128x128.png differ diff --git a/icons/icons/png/16x16.png b/icons/icons/png/16x16.png index 66ea3f137..e1f45c1f4 100644 Binary files a/icons/icons/png/16x16.png and b/icons/icons/png/16x16.png differ diff --git a/icons/icons/png/24x24.png b/icons/icons/png/24x24.png index b56ffe899..e146b337a 100644 Binary files a/icons/icons/png/24x24.png and b/icons/icons/png/24x24.png differ diff --git a/icons/icons/png/256x256.png b/icons/icons/png/256x256.png index 6d6eb25e7..0592161ab 100644 Binary files a/icons/icons/png/256x256.png and b/icons/icons/png/256x256.png differ diff --git a/icons/icons/png/32x32.png b/icons/icons/png/32x32.png index dee1d807a..6fb056f8a 100644 Binary files a/icons/icons/png/32x32.png and b/icons/icons/png/32x32.png differ diff --git a/icons/icons/png/48x48.png b/icons/icons/png/48x48.png index 78725c5a5..48af277bf 100644 Binary files a/icons/icons/png/48x48.png and b/icons/icons/png/48x48.png differ diff --git a/icons/icons/png/512x512.png b/icons/icons/png/512x512.png index 1d1a8fa60..4016ff592 100644 Binary files a/icons/icons/png/512x512.png and b/icons/icons/png/512x512.png differ diff --git a/icons/icons/png/64x64.png b/icons/icons/png/64x64.png index fc3c7a673..e132d69a4 100644 Binary files a/icons/icons/png/64x64.png and b/icons/icons/png/64x64.png differ diff --git a/icons/icons/win/icon.ico b/icons/icons/win/icon.ico index 6a327f3d9..52e47c894 100644 Binary files a/icons/icons/win/icon.ico and b/icons/icons/win/icon.ico differ diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts index 7df341f95..a362c1267 100644 --- a/src/lib/exporter/modernFrameRenderer.test.ts +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -304,6 +304,28 @@ describe("ModernFrameRenderer blur export path", () => { }); describe("ModernFrameRenderer webcam frame cache", () => { + it("stages webcam video frames on WebGPU instead of using retained frame uploads", () => { + const renderer = createRenderer() as any; + renderer.rendererBackend = "webgpu"; + + const frame = { + displayWidth: 320, + displayHeight: 180, + timestamp: 0, + } as VideoFrame; + + const result = renderer.stageVideoFrameForTexture(frame, "webcam", 640, 360); + + expect(result).toBe(renderer.webcamVideoFrameStagingCanvas); + expect(renderer.webcamVideoFrameStagingCtx.drawImage).toHaveBeenCalledWith( + frame, + 0, + 0, + 320, + 180, + ); + }); + it("uses staging canvas instead of recursing when WebGPU frame retention fails", () => { const renderer = createRenderer() as any; const originalVideoFrame = (globalThis as any).VideoFrame; @@ -355,7 +377,12 @@ describe("ModernFrameRenderer webcam frame cache", () => { it("bypasses the refresh throttle for cropped webcam regions", () => { const renderer = createRenderer() as any; - renderer.config.webcam.cropRegion = { x: 0.25, y: 0, width: 0.5, height: 1 }; + renderer.config.webcam.cropRegion = { + x: 0.25, + y: 0, + width: 0.5, + height: 1, + }; renderer.webcamFrameCacheCanvas = { width: 640, height: 720 }; renderer.lastWebcamCacheRefreshTime = 10; renderer.currentVideoTime = 10.1; @@ -506,4 +533,221 @@ describe("ModernFrameRenderer webcam export fallback", () => { vi.useRealTimers(); } }); + + it("keeps the webcam live when sync uses an offset timeline", () => { + const renderer = createRenderer() as any; + renderer.config.webcam = { + ...DEFAULT_WEBCAM_OVERLAY, + enabled: true, + timeOffsetMs: 500, + }; + renderer.currentVideoTime = 10; + renderer.lastSyncedWebcamTime = 9.5; + renderer.webcamVideoElement = { + readyState: 2, + seeking: false, + videoWidth: 640, + videoHeight: 360, + duration: Number.NaN, + }; + renderer.webcamRootContainer = { + visible: false, + position: { set: vi.fn() }, + }; + renderer.webcamContainer = { + addChildAt: vi.fn(), + }; + renderer.webcamMaskGraphics = { + clear: vi.fn(), + moveTo: vi.fn(), + lineTo: vi.fn(), + closePath: vi.fn(), + fill: vi.fn(), + }; + renderer.webcamShadowLayers = []; + renderer.animationState = { + appliedScale: 1, + }; + + renderer.updateWebcamOverlay(); + + expect(renderer.webcamRootContainer.visible).toBe(true); + expect(renderer.webcamSprite).toBeTruthy(); + }); + + it("keeps the webcam live when the media element time is current but lastSyncedWebcamTime is stale", () => { + const renderer = createRenderer() as any; + renderer.config.webcam = { + ...DEFAULT_WEBCAM_OVERLAY, + enabled: true, + timeOffsetMs: 500, + }; + renderer.currentVideoTime = 10; + renderer.lastSyncedWebcamTime = 8; + renderer.webcamVideoElement = { + currentTime: 9.5, + readyState: 2, + seeking: false, + videoWidth: 640, + videoHeight: 360, + duration: Number.NaN, + }; + renderer.webcamRootContainer = { + visible: false, + position: { set: vi.fn() }, + }; + renderer.webcamContainer = { + addChildAt: vi.fn(), + }; + renderer.webcamMaskGraphics = { + clear: vi.fn(), + moveTo: vi.fn(), + lineTo: vi.fn(), + closePath: vi.fn(), + fill: vi.fn(), + }; + renderer.webcamShadowLayers = []; + renderer.animationState = { + appliedScale: 1, + }; + + renderer.updateWebcamOverlay(); + + expect(renderer.webcamRootContainer.visible).toBe(true); + expect(renderer.webcamSprite).toBeTruthy(); + }); + + it("snapshots media-element webcam frames into the cache before rendering", () => { + const renderer = createRenderer() as any; + renderer.config.webcam = { + ...DEFAULT_WEBCAM_OVERLAY, + enabled: true, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + }; + renderer.currentVideoTime = 4; + const previousHtmlVideoElement = ( + globalThis as typeof globalThis & { HTMLVideoElement?: unknown } + ).HTMLVideoElement; + class MockHtmlVideoElement { + currentTime = 4; + readyState = 2; + seeking = false; + videoWidth = 640; + videoHeight = 360; + duration = Number.NaN; + } + Object.assign(globalThis, { + HTMLVideoElement: MockHtmlVideoElement, + }); + + const webcamVideoElement = new MockHtmlVideoElement(); + + try { + const renderableSource = renderer.resolveRenderableWebcamSource( + webcamVideoElement, + 640, + 360, + true, + ); + + expect(renderableSource?.source).toBe(renderer.webcamFrameCacheCanvas); + expect(renderer.webcamFrameCacheCtx.drawImage).toHaveBeenCalledWith( + webcamVideoElement, + 0, + 0, + 640, + 360, + 0, + 0, + 640, + 360, + ); + } finally { + Object.assign(globalThis, { + HTMLVideoElement: previousHtmlVideoElement, + }); + } + }); + + it("renders decoder-backed webcam frames directly for the default crop region", () => { + const renderer = createRenderer() as any; + renderer.config.webcam = { + ...DEFAULT_WEBCAM_OVERLAY, + enabled: true, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + }; + + const previousVideoFrame = (globalThis as typeof globalThis & { VideoFrame?: unknown }) + .VideoFrame; + class MockVideoFrame {} + Object.assign(globalThis, { + VideoFrame: MockVideoFrame, + }); + + try { + const webcamFrame = new MockVideoFrame(); + const renderableSource = renderer.resolveRenderableWebcamSource( + webcamFrame, + 640, + 360, + true, + ); + + expect(renderableSource?.source).toBe(webcamFrame); + expect(renderer.webcamFrameCacheCanvas).toBeNull(); + } finally { + Object.assign(globalThis, { + VideoFrame: previousVideoFrame, + }); + } + }); +}); + +describe("ModernFrameRenderer temporal webcam sync", () => { + it("pins webcam sync to the frame center during temporal blur sampling", async () => { + const renderer = createRenderer() as any; + renderer.config.annotationRegions = []; + renderer.config.zoomTemporalMotionBlur = 1; + renderer.config.zoomMotionBlurSampleCount = 3; + renderer.config.zoomMotionBlurShutterFraction = 0.5; + renderer.app = { canvas: createMockCanvas() }; + renderer.updateCaptionLayer = vi.fn(); + renderer.renderSceneSample = vi.fn(async (sampleTimestamp: number) => ({ + timeMs: sampleTimestamp / 1000, + cursorTimeMs: sampleTimestamp / 1000, + backgroundTimelineTimeMs: sampleTimestamp / 1000, + sceneTransform: { scale: 1, x: 0, y: 0 }, + zoom: { scale: 1, focusX: 0.5, focusY: 0.5, progress: 0 }, + })); + + await renderer.renderTemporalMotionBlurFrame( + 1_000_000, + 1_000_000, + 1_000_000, + 33_333, + { + stageSize: { width: 1920, height: 1080 }, + videoSize: { width: 1920, height: 1080 }, + baseScale: 1, + baseOffset: { x: 0, y: 0 }, + maskRect: { + x: 0, + y: 0, + width: 1920, + height: 1080, + sourceCrop: { x: 0, y: 0, width: 1, height: 1 }, + }, + }, + ); + + expect(renderer.renderSceneSample).toHaveBeenCalledTimes(3); + expect(renderer.renderSceneSample.mock.calls.map((call: unknown[]) => call[6])).toEqual([ + 1, + 1, + 1, + ]); + expect( + new Set(renderer.renderSceneSample.mock.calls.map((call: unknown[]) => call[0])).size, + ).toBeGreaterThan(1); + }); }); diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index b71efe81e..a6819f6c7 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -450,9 +450,15 @@ export class FrameRenderer { private captionRenderKey: string | null = null; private frameSprite: Sprite | null = null; private frameImage: HTMLImageElement | null = null; - private frameDraw: ((ctx: CanvasRenderingContext2D, width: number, height: number) => void) | null = - null; - private frameInsets: { top: number; right: number; bottom: number; left: number } | null = null; + private frameDraw: + | ((ctx: CanvasRenderingContext2D, width: number, height: number) => void) + | null = null; + private frameInsets: { + top: number; + right: number; + bottom: number; + left: number; + } | null = null; private frameRasterCanvas: HTMLCanvasElement | null = null; private frameRasterWidth = 0; private frameRasterHeight = 0; @@ -534,7 +540,9 @@ export class FrameRenderer { canvas.height = this.config.height; try { - const exportCanvas = canvas as HTMLCanvasElement & { colorSpace?: string }; + const exportCanvas = canvas as HTMLCanvasElement & { + colorSpace?: string; + }; if ("colorSpace" in exportCanvas) { exportCanvas.colorSpace = "srgb"; } @@ -628,7 +636,10 @@ export class FrameRenderer { this.setupCaptionResources(); if (this.shouldUseZoomMotionBlur()) { - this.zoomBlurFilter = new ZoomBlurFilter({ strength: 0, maxKernelSize: 13 }); + this.zoomBlurFilter = new ZoomBlurFilter({ + strength: 0, + maxKernelSize: 13, + }); this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); } @@ -732,7 +743,11 @@ export class FrameRenderer { private createShadowLayers( parent: Container, - configs: ReadonlyArray<{ offsetScale: number; alphaScale: number; blurScale: number }>, + configs: ReadonlyArray<{ + offsetScale: number; + alphaScale: number; + blurScale: number; + }>, ): ShadowLayer[] { return configs.map((config) => { const container = new Container(); @@ -1050,6 +1065,13 @@ export class FrameRenderer { fallbackWidth: number, fallbackHeight: number, ): CanvasImageSource | VideoFrame { + // Keep webcam uploads on the older canvas-staged path. The newer + // retained-VideoFrame upload path is fine for the main scene/background, + // but it has produced unstable webcam overlays in Lightning exports. + if (kind === "webcam") { + return this.stageVideoFrameOnCanvas(frame, kind, fallbackWidth, fallbackHeight); + } + if (this.rendererBackend === "webgpu") { return this.resolveRetainedVideoFrameSource(frame, kind, fallbackWidth, fallbackHeight); } @@ -2004,8 +2026,12 @@ export class FrameRenderer { } }; - video.addEventListener("seeked", waitForPresentedFrame, { once: true }); - video.addEventListener("loadeddata", handleMediaReady, { once: true }); + video.addEventListener("seeked", waitForPresentedFrame, { + once: true, + }); + video.addEventListener("loadeddata", handleMediaReady, { + once: true, + }); video.addEventListener("canplay", handleMediaReady, { once: true }); video.addEventListener("error", finish, { once: true }); @@ -2111,7 +2137,11 @@ export class FrameRenderer { await new Promise((resolve, reject) => { image.onload = () => resolve(); image.onerror = () => - reject(new Error(`[ModernFrameRenderer] Failed to load device frame image: ${frameId}`)); + reject( + new Error( + `[ModernFrameRenderer] Failed to load device frame image: ${frameId}`, + ), + ); image.src = frame.filePath; }); this.frameImage = image; @@ -2474,6 +2504,7 @@ export class FrameRenderer { source: CanvasImageSource | VideoFrame, width: number, height: number, + referenceTimeSeconds = this.currentVideoTime, ): boolean { const sourceRect = getWebcamCropSourceRect(this.config.webcam?.cropRegion, width, height); if (!this.ensureWebcamFrameCache(sourceRect.sw, sourceRect.sh)) { @@ -2501,7 +2532,7 @@ export class FrameRenderer { this.webcamFrameCacheCanvas.width, this.webcamFrameCacheCanvas.height, ); - this.lastWebcamCacheRefreshTime = this.currentVideoTime; + this.lastWebcamCacheRefreshTime = referenceTimeSeconds; return true; } @@ -2527,18 +2558,29 @@ export class FrameRenderer { liveSourceWidth: number, liveSourceHeight: number, canUseLiveSource: boolean, + referenceTimeSeconds = this.currentVideoTime, ): WebcamRenderSource | null { if (canUseLiveSource && liveSource && liveSourceWidth > 0 && liveSourceHeight > 0) { - if (this.shouldRefreshWebcamFrameCache(liveSourceWidth, liveSourceHeight)) { - this.refreshWebcamFrameCache(liveSource, liveSourceWidth, liveSourceHeight); - } - if (!isWebcamCropRegionDefault(this.config.webcam?.cropRegion)) { + const usesDefaultCropRegion = isWebcamCropRegionDefault(this.config.webcam?.cropRegion); + const needsCacheBackedSource = + !usesDefaultCropRegion || + (typeof HTMLVideoElement !== "undefined" && + liveSource instanceof HTMLVideoElement); + + if (needsCacheBackedSource) { + this.refreshWebcamFrameCache( + liveSource, + liveSourceWidth, + liveSourceHeight, + referenceTimeSeconds, + ); const cachedSource = this.getCachedWebcamRenderSource(); if (cachedSource) { this.setWebcamRenderMode("live"); return cachedSource; } } + this.setWebcamRenderMode("live"); return { source: liveSource, @@ -2636,14 +2678,18 @@ export class FrameRenderer { this.webcamLayoutCache = { ...nextLayout }; } - private async syncWebcamFrame(targetTime: number): Promise { - const webcamTargetTime = getWebcamMediaTargetTimeSeconds({ + private getExpectedWebcamTargetTimeSeconds(targetTime: number): number { + return getWebcamMediaTargetTimeSeconds({ currentTime: targetTime, webcamDuration: Number.isFinite(this.webcamVideoElement?.duration) ? this.webcamVideoElement?.duration : null, timeOffsetMs: this.config.webcam?.timeOffsetMs, }); + } + + private async syncWebcamFrame(targetTime: number): Promise { + const webcamTargetTime = this.getExpectedWebcamTargetTimeSeconds(targetTime); if (this.webcamForwardFrameSource) { const clampedTime = clampMediaTimeToDuration(webcamTargetTime, null); @@ -2779,8 +2825,12 @@ export class FrameRenderer { } }; - webcamVideo.addEventListener("seeked", waitForPresentedFrame, { once: true }); - webcamVideo.addEventListener("loadeddata", handleMediaReady, { once: true }); + webcamVideo.addEventListener("seeked", waitForPresentedFrame, { + once: true, + }); + webcamVideo.addEventListener("loadeddata", handleMediaReady, { + once: true, + }); webcamVideo.addEventListener("canplay", handleMediaReady, { once: true }); webcamVideo.addEventListener("error", finish, { once: true }); @@ -2811,7 +2861,7 @@ export class FrameRenderer { } } - private updateWebcamOverlay(): void { + private updateWebcamOverlay(referenceTimeSeconds = this.currentVideoTime): void { const webcam = this.config.webcam; if (!webcam?.enabled || !this.webcamRootContainer || !this.webcamMaskGraphics) { if (this.webcamRootContainer) { @@ -2828,10 +2878,16 @@ export class FrameRenderer { : { width: 0, height: 0 }; const activeWebcamVideoElement = webcamSource === this.webcamVideoElement ? this.webcamVideoElement : null; + const expectedWebcamTargetTime = + this.getExpectedWebcamTargetTimeSeconds(referenceTimeSeconds); + const measuredWebcamTime = + activeWebcamVideoElement && Number.isFinite(activeWebcamVideoElement.currentTime) + ? activeWebcamVideoElement.currentTime + : this.lastSyncedWebcamTime; const webcamTimeDrift = - this.lastSyncedWebcamTime === null + measuredWebcamTime === null ? 0 - : Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime); + : Math.abs(measuredWebcamTime - expectedWebcamTargetTime); const canUseLiveSource = !!webcamSource && liveSourceDimensions.width > 0 && @@ -2845,6 +2901,7 @@ export class FrameRenderer { liveSourceDimensions.width, liveSourceDimensions.height, canUseLiveSource, + referenceTimeSeconds, ); if (!renderableWebcamSource) { @@ -2910,15 +2967,20 @@ export class FrameRenderer { layoutCache: LayoutCache, useVelocityMotionBlur: boolean, includeOverlayLayers = true, + webcamTimeSecondsOverride?: number, ): Promise { if (!this.app || !this.cameraContainer || !this.videoMaskGraphics) { throw new Error("Renderer not initialized"); } this.currentVideoTime = timestamp / 1_000_000; + const webcamRenderTimeSeconds = Math.max( + 0, + webcamTimeSecondsOverride ?? this.currentVideoTime, + ); if (this.webcamForwardFrameSource || this.webcamVideoElement) { - await this.syncWebcamFrame(Math.max(0, this.currentVideoTime)); + await this.syncWebcamFrame(webcamRenderTimeSeconds); } if (this.backgroundForwardFrameSource || this.backgroundVideoElement) { @@ -2966,7 +3028,7 @@ export class FrameRenderer { this.updateAnnotationLayer(timeMs); this.updateCaptionLayer(timeMs); } - this.updateWebcamOverlay(); + this.updateWebcamOverlay(webcamRenderTimeSeconds); const annotationContainerVisible = this.annotationContainer?.visible ?? true; const captionContainerVisible = this.captionContainer?.visible ?? true; @@ -3033,6 +3095,7 @@ export class FrameRenderer { } const samplePlan = buildTemporalSamplePlanUs(frameDurationUs, blurConfig); + const webcamReferenceTimeSeconds = Math.max(0, timestamp / 1_000_000); compositeState.context.clearRect( 0, @@ -3058,6 +3121,7 @@ export class FrameRenderer { layoutCache, false, false, + webcamReferenceTimeSeconds, ); lastSnapshot = snapshot; if (Math.abs(sampleOffsetUs) < 0.0001) { diff --git a/src/lib/exporter/modernVideoExporter.fallback.test.ts b/src/lib/exporter/modernVideoExporter.fallback.test.ts index 80125266b..95f06a997 100644 --- a/src/lib/exporter/modernVideoExporter.fallback.test.ts +++ b/src/lib/exporter/modernVideoExporter.fallback.test.ts @@ -117,4 +117,57 @@ describe("ModernVideoExporter native fallback routing", () => { expect(initializeEncoder).toHaveBeenCalledTimes(1); expect(mocks.muxerFinalize).toHaveBeenCalledTimes(1); }, 15_000); + + it("retries the main decode path once with a readable file-backed source", async () => { + const { ModernVideoExporter } = await import("./modernVideoExporter"); + mocks.streamingDecoderGetEffectiveDuration.mockReturnValue(1); + mocks.streamingDecoderDecodeAll + .mockRejectedValueOnce( + new Error("readAVPacket pipeline failed: Failed after 3 attempts"), + ) + .mockResolvedValueOnce(undefined); + + const exporter = new ModernVideoExporter({ + videoUrl: "file:///recording.mp4", + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + wallpaper: "#101010", + padding: 0, + borderRadius: 0, + backgroundBlur: 0, + shadowIntensity: 0, + showShadow: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + backendPreference: "webcodecs", + } as never) as unknown as { + export: () => Promise<{ success: boolean; blob?: Blob; error?: string }>; + initializeEncoder: () => Promise; + }; + + vi.spyOn(exporter, "initializeEncoder").mockResolvedValue({ + codec: "avc1.640034", + hardwareAcceleration: "prefer-hardware", + }); + + const result = await exporter.export(); + + expect(result.success).toBe(true); + expect(mocks.streamingDecoderLoadMetadata).toHaveBeenCalledTimes(2); + expect(mocks.streamingDecoderLoadMetadata.mock.calls[0]).toEqual([ + "file:///recording.mp4", + { + forceReadableFileSource: false, + }, + ]); + expect(mocks.streamingDecoderLoadMetadata.mock.calls[1]).toEqual([ + "file:///recording.mp4", + { + forceReadableFileSource: true, + }, + ]); + expect(mocks.streamingDecoderDecodeAll).toHaveBeenCalledTimes(2); + expect(mocks.muxerFinalize).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 55f47923e..679c9a9da 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -167,7 +167,11 @@ type NativeAudioPlan = audioSourcePath: string; audioSourceCodec?: string; audioSourceSampleRate: number; - editedTrackSegments: Array<{ startMs: number; endMs: number; speed: number }>; + editedTrackSegments: Array<{ + startMs: number; + endMs: number; + speed: number; + }>; }; const FILTERGRAPH_FALLBACK_AUDIO_SAMPLE_RATE = 48_000; @@ -271,6 +275,13 @@ type NativeStaticLayoutZoomSample = { }; const NATIVE_EXPORT_ENGINE_NAME = "Breeze"; +const READABLE_SOURCE_RETRY_ERROR_TOKENS = [ + "readavpacket", + "get_media_info", + "avfoundation", + "failed after 3 attempts", + "pipeline failed", +]; const LIGHTNING_PIPELINE_NAME = "Lightning (Beta)"; const STATIC_LAYOUT_CHUNK_DURATION_SEC = 120; const MISSING_NATIVE_WALLPAPER_FALLBACK_COLOR = "#ffffff"; @@ -347,128 +358,177 @@ export class ModernVideoExporter { } async export(): Promise { - try { - this.cleanup(); - this.cancelled = false; - this.encoderError = null; - this.nativeEncoderError = null; - this.nativeStaticLayoutSkipReason = null; - this.nativeStaticLayoutSkipReasons = []; - this.nativeStaticLayoutBackgroundSkipReason = null; - this.totalExportStartTimeMs = this.getNowMs(); - const backendPreference = this.config.backendPreference ?? "auto"; - const runtimePlatform = this.getRuntimePlatform(); - let useNativeEncoder = false; - let triedNativeStaticLayoutWithProbe = false; - let shouldDeferNativeEncoderStart = backendPreference === "breeze"; - this.lastNativeExportError = null; - - let stageStartedAt = this.getNowMs(); - if (backendPreference === "breeze") { - // Defer the streaming native encoder until after metadata is known. - // Static-layout exports can then use the faster Windows D3D compositor - // instead of unnecessarily rendering every frame through JS first. - } else if ( - backendPreference === "auto" && - shouldPreferNativeAutoBackend(runtimePlatform) - ) { - stageStartedAt = this.getNowMs(); - useNativeEncoder = await this.tryStartNativeVideoExport(); - this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; + let preferReadableFileSource = false; + let retriedWithReadableFileSource = false; - if (!useNativeEncoder) { - console.warn( - `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} auto-preferred native export was unavailable; falling back to WebCodecs.`, - this.lastNativeExportError, - ); + while (true) { + let shouldRetryWithReadableFileSource = false; + try { + this.cleanup(); + this.cancelled = false; + this.encoderError = null; + this.nativeEncoderError = null; + this.nativeStaticLayoutSkipReason = null; + this.nativeStaticLayoutSkipReasons = []; + this.nativeStaticLayoutBackgroundSkipReason = null; + this.totalExportStartTimeMs = this.getNowMs(); + const backendPreference = this.config.backendPreference ?? "auto"; + const runtimePlatform = this.getRuntimePlatform(); + let useNativeEncoder = false; + let triedNativeStaticLayoutWithProbe = false; + let shouldDeferNativeEncoderStart = backendPreference === "breeze"; + this.lastNativeExportError = null; + + let stageStartedAt = this.getNowMs(); + if (backendPreference === "breeze") { + // Defer the streaming native encoder until after metadata is known. + // Static-layout exports can then use the faster Windows D3D compositor + // instead of unnecessarily rendering every frame through JS first. + } else if ( + backendPreference === "auto" && + shouldPreferNativeAutoBackend(runtimePlatform) + ) { stageStartedAt = this.getNowMs(); - await this.initializeEncoder(); - } - } else { - try { - const configuredWebCodecsPath = await this.initializeEncoder(); - if ( - backendPreference === "auto" && - configuredWebCodecsPath.hardwareAcceleration === "prefer-software" - ) { + useNativeEncoder = await this.tryStartNativeVideoExport(); + this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; + + if (!useNativeEncoder) { console.warn( - "[VideoExporter] Auto backend resolved to a software WebCodecs encoder; trying Breeze native export instead.", + `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} auto-preferred native export was unavailable; falling back to WebCodecs.`, + this.lastNativeExportError, ); + stageStartedAt = this.getNowMs(); + await this.initializeEncoder(); + } + } else { + try { + const configuredWebCodecsPath = await this.initializeEncoder(); + if ( + backendPreference === "auto" && + configuredWebCodecsPath.hardwareAcceleration === "prefer-software" + ) { + console.warn( + "[VideoExporter] Auto backend resolved to a software WebCodecs encoder; trying Breeze native export instead.", + ); + stageStartedAt = this.getNowMs(); + useNativeEncoder = await this.tryStartNativeVideoExport(); + this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; + if (useNativeEncoder) { + this.disposeEncoder(); + } + } + } catch (error) { + const webCodecsError = + error instanceof Error ? error : new Error(String(error)); + if (backendPreference === "webcodecs") { + throw webCodecsError; + } + + console.warn( + `[VideoExporter] WebCodecs encoder unavailable, trying ${NATIVE_EXPORT_ENGINE_NAME} native export fallback`, + webCodecsError, + ); + this.disposeEncoder(); + stageStartedAt = this.getNowMs(); useNativeEncoder = await this.tryStartNativeVideoExport(); this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; - if (useNativeEncoder) { - this.disposeEncoder(); + + if (!useNativeEncoder) { + throw webCodecsError; } } - } catch (error) { - const webCodecsError = - error instanceof Error ? error : new Error(String(error)); - if (backendPreference === "webcodecs") { - throw webCodecsError; - } - - console.warn( - `[VideoExporter] WebCodecs encoder unavailable, trying ${NATIVE_EXPORT_ENGINE_NAME} native export fallback`, - webCodecsError, - ); - this.disposeEncoder(); + } - stageStartedAt = this.getNowMs(); - useNativeEncoder = await this.tryStartNativeVideoExport(); - this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; + this.backpressureProfile = getExportBackpressureProfile({ + encodeBackend: + shouldDeferNativeEncoderStart || useNativeEncoder ? "ffmpeg" : "webcodecs", + width: this.config.width, + height: this.config.height, + frameRate: this.config.frameRate, + encodingMode: this.config.encodingMode, + }); + this.maxNativeWriteInFlight = useNativeEncoder + ? Math.max( + 1, + Math.floor( + this.config.maxInFlightNativeWrites ?? + this.backpressureProfile.maxInFlightNativeWrites, + ), + ) + : 1; + + console.log("[VideoExporter] Backpressure profile", { + profile: this.backpressureProfile.name, + encodeBackend: + shouldDeferNativeEncoderStart || useNativeEncoder ? "ffmpeg" : "webcodecs", + maxEncodeQueue: + this.config.maxEncodeQueue ?? this.backpressureProfile.maxEncodeQueue, + maxDecodeQueue: + this.config.maxDecodeQueue ?? this.backpressureProfile.maxDecodeQueue, + maxPendingFrames: + this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames, + maxInFlightNativeWrites: this.maxNativeWriteInFlight, + }); - if (!useNativeEncoder) { - throw webCodecsError; + if ( + (backendPreference === "auto" || backendPreference === "breeze") && + !useNativeEncoder + ) { + const nativeVideoInfo = await this.loadNativeStaticLayoutVideoInfo(); + if (nativeVideoInfo) { + triedNativeStaticLayoutWithProbe = true; + const nativeAudioPlan = this.buildNativeAudioPlan(nativeVideoInfo); + const effectiveDuration = + this.getNativeStaticLayoutEffectiveDuration(nativeVideoInfo); + this.effectiveDurationSec = effectiveDuration; + const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); + const staticLayoutResult = await this.tryExportNativeStaticLayout( + nativeVideoInfo, + nativeAudioPlan, + effectiveDuration, + totalFrames, + ); + if (staticLayoutResult) { + this.disposeEncoder(); + return staticLayoutResult; + } } } - } - this.backpressureProfile = getExportBackpressureProfile({ - encodeBackend: - shouldDeferNativeEncoderStart || useNativeEncoder ? "ffmpeg" : "webcodecs", - width: this.config.width, - height: this.config.height, - frameRate: this.config.frameRate, - encodingMode: this.config.encodingMode, - }); - this.maxNativeWriteInFlight = useNativeEncoder - ? Math.max( - 1, - Math.floor( - this.config.maxInFlightNativeWrites ?? - this.backpressureProfile.maxInFlightNativeWrites, - ), - ) - : 1; - - console.log("[VideoExporter] Backpressure profile", { - profile: this.backpressureProfile.name, - encodeBackend: - shouldDeferNativeEncoderStart || useNativeEncoder ? "ffmpeg" : "webcodecs", - maxEncodeQueue: - this.config.maxEncodeQueue ?? this.backpressureProfile.maxEncodeQueue, - maxDecodeQueue: - this.config.maxDecodeQueue ?? this.backpressureProfile.maxDecodeQueue, - maxPendingFrames: - this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames, - maxInFlightNativeWrites: this.maxNativeWriteInFlight, - }); + this.streamingDecoder = new StreamingVideoDecoder({ + maxDecodeQueue: + this.config.maxDecodeQueue ?? this.backpressureProfile.maxDecodeQueue, + maxPendingFrames: + this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames, + }); + stageStartedAt = this.getNowMs(); + const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl, { + forceReadableFileSource: preferReadableFileSource, + }); + this.metadataLoadTimeMs = this.getNowMs() - stageStartedAt; + const nativeAudioPlan = this.buildNativeAudioPlan(videoInfo); + const shouldUsePitchPreservingFfmpegAudio = + nativeAudioPlan.audioMode === "edited-track" && + nativeAudioPlan.strategy === "filtergraph-fast-path"; + const shouldUseFfmpegAudioFallback = + !useNativeEncoder && + nativeAudioPlan.audioMode !== "none" && + (shouldUsePitchPreservingFfmpegAudio || !(await isAacAudioEncodingSupported())); + const effectiveDuration = this.streamingDecoder.getEffectiveDuration( + this.config.trimRegions, + this.config.speedRegions, + ); + this.effectiveDurationSec = effectiveDuration; + const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); - if ( - (backendPreference === "auto" || backendPreference === "breeze") && - !useNativeEncoder - ) { - const nativeVideoInfo = await this.loadNativeStaticLayoutVideoInfo(); - if (nativeVideoInfo) { - triedNativeStaticLayoutWithProbe = true; - const nativeAudioPlan = this.buildNativeAudioPlan(nativeVideoInfo); - const effectiveDuration = - this.getNativeStaticLayoutEffectiveDuration(nativeVideoInfo); - this.effectiveDurationSec = effectiveDuration; - const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); + if ( + (backendPreference === "auto" || backendPreference === "breeze") && + !useNativeEncoder && + !triedNativeStaticLayoutWithProbe + ) { const staticLayoutResult = await this.tryExportNativeStaticLayout( - nativeVideoInfo, + videoInfo, nativeAudioPlan, effectiveDuration, totalFrames, @@ -478,382 +538,390 @@ export class ModernVideoExporter { return staticLayoutResult; } } - } - this.streamingDecoder = new StreamingVideoDecoder({ - maxDecodeQueue: - this.config.maxDecodeQueue ?? this.backpressureProfile.maxDecodeQueue, - maxPendingFrames: - this.config.maxPendingFrames ?? this.backpressureProfile.maxPendingFrames, - }); - stageStartedAt = this.getNowMs(); - const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl); - this.metadataLoadTimeMs = this.getNowMs() - stageStartedAt; - const nativeAudioPlan = this.buildNativeAudioPlan(videoInfo); - const shouldUsePitchPreservingFfmpegAudio = - nativeAudioPlan.audioMode === "edited-track" && - nativeAudioPlan.strategy === "filtergraph-fast-path"; - const shouldUseFfmpegAudioFallback = - !useNativeEncoder && - nativeAudioPlan.audioMode !== "none" && - (shouldUsePitchPreservingFfmpegAudio || !(await isAacAudioEncodingSupported())); - const effectiveDuration = this.streamingDecoder.getEffectiveDuration( - this.config.trimRegions, - this.config.speedRegions, - ); - this.effectiveDurationSec = effectiveDuration; - const totalFrames = Math.ceil(effectiveDuration * this.config.frameRate); - - if ( - (backendPreference === "auto" || backendPreference === "breeze") && - !useNativeEncoder && - !triedNativeStaticLayoutWithProbe - ) { - const staticLayoutResult = await this.tryExportNativeStaticLayout( - videoInfo, - nativeAudioPlan, - effectiveDuration, - totalFrames, - ); - if (staticLayoutResult) { - this.disposeEncoder(); - return staticLayoutResult; + if (shouldDeferNativeEncoderStart && !useNativeEncoder) { + stageStartedAt = this.getNowMs(); + useNativeEncoder = await this.tryStartNativeVideoExport(); + this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; + if (!useNativeEncoder) { + const nativeFailure = + this.lastNativeExportError ?? + `${NATIVE_EXPORT_ENGINE_NAME} export is unavailable for this output profile on this system.`; + console.warn( + `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} native export unavailable after static-layout fallback; falling back to WebCodecs.`, + nativeFailure, + ); + shouldDeferNativeEncoderStart = false; + this.backpressureProfile = getExportBackpressureProfile({ + encodeBackend: "webcodecs", + width: this.config.width, + height: this.config.height, + frameRate: this.config.frameRate, + encodingMode: this.config.encodingMode, + }); + this.maxNativeWriteInFlight = 1; + await this.initializeEncoder(); + } } - } - if (shouldDeferNativeEncoderStart && !useNativeEncoder) { stageStartedAt = this.getNowMs(); - useNativeEncoder = await this.tryStartNativeVideoExport(); - this.nativeSessionStartTimeMs = this.getNowMs() - stageStartedAt; + this.renderer = new ModernFrameRenderer({ + width: this.config.width, + height: this.config.height, + preferredRenderBackend: undefined, + wallpaper: this.config.wallpaper, + zoomRegions: this.config.zoomRegions, + showShadow: this.config.showShadow, + shadowIntensity: this.config.shadowIntensity, + backgroundBlur: this.config.backgroundBlur, + zoomMotionBlur: this.config.zoomMotionBlur, + zoomMotionBlurTuning: this.config.zoomMotionBlurTuning, + zoomTemporalMotionBlur: this.config.zoomTemporalMotionBlur, + zoomMotionBlurSampleCount: this.config.zoomMotionBlurSampleCount, + zoomMotionBlurShutterFraction: this.config.zoomMotionBlurShutterFraction, + connectZooms: this.config.connectZooms, + zoomInDurationMs: this.config.zoomInDurationMs, + zoomInOverlapMs: this.config.zoomInOverlapMs, + zoomOutDurationMs: this.config.zoomOutDurationMs, + connectedZoomGapMs: this.config.connectedZoomGapMs, + connectedZoomDurationMs: this.config.connectedZoomDurationMs, + zoomInEasing: this.config.zoomInEasing, + zoomOutEasing: this.config.zoomOutEasing, + connectedZoomEasing: this.config.connectedZoomEasing, + borderRadius: this.config.borderRadius, + padding: this.config.padding, + cropRegion: this.config.cropRegion, + webcam: this.config.webcam, + webcamUrl: this.config.webcamUrl, + videoWidth: videoInfo.width, + videoHeight: videoInfo.height, + annotationRegions: this.config.annotationRegions, + autoCaptions: this.config.autoCaptions, + autoCaptionSettings: this.config.autoCaptionSettings, + speedRegions: this.config.speedRegions, + previewWidth: this.config.previewWidth, + previewHeight: this.config.previewHeight, + cursorTelemetry: this.config.cursorTelemetry, + showCursor: this.config.showCursor, + cursorStyle: this.config.cursorStyle, + cursorSize: this.config.cursorSize, + cursorSmoothing: this.config.cursorSmoothing, + cursorSpringStiffnessMultiplier: this.config.cursorSpringStiffnessMultiplier, + cursorSpringDampingMultiplier: this.config.cursorSpringDampingMultiplier, + cursorSpringMassMultiplier: this.config.cursorSpringMassMultiplier, + cameraSpringStiffnessMultiplier: this.config.cameraSpringStiffnessMultiplier, + cameraSpringDampingMultiplier: this.config.cameraSpringDampingMultiplier, + cameraSpringMassMultiplier: this.config.cameraSpringMassMultiplier, + cursorMotionBlur: this.config.cursorMotionBlur, + cursorClickBounce: this.config.cursorClickBounce, + cursorClickBounceDuration: this.config.cursorClickBounceDuration, + cursorSway: this.config.cursorSway, + zoomSmoothness: this.config.zoomSmoothness, + zoomClassicMode: this.config.zoomClassicMode, + frame: this.config.frame, + }); + await this.renderer.initialize(); + this.rendererInitTimeMs = this.getNowMs() - stageStartedAt; + this.renderBackend = this.renderer.getRendererBackend(); + console.log(`[VideoExporter] Using ${this.renderBackend} render backend`); + if (!useNativeEncoder) { - const nativeFailure = - this.lastNativeExportError ?? - `${NATIVE_EXPORT_ENGINE_NAME} export is unavailable for this output profile on this system.`; - console.warn( - `[VideoExporter] ${NATIVE_EXPORT_ENGINE_NAME} native export unavailable after static-layout fallback; falling back to WebCodecs.`, - nativeFailure, + const hasAudio = nativeAudioPlan.audioMode !== "none"; + this.muxer = new VideoMuxer( + this.config, + hasAudio && !shouldUseFfmpegAudioFallback, ); - shouldDeferNativeEncoderStart = false; - this.backpressureProfile = getExportBackpressureProfile({ - encodeBackend: "webcodecs", - width: this.config.width, - height: this.config.height, - frameRate: this.config.frameRate, - encodingMode: this.config.encodingMode, - }); - this.maxNativeWriteInFlight = 1; - await this.initializeEncoder(); + await this.muxer.initialize(); } - } - - stageStartedAt = this.getNowMs(); - this.renderer = new ModernFrameRenderer({ - width: this.config.width, - height: this.config.height, - preferredRenderBackend: undefined, - wallpaper: this.config.wallpaper, - zoomRegions: this.config.zoomRegions, - showShadow: this.config.showShadow, - shadowIntensity: this.config.shadowIntensity, - backgroundBlur: this.config.backgroundBlur, - zoomMotionBlur: this.config.zoomMotionBlur, - zoomMotionBlurTuning: this.config.zoomMotionBlurTuning, - zoomTemporalMotionBlur: this.config.zoomTemporalMotionBlur, - zoomMotionBlurSampleCount: this.config.zoomMotionBlurSampleCount, - zoomMotionBlurShutterFraction: this.config.zoomMotionBlurShutterFraction, - connectZooms: this.config.connectZooms, - zoomInDurationMs: this.config.zoomInDurationMs, - zoomInOverlapMs: this.config.zoomInOverlapMs, - zoomOutDurationMs: this.config.zoomOutDurationMs, - connectedZoomGapMs: this.config.connectedZoomGapMs, - connectedZoomDurationMs: this.config.connectedZoomDurationMs, - zoomInEasing: this.config.zoomInEasing, - zoomOutEasing: this.config.zoomOutEasing, - connectedZoomEasing: this.config.connectedZoomEasing, - borderRadius: this.config.borderRadius, - padding: this.config.padding, - cropRegion: this.config.cropRegion, - webcam: this.config.webcam, - webcamUrl: this.config.webcamUrl, - videoWidth: videoInfo.width, - videoHeight: videoInfo.height, - annotationRegions: this.config.annotationRegions, - autoCaptions: this.config.autoCaptions, - autoCaptionSettings: this.config.autoCaptionSettings, - speedRegions: this.config.speedRegions, - previewWidth: this.config.previewWidth, - previewHeight: this.config.previewHeight, - cursorTelemetry: this.config.cursorTelemetry, - showCursor: this.config.showCursor, - cursorStyle: this.config.cursorStyle, - cursorSize: this.config.cursorSize, - cursorSmoothing: this.config.cursorSmoothing, - cursorSpringStiffnessMultiplier: this.config.cursorSpringStiffnessMultiplier, - cursorSpringDampingMultiplier: this.config.cursorSpringDampingMultiplier, - cursorSpringMassMultiplier: this.config.cursorSpringMassMultiplier, - cameraSpringStiffnessMultiplier: this.config.cameraSpringStiffnessMultiplier, - cameraSpringDampingMultiplier: this.config.cameraSpringDampingMultiplier, - cameraSpringMassMultiplier: this.config.cameraSpringMassMultiplier, - cursorMotionBlur: this.config.cursorMotionBlur, - cursorClickBounce: this.config.cursorClickBounce, - cursorClickBounceDuration: this.config.cursorClickBounceDuration, - cursorSway: this.config.cursorSway, - zoomSmoothness: this.config.zoomSmoothness, - zoomClassicMode: this.config.zoomClassicMode, - frame: this.config.frame, - }); - await this.renderer.initialize(); - this.rendererInitTimeMs = this.getNowMs() - stageStartedAt; - this.renderBackend = this.renderer.getRendererBackend(); - console.log(`[VideoExporter] Using ${this.renderBackend} render backend`); - - if (!useNativeEncoder) { - const hasAudio = nativeAudioPlan.audioMode !== "none"; - this.muxer = new VideoMuxer(this.config, hasAudio && !shouldUseFfmpegAudioFallback); - await this.muxer.initialize(); - } - console.log("[VideoExporter] Original duration:", videoInfo.duration, "s"); - console.log("[VideoExporter] Effective duration:", effectiveDuration, "s"); - console.log("[VideoExporter] Total frames to export:", totalFrames); - console.log( - `[VideoExporter] Using ${useNativeEncoder ? `${NATIVE_EXPORT_ENGINE_NAME} native` : "WebCodecs"} encode path`, - ); - - const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds - let frameIndex = 0; - this.exportStartTimeMs = this.getNowMs(); - this.lastThroughputLogTimeMs = this.exportStartTimeMs; - this.lastProgressSampleTimeMs = this.exportStartTimeMs; - this.lastProgressSampleFrame = 0; - this.displayedRenderFps = 0; - const decodeLoopStartedAt = this.getNowMs(); - - await this.streamingDecoder.decodeAll( - this.config.frameRate, - this.config.trimRegions, - this.config.speedRegions, - async (videoFrame, _exportTimestampUs, sourceTimestampMs, cursorTimestampMs) => { - const callbackStartedAt = this.getNowMs(); - if (this.cancelled) { - return; - } + console.log("[VideoExporter] Original duration:", videoInfo.duration, "s"); + console.log("[VideoExporter] Effective duration:", effectiveDuration, "s"); + console.log("[VideoExporter] Total frames to export:", totalFrames); + console.log( + `[VideoExporter] Using ${useNativeEncoder ? `${NATIVE_EXPORT_ENGINE_NAME} native` : "WebCodecs"} encode path`, + ); - const timestamp = frameIndex * frameDuration; - const sourceTimestampUs = sourceTimestampMs * 1000; - const cursorTimestampUs = cursorTimestampMs * 1000; - const renderStartedAt = this.getNowMs(); - await this.renderer!.renderFrame( + const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds + let frameIndex = 0; + this.exportStartTimeMs = this.getNowMs(); + this.lastThroughputLogTimeMs = this.exportStartTimeMs; + this.lastProgressSampleTimeMs = this.exportStartTimeMs; + this.lastProgressSampleFrame = 0; + this.displayedRenderFps = 0; + const decodeLoopStartedAt = this.getNowMs(); + + await this.streamingDecoder.decodeAll( + this.config.frameRate, + this.config.trimRegions, + this.config.speedRegions, + async ( videoFrame, - sourceTimestampUs, - cursorTimestampUs, - frameDuration, - timestamp, - ); - this.renderFrameTimeMs += this.getNowMs() - renderStartedAt; + _exportTimestampUs, + sourceTimestampMs, + cursorTimestampMs, + ) => { + const callbackStartedAt = this.getNowMs(); + if (this.cancelled) { + return; + } - if (this.cancelled) { - return; - } + const timestamp = frameIndex * frameDuration; + const sourceTimestampUs = sourceTimestampMs * 1000; + const cursorTimestampUs = cursorTimestampMs * 1000; + const renderStartedAt = this.getNowMs(); + await this.renderer!.renderFrame( + videoFrame, + sourceTimestampUs, + cursorTimestampUs, + frameDuration, + timestamp, + ); + this.renderFrameTimeMs += this.getNowMs() - renderStartedAt; - if (useNativeEncoder) { - await this.encodeRenderedFrameNative(timestamp, frameDuration, frameIndex); - } else { - await this.encodeRenderedFrame(timestamp, frameDuration, frameIndex); + if (this.cancelled) { + return; + } + + if (useNativeEncoder) { + await this.encodeRenderedFrameNative( + timestamp, + frameDuration, + frameIndex, + ); + } else { + await this.encodeRenderedFrame(timestamp, frameDuration, frameIndex); + } + this.frameCallbackTimeMs += this.getNowMs() - callbackStartedAt; + frameIndex++; + this.processedFrameCount = frameIndex; + this.reportProgress(frameIndex, totalFrames, "extracting"); + extensionHost.emitEvent({ + type: "export:frame", + data: { frameIndex, totalFrames }, + }); + }, + ); + this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt; + + if (this.cancelled) { + if (this.encoderError) { + return { + success: false, + error: this.buildLightningExportError(this.encoderError), + metrics: this.buildExportMetrics(), + }; } - this.frameCallbackTimeMs += this.getNowMs() - callbackStartedAt; - frameIndex++; - this.processedFrameCount = frameIndex; - this.reportProgress(frameIndex, totalFrames, "extracting"); - extensionHost.emitEvent({ - type: "export:frame", - data: { frameIndex, totalFrames }, - }); - }, - ); - this.decodeLoopTimeMs = this.getNowMs() - decodeLoopStartedAt; - if (this.cancelled) { - if (this.encoderError) { return { success: false, - error: this.buildLightningExportError(this.encoderError), + error: "Export cancelled", metrics: this.buildExportMetrics(), }; } - return { - success: false, - error: "Export cancelled", - metrics: this.buildExportMetrics(), - }; - } + this.reportFinalizingProgress(totalFrames, 96); - this.reportFinalizingProgress(totalFrames, 96); + if (useNativeEncoder) { + stageStartedAt = this.getNowMs(); + this.reportFinalizingProgress(totalFrames, 99); + if (this.nativeH264Encoder) { + await this.measureFinalizationStage("nativeEncoderFlushMs", async () => { + await this.nativeH264Encoder!.flush(); + }); + } + const finishResult = await this.finishNativeVideoExport(nativeAudioPlan); + this.finalizationTimeMs = this.getNowMs() - stageStartedAt; + if ( + !finishResult.success || + (!finishResult.tempFilePath && !finishResult.blob) + ) { + return { + success: false, + error: + finishResult.error || `${NATIVE_EXPORT_ENGINE_NAME} export failed`, + metrics: this.buildExportMetrics(), + }; + } - if (useNativeEncoder) { - stageStartedAt = this.getNowMs(); - this.reportFinalizingProgress(totalFrames, 99); - if (this.nativeH264Encoder) { - await this.measureFinalizationStage("nativeEncoderFlushMs", async () => { - await this.nativeH264Encoder!.flush(); - }); - } - const finishResult = await this.finishNativeVideoExport(nativeAudioPlan); - this.finalizationTimeMs = this.getNowMs() - stageStartedAt; - if (!finishResult.success || (!finishResult.tempFilePath && !finishResult.blob)) { return { - success: false, - error: finishResult.error || `${NATIVE_EXPORT_ENGINE_NAME} export failed`, + success: true, + tempFilePath: finishResult.tempFilePath, + blob: finishResult.blob, metrics: this.buildExportMetrics(), }; } - return { - success: true, - tempFilePath: finishResult.tempFilePath, - blob: finishResult.blob, - metrics: this.buildExportMetrics(), - }; - } + stageStartedAt = this.getNowMs(); + if (this.encoder && this.encoder.state === "configured") { + this.reportFinalizingProgress(totalFrames, 97); + await this.measureFinalizationStage("encoderFlushMs", async () => { + await this.awaitWithFinalizationTimeout( + this.encoder!.flush(), + "encoder flush", + ); + }); + } - stageStartedAt = this.getNowMs(); - if (this.encoder && this.encoder.state === "configured") { - this.reportFinalizingProgress(totalFrames, 97); - await this.measureFinalizationStage("encoderFlushMs", async () => { - await this.awaitWithFinalizationTimeout(this.encoder!.flush(), "encoder flush"); + this.reportFinalizingProgress(totalFrames, 98); + await this.measureFinalizationStage("queuedMuxingMs", async () => { + await this.awaitWithFinalizationTimeout( + this.pendingMuxing, + "muxing queued video chunks", + ); }); - } - - this.reportFinalizingProgress(totalFrames, 98); - await this.measureFinalizationStage("queuedMuxingMs", async () => { - await this.awaitWithFinalizationTimeout( - this.pendingMuxing, - "muxing queued video chunks", - ); - }); - // Surface muxing errors before proceeding with finalization - if (this.encoderError) { - throw this.encoderError; - } + // Surface muxing errors before proceeding with finalization + if (this.encoderError) { + throw this.encoderError; + } - if ( - nativeAudioPlan.audioMode !== "none" && - !shouldUseFfmpegAudioFallback && - !this.cancelled - ) { - const demuxer = this.streamingDecoder.getDemuxer(); if ( - demuxer || - (this.config.audioRegions ?? []).length > 0 || - (this.config.sourceAudioFallbackPaths ?? []).length > 0 + nativeAudioPlan.audioMode !== "none" && + !shouldUseFfmpegAudioFallback && + !this.cancelled ) { - this.audioProcessor = new AudioProcessor(); - this.audioProcessor.setOnProgress((progress) => { - this.reportFinalizingProgress(totalFrames, 99, progress); - }); - this.reportFinalizingProgress(totalFrames, 99); - await this.measureFinalizationStage("audioProcessingMs", async () => { - await this.awaitWithFinalizationTimeout( - this.audioProcessor!.process( - demuxer, - this.muxer!, - this.config.videoUrl, - this.config.trimRegions, - this.config.speedRegions, - undefined, - this.config.audioRegions, - this.config.sourceAudioFallbackPaths, - this.config.sourceAudioFallbackStartDelayMsByPath, - this.config.sourceAudioTrackSettings, - this.config.clipRegions, - ), - "audio processing", - "audio", - true, - ); - }); + const demuxer = this.streamingDecoder.getDemuxer(); + if ( + demuxer || + (this.config.audioRegions ?? []).length > 0 || + (this.config.sourceAudioFallbackPaths ?? []).length > 0 + ) { + this.audioProcessor = new AudioProcessor(); + this.audioProcessor.setOnProgress((progress) => { + this.reportFinalizingProgress(totalFrames, 99, progress); + }); + this.reportFinalizingProgress(totalFrames, 99); + await this.measureFinalizationStage("audioProcessingMs", async () => { + await this.awaitWithFinalizationTimeout( + this.audioProcessor!.process( + demuxer, + this.muxer!, + this.config.videoUrl, + this.config.trimRegions, + this.config.speedRegions, + undefined, + this.config.audioRegions, + this.config.sourceAudioFallbackPaths, + this.config.sourceAudioFallbackStartDelayMsByPath, + this.config.sourceAudioTrackSettings, + this.config.clipRegions, + ), + "audio processing", + "audio", + true, + ); + }); + } } - } - - this.reportFinalizingProgress(totalFrames, 99); - const muxerResult = await this.measureFinalizationStage("muxerFinalizeMs", async () => - this.awaitWithFinalizationTimeout( - this.muxer!.finalize(), - "muxer finalization", - nativeAudioPlan.audioMode !== "none" && !shouldUseFfmpegAudioFallback - ? "audio" - : "default", - ), - ); - if (shouldUseFfmpegAudioFallback) { - console.warn( - shouldUsePitchPreservingFfmpegAudio - ? "[VideoExporter] Using FFmpeg audio muxing for pitch-preserving speed edits." - : "[VideoExporter] Browser AAC encoding is unavailable; falling back to FFmpeg audio muxing.", - ); - const muxedResult = await this.finalizeExportWithFfmpegAudio( - muxerResult, - nativeAudioPlan, + this.reportFinalizingProgress(totalFrames, 99); + const muxerResult = await this.measureFinalizationStage( + "muxerFinalizeMs", + async () => + this.awaitWithFinalizationTimeout( + this.muxer!.finalize(), + "muxer finalization", + nativeAudioPlan.audioMode !== "none" && !shouldUseFfmpegAudioFallback + ? "audio" + : "default", + ), ); + + if (shouldUseFfmpegAudioFallback) { + console.warn( + shouldUsePitchPreservingFfmpegAudio + ? "[VideoExporter] Using FFmpeg audio muxing for pitch-preserving speed edits." + : "[VideoExporter] Browser AAC encoding is unavailable; falling back to FFmpeg audio muxing.", + ); + const muxedResult = await this.finalizeExportWithFfmpegAudio( + muxerResult, + nativeAudioPlan, + ); + this.finalizationTimeMs = this.getNowMs() - stageStartedAt; + if (!muxedResult.success || (!muxedResult.blob && !muxedResult.tempFilePath)) { + return { + success: false, + error: muxedResult.error || "Failed to mux audio with FFmpeg", + metrics: this.buildExportMetrics(), + }; + } + + return { + success: true, + blob: muxedResult.blob, + tempFilePath: muxedResult.tempFilePath, + metrics: muxedResult.metrics ?? this.buildExportMetrics(), + }; + } + this.finalizationTimeMs = this.getNowMs() - stageStartedAt; - if (!muxedResult.success || (!muxedResult.blob && !muxedResult.tempFilePath)) { + if (muxerResult.mode === "stream") { return { - success: false, - error: muxedResult.error || "Failed to mux audio with FFmpeg", + success: true, + tempFilePath: muxerResult.tempFilePath, metrics: this.buildExportMetrics(), }; } - return { success: true, - blob: muxedResult.blob, - tempFilePath: muxedResult.tempFilePath, - metrics: muxedResult.metrics ?? this.buildExportMetrics(), - }; - } - - this.finalizationTimeMs = this.getNowMs() - stageStartedAt; - if (muxerResult.mode === "stream") { - return { - success: true, - tempFilePath: muxerResult.tempFilePath, - metrics: this.buildExportMetrics(), - }; - } - return { - success: true, - blob: muxerResult.blob, - metrics: this.buildExportMetrics(), - }; - } catch (error) { - if (this.cancelled && !this.encoderError) { - return { - success: false, - error: "Export cancelled", + blob: muxerResult.blob, metrics: this.buildExportMetrics(), }; + } catch (error) { + if ( + !preferReadableFileSource && + !retriedWithReadableFileSource && + this.shouldRetryWithReadableFileSource(error) + ) { + retriedWithReadableFileSource = true; + preferReadableFileSource = true; + shouldRetryWithReadableFileSource = true; + console.warn( + "[VideoExporter] Primary decode path failed; retrying export once with a readable file-backed media source.", + error, + ); + } else { + if (this.cancelled && !this.encoderError) { + return { + success: false, + error: "Export cancelled", + metrics: this.buildExportMetrics(), + }; + } + + const resolvedError = this.encoderError ?? error; + console.error("Export error:", error); + return { + success: false, + error: this.buildLightningExportError(resolvedError), + metrics: this.buildExportMetrics(), + }; + } + } finally { + if (!shouldRetryWithReadableFileSource && this.totalExportStartTimeMs > 0) { + console.log( + `[VideoExporter] Final metrics ${JSON.stringify(this.buildExportMetrics())}`, + ); + } + this.cleanup(); } - const resolvedError = this.encoderError ?? error; - console.error("Export error:", error); - return { - success: false, - error: this.buildLightningExportError(resolvedError), - metrics: this.buildExportMetrics(), - }; - } finally { - if (this.totalExportStartTimeMs > 0) { - console.log( - `[VideoExporter] Final metrics ${JSON.stringify(this.buildExportMetrics())}`, - ); + if (shouldRetryWithReadableFileSource) { + continue; } - this.cleanup(); } } + private shouldRetryWithReadableFileSource(error: unknown): boolean { + const resolvedError = this.encoderError ?? error; + const message = + resolvedError instanceof Error ? resolvedError.message : String(resolvedError); + const normalizedMessage = message.toLowerCase(); + return READABLE_SOURCE_RETRY_ERROR_TOKENS.some((token) => + normalizedMessage.includes(token), + ); + } + private getPlatformLabel(): string { switch (this.getRuntimePlatform()) { case "win32": @@ -1100,7 +1168,11 @@ export class ModernVideoExporter { speed: region.speed, })) .filter((region) => region.endMs - region.startMs > 0.5); - const sourceSegments: Array<{ startMs: number; endMs: number; speed: number }> = []; + const sourceSegments: Array<{ + startMs: number; + endMs: number; + speed: number; + }> = []; for (const keptRange of this.buildNativeTrimSegments(sourceDurationMs)) { const boundaries = new Set([keptRange.startMs, keptRange.endMs]); @@ -2635,7 +2707,10 @@ export class ModernVideoExporter { if (this.nativeEncoderError) throw this.nativeEncoderError; } const canvas = this.renderer!.getCanvas(); - const frame = new VideoFrame(canvas, { timestamp, duration: frameDuration }); + const frame = new VideoFrame(canvas, { + timestamp, + duration: frameDuration, + }); this.nativeH264Encoder.encode(frame, { keyFrame: frameIndex % 300 === 0 }); frame.close(); } diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index e0f5a18ce..861d6720a 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -1,10 +1,7 @@ import { WebDemuxer } from "web-demuxer"; import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; import { getEffectiveVideoStreamDurationSeconds } from "@/lib/mediaTiming"; -import { - createReadableMediaResourceFile, - resolveMediaResourceUrl, -} from "./localMediaSource"; +import { createReadableMediaResourceFile, resolveMediaResourceUrl } from "./localMediaSource"; const DEFAULT_MAX_DECODE_QUEUE = 12; const DEFAULT_MAX_PENDING_FRAMES = 32; @@ -26,6 +23,10 @@ export interface DecodedVideoInfo { audioSampleRate?: number; } +interface StreamingVideoDecoderLoadOptions { + forceReadableFileSource?: boolean; +} + /** Decoder retains ownership of the VideoFrame and closes it after use. */ type OnFrameCallback = ( frame: VideoFrame, @@ -76,7 +77,10 @@ export class StreamingVideoDecoder { private readonly maxDecodeQueue: number; private readonly maxPendingFrames: number; - constructor(options?: { maxDecodeQueue?: number; maxPendingFrames?: number }) { + constructor(options?: { + maxDecodeQueue?: number; + maxPendingFrames?: number; + }) { this.maxDecodeQueue = Math.max( 1, Math.floor(options?.maxDecodeQueue ?? DEFAULT_MAX_DECODE_QUEUE), @@ -87,7 +91,10 @@ export class StreamingVideoDecoder { ); } - async loadMetadata(videoUrl: string): Promise { + async loadMetadata( + videoUrl: string, + options: StreamingVideoDecoderLoadOptions = {}, + ): Promise { if (this.decoder) { try { if (this.decoder.state === "configured") { @@ -119,22 +126,26 @@ export class StreamingVideoDecoder { }; let mediaInfo; - try { - mediaInfo = await loadMediaInfo(resourceUrl); - } catch (error) { - console.warn( - "[StreamingVideoDecoder] Direct source load failed, retrying with file fallback:", - error, - ); - const currentDemuxer = this.demuxer; - if (currentDemuxer) { - try { - (currentDemuxer as unknown as { destroy: () => void }).destroy(); - } catch { - // Ignore cleanup errors before fallback re-init. + if (options.forceReadableFileSource) { + mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl)); + } else { + try { + mediaInfo = await loadMediaInfo(resourceUrl); + } catch (error) { + console.warn( + "[StreamingVideoDecoder] Direct source load failed, retrying with file fallback:", + error, + ); + const currentDemuxer = this.demuxer; + if (currentDemuxer) { + try { + (currentDemuxer as unknown as { destroy: () => void }).destroy(); + } catch { + // Ignore cleanup errors before fallback re-init. + } } + mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl)); } - mediaInfo = await loadMediaInfo(await createReadableMediaResourceFile(videoUrl)); } const videoStream = mediaInfo.streams.find((s) => s.codec_type_string === "video"); @@ -628,7 +639,11 @@ export class StreamingVideoDecoder { } const effectiveStart = Math.max(cursor, srStart); if (srEnd > effectiveStart) { - result.push({ startSec: effectiveStart, endSec: srEnd, speed: sr.speed }); + result.push({ + startSec: effectiveStart, + endSec: srEnd, + speed: sr.speed, + }); } cursor = Math.max(cursor, srEnd); }