fix(export): make audio finalization timeouts progress-aware

src/lib/exporter/finalizationTimeout.test.ts

@@ -0,0 +1,212 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+	advanceFinalizationProgress,
+	type FinalizationProgressWatchdog,
+	getExportFinalizationIdleTimeoutMs,
+	getExportFinalizationTimeoutMs,
+	INITIAL_FINALIZATION_PROGRESS_STATE,
+	withFinalizationTimeout,
+} from "./finalizationTimeout";
+
+describe("finalizationTimeout", () => {
+	it("keeps non-audio finalization on the existing 10 minute timeout", () => {
+		expect(getExportFinalizationTimeoutMs({ workload: "default" })).toBe(600_000);
+		expect(
+			getExportFinalizationTimeoutMs({
+				workload: "default",
+				effectiveDurationSec: 7_200,
+			}),
+		).toBe(600_000);
+	});
+
+	it("gives audio finalization more headroom on longer exports", () => {
+		expect(
+			getExportFinalizationTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 1_200,
+			}),
+		).toBe(1_200_000);
+		expect(
+			getExportFinalizationTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 2_700,
+			}),
+		).toBe(1_950_000);
+	});
+
+	it("caps adaptive audio timeout growth", () => {
+		expect(
+			getExportFinalizationTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 10_800,
+			}),
+		).toBe(2_700_000);
+	});
+
+	it("falls back to the base timeout for invalid audio durations", () => {
+		expect(
+			getExportFinalizationTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 0,
+			}),
+		).toBe(600_000);
+		expect(
+			getExportFinalizationTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: Number.NaN,
+			}),
+		).toBe(600_000);
+	});
+
+	it("derives a bounded idle watchdog window from the total timeout", () => {
+		expect(
+			getExportFinalizationIdleTimeoutMs({
+				workload: "default",
+			}),
+		).toBe(150_000);
+		expect(
+			getExportFinalizationIdleTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 1_200,
+			}),
+		).toBe(300_000);
+		expect(
+			getExportFinalizationIdleTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 2_700,
+			}),
+		).toBe(300_000);
+		expect(
+			getExportFinalizationIdleTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 0,
+			}),
+		).toBe(150_000);
+		expect(
+			getExportFinalizationIdleTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: Number.NaN,
+			}),
+		).toBe(150_000);
+	});
+
+	it("rejects when a progress-aware finalization stage stops reporting progress", async () => {
+		vi.useFakeTimers();
+
+		try {
+			const idleTimeoutMs = getExportFinalizationIdleTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 1_200,
+			});
+
+			const pendingStage = withFinalizationTimeout({
+				promise: new Promise<never>(() => {}),
+				stage: "audio processing",
+				workload: "audio",
+				effectiveDurationSec: 1_200,
+				progressAware: true,
+			});
+
+			const rejection = pendingStage.then(
+				() => null,
+				(error) => (error instanceof Error ? error.message : String(error)),
+			);
+
+			await vi.advanceTimersByTimeAsync(idleTimeoutMs + 1);
+
+			await expect(rejection).resolves.toContain("without observable progress");
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it("resets the idle watchdog when finalization progress continues", async () => {
+		vi.useFakeTimers();
+
+		try {
+			const idleTimeoutMs = getExportFinalizationIdleTimeoutMs({
+				workload: "audio",
+				effectiveDurationSec: 1_200,
+			});
+			let watchdog: FinalizationProgressWatchdog | null = null;
+			const pendingStage = withFinalizationTimeout({
+				promise: new Promise<never>(() => {}),
+				stage: "audio processing",
+				workload: "audio",
+				effectiveDurationSec: 1_200,
+				progressAware: true,
+				onWatchdogChanged: (nextWatchdog) => {
+					watchdog = nextWatchdog;
+				},
+			});
+			const rejection = pendingStage.then(
+				() => null,
+				(error) => (error instanceof Error ? error.message : String(error)),
+			);
+
+			await vi.advanceTimersByTimeAsync(idleTimeoutMs - 1_000);
+			expect(watchdog).not.toBeNull();
+
+			watchdog?.refreshProgress();
+			await vi.advanceTimersByTimeAsync(idleTimeoutMs - 1_000);
+
+			const pendingSentinel = Symbol("pending");
+			await expect(Promise.race([rejection, Promise.resolve(pendingSentinel)])).resolves.toBe(
+				pendingSentinel,
+			);
+
+			await vi.advanceTimersByTimeAsync(1_001);
+			await expect(rejection).resolves.toContain("without observable progress");
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it("only marks finalization as progressed when normalized progress increases", () => {
+		const initial = advanceFinalizationProgress({
+			renderProgress: 99,
+			audioProgress: 0.5,
+			state: INITIAL_FINALIZATION_PROGRESS_STATE,
+		});
+		expect(initial.progressed).toBe(true);
+		expect(initial.lastRenderProgress).toBe(99);
+		expect(initial.lastAudioProgress).toBe(0.5);
+
+		const repeated = advanceFinalizationProgress({
+			renderProgress: 99,
+			audioProgress: 0.5,
+			state: initial,
+		});
+		expect(repeated.progressed).toBe(false);
+
+		const advanced = advanceFinalizationProgress({
+			renderProgress: 100,
+			audioProgress: 0.5,
+			state: repeated,
+		});
+		expect(advanced.progressed).toBe(true);
+		expect(advanced.lastRenderProgress).toBe(100);
+		expect(advanced.lastAudioProgress).toBe(0.5);
+	});
+
+	it("ignores non-finite render progress without poisoning later updates", () => {
+		const invalid = advanceFinalizationProgress({
+			renderProgress: Number.NaN,
+			audioProgress: 0.25,
+			state: INITIAL_FINALIZATION_PROGRESS_STATE,
+		});
+		expect(invalid.progressed).toBe(true);
+		expect(invalid.lastRenderProgress).toBe(-1);
+		expect(invalid.lastAudioProgress).toBe(0.25);
+
+		const recovered = advanceFinalizationProgress({
+			renderProgress: 99,
+			audioProgress: 0.25,
+			state: invalid,
+		});
+		expect(recovered.progressed).toBe(true);
+		expect(recovered.lastRenderProgress).toBe(99);
+		expect(recovered.lastAudioProgress).toBe(0.25);
+	});
+});

src/lib/exporter/finalizationTimeout.ts

@@ -0,0 +1,181 @@
+export type FinalizationTimeoutWorkload = "default" | "audio";
+export type FinalizationProgressWatchdog = {
+	refreshProgress: () => void;
+};
+export type FinalizationProgressState = {
+	lastRenderProgress: number;
+	lastAudioProgress: number;
+};
+
+const BASE_FINALIZATION_TIMEOUT_MS = 10 * 60_000;
+const AUDIO_TIMEOUT_HEADROOM_PER_OUTPUT_SECOND_MS = 500;
+const MAX_AUDIO_FINALIZATION_TIMEOUT_MS = 45 * 60_000;
+const MIN_PROGRESS_IDLE_TIMEOUT_MS = 90_000;
+const MAX_PROGRESS_IDLE_TIMEOUT_MS = 5 * 60_000;
+const PROGRESS_IDLE_TIMEOUT_FRACTION = 0.25;
+
+export const INITIAL_FINALIZATION_PROGRESS_STATE: FinalizationProgressState = {
+	lastRenderProgress: -1,
+	lastAudioProgress: -1,
+};
+
+export function getExportFinalizationTimeoutMs({
+	effectiveDurationSec,
+	workload = "default",
+}: {
+	effectiveDurationSec?: number | null;
+	workload?: FinalizationTimeoutWorkload;
+}): number {
+	if (workload !== "audio") {
+		return BASE_FINALIZATION_TIMEOUT_MS;
+	}
+
+	const safeEffectiveDurationSec =
+		typeof effectiveDurationSec === "number" ? effectiveDurationSec : Number.NaN;
+	if (!Number.isFinite(safeEffectiveDurationSec) || safeEffectiveDurationSec <= 0) {
+		return BASE_FINALIZATION_TIMEOUT_MS;
+	}
+
+	// Audio finalization work scales with the output timeline, so long exports need
+	// more headroom without making unrelated finalization hangs wait longer.
+	const adaptiveTimeoutMs =
+		BASE_FINALIZATION_TIMEOUT_MS +
+		safeEffectiveDurationSec * AUDIO_TIMEOUT_HEADROOM_PER_OUTPUT_SECOND_MS;
+
+	return Math.min(adaptiveTimeoutMs, MAX_AUDIO_FINALIZATION_TIMEOUT_MS);
+}
+
+export function getExportFinalizationIdleTimeoutMs({
+	effectiveDurationSec,
+	workload = "default",
+}: {
+	effectiveDurationSec?: number | null;
+	workload?: FinalizationTimeoutWorkload;
+}): number {
+	const totalTimeoutMs = getExportFinalizationTimeoutMs({
+		effectiveDurationSec,
+		workload,
+	});
+
+	return Math.min(
+		Math.max(
+			Math.floor(totalTimeoutMs * PROGRESS_IDLE_TIMEOUT_FRACTION),
+			MIN_PROGRESS_IDLE_TIMEOUT_MS,
+		),
+		MAX_PROGRESS_IDLE_TIMEOUT_MS,
+	);
+}
+
+export function advanceFinalizationProgress({
+	renderProgress,
+	audioProgress,
+	state,
+}: {
+	renderProgress: number;
+	audioProgress?: number;
+	state: FinalizationProgressState;
+}): FinalizationProgressState & { progressed: boolean } {
+	const normalizedRenderProgress =
+		typeof renderProgress === "number" && Number.isFinite(renderProgress)
+			? Math.max(0, Math.min(renderProgress, 100))
+			: null;
+	const normalizedAudioProgress =
+		typeof audioProgress === "number" && Number.isFinite(audioProgress)
+			? Math.max(0, Math.min(audioProgress, 1))
+			: null;
+	const nextRenderProgress =
+		normalizedRenderProgress === null
+			? state.lastRenderProgress
+			: Math.max(state.lastRenderProgress, normalizedRenderProgress);
+	const nextAudioProgress =
+		normalizedAudioProgress === null
+			? state.lastAudioProgress
+			: Math.max(state.lastAudioProgress, normalizedAudioProgress);
+
+	return {
+		progressed:
+			nextRenderProgress > state.lastRenderProgress ||
+			nextAudioProgress > state.lastAudioProgress,
+		lastRenderProgress: nextRenderProgress,
+		lastAudioProgress: nextAudioProgress,
+	};
+}
+
+export async function withFinalizationTimeout<T>({
+	promise,
+	stage,
+	effectiveDurationSec,
+	workload = "default",
+	progressAware = false,
+	onWatchdogChanged,
+}: {
+	promise: Promise<T>;
+	stage: string;
+	effectiveDurationSec?: number | null;
+	workload?: FinalizationTimeoutWorkload;
+	progressAware?: boolean;
+	onWatchdogChanged?: (watchdog: FinalizationProgressWatchdog | null) => void;
+}): Promise<T> {
+	let timeoutId: ReturnType<typeof setTimeout> | null = null;
+	let idleTimeoutId: ReturnType<typeof setTimeout> | null = null;
+	const timeoutMs = getExportFinalizationTimeoutMs({
+		effectiveDurationSec,
+		workload,
+	});
+	const idleTimeoutMs = progressAware
+		? getExportFinalizationIdleTimeoutMs({
+				effectiveDurationSec,
+				workload,
+			})
+		: null;
+	const hasIdleWatchdog =
+		progressAware &&
+		typeof idleTimeoutMs === "number" &&
+		Number.isFinite(idleTimeoutMs) &&
+		idleTimeoutMs >= 0;
+	const resolvedIdleTimeoutMs = hasIdleWatchdog ? idleTimeoutMs : null;
+	const watchdog: FinalizationProgressWatchdog | null = hasIdleWatchdog
+		? {
+				refreshProgress: () => undefined,
+			}
+		: null;
+
+	try {
+		return await Promise.race([
+			promise,
+			new Promise<T>((_, reject) => {
+				const rejectWithMessage = (message: string) => {
+					reject(new Error(message));
+				};
+				if (watchdog && resolvedIdleTimeoutMs !== null) {
+					const refreshProgress = () => {
+						if (idleTimeoutId) {
+							clearTimeout(idleTimeoutId);
+						}
+						idleTimeoutId = setTimeout(() => {
+							rejectWithMessage(
+								`Export timed out during ${stage} after ${Math.ceil(resolvedIdleTimeoutMs / 1000)} seconds without observable progress`,
+							);
+						}, resolvedIdleTimeoutMs);
+					};
+					watchdog.refreshProgress = refreshProgress;
+					onWatchdogChanged?.(watchdog);
+					refreshProgress();
+				}
+				timeoutId = setTimeout(() => {
+					rejectWithMessage(
+						`Export timed out during ${stage} after ${Math.ceil(timeoutMs / 60_000)} minutes`,
+					);
+				}, timeoutMs);
+			}),
+		]);
+	} finally {
+		if (timeoutId) {
+			clearTimeout(timeoutId);
+		}
+		if (idleTimeoutId) {
+			clearTimeout(idleTimeoutId);
+		}
+		onWatchdogChanged?.(null);
+	}
+}