Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,21 @@ describe("compileForRender natural media duration parity", () => {
}
});

it("marks only variable-bound media whose natural duration was inferred", async () => {
const { document } = await compile(`
<audio id="inferred" src="ten-seconds.wav" data-var-src="track"></audio>
<audio id="authored" src="ten-seconds.wav" data-var-src="track" data-duration="4"></audio>
<audio id="plain" src="ten-seconds.wav"></audio>`);

expect(document.getElementById("inferred")?.hasAttribute("data-hf-inferred-duration")).toBe(
true,
);
expect(document.getElementById("authored")?.hasAttribute("data-hf-inferred-duration")).toBe(
false,
);
expect(document.getElementById("plain")?.hasAttribute("data-hf-inferred-duration")).toBe(false);
});

it("uses shared playback-start precedence and fallback semantics", async () => {
const cases = [
["precedence", 'data-playback-start="2" data-media-start="7"', 8],
Expand Down
20 changes: 20 additions & 0 deletions packages/producer/src/services/htmlCompiler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,17 @@ describe("discoverMediaFromBrowser", () => {
html: string,
currentSrcById: Record<string, string>,
serializeCallback = false,
intrinsicDurationById: Record<string, number> = {},
) {
const { document } = parseHTML(html);
for (const [id, currentSrc] of Object.entries(currentSrcById)) {
const element = document.getElementById(id);
if (element) Object.defineProperty(element, "currentSrc", { value: currentSrc });
}
for (const [id, duration] of Object.entries(intrinsicDurationById)) {
const element = document.getElementById(id);
if (element) Object.defineProperty(element, "duration", { value: duration });
}
const previousDocument = Reflect.get(globalThis, "document");
Reflect.set(globalThis, "document", document);
try {
Expand Down Expand Up @@ -82,6 +87,21 @@ describe("discoverMediaFromBrowser", () => {
expect(media[0]).toMatchObject({ id: "hf-img-1", tagName: "image" });
});

it("uses intrinsic duration only for variable media with an inferred duration", async () => {
const media = await discover(
`<audio id="inferred" src="fallback.wav" data-start="0" data-duration="3" data-end="3" data-var-src="track" data-hf-inferred-duration></audio>
<audio id="authored" src="fallback.wav" data-start="0" data-duration="3" data-end="3" data-var-src="track"></audio>`,
{ inferred: "selected.wav", authored: "selected.wav" },
false,
{ inferred: 6.530612, authored: 6.530612 },
);

expect(media.find((item) => item.id === "inferred")?.duration).toBe(6.530612);
expect(media.find((item) => item.id === "inferred")?.durationInferred).toBe(true);
expect(media.find((item) => item.id === "authored")?.duration).toBe(3);
expect(media.find((item) => item.id === "authored")?.durationInferred).toBe(false);
});

it("discovers the owning image for a variable-bound picture source", async () => {
const media = await discover(
`<picture>
Expand Down
59 changes: 47 additions & 12 deletions packages/producer/src/services/htmlCompiler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ export interface CompiledComposition {
hasAncestorBackgroundImage: boolean;
}

const INFERRED_MEDIA_DURATION_ATTR = "data-hf-inferred-duration";

/** Adapts linkedom's `parseHTML` to the `checkSubCompositionUsability` contract. */
function parseSubCompHtmlForValidity(html: string): ParsableDocumentLike {
return parseHTML(html).document as unknown as ParsableDocumentLike;
Expand Down Expand Up @@ -534,6 +536,23 @@ async function resolveMediaDuration(
* Compile a single HTML file: static pass + ffprobe for unresolved media.
* Returns compiled HTML and any unresolved composition elements that need browser resolution.
*/
function markInferredVariableMediaDurations(
html: string,
unresolvedMedia: readonly UnresolvedElement[],
): string {
const unresolvedIds = new Set(unresolvedMedia.map((element) => element.id));
if (unresolvedIds.size === 0) return html;

const { document } = parseHTML(html);
let changed = false;
for (const element of document.querySelectorAll("video[data-var-src], audio[data-var-src]")) {
if (!unresolvedIds.has(element.id)) continue;
element.setAttribute(INFERRED_MEDIA_DURATION_ATTR, "");
changed = true;
}
return changed ? document.toString() : html;
}

async function compileHtmlFile(
html: string,
baseDir: string,
Expand Down Expand Up @@ -567,8 +586,9 @@ async function compileHtmlFile(
(r): r is ResolvedDuration => r.duration != null && Number.isFinite(r.duration),
);

const markedStaticHtml = markInferredVariableMediaDurations(staticCompiled, mediaUnresolved);
let compiledHtml =
resolutions.length > 0 ? injectDurations(staticCompiled, resolutions) : staticCompiled;
resolutions.length > 0 ? injectDurations(markedStaticHtml, resolutions) : markedStaticHtml;

// Phase 2: Bound authored audio to playable source (parallel ffprobe).
// Explicit video slots may outlive their source and hold the final frame.
Expand Down Expand Up @@ -2101,6 +2121,8 @@ export interface BrowserMediaElement {
start: number;
end: number;
duration: number;
/** True when compilation inferred duration from the fallback source. */
durationInferred: boolean;
mediaStart: number;
loop: boolean;
hasAudio: boolean;
Expand All @@ -2123,6 +2145,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
start: number;
endRaw: string | null;
durationRaw: string | null;
intrinsicDuration: number;
durationInferred: boolean;
playbackStartRaw: string | null;
mediaStartRaw: string | null;
loop: boolean;
Expand Down Expand Up @@ -2169,6 +2193,10 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
const start = parseFloat(htmlEl.getAttribute("data-start") || "0");
const endRaw = htmlEl.getAttribute("data-end");
const durationRaw = htmlEl.getAttribute("data-duration");
const durationInferred = htmlEl.hasAttribute("data-hf-inferred-duration");
const intrinsicDuration = isImage
? 0
: (htmlEl as HTMLVideoElement | HTMLAudioElement).duration;
const playbackStartRaw = htmlEl.getAttribute("data-playback-start");
const mediaStartRaw = htmlEl.getAttribute("data-media-start");
const loop = htmlEl.hasAttribute("loop");
Expand All @@ -2185,6 +2213,8 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
start,
endRaw,
durationRaw,
intrinsicDuration,
durationInferred,
playbackStartRaw,
mediaStartRaw,
loop,
Expand All @@ -2197,18 +2227,23 @@ export async function discoverMediaFromBrowser(page: Page): Promise<BrowserMedia
return results;
});

return elements.map(({ endRaw, durationRaw, playbackStartRaw, mediaStartRaw, ...element }) => ({
...element,
end: parseStrictFiniteTimingNumber(endRaw) ?? 0,
duration: parseStrictFiniteTimingNumber(durationRaw) ?? 0,
mediaStart: readMediaStart({
getAttribute(name: string) {
if (name === "data-playback-start") return playbackStartRaw;
if (name === "data-media-start") return mediaStartRaw;
return null;
},
return elements.map(
({ endRaw, durationRaw, intrinsicDuration, playbackStartRaw, mediaStartRaw, ...element }) => ({
...element,
end: parseStrictFiniteTimingNumber(endRaw) ?? 0,
duration:
element.durationInferred && Number.isFinite(intrinsicDuration) && intrinsicDuration > 0
? intrinsicDuration
: (parseStrictFiniteTimingNumber(durationRaw) ?? 0),
mediaStart: readMediaStart({
getAttribute(name: string) {
if (name === "data-playback-start") return playbackStartRaw;
if (name === "data-media-start") return mediaStartRaw;
return null;
},
}),
}),
}));
);
}

export async function discoverAudioVolumeAutomationFromTimeline(
Expand Down
73 changes: 73 additions & 0 deletions packages/producer/src/services/render/stages/probeStage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,79 @@ describe("runProbeStage — forceScreenshot threading", () => {
expect(mediaPreflightComposition).toBe(input.composition);
});

it("uses selected intrinsic duration only when the variable-bound duration was inferred", async () => {
resetRetryMocks();
const media = (
id: string,
duration: number,
durationInferred: boolean,
): Record<string, unknown> => ({
id,
tagName: "audio",
src: `${id}-selected.wav`,
start: 0,
end: duration,
duration,
durationInferred,
mediaStart: 0,
loop: false,
hasAudio: true,
volume: 1,
muted: false,
});
browserMediaResults = [
media("longer-inferred", 6.530612, true),
media("longer-authored", 6.530612, false),
media("shorter-inferred", 3.836939, true),
];
const { runProbeStage } = await import("./probeStage.js");
const input = makeProbeInput({});
input.composition.duration = 7;
input.composition.audios.push(
{
id: "longer-inferred",
src: "short.wav",
start: 0,
end: 3.836939,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
{
id: "longer-authored",
src: "short.wav",
start: 0,
end: 3.836939,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
{
id: "shorter-inferred",
src: "long.wav",
start: 0,
end: 6.530612,
mediaStart: 0,
layer: 0,
volume: 1,
type: "audio",
},
);
input.compiled.html = `
<audio id="longer-inferred" src="short.wav" data-var-src="a" data-hf-inferred-duration></audio>
<audio id="longer-authored" src="short.wav" data-var-src="b" data-duration="3.836939"></audio>
<audio id="shorter-inferred" src="long.wav" data-var-src="c" data-hf-inferred-duration></audio>`;
input.job.config.variables = { a: "a.wav", b: "b.wav", c: "c.wav" };

await runProbeStage(input);

expect(input.composition.audios.map((audio) => audio.end)).toEqual([
6.530612, 3.836939, 3.836939,
]);
});

it("passes cancellation through and closes probe-owned resources when preflight rejects", async () => {
resetRetryMocks();
mediaPreflightError = new Error("ASSET_MEDIA_TYPE_MISMATCH");
Expand Down
37 changes: 27 additions & 10 deletions packages/producer/src/services/render/stages/probeStage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,17 @@ export function hasVariableBoundMedia(
});
}

function reconcileBrowserMediaEnd(
existingEnd: number,
projectedEnd: number,
sourceChanged: boolean,
durationInferred: boolean,
): number {
if (projectedEnd <= 0) return existingEnd;
if (sourceChanged && durationInferred) return projectedEnd;
return existingEnd <= 0 ? projectedEnd : Math.min(existingEnd, projectedEnd);
}

/**
* Runtime-created media does not exist when the static compiler scans the HTML.
* Launch a browser probe so discoverMediaFromBrowser can reconcile it before
Expand Down Expand Up @@ -500,18 +511,21 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
// Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
const existing = composition.videos.find((v) => v.id === el.id);
if (existing) {
if (existing.src !== src) {
const sourceChanged = existing.src !== src;
if (sourceChanged) {
existing.src = src;
}
const projectedEnd = projectBrowserEndToCompositionTimeline(
existing.start,
el.start,
resolveBrowserMediaEnd(el.start, el.end, el.duration),
);
if (projectedEnd > 0) {
existing.end =
existing.end <= 0 ? projectedEnd : Math.min(existing.end, projectedEnd);
}
existing.end = reconcileBrowserMediaEnd(
existing.end,
projectedEnd,
sourceChanged,
el.durationInferred,
);
if (
el.mediaStart > 0 &&
(existing.mediaStart <= 0 ||
Expand Down Expand Up @@ -544,18 +558,21 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
if (existingAudioIds.has(el.id)) {
const existing = composition.audios.find((a) => a.id === el.id);
if (existing) {
if (existing.src !== src) {
const sourceChanged = existing.src !== src;
if (sourceChanged) {
existing.src = src;
}
const projectedEnd = projectBrowserEndToCompositionTimeline(
existing.start,
el.start,
resolveBrowserMediaEnd(el.start, el.end, el.duration),
);
if (projectedEnd > 0) {
existing.end =
existing.end <= 0 ? projectedEnd : Math.min(existing.end, projectedEnd);
}
existing.end = reconcileBrowserMediaEnd(
existing.end,
projectedEnd,
sourceChanged,
el.durationInferred,
);
if (
el.mediaStart > 0 &&
(existing.mediaStart <= 0 ||
Expand Down
Loading