From 7bd3229ff243a99c6292d5561625f4e93f891ce5 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Wed, 5 Aug 2026 22:28:56 -0400 Subject: [PATCH 1/2] fix(video): Concatenate Videos preserves audio (cut and transitions) The concat node decoded and re-encoded video frames only, so any input with an audio track (MiniMax H3 clips carry AAC; Wan clips are silent) produced a silent output. Extending an H3 video therefore always lost the soundtrack. The node now rebuilds the soundtrack on the emitted timeline whenever at least one input carries audio: - audio decode via the bundled imageio-ffmpeg binary (new app/util/video_audio helpers: extract_audio_pcm, resample_linear, mux_audio_into_video); - per-clip mapping onto the output timeline with a single linear resample, covering sample-rate unification (to the first audible clip's rate) and fps-override retiming (audio speed/pitch follows the video's retime); - silent inputs contribute silence; all-silent inputs keep the old behavior (no audio stream); - 'cut' splices sample-accurately against the per-clip frame counts actually decoded; 'crossfade' blends the boundary equal-power; 'fade_through_black' ramps out to silence and back in, mirroring the video, with the same asymmetric odd-tf split as the frame path; - the finished track muxes into the already-encoded MP4 with a video stream copy (no re-encode). Node version 1.1.0; openapi.json/schema.ts regenerated. Co-Authored-By: Claude Fable 5 --- invokeai/app/invocations/video_concat.py | 184 ++++++++++- invokeai/app/util/video_audio.py | 147 +++++++++ invokeai/frontend/web/openapi.json | 4 +- .../frontend/web/src/services/api/schema.ts | 7 + .../invocations/test_video_concat_audio.py | 299 ++++++++++++++++++ 5 files changed, 630 insertions(+), 11 deletions(-) create mode 100644 invokeai/app/util/video_audio.py create mode 100644 tests/app/invocations/test_video_concat_audio.py diff --git a/invokeai/app/invocations/video_concat.py b/invokeai/app/invocations/video_concat.py index 93b01cc9abc..bd8db7f9b58 100644 --- a/invokeai/app/invocations/video_concat.py +++ b/invokeai/app/invocations/video_concat.py @@ -10,6 +10,14 @@ at a time, buffering only the transition windows, so peak memory stays O(transition_frames) even when the inputs are long uploads (the upload cap admits files whose decoded frames would run to tens of gigabytes). + +The AUDIO path is a deliberate exception to that bound: soundtracks buffer in +full (float32 stereo PCM of the emitted timeline, roughly 23 MB per minute at +48 kHz). Audio is ~3 orders of magnitude smaller per second than decoded +frames, so even upload-cap-length inputs stay within a few hundred MB — but it +is O(total duration), not O(transition_frames). Audio problems fail the node +loudly rather than emit another silent video: a silent output masquerading as +success is the exact bug this path exists to fix. """ import math @@ -30,7 +38,8 @@ from invokeai.app.invocations.primitives import VideoOutput from invokeai.app.services.session_processor.session_processor_common import CanceledException from invokeai.app.services.shared.invocation_context import InvocationContext -from invokeai.app.util.video_encoding import make_mp4_writer +from invokeai.app.util.video_audio import extract_audio_pcm, mux_audio_into_video, resample_linear +from invokeai.app.util.video_encoding import make_mp4_writer, write_stereo_wav from invokeai.app.util.video_thumbnails import iter_video_frames, probe_video TransitionMode = Literal["cut", "crossfade", "fade_through_black"] @@ -69,12 +78,19 @@ def _fade_through_black(a_tail: list[np.ndarray], b_head: list[np.ndarray]) -> I title="Concatenate Videos", tags=["video", "concat", "transition"], category="video", - version="1.0.0", + version="1.1.0", classification=Classification.Prototype, ) class VideoConcatInvocation(BaseInvocation, WithMetadata, WithBoard): """Join two or more videos into a single MP4. + Audio: if any input carries an audio track, the output gets an AAC track assembled on + the emitted timeline — silent inputs contribute silence, `cut` splices the tracks, + `crossfade` blends them equal-power over the transition window, and + `fade_through_black` fades the outgoing track to silence and the incoming one up from + it, mirroring the video. An fps override retimes audio with the video (a deliberate + speed/pitch change). If no input has audio, the output has no audio stream. + Transitions: * ``cut`` — hard splice, no blending. Fastest; total length is the sum of inputs. @@ -143,14 +159,19 @@ def invoke(self, context: InvocationContext) -> VideoOutput: tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_concat_", suffix=".mp4", delete=False) tmp.close() tmp_path = Path(tmp.name) + wav_path: Path | None = None + muxed_path: Path | None = None try: # Frames stream from the decoders straight into the encoder; only the # transition windows are buffered. See _iter_joined_frames. writer = make_mp4_writer(tmp_path, output_fps) num_frames = 0 + frame_counts: list[int] = [] try: clip_iters = [iter_video_frames(p, is_canceled=context.util.is_canceled) for p in paths] - for frame in self._iter_joined_frames(clip_iters, is_canceled=context.util.is_canceled): + for frame in self._iter_joined_frames( + clip_iters, is_canceled=context.util.is_canceled, frame_counts=frame_counts + ): writer.append_data(frame) num_frames += 1 finally: @@ -159,13 +180,39 @@ def invoke(self, context: InvocationContext) -> VideoOutput: if num_frames == 0: raise ValueError("Concatenation produced zero output frames.") + # Rebuild the soundtrack on the emitted timeline. Inputs without an audio + # track (e.g. Wan clips) contribute silence; if none carries audio, the + # output has no audio stream, as before. + source_path = tmp_path + audio = self._build_audio_track( + context=context, + paths=paths, + frame_counts=frame_counts, + native_rates=[probe[3] for probe in probes], + native_durations=[probe[2] for probe in probes], + output_fps=output_fps, + num_output_frames=num_frames, + ) + if audio is not None: + pcm, rate = audio + context.util.signal_progress("Muxing audio") + wav_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_concat_", suffix=".wav", delete=False) + wav_tmp.close() + wav_path = Path(wav_tmp.name) + write_stereo_wav(wav_path, pcm, rate) + mux_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_concat_", suffix=".mp4", delete=False) + mux_tmp.close() + muxed_path = Path(mux_tmp.name) + mux_audio_into_video(tmp_path, wav_path, muxed_path) + source_path = muxed_path + duration = num_frames / output_fps context.logger.info( f"Encoded concatenated MP4: {num_frames} frames @ {output_fps:.2f} fps " - f"({duration:.2f}s) at {width}x{height}" + f"({duration:.2f}s) at {width}x{height}" + (", with AAC audio" if audio is not None else ", silent") ) video_dto = context.videos.save( - source_path=tmp_path, + source_path=source_path, width=width, height=height, duration=duration, @@ -174,10 +221,13 @@ def invoke(self, context: InvocationContext) -> VideoOutput: context.logger.info(f"Saved concatenated video: {video_dto.video_name}") return VideoOutput.build(video_dto) finally: - try: - tmp_path.unlink(missing_ok=True) - except Exception: - pass + for cleanup in (tmp_path, wav_path, muxed_path): + if cleanup is None: + continue + try: + cleanup.unlink(missing_ok=True) + except Exception: + pass def _estimate_transition_memory(self, width: int, height: int) -> int: if self.transition == "cut" or self.transition_frames == 0: @@ -212,10 +262,124 @@ def _validate_transition_memory(self, width: int, height: int) -> None: "Lower transition_frames or use lower-resolution clips." ) + def _build_audio_track( + self, + context: InvocationContext, + paths: list[Path], + frame_counts: list[int], + native_rates: list[Optional[float]], + native_durations: list[Optional[float]], + output_fps: float, + num_output_frames: int, + ) -> Optional[tuple[np.ndarray, int]]: + """Assemble the output soundtrack on the emitted timeline. + + Returns ``(pcm, sample_rate)`` — ``pcm`` shaped ``(2, n)``, float in [-1, 1] — or + ``None`` when no input carries audio. The frame bookkeeping (which frames of each + clip are emitted directly vs consumed into a boundary) mirrors + ``_iter_joined_frames`` exactly; ``frame_counts`` must be that pass's per-clip + decoded frame counts so the audio is cut against the frames actually emitted. + + Every clip's audio is mapped onto the emitted timeline with one linear resample: + the clip's video spans ``n_i / native_fps`` seconds natively and ``n_i / + output_fps`` seconds in the output, so this single step covers both sample-rate + unification (to the first audible clip's rate) and any fps-override retime. + """ + context.util.signal_progress("Extracting audio") + extracted: list[Optional[tuple[np.ndarray, int]]] = [] + for path in paths: + if context.util.is_canceled(): + raise CanceledException + extracted.append(extract_audio_pcm(path)) + if all(item is None for item in extracted): + return None + if context.util.is_canceled(): + raise CanceledException + rate = next(item[1] for item in extracted if item is not None) + + def s(frames: int) -> int: + """Frame count -> sample count on the emitted timeline.""" + return round(frames * rate / output_fps) + + # Boundary bookkeeping — must mirror _iter_joined_frames. + if self.transition == "crossfade" and self.transition_frames > 0: + tail_need = head_need = self.transition_frames + elif self.transition == "fade_through_black" and self.transition_frames > 0: + tail_need = self.transition_frames // 2 + head_need = self.transition_frames - tail_need + else: + tail_need = head_need = 0 + + clips: list[np.ndarray] = [] + for i, (item, n_frames, native_fps) in enumerate(zip(extracted, frame_counts, native_rates, strict=True)): + want = s(n_frames) + if item is None: + clips.append(np.zeros((2, want), dtype=np.float32)) + continue + pcm, src_rate = item + # The clip's audio must be sliced to the span its video occupies natively + # (trimming AAC end-padding) before the resample maps it onto the emitted + # timeline. Preference order for that native span: the probed fps; the + # probed duration (fps can be None for VFR-flagged containers, and under an + # fps override guessing output_fps would skip the retime — the audio would + # play at the wrong speed and truncate or pad); the extracted length itself, + # which approximates the native span to within the codec padding. + if native_fps is not None and native_fps > 0: + src_want = round(n_frames / native_fps * src_rate) + elif native_durations[i] is not None and native_durations[i] > 0: + src_want = round(native_durations[i] * src_rate) + else: + src_want = pcm.shape[1] + if pcm.shape[1] < src_want: + pcm = np.pad(pcm, ((0, 0), (0, src_want - pcm.shape[1]))) + else: + pcm = pcm[:, :src_want] + clips.append(resample_linear(pcm, want)) + extracted[i] = None # drop the pre-resample copy; peak memory is O(total audio) + + # A boundary exists between consecutive clips whenever the transition consumes + # any frames — even when only ONE side contributes (fade_through_black with + # transition_frames=1 has an empty tail window but still fades the head in). + has_boundary = (head_need + tail_need) > 0 + + segments: list[np.ndarray] = [] + prev_tail: Optional[np.ndarray] = None + for i, pcm in enumerate(clips): + n_i = frame_counts[i] + head_want = 0 if i == 0 else head_need + tail_keep = 0 if i == len(clips) - 1 else tail_need + head_end = s(head_want) + tail_start = s(n_i - tail_keep) if tail_keep else pcm.shape[1] + if prev_tail is not None: + head = pcm[:, :head_end] + if self.transition == "crossfade": + # Equal-power blend: constant perceived loudness through the overlap. + n = min(prev_tail.shape[1], head.shape[1]) + theta = np.linspace(0.0, np.pi / 2.0, n, dtype=np.float32) + segments.append(prev_tail[:, :n] * np.cos(theta) + head[:, :n] * np.sin(theta)) + else: # fade_through_black: out to silence, then in from silence. + segments.append(prev_tail * np.linspace(1.0, 0.0, prev_tail.shape[1], dtype=np.float32)) + segments.append(head * np.linspace(0.0, 1.0, head.shape[1], dtype=np.float32)) + segments.append(pcm[:, head_end:tail_start]) + # An empty tail window (shape (2, 0)) still marks the boundary as pending so + # the next clip's head is faded in rather than silently dropped. + prev_tail = pcm[:, tail_start:] if (has_boundary and i < len(clips) - 1) else None + + pcm_out = np.concatenate(segments, axis=1) + # Per-clip rounding can drift the total by a sample per boundary; pin the track + # to the emitted video duration exactly. + target = s(num_output_frames) + if pcm_out.shape[1] < target: + pcm_out = np.pad(pcm_out, ((0, 0), (0, target - pcm_out.shape[1]))) + else: + pcm_out = pcm_out[:, :target] + return pcm_out.astype(np.float32, copy=False), rate + def _iter_joined_frames( self, clips: list[Iterable[np.ndarray]], is_canceled: Optional[Callable[[], bool]] = None, + frame_counts: Optional[list[int]] = None, ) -> Iterator[np.ndarray]: """Yields the joined output frames, pulling lazily from each clip's frame iterator. @@ -275,6 +439,8 @@ def _iter_joined_frames( tail_buf.append(frame) if len(tail_buf) > tail_keep: yield tail_buf.popleft() + if frame_counts is not None: + frame_counts.append(n_frames) if n_frames == 0: raise ValueError(f"Input video {i} ({self.videos[i].video_name}) decoded to zero frames.") if n_frames < head_want + tail_keep: diff --git a/invokeai/app/util/video_audio.py b/invokeai/app/util/video_audio.py new file mode 100644 index 00000000000..33f4c94f947 --- /dev/null +++ b/invokeai/app/util/video_audio.py @@ -0,0 +1,147 @@ +"""Audio extraction and remuxing helpers for video-editing invocations. + +The decode side of the app's video plumbing (imageio's FFMPEG reader) exposes only video +frames, so audio work shells out to the same bundled ffmpeg binary imageio uses. Two +operations are provided: + +- :func:`extract_audio_pcm` — decode a container's audio track to float stereo PCM + (returns ``None`` for silent containers, which is how Wan clips present). +- :func:`mux_audio_into_video` — attach a WAV to an already-encoded MP4 by stream-copying + the video (no re-encode) and encoding the audio to AAC-LC, the browser-safe codec used + everywhere else in the app. + +Plus :func:`resample_linear`, a dependency-free linear resampler used both to unify +sample rates across inputs and to retime audio when a clip's video is retimed by an +output-fps override (a deliberate speed/pitch change, matching what retiming does to the +video). +""" + +import subprocess +import tempfile +import wave +from pathlib import Path + +import numpy as np + + +def _ffmpeg_exe() -> str: + import imageio_ffmpeg + + return imageio_ffmpeg.get_ffmpeg_exe() + + +class AudioExtractionError(RuntimeError): + """ffmpeg failed to decode an audio track for a reason other than 'no audio stream'.""" + + +def extract_audio_pcm(video_path: Path) -> tuple[np.ndarray, int] | None: + """Decode ``video_path``'s audio track to float32 stereo PCM at its native sample rate. + + Returns ``(samples, sample_rate)`` with ``samples`` shaped ``(2, n)`` in [-1, 1], or + ``None`` when the container has no audio stream. Mono sources are upmixed to stereo by + ffmpeg; multi-channel sources are downmixed. + """ + with tempfile.NamedTemporaryFile(prefix="invokeai_audio_extract_", suffix=".wav", delete=False) as tmp: + wav_path = Path(tmp.name) + try: + try: + proc = subprocess.run( + [ + _ffmpeg_exe(), + "-y", + "-loglevel", + "error", + "-i", + str(video_path), + "-vn", + "-ac", + "2", + "-acodec", + "pcm_s16le", + str(wav_path), + ], + capture_output=True, + timeout=600, + ) + except subprocess.TimeoutExpired as e: + raise AudioExtractionError(f"ffmpeg timed out extracting audio from {video_path.name}") from e + if proc.returncode != 0: + stderr = proc.stderr.decode("utf-8", errors="replace") + # ffmpeg's phrasing for an input with no audio track ("Output file #0 does not + # contain any stream" / newer "does not contain any stream" variants). + if "does not contain any stream" in stderr: + return None + raise AudioExtractionError( + f"ffmpeg could not extract audio from {video_path.name}: {stderr.strip()[-500:]}" + ) + with wave.open(str(wav_path), "rb") as wav: + rate = wav.getframerate() + n = wav.getnframes() + raw = wav.readframes(n) + if n == 0: + return None + data = np.frombuffer(raw, dtype=np.int16).reshape(-1, 2).T + return data.astype(np.float32) / 32768.0, rate + finally: + wav_path.unlink(missing_ok=True) + + +def resample_linear(samples: np.ndarray, num_output_samples: int) -> np.ndarray: + """Linearly resample ``(2, n)`` PCM to ``(2, num_output_samples)``. + + Used for sample-rate unification and for retiming (where the accompanying video is + being speed-changed, so the pitch shift this introduces is the correct behavior, not + an artifact). An empty input yields silence. + """ + if samples.ndim != 2 or samples.shape[0] != 2: + raise ValueError(f"Expected samples shaped (2, n), got {samples.shape}") + n_in = samples.shape[1] + if num_output_samples <= 0: + return np.zeros((2, 0), dtype=np.float32) + if n_in == 0: + return np.zeros((2, num_output_samples), dtype=np.float32) + if n_in == num_output_samples: + return samples.astype(np.float32, copy=False) + positions = np.linspace(0.0, n_in - 1, num_output_samples) + grid = np.arange(n_in, dtype=np.float64) + return np.stack([np.interp(positions, grid, samples[c]) for c in range(2)]).astype(np.float32) + + +def mux_audio_into_video(video_path: Path, wav_path: Path, output_path: Path) -> None: + """Produce ``output_path`` = ``video_path``'s streams with ``wav_path`` as an AAC track. + + The video stream is copied bit-exactly (no re-encode); only the audio is encoded. + """ + try: + proc = _run_mux(video_path, wav_path, output_path) + except subprocess.TimeoutExpired as e: + raise AudioExtractionError(f"ffmpeg timed out muxing audio into {output_path.name}") from e + if proc.returncode != 0: + stderr = proc.stderr.decode("utf-8", errors="replace") + raise AudioExtractionError(f"ffmpeg could not mux audio into {output_path.name}: {stderr.strip()[-500:]}") + + +def _run_mux(video_path: Path, wav_path: Path, output_path: Path) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [ + _ffmpeg_exe(), + "-y", + "-loglevel", + "error", + "-i", + str(video_path), + "-i", + str(wav_path), + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c:v", + "copy", + "-c:a", + "aac", + str(output_path), + ], + capture_output=True, + timeout=600, + ) diff --git a/invokeai/frontend/web/openapi.json b/invokeai/frontend/web/openapi.json index 1bfae941c15..80c72b4d16b 100644 --- a/invokeai/frontend/web/openapi.json +++ b/invokeai/frontend/web/openapi.json @@ -87053,7 +87053,7 @@ "category": "video", "class": "invocation", "classification": "prototype", - "description": "Join two or more videos into a single MP4.\n\nTransitions:\n\n* ``cut`` \u2014 hard splice, no blending. Fastest; total length is the sum of inputs.\n* ``crossfade`` \u2014 linear A\u2192B cross-dissolve over ``transition_frames``. Each boundary\n consumes ``transition_frames`` from both adjacent clips, so total length is\n ``sum(inputs) - transition_frames * (n - 1)``.\n* ``fade_through_black`` \u2014 A fades to black, then B fades in from black. Each boundary\n consumes ``transition_frames // 2`` frames from the preceding clip's tail and the\n remainder (``transition_frames - transition_frames // 2``) from the next clip's head,\n so the total emitted is exactly ``transition_frames`` per boundary \u2014 even for odd\n ``transition_frames`` \u2014 and the overall length equals the sum of inputs.\n\nAll inputs must share the same pixel dimensions. Output frame rate defaults to the\nfirst input's fps; override with ``fps`` to force a specific rate (the frames are not\nresampled, only the container is encoded at the new rate).", + "description": "Join two or more videos into a single MP4.\n\nAudio: if any input carries an audio track, the output gets an AAC track assembled on\nthe emitted timeline \u2014 silent inputs contribute silence, `cut` splices the tracks,\n`crossfade` blends them equal-power over the transition window, and\n`fade_through_black` fades the outgoing track to silence and the incoming one up from\nit, mirroring the video. An fps override retimes audio with the video (a deliberate\nspeed/pitch change). If no input has audio, the output has no audio stream.\n\nTransitions:\n\n* ``cut`` \u2014 hard splice, no blending. Fastest; total length is the sum of inputs.\n* ``crossfade`` \u2014 linear A\u2192B cross-dissolve over ``transition_frames``. Each boundary\n consumes ``transition_frames`` from both adjacent clips, so total length is\n ``sum(inputs) - transition_frames * (n - 1)``.\n* ``fade_through_black`` \u2014 A fades to black, then B fades in from black. Each boundary\n consumes ``transition_frames // 2`` frames from the preceding clip's tail and the\n remainder (``transition_frames - transition_frames // 2``) from the next clip's head,\n so the total emitted is exactly ``transition_frames`` per boundary \u2014 even for odd\n ``transition_frames`` \u2014 and the overall length equals the sum of inputs.\n\nAll inputs must share the same pixel dimensions. Output frame rate defaults to the\nfirst input's fps; override with ``fps`` to force a specific rate (the frames are not\nresampled, only the container is encoded at the new rate).", "node_pack": "invokeai", "properties": { "board": { @@ -87186,7 +87186,7 @@ "tags": ["video", "concat", "transition"], "title": "Concatenate Videos", "type": "object", - "version": "1.0.0", + "version": "1.1.0", "output": { "$ref": "#/components/schemas/VideoOutput" } diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index efa8f6bd378..e6f79beaf25 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -38429,6 +38429,13 @@ export type components = { * Concatenate Videos * @description Join two or more videos into a single MP4. * + * Audio: if any input carries an audio track, the output gets an AAC track assembled on + * the emitted timeline — silent inputs contribute silence, `cut` splices the tracks, + * `crossfade` blends them equal-power over the transition window, and + * `fade_through_black` fades the outgoing track to silence and the incoming one up from + * it, mirroring the video. An fps override retimes audio with the video (a deliberate + * speed/pitch change). If no input has audio, the output has no audio stream. + * * Transitions: * * * ``cut`` — hard splice, no blending. Fastest; total length is the sum of inputs. diff --git a/tests/app/invocations/test_video_concat_audio.py b/tests/app/invocations/test_video_concat_audio.py new file mode 100644 index 00000000000..e3b59c5d373 --- /dev/null +++ b/tests/app/invocations/test_video_concat_audio.py @@ -0,0 +1,299 @@ +"""Audio-preservation tests for VideoConcatInvocation. + +The concat node originally decoded and re-encoded video frames only, silently dropping +the audio track of any input that had one (H3 clips carry AAC; Wan clips are silent). +These tests pin the rebuilt soundtrack path: extraction, per-clip timeline mapping, +boundary blending per transition mode, silent-input handling, fps-override retiming, and +the final mux. + +Real ffmpeg (the imageio-ffmpeg bundled binary) encodes and decodes throughout — tone +clips are tiny (64x64) so the suite stays fast. +""" + +from pathlib import Path + +import numpy as np +import pytest + +from invokeai.app.invocations.fields import VideoField +from invokeai.app.invocations.video_concat import VideoConcatInvocation +from invokeai.app.util.video_audio import extract_audio_pcm, mux_audio_into_video, resample_linear +from invokeai.app.util.video_encoding import make_mp4_writer, write_stereo_wav + +RATE = 32000 +FPS = 16.0 +SIZE = 64 + + +class _Util: + def signal_progress(self, *args, **kwargs) -> None: + pass + + def is_canceled(self) -> bool: + return False + + +class _Ctx: + util = _Util() + + +def _tone(freq: float, seconds: float, rate: int = RATE) -> np.ndarray: + t = np.arange(round(seconds * rate)) / rate + mono = (0.5 * np.sin(2 * np.pi * freq * t)).astype(np.float32) + return np.stack([mono, mono]) + + +def _make_clip(dirpath: Path, name: str, n_frames: int, freq: float | None, gray: int = 128, fps: float = FPS) -> Path: + path = dirpath / f"{name}.mp4" + audio_path = None + if freq is not None: + audio_path = dirpath / f"{name}.wav" + write_stereo_wav(audio_path, _tone(freq, n_frames / fps), RATE) + writer = make_mp4_writer(path, fps, audio_path=audio_path) + frame = np.full((SIZE, SIZE, 3), gray, dtype=np.uint8) + for _ in range(n_frames): + writer.append_data(frame) + writer.close() + return path + + +def _dominant_freq(pcm: np.ndarray, rate: int) -> float: + spectrum = np.abs(np.fft.rfft(pcm[0])) + spectrum[0] = 0.0 # ignore DC + return float(np.fft.rfftfreq(pcm.shape[1], 1.0 / rate)[int(np.argmax(spectrum))]) + + +def _invocation(n_videos: int = 2, transition: str = "cut", transition_frames: int = 8) -> VideoConcatInvocation: + return VideoConcatInvocation( + videos=[VideoField(video_name=f"clip{i}") for i in range(n_videos)], + transition=transition, # type: ignore[arg-type] + transition_frames=transition_frames, + ) + + +class TestExtraction: + def test_roundtrip_tone(self, tmp_path): + clip = _make_clip(tmp_path, "a", 32, freq=440.0) + extracted = extract_audio_pcm(clip) + assert extracted is not None + pcm, rate = extracted + assert rate == RATE + assert pcm.shape[0] == 2 + assert abs(pcm.shape[1] / rate - 2.0) < 0.15 # ~2 s, AAC padding tolerated + assert abs(_dominant_freq(pcm, rate) - 440.0) < 10.0 + + def test_silent_clip_returns_none(self, tmp_path): + clip = _make_clip(tmp_path, "s", 16, freq=None) + assert extract_audio_pcm(clip) is None + + +class TestResample: + def test_lengths_and_identity(self): + pcm = _tone(440.0, 1.0) + assert resample_linear(pcm, pcm.shape[1]).shape == pcm.shape + assert resample_linear(pcm, 1234).shape == (2, 1234) + assert resample_linear(np.zeros((2, 0), dtype=np.float32), 100).shape == (2, 100) + + def test_rejects_bad_shape(self): + with pytest.raises(ValueError, match="shaped"): + resample_linear(np.zeros(10, dtype=np.float32), 5) + + +class TestBuildAudioTrack: + def _build(self, paths, frame_counts, invocation=None, output_fps=FPS, native=None, durations=None): + inv = invocation or _invocation(n_videos=len(paths)) + return inv._build_audio_track( + context=_Ctx(), # type: ignore[arg-type] + paths=paths, + frame_counts=frame_counts, + native_rates=native if native is not None else [FPS] * len(paths), + native_durations=durations if durations is not None else [None] * len(paths), + output_fps=output_fps, + num_output_frames=sum(frame_counts) + if (invocation or _invocation()).transition != "crossfade" + else sum(frame_counts) - (len(paths) - 1) * (invocation or _invocation()).transition_frames, + ) + + def test_cut_splices_tracks(self, tmp_path): + a = _make_clip(tmp_path, "a", 32, freq=440.0) + b = _make_clip(tmp_path, "b", 32, freq=880.0) + pcm, rate = self._build([a, b], [32, 32]) + assert rate == RATE + assert pcm.shape[1] == round(64 * RATE / FPS) + half = pcm.shape[1] // 2 + assert abs(_dominant_freq(pcm[:, :half], rate) - 440.0) < 10.0 + assert abs(_dominant_freq(pcm[:, half:], rate) - 880.0) < 10.0 + + def test_silent_input_contributes_silence(self, tmp_path): + a = _make_clip(tmp_path, "a", 32, freq=None) + b = _make_clip(tmp_path, "b", 32, freq=880.0) + pcm, rate = self._build([a, b], [32, 32]) + half = pcm.shape[1] // 2 + assert np.abs(pcm[:, : half - RATE // 10]).max() == 0.0 + assert np.abs(pcm[:, half:]).max() > 0.2 + + def test_all_silent_returns_none(self, tmp_path): + a = _make_clip(tmp_path, "a", 16, freq=None) + b = _make_clip(tmp_path, "b", 16, freq=None) + assert self._build([a, b], [16, 16]) is None + + def test_crossfade_length_and_blend(self, tmp_path): + tf = 8 + inv = _invocation(transition="crossfade", transition_frames=tf) + a = _make_clip(tmp_path, "a", 32, freq=440.0) + b = _make_clip(tmp_path, "b", 32, freq=880.0) + pcm, rate = inv._build_audio_track( + context=_Ctx(), # type: ignore[arg-type] + paths=[a, b], + frame_counts=[32, 32], + native_rates=[FPS, FPS], + native_durations=[None, None], + output_fps=FPS, + num_output_frames=32 + 32 - tf, + ) + assert pcm.shape[1] == round((64 - tf) * RATE / FPS) + # Ends stay pure. + quarter = pcm.shape[1] // 4 + assert abs(_dominant_freq(pcm[:, :quarter], rate) - 440.0) < 10.0 + assert abs(_dominant_freq(pcm[:, -quarter:], rate) - 880.0) < 10.0 + # The overlap window contains BOTH tones (equal-power blend, not a hard cut). + window = round(tf * RATE / FPS) + start = round((32 - tf) * RATE / FPS) + overlap = pcm[:, start : start + window] + spectrum = np.abs(np.fft.rfft(overlap[0])) + freqs = np.fft.rfftfreq(overlap.shape[1], 1.0 / rate) + e440 = spectrum[(np.abs(freqs - 440.0) < 30)].max() + e880 = spectrum[(np.abs(freqs - 880.0) < 30)].max() + assert e440 > 0.1 * e880 and e880 > 0.1 * e440 + + # Orientation: the outgoing tone dominates early in the overlap, the incoming + # tone late — a swapped tail/head blend fails these. + def band_energy(seg, freq): + spec = np.abs(np.fft.rfft(seg[0])) + f = np.fft.rfftfreq(seg.shape[1], 1.0 / rate) + return spec[np.abs(f - freq) < 60].max() + + third = window // 3 + early, late = overlap[:, :third], overlap[:, -third:] + assert band_energy(early, 440.0) > band_energy(early, 880.0) + assert band_energy(late, 880.0) > band_energy(late, 440.0) + + def test_fade_through_black_preserves_length_and_dips(self, tmp_path): + tf = 8 + inv = _invocation(transition="fade_through_black", transition_frames=tf) + a = _make_clip(tmp_path, "a", 32, freq=440.0) + b = _make_clip(tmp_path, "b", 32, freq=880.0) + pcm, rate = inv._build_audio_track( + context=_Ctx(), # type: ignore[arg-type] + paths=[a, b], + frame_counts=[32, 32], + native_rates=[FPS, FPS], + native_durations=[None, None], + output_fps=FPS, + num_output_frames=64, + ) + assert pcm.shape[1] == round(64 * RATE / FPS) + boundary = round(32 * RATE / FPS) + window = round(1 * RATE / FPS) # one frame around the silence point + rms_boundary = float(np.sqrt(np.mean(pcm[:, boundary - window : boundary + window] ** 2))) + rms_mid = float(np.sqrt(np.mean(pcm[:, boundary // 2 : boundary // 2 + 2 * window] ** 2))) + assert rms_boundary < 0.25 * rms_mid + + def test_fps_override_retimes_audio(self, tmp_path): + a = _make_clip(tmp_path, "a", 32, freq=440.0) + b = _make_clip(tmp_path, "b", 32, freq=440.0) + pcm, rate = self._build([a, b], [32, 32], output_fps=2 * FPS) + # Video plays twice as fast -> audio spans half the wall clock (and pitches up). + assert pcm.shape[1] == round(64 * RATE / (2 * FPS)) + assert abs(_dominant_freq(pcm, rate) - 880.0) < 20.0 + + def test_fade_through_black_tf1_keeps_audio_aligned(self, tmp_path): + """Regression: tf=1 has an EMPTY tail window (1 // 2 == 0) but still a boundary. + + The head frame's audio used to be silently dropped, shifting everything after the + boundary earlier and leaving trailing silence masked by the final pad.""" + inv = _invocation(transition="fade_through_black", transition_frames=1) + a = _make_clip(tmp_path, "a", 32, freq=440.0) + b = _make_clip(tmp_path, "b", 32, freq=880.0) + pcm, rate = inv._build_audio_track( + context=_Ctx(), # type: ignore[arg-type] + paths=[a, b], + frame_counts=[32, 32], + native_rates=[FPS, FPS], + native_durations=[None, None], + output_fps=FPS, + num_output_frames=64, + ) + assert pcm.shape[1] == round(64 * RATE / FPS) + # No trailing silence: the final one-frame window carries the full 880 Hz tone. + one_frame = round(RATE / FPS) + assert float(np.sqrt(np.mean(pcm[:, -one_frame:] ** 2))) > 0.2 + # And the boundary is where it belongs: 880 Hz dominates right after it. + boundary = round(32 * RATE / FPS) + after = pcm[:, boundary + one_frame : boundary + 5 * one_frame] + assert abs(_dominant_freq(after, rate) - 880.0) < 10.0 + + def test_unknown_native_fps_uses_probed_duration_for_retime(self, tmp_path): + """Regression: with probe fps None and an fps override, the retime used to be + skipped entirely — audio played at the wrong speed and truncated/padded.""" + a = _make_clip(tmp_path, "a", 32, freq=440.0) + b = _make_clip(tmp_path, "b", 32, freq=440.0) + pcm, rate = self._build([a, b], [32, 32], output_fps=2 * FPS, native=[None, None], durations=[2.0, 2.0]) + assert pcm.shape[1] == round(64 * RATE / (2 * FPS)) + assert abs(_dominant_freq(pcm, rate) - 880.0) < 20.0 + # Both halves stay audible — the broken path left the second half silent. + half = pcm.shape[1] // 2 + assert float(np.sqrt(np.mean(pcm[:, :half] ** 2))) > 0.2 + assert float(np.sqrt(np.mean(pcm[:, half:] ** 2))) > 0.2 + + def test_unknown_fps_and_duration_falls_back_to_extracted_length(self, tmp_path): + a = _make_clip(tmp_path, "a", 32, freq=440.0) + b = _make_clip(tmp_path, "b", 32, freq=880.0) + pcm, rate = self._build([a, b], [32, 32], native=[None, None], durations=[None, None]) + assert pcm.shape[1] == round(64 * RATE / FPS) + half = pcm.shape[1] // 2 + assert abs(_dominant_freq(pcm[:, : half - RATE // 4], rate) - 440.0) < 15.0 + assert abs(_dominant_freq(pcm[:, half + RATE // 4 :], rate) - 880.0) < 15.0 + + +class TestMux: + def test_end_to_end_cut_produces_audible_mp4(self, tmp_path): + """The full pipeline the node runs: silent video encode -> audio build -> mux.""" + inv = _invocation() + a = _make_clip(tmp_path, "a", 32, freq=440.0, gray=60) + b = _make_clip(tmp_path, "b", 32, freq=880.0, gray=200) + + from invokeai.app.util.video_thumbnails import iter_video_frames, probe_video + + video_only = tmp_path / "joined_video_only.mp4" + writer = make_mp4_writer(video_only, FPS) + frame_counts: list[int] = [] + for frame in inv._iter_joined_frames([iter_video_frames(a), iter_video_frames(b)], frame_counts=frame_counts): + writer.append_data(frame) + writer.close() + assert frame_counts == [32, 32] + assert extract_audio_pcm(video_only) is None + + pcm, rate = inv._build_audio_track( + context=_Ctx(), # type: ignore[arg-type] + paths=[a, b], + frame_counts=frame_counts, + native_rates=[FPS, FPS], + native_durations=[None, None], + output_fps=FPS, + num_output_frames=64, + ) + wav = tmp_path / "joined.wav" + write_stereo_wav(wav, pcm, rate) + out = tmp_path / "joined.mp4" + mux_audio_into_video(video_only, wav, out) + + width, height, _, fps = probe_video(out) + assert (width, height) == (SIZE, SIZE) + extracted = extract_audio_pcm(out) + assert extracted is not None + out_pcm, out_rate = extracted + assert abs(out_pcm.shape[1] / out_rate - 4.0) < 0.2 + half = out_pcm.shape[1] // 2 + assert abs(_dominant_freq(out_pcm[:, : half - out_rate // 4], out_rate) - 440.0) < 10.0 + assert abs(_dominant_freq(out_pcm[:, half + out_rate // 4 :], out_rate) - 880.0) < 10.0 From 5b9322864fed05bb133dc37b2e50ea38a94bbc00 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 6 Aug 2026 08:55:02 -0400 Subject: [PATCH 2/2] feat(nodes): preserve audio in Frame Range from Video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extract_video_range node had the same defect just fixed in video_concat: it re-encoded video frames only, so trimming a clip silently stripped its soundtrack. In the extend-video workflow the trimmed source clip feeds Concatenate Videos as clip 1, which then correctly rendered its span as silence — the source's audio never survived the trim. The node now slices the matching span of the source's PCM (same fps -> probed-duration -> extracted-length fallback chain as concat), zero-pads short audio tracks to keep temporal alignment, retimes the audio alongside the video under an fps override, and muxes it into the trimmed MP4 with a video stream copy. Silent sources keep the old video-only output. Audio failures fail the node loudly, matching the concat node's policy. Node version 1.2.0; openapi.json/schema.ts regenerated. Co-Authored-By: Claude Fable 5 --- .../invocations/video_frame_extract_range.py | 109 ++++++++++- invokeai/frontend/web/openapi.json | 4 +- .../frontend/web/src/services/api/schema.ts | 5 +- .../test_video_frame_extract_range.py | 1 + .../test_video_frame_extract_range_audio.py | 183 ++++++++++++++++++ 5 files changed, 290 insertions(+), 12 deletions(-) create mode 100644 tests/app/invocations/test_video_frame_extract_range_audio.py diff --git a/invokeai/app/invocations/video_frame_extract_range.py b/invokeai/app/invocations/video_frame_extract_range.py index b9449e5201c..2c856980449 100644 --- a/invokeai/app/invocations/video_frame_extract_range.py +++ b/invokeai/app/invocations/video_frame_extract_range.py @@ -5,6 +5,13 @@ video and emits a new MP4, so the output can be fed straight into Concatenate Videos to splice clips together — e.g. trim a generated clip to a usable middle section before chaining it to another shot. + +If the source carries an audio track, the matching slice of it is carried +through (and retimed alongside the video when the output fps differs from +the source's). Like ``video_concat``, the audio path buffers the source +soundtrack in full — float32 stereo PCM, ~3 orders of magnitude smaller +per second than decoded frames — and audio failures fail the node loudly +rather than emit a silently-stripped clip. """ import tempfile @@ -32,7 +39,8 @@ from invokeai.app.invocations.primitives import VideoOutput from invokeai.app.services.session_processor.session_processor_common import CanceledException from invokeai.app.services.shared.invocation_context import InvocationContext -from invokeai.app.util.video_encoding import make_mp4_writer +from invokeai.app.util.video_audio import extract_audio_pcm, mux_audio_into_video, resample_linear +from invokeai.app.util.video_encoding import make_mp4_writer, write_stereo_wav from invokeai.app.util.video_thumbnails import decoder_frame_count, iter_video_frames, probe_video @@ -104,7 +112,7 @@ class ExtractVideoRangeOutput(BaseInvocationOutput): title="Frame Range from Video", tags=["video", "trim", "range", "frames"], category="video", - version="1.1.0", + version="1.2.0", classification=Classification.Prototype, ) class ExtractVideoRangeInvocation(BaseInvocation, WithMetadata, WithBoard): @@ -114,7 +122,10 @@ class ExtractVideoRangeInvocation(BaseInvocation, WithMetadata, WithBoard): emits 41 frames. Negative indices count from the end (``end_frame=-1`` is the final frame), matching ``video_frame_extract``. The output frame rate defaults to the source video's frame rate; set ``fps=0`` to inherit - it (or 16 fps if the source rate can't be probed). + it (or 16 fps if the source rate can't be probed). If the source has an + audio track, the same range of it is carried into the output (retimed + with the video when the output fps changes playback speed); silent + sources stay silent. The resolved (positive) ``start_frame`` and ``end_frame`` are also emitted as outputs, so chained workflows can re-use the boundary indices — e.g. feeding @@ -183,6 +194,8 @@ def invoke(self, context: InvocationContext) -> ExtractVideoRangeOutput: tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".mp4", delete=False) tmp.close() tmp_path = Path(tmp.name) + wav_path: Optional[Path] = None + muxed_path: Optional[Path] = None try: # imageio's iter_index isn't exposed by iio.imiter, so we enumerate and skip. # Frames stream straight from the decoder into the encoder; see _write_frame_range. @@ -206,13 +219,40 @@ def invoke(self, context: InvocationContext) -> ExtractVideoRangeOutput: f"(probed {n_frames} frames). The container's metadata may be inaccurate." ) + # Carry the source's audio (if any) through: slice the matching span of + # its soundtrack and map it onto the output timeline. Silent sources + # keep the old behavior (no audio stream). + source_path = tmp_path + audio = self._build_audio_track( + context=context, + video_path=video_path, + start=start, + end=end, + n_frames=n_frames, + source_fps=source_fps, + source_duration=duration, + output_fps=output_fps, + ) + if audio is not None: + pcm, rate = audio + context.util.signal_progress("Muxing audio") + wav_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".wav", delete=False) + wav_tmp.close() + wav_path = Path(wav_tmp.name) + write_stereo_wav(wav_path, pcm, rate) + mux_tmp = tempfile.NamedTemporaryFile(prefix="invokeai_video_range_", suffix=".mp4", delete=False) + mux_tmp.close() + muxed_path = Path(mux_tmp.name) + mux_audio_into_video(tmp_path, wav_path, muxed_path) + source_path = muxed_path + out_duration = num_frames / output_fps context.logger.info( f"Encoded trimmed MP4: {num_frames} frames @ {output_fps:.2f} fps " - f"({out_duration:.2f}s) at {width}x{height}" + f"({out_duration:.2f}s) at {width}x{height}" + (", with AAC audio" if audio is not None else ", silent") ) video_dto = context.videos.save( - source_path=tmp_path, + source_path=source_path, width=width, height=height, duration=out_duration, @@ -231,10 +271,61 @@ def invoke(self, context: InvocationContext) -> ExtractVideoRangeOutput: end_frame=end, ) finally: - try: - tmp_path.unlink(missing_ok=True) - except Exception: - pass + for cleanup in (tmp_path, wav_path, muxed_path): + if cleanup is None: + continue + try: + cleanup.unlink(missing_ok=True) + except Exception: + pass + + def _build_audio_track( + self, + context: InvocationContext, + video_path: Path, + start: int, + end: int, + n_frames: int, + source_fps: Optional[float], + source_duration: Optional[float], + output_fps: float, + ) -> Optional[tuple[np.ndarray, int]]: + """Slice the source soundtrack to frames [start, end] and map it onto the output timeline. + + Returns ``(pcm, sample_rate)`` — ``pcm`` shaped ``(2, n)``, float32 in [-1, 1] — + or ``None`` when the source has no audio track. + + Frame-to-sample mapping needs the source's effective frame rate. Preference + order (mirroring ``video_concat``): the probed fps; the probed duration (fps can + be None for VFR-flagged containers); the extracted length itself, which + approximates the native span to within the codec padding. The final resample + maps the sliced window onto the span the trimmed video occupies in the output — + a no-op at matching fps, a deliberate speed/pitch retime under an fps override + (matching what the retime does to the video). + """ + context.util.signal_progress("Extracting audio") + extracted = extract_audio_pcm(video_path) + if extracted is None: + return None + if context.util.is_canceled(): + raise CanceledException + pcm, rate = extracted + if source_fps is not None and source_fps > 0: + eff_fps = float(source_fps) + elif source_duration is not None and source_duration > 0: + eff_fps = n_frames / float(source_duration) + else: + # extract_audio_pcm returns None for zero-sample tracks, so pcm is non-empty. + eff_fps = n_frames * rate / pcm.shape[1] + window_start = round(start * rate / eff_fps) + window_end = round((end + 1) * rate / eff_fps) + window = pcm[:, window_start:window_end] + if window.shape[1] < window_end - window_start: + # Audio track shorter than the video (or the range lies past its end): + # keep temporal alignment by padding the shortfall with silence. + window = np.pad(window, ((0, 0), (0, (window_end - window_start) - window.shape[1]))) + target = round((end - start + 1) / output_fps * rate) + return resample_linear(window, target), rate @staticmethod def _resolve_index(value: int, n_frames: int, field_name: str) -> int: diff --git a/invokeai/frontend/web/openapi.json b/invokeai/frontend/web/openapi.json index 80c72b4d16b..16fc1725ec6 100644 --- a/invokeai/frontend/web/openapi.json +++ b/invokeai/frontend/web/openapi.json @@ -27065,7 +27065,7 @@ "category": "video", "class": "invocation", "classification": "prototype", - "description": "Trim a video to a contiguous frame range and re-encode as MP4.\n\nBoth bounds are inclusive and 0-based \u2014 ``start_frame=10, end_frame=50``\nemits 41 frames. Negative indices count from the end (``end_frame=-1``\nis the final frame), matching ``video_frame_extract``. The output frame\nrate defaults to the source video's frame rate; set ``fps=0`` to inherit\nit (or 16 fps if the source rate can't be probed).\n\nThe resolved (positive) ``start_frame`` and ``end_frame`` are also emitted as\noutputs, so chained workflows can re-use the boundary indices \u2014 e.g. feeding\nthem into a downstream Frame from Video to extract the same boundary frame.", + "description": "Trim a video to a contiguous frame range and re-encode as MP4.\n\nBoth bounds are inclusive and 0-based \u2014 ``start_frame=10, end_frame=50``\nemits 41 frames. Negative indices count from the end (``end_frame=-1``\nis the final frame), matching ``video_frame_extract``. The output frame\nrate defaults to the source video's frame rate; set ``fps=0`` to inherit\nit (or 16 fps if the source rate can't be probed). If the source has an\naudio track, the same range of it is carried into the output (retimed\nwith the video when the output fps changes playback speed); silent\nsources stay silent.\n\nThe resolved (positive) ``start_frame`` and ``end_frame`` are also emitted as\noutputs, so chained workflows can re-use the boundary indices \u2014 e.g. feeding\nthem into a downstream Frame from Video to extract the same boundary frame.", "node_pack": "invokeai", "properties": { "board": { @@ -27185,7 +27185,7 @@ "tags": ["video", "trim", "range", "frames"], "title": "Frame Range from Video", "type": "object", - "version": "1.1.0", + "version": "1.2.0", "output": { "$ref": "#/components/schemas/ExtractVideoRangeOutput" } diff --git a/invokeai/frontend/web/src/services/api/schema.ts b/invokeai/frontend/web/src/services/api/schema.ts index e6f79beaf25..7f882e8dae4 100644 --- a/invokeai/frontend/web/src/services/api/schema.ts +++ b/invokeai/frontend/web/src/services/api/schema.ts @@ -10702,7 +10702,10 @@ export type components = { * emits 41 frames. Negative indices count from the end (``end_frame=-1`` * is the final frame), matching ``video_frame_extract``. The output frame * rate defaults to the source video's frame rate; set ``fps=0`` to inherit - * it (or 16 fps if the source rate can't be probed). + * it (or 16 fps if the source rate can't be probed). If the source has an + * audio track, the same range of it is carried into the output (retimed + * with the video when the output fps changes playback speed); silent + * sources stay silent. * * The resolved (positive) ``start_frame`` and ``end_frame`` are also emitted as * outputs, so chained workflows can re-use the boundary indices — e.g. feeding diff --git a/tests/app/invocations/test_video_frame_extract_range.py b/tests/app/invocations/test_video_frame_extract_range.py index 52945ca7d1a..a62590ec818 100644 --- a/tests/app/invocations/test_video_frame_extract_range.py +++ b/tests/app/invocations/test_video_frame_extract_range.py @@ -101,6 +101,7 @@ def test_invocation_only_saves_complete_requested_range(written: int, should_rai patch("invokeai.app.invocations.video_frame_extract_range.decoder_frame_count", return_value=5), patch("invokeai.app.invocations.video_frame_extract_range.make_mp4_writer", return_value=MagicMock()), patch("invokeai.app.invocations.video_frame_extract_range._write_frame_range", return_value=written), + patch("invokeai.app.invocations.video_frame_extract_range.extract_audio_pcm", return_value=None), patch("invokeai.app.invocations.video_frame_extract_range.VideoOutput.build", return_value=base_output), ): if should_raise: diff --git a/tests/app/invocations/test_video_frame_extract_range_audio.py b/tests/app/invocations/test_video_frame_extract_range_audio.py new file mode 100644 index 00000000000..fe754366da5 --- /dev/null +++ b/tests/app/invocations/test_video_frame_extract_range_audio.py @@ -0,0 +1,183 @@ +"""Audio-preservation tests for ExtractVideoRangeInvocation. + +The node originally re-encoded video frames only, so trimming a clip silently stripped +its soundtrack — feeding a trimmed clip into Concatenate Videos then produced correct +silence for its span (the "extend video" workflow lost the source's audio). These tests +pin the carried-through audio: slice alignment against the frame range, fps-override +retiming, the frame-rate fallback chain, and silent-source passthrough. + +Real ffmpeg (the imageio-ffmpeg bundled binary) encodes and decodes throughout — tone +clips are tiny (64x64) so the suite stays fast. +""" + +import shutil +from pathlib import Path +from unittest.mock import MagicMock, patch + +import numpy as np + +from invokeai.app.invocations.fields import VideoField +from invokeai.app.invocations.video_frame_extract_range import ExtractVideoRangeInvocation +from invokeai.app.util.video_audio import extract_audio_pcm +from invokeai.app.util.video_encoding import make_mp4_writer, write_stereo_wav + +RATE = 32000 +FPS = 16.0 +SIZE = 64 + + +class _Util: + def signal_progress(self, *args, **kwargs) -> None: + pass + + def is_canceled(self) -> bool: + return False + + +class _Ctx: + util = _Util() + + +def _tone(freq: float, seconds: float, rate: int = RATE) -> np.ndarray: + t = np.arange(round(seconds * rate)) / rate + mono = (0.5 * np.sin(2 * np.pi * freq * t)).astype(np.float32) + return np.stack([mono, mono]) + + +def _dominant_freq(pcm: np.ndarray, rate: int) -> float: + spectrum = np.abs(np.fft.rfft(pcm[0])) + spectrum[0] = 0.0 # ignore DC + return float(np.fft.rfftfreq(pcm.shape[1], 1.0 / rate)[int(np.argmax(spectrum))]) + + +def _make_clip(dirpath: Path, name: str, n_frames: int, audio: np.ndarray | None, fps: float = FPS) -> Path: + path = dirpath / f"{name}.mp4" + audio_path = None + if audio is not None: + audio_path = dirpath / f"{name}.wav" + write_stereo_wav(audio_path, audio, RATE) + writer = make_mp4_writer(path, fps, audio_path=audio_path) + frame = np.full((SIZE, SIZE, 3), 128, dtype=np.uint8) + for _ in range(n_frames): + writer.append_data(frame) + writer.close() + return path + + +def _invoke_and_capture(invocation: ExtractVideoRangeInvocation, clip_path: Path, out_dir: Path) -> Path: + """Runs invoke() with a mocked context and returns a copy of the saved MP4. + + The node unlinks its temp files in a finally block, so ``videos.save`` copies the + file out before returning. + """ + saved = out_dir / "saved.mp4" + + def _save(source_path: Path, **kwargs) -> MagicMock: + shutil.copyfile(source_path, saved) + return MagicMock() + + context = MagicMock() + context.videos.get_path.return_value = clip_path + context.util.is_canceled.return_value = False + context.videos.save.side_effect = _save + base_output = MagicMock( + video=VideoField(video_name="saved.mp4"), width=SIZE, height=SIZE, num_frames=1, fps=FPS, duration=1.0 + ) + with patch("invokeai.app.invocations.video_frame_extract_range.VideoOutput.build", return_value=base_output): + invocation.invoke(context) + assert saved.exists() + return saved + + +class TestRangeAudio: + def test_trim_keeps_the_matching_audio_slice(self, tmp_path): + # 2 s clip: first second 440 Hz, second second 880 Hz. Trimming the back half + # must carry the 880 Hz second, not the 440 Hz one (slice offset correctness). + audio = np.concatenate([_tone(440.0, 1.0), _tone(880.0, 1.0)], axis=1) + clip = _make_clip(tmp_path, "twotone", n_frames=32, audio=audio) + node = ExtractVideoRangeInvocation(video=VideoField(video_name="twotone"), start_frame=16, end_frame=31) + saved = _invoke_and_capture(node, clip, tmp_path) + + extracted = extract_audio_pcm(saved) + assert extracted is not None + pcm, rate = extracted + assert abs(pcm.shape[1] / rate - 1.0) < 0.05 # ~1 s of audio for 16 frames @ 16 fps + assert abs(_dominant_freq(pcm, rate) - 880.0) < 20.0 + + def test_full_range_round_trips_audio(self, tmp_path): + clip = _make_clip(tmp_path, "tone", n_frames=32, audio=_tone(440.0, 2.0)) + node = ExtractVideoRangeInvocation(video=VideoField(video_name="tone"), start_frame=0, end_frame=-1) + saved = _invoke_and_capture(node, clip, tmp_path) + + extracted = extract_audio_pcm(saved) + assert extracted is not None + pcm, rate = extracted + assert abs(pcm.shape[1] / rate - 2.0) < 0.05 + assert abs(_dominant_freq(pcm, rate) - 440.0) < 20.0 + + def test_silent_source_stays_silent(self, tmp_path): + clip = _make_clip(tmp_path, "silent", n_frames=16, audio=None) + node = ExtractVideoRangeInvocation(video=VideoField(video_name="silent"), start_frame=0, end_frame=7) + saved = _invoke_and_capture(node, clip, tmp_path) + assert extract_audio_pcm(saved) is None + + def test_fps_override_retimes_audio_with_video(self, tmp_path): + # Doubling the frame rate halves the duration; the audio must follow the video's + # retime (half the samples, pitch doubled) rather than play at original speed. + clip = _make_clip(tmp_path, "tone", n_frames=32, audio=_tone(440.0, 2.0)) + node = ExtractVideoRangeInvocation(video=VideoField(video_name="tone"), start_frame=0, end_frame=-1, fps=32) + saved = _invoke_and_capture(node, clip, tmp_path) + + extracted = extract_audio_pcm(saved) + assert extracted is not None + pcm, rate = extracted + assert abs(pcm.shape[1] / rate - 1.0) < 0.05 + assert abs(_dominant_freq(pcm, rate) - 880.0) < 25.0 + + +class TestBuildAudioTrackFallbacks: + """The frame->sample mapping's frame-rate fallback chain, with synthetic extraction.""" + + def _build(self, node, source_fps, source_duration, pcm, start, end, n_frames, output_fps=FPS): + with patch( + "invokeai.app.invocations.video_frame_extract_range.extract_audio_pcm", + return_value=(pcm, RATE), + ): + return node._build_audio_track( + context=_Ctx(), # type: ignore[arg-type] + video_path=Path("unused.mp4"), + start=start, + end=end, + n_frames=n_frames, + source_fps=source_fps, + source_duration=source_duration, + output_fps=output_fps, + ) + + def test_unknown_fps_uses_probed_duration(self): + node = ExtractVideoRangeInvocation(video=VideoField(video_name="x")) + pcm = np.concatenate([_tone(440.0, 1.0), _tone(880.0, 1.0)], axis=1) + result = self._build(node, source_fps=None, source_duration=2.0, pcm=pcm, start=16, end=31, n_frames=32) + assert result is not None + out, rate = result + assert out.shape[1] == round(16 / FPS * RATE) + assert abs(_dominant_freq(out, rate) - 880.0) < 20.0 + + def test_unknown_fps_and_duration_falls_back_to_extracted_length(self): + node = ExtractVideoRangeInvocation(video=VideoField(video_name="x")) + pcm = np.concatenate([_tone(440.0, 1.0), _tone(880.0, 1.0)], axis=1) + result = self._build(node, source_fps=None, source_duration=None, pcm=pcm, start=16, end=31, n_frames=32) + assert result is not None + out, rate = result + assert abs(_dominant_freq(out, rate) - 880.0) < 20.0 + + def test_short_audio_track_is_padded_not_stretched(self): + # Audio covers only the first second of a 2 s video; trimming the back half must + # yield silence (aligned), not the front half's audio stretched over the range. + node = ExtractVideoRangeInvocation(video=VideoField(video_name="x")) + pcm = _tone(440.0, 1.0) + result = self._build(node, source_fps=FPS, source_duration=2.0, pcm=pcm, start=16, end=31, n_frames=32) + assert result is not None + out, _rate = result + assert out.shape[1] == round(16 / FPS * RATE) + assert float(np.abs(out).max()) < 1e-6