feat(web-ui): robust MP2 software decode with frame-accurate demux and WSOLA rate matching

web-ui/src/mpegts/audio/wasm-stretcher.ts

@@ -0,0 +1,176 @@
+/*
+ * WASM time stretcher (WSOLA)
+ *
+ * Thin main-thread wrapper around the wsola_* exports of mp2_decoder.wasm
+ * (the same module used for MP2 decoding in the worker — instantiated again
+ * here because the stretcher runs on the main thread next to the video clock).
+ *
+ * Pitch-preserving tempo change lets software-decoded audio follow
+ * video.playbackRate during live-sync catch-up and absorb small clock drift.
+ */
+
+import Log from "../utils/logger";
+
+const TAG = "WasmStretcher";
+
+export interface Stretcher {
+  readonly sampleRate: number;
+  readonly channels: number;
+  /** Input frames consumed for emitted output since reset (fractional). */
+  readonly position: number;
+  setRatio(ratio: number): void;
+  /** Feed interleaved PCM, get stretched interleaved PCM (may be empty). */
+  process(input: Float32Array): Float32Array;
+  reset(): void;
+  destroy(): void;
+}
+
+/**
+ * Fallback when WASM is unavailable on the main thread: identity passthrough.
+ * Sync then degrades to occasional hard resyncs instead of smooth rate matching,
+ * but the scheduling chain stays gapless.
+ */
+export class PassthroughStretcher implements Stretcher {
+  readonly sampleRate: number;
+  readonly channels: number;
+  position = 0;
+
+  constructor(sampleRate: number, channels: number) {
+    this.sampleRate = sampleRate;
+    this.channels = channels;
+  }
+
+  setRatio(_ratio: number): void {}
+
+  process(input: Float32Array): Float32Array {
+    this.position += input.length / this.channels;
+    return input;
+  }
+
+  reset(): void {
+    this.position = 0;
+  }
+
+  destroy(): void {}
+}
+
+interface WsolaExports {
+  memory: WebAssembly.Memory;
+  _initialize: () => void;
+  malloc: (size: number) => number;
+  free: (ptr: number) => void;
+  wsola_create: (sampleRate: number, channels: number) => number;
+  wsola_destroy: (ptr: number) => void;
+  wsola_reset: (ptr: number) => void;
+  wsola_set_ratio: (ptr: number, ratio: number) => void;
+  wsola_position: (ptr: number) => number;
+  wsola_process: (ptr: number, input: number, inFrames: number, output: number, outCapacity: number) => number;
+}
+
+export class WasmStretcher implements Stretcher {
+  readonly sampleRate: number;
+  readonly channels: number;
+
+  private exports: WsolaExports;
+  private handle: number;
+  private inPtr = 0;
+  private inCapacityFrames = 0;
+  private outPtr = 0;
+  private outCapacityFrames = 0;
+
+  private constructor(exports: WsolaExports, handle: number, sampleRate: number, channels: number) {
+    this.exports = exports;
+    this.handle = handle;
+    this.sampleRate = sampleRate;
+    this.channels = channels;
+  }
+
+  static async create(wasmUrl: string, sampleRate: number, channels: number): Promise<WasmStretcher> {
+    const imports = {
+      env: {
+        emscripten_notify_memory_growth: () => {},
+      },
+    };
+    const { instance } = await WebAssembly.instantiateStreaming(fetch(wasmUrl), imports);
+    const ex = instance.exports as unknown as WsolaExports;
+    ex._initialize();
+    const handle = ex.wsola_create(sampleRate, channels);
+    if (!handle) {
+      throw new Error("wsola_create failed");
+    }
+    Log.v(TAG, `WSOLA stretcher created: ${sampleRate}Hz, ${channels}ch`);
+    return new WasmStretcher(ex, handle, sampleRate, channels);
+  }
+
+  get position(): number {
+    return this.handle ? this.exports.wsola_position(this.handle) : 0;
+  }
+
+  setRatio(ratio: number): void {
+    if (this.handle) {
+      this.exports.wsola_set_ratio(this.handle, ratio);
+    }
+  }
+
+  process(input: Float32Array): Float32Array {
+    if (!this.handle) {
+      return new Float32Array(0);
+    }
+    const ch = this.channels;
+    const inFrames = Math.floor(input.length / ch);
+
+    if (inFrames > this.inCapacityFrames) {
+      if (this.inPtr) this.exports.free(this.inPtr);
+      this.inCapacityFrames = Math.max(inFrames, 4096);
+      this.inPtr = this.exports.malloc(this.inCapacityFrames * ch * 4);
+    }
+    // Output bound: ratio >= 0.5 doubles duration at most, plus internal
+    // pending input (~one synthesis cycle) flushed in the same call.
+    const outNeeded = inFrames * 2 + 8192;
+    if (outNeeded > this.outCapacityFrames) {
+      if (this.outPtr) this.exports.free(this.outPtr);
+      this.outCapacityFrames = outNeeded;
+      this.outPtr = this.exports.malloc(this.outCapacityFrames * ch * 4);
+    }
+
+    // Views must be created after malloc (memory growth detaches buffers)
+    new Float32Array(this.exports.memory.buffer, this.inPtr, input.length).set(input);
+
+    const outFrames = this.exports.wsola_process(
+      this.handle,
+      this.inPtr,
+      inFrames,
+      this.outPtr,
+      this.outCapacityFrames,
+    );
+    if (outFrames <= 0) {
+      return new Float32Array(0);
+    }
+
+    const view = new Float32Array(this.exports.memory.buffer, this.outPtr, outFrames * ch);
+    const out = new Float32Array(outFrames * ch);
+    out.set(view);
+    return out;
+  }
+
+  reset(): void {
+    if (this.handle) {
+      this.exports.wsola_reset(this.handle);
+    }
+  }
+
+  destroy(): void {
+    if (this.handle) {
+      this.exports.wsola_destroy(this.handle);
+      this.handle = 0;
+    }
+    if (this.inPtr) {
+      this.exports.free(this.inPtr);
+      this.inPtr = 0;
+    }
+    if (this.outPtr) {
+      this.exports.free(this.outPtr);
+      this.outPtr = 0;
+    }
+  }
+}

web-ui/src/mpegts/decoder/mpeg-audio-decoder.ts

@@ -4,22 +4,35 @@
  * Minimal WASM wrapper for minimp3 — directly calls WASM exports
  * without the Emscripten JS glue.
  *
+ * Decodes whole PES payloads: the WASM side loops over all complete frames
+ * and keeps trailing partial frames in an internal carry buffer, so frames
+ * split across PES packets are handled transparently.
+ *
  * The WASM is built as standalone (`-o .wasm`) with -O2 to preserve
  * readable export/import names.
  */
 
-// Maximum samples per frame for MPEG audio
-const MAX_SAMPLES_PER_FRAME = 1152 * 2; // 1152 samples × 2 channels
+// Maximum samples per frame for MPEG audio (all channels interleaved)
+const MAX_SAMPLES_PER_FRAME = 1152 * 2;
+
+// Smallest valid Layer II frame (32kbps @ 48kHz): used to bound the
+// number of frames a payload can contain when sizing the output buffer.
+const MIN_FRAME_BYTES = 96;
+
+// Carry buffer size on the WASM side (one partial frame at most)
+const CARRY_MAX = 2048;
 
-// Info array layout (6 × i32): [samples, sampleRate, channels, layer, bitrate, frameBytes]
+// Info array layout (6 × i32):
+// [samplesPerChannel, sampleRate, channels, frames, carryBytes, consumedBytes]
 const INFO_SAMPLES = 0;
 const INFO_SAMPLE_RATE = 1;
 const INFO_CHANNELS = 2;
 const INFO_I32_COUNT = 6;
 
-interface DecodedAudio {
+export interface DecodedAudio {
+  /** Interleaved float32 PCM for all decoded frames. */
   pcm: Float32Array;
-  samples: number;
+  samplesPerChannel: number;
   sampleRate: number;
   channels: number;
 }
@@ -43,6 +56,7 @@ export class MpegAudioDecoder {
   private outputPtr = 0;
   private infoPtr = 0;
   private inputBufSize = 0;
+  private outputBufFloats = 0;
 
   private _ready: Promise<void>;
   private _isReady = false;
@@ -77,8 +91,7 @@ export class MpegAudioDecoder {
     }
 
     const malloc = ex.malloc as (size: number) => number;
-    this.outputPtr = malloc(MAX_SAMPLES_PER_FRAME * 2); // int16 output
-    this.infoPtr = malloc(INFO_I32_COUNT * 4); // int32 array
+    this.infoPtr = malloc(INFO_I32_COUNT * 4);
 
     this._isReady = true;
   }
@@ -96,37 +109,51 @@ export class MpegAudioDecoder {
       this.inputPtr = malloc(this.inputBufSize);
     }
 
+    // Grow output buffer to hold every frame the payload could contain
+    const maxFrames = Math.floor((CARRY_MAX + input.length) / MIN_FRAME_BYTES) + 2;
+    const neededFloats = maxFrames * MAX_SAMPLES_PER_FRAME;
+    if (neededFloats > this.outputBufFloats) {
+      if (this.outputPtr) free(this.outputPtr);
+      this.outputBufFloats = neededFloats;
+      this.outputPtr = malloc(neededFloats * 4);
+    }
+
     // Copy input into WASM memory
     const heap = new Uint8Array(this.memoryRef.memory.buffer);
     heap.set(input, this.inputPtr);
 
-    const decodeFn = this.exports.mpeg_audio_decode_frame as (
+    const decodeFn = this.exports.mpeg_audio_decode_payload as (
       dec: number,
       inp: number,
       inpSz: number,
       out: number,
+      outCap: number,
       info: number,
     ) => number;
-    const samples = decodeFn(this.decoderPtr, this.inputPtr, input.length, this.outputPtr, this.infoPtr);
+    const samples = decodeFn(
+      this.decoderPtr,
+      this.inputPtr,
+      input.length,
+      this.outputPtr,
+      this.outputBufFloats,
+      this.infoPtr,
+    );
     if (samples <= 0) return null;
 
     // Read info from WASM memory (may have changed due to memory growth)
     const i32 = new Int32Array(this.memoryRef.memory.buffer);
     const infoBase = this.infoPtr >> 2;
-    const infoSamples = i32[infoBase + INFO_SAMPLES];
+    const samplesPerChannel = i32[infoBase + INFO_SAMPLES];
     const sampleRate = i32[infoBase + INFO_SAMPLE_RATE];
     const channels = i32[infoBase + INFO_CHANNELS];
 
-    // Read int16 output and convert to float32
-    const totalSamples = infoSamples * channels;
-    const i16 = new Int16Array(this.memoryRef.memory.buffer, this.outputPtr, totalSamples);
-    const pcm = new Float32Array(totalSamples);
-    const scale = 1.0 / 32768.0;
-    for (let j = 0; j < totalSamples; j++) {
-      pcm[j] = i16[j] * scale;
-    }
+    // Copy float32 PCM out of WASM memory
+    const totalFloats = samplesPerChannel * channels;
+    const view = new Float32Array(this.memoryRef.memory.buffer, this.outputPtr, totalFloats);
+    const pcm = new Float32Array(totalFloats);
+    pcm.set(view);
 
-    return { pcm, samples: infoSamples, sampleRate, channels };
+    return { pcm, samplesPerChannel, sampleRate, channels };
   }
 
   /** Reset decoder state (call on stream switch to avoid stale mdct/qmf state) */

web-ui/src/mpegts/decoder/worker-audio-decoder.ts

@@ -6,17 +6,10 @@
  */
 
 import Log from "../utils/logger";
-import { MpegAudioDecoder } from "./mpeg-audio-decoder";
+import { type DecodedAudio, MpegAudioDecoder } from "./mpeg-audio-decoder";
 
 const TAG = "WorkerAudioDecoder";
 
-interface PCMAudioData {
-  pcm: Float32Array;
-  channels: number;
-  sampleRate: number;
-  pts: number; // PTS in milliseconds
-}
-
 /**
  * Audio decoder for use in Web Worker (MP2 only).
  * The consumer provides the WASM URL via config — the library does NOT bundle WASM.
@@ -50,18 +43,10 @@ export class WorkerAudioDecoder {
     }
   }
 
-  decode(data: Uint8Array, pts: number): PCMAudioData | null {
+  /** Decode all complete frames in a PES payload (partial frames are carried over). */
+  decode(data: Uint8Array): DecodedAudio | null {
     if (!this.decoder?.isReady) return null;
-
-    const result = this.decoder.decode(data);
-    if (!result) return null;
-
-    return {
-      pcm: result.pcm,
-      channels: result.channels,
-      sampleRate: result.sampleRate,
-      pts,
-    };
+    return this.decoder.decode(data);
   }
 
   reset(): void {