From 89ee1e36d7a65c1638d8397f9b94233fbed15951 Mon Sep 17 00:00:00 2001
From: James <james.russo@heygen.com>
Date: Mon, 11 May 2026 13:59:19 +0000
Subject: [PATCH 1/2] fix(core): thread playback rate into WebAudio audio
 sources

WebAudioTransport scheduled AudioBufferSourceNodes with the implicit
default playbackRate of 1, so non-1x transport rates desynced visuals
from audio: GSAP timelines, the transport clock, and native <video>
all sped up while WebAudio-routed <audio> clips kept playing at 1x.

- schedulePlayback now accepts a rate, sets sourceNode.playbackRate,
  and scales the future-clip start delay by the rate (the in-progress
  buffer offset stays elapsed + mediaStart, which is rate-independent).
- New setRate() updates active sources in place and rebases the
  getTime() reference frame so the audio-master clock stays continuous
  across mid-playback rate changes.
- Runtime onSetPlaybackRate now forwards into webAudio.setRate, and
  player.play() schedules each clip with state.playbackRate.

Fixes #713
---
 packages/core/src/runtime/init.ts             |   2 +
 .../src/runtime/webAudioTransport.test.ts     | 149 ++++++++++++++++++
 .../core/src/runtime/webAudioTransport.ts     |  57 ++++++-
 3 files changed, 204 insertions(+), 4 deletions(-)

diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 1493b3d8c..91aa2b23b 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -1589,6 +1589,7 @@ export function initSandboxRuntimeModular(): void {
     onSetPlaybackRate: (rate) => {
       applyPlaybackRate(rate);
       if (state.transportClock) state.transportClock.setRate(state.playbackRate);
+      webAudio.setRate(state.playbackRate);
     },
     onEnablePickMode: () => picker.enablePickMode(),
     onDisablePickMode: () => picker.disablePickMode(),
@@ -1858,6 +1859,7 @@ export function initSandboxRuntimeModular(): void {
             clock.now(),
             vol * state.bridgeVolume,
             gen,
+            state.playbackRate,
           );
         });
       }
diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts
index d34ac7593..6bccdf734 100644
--- a/packages/core/src/runtime/webAudioTransport.test.ts
+++ b/packages/core/src/runtime/webAudioTransport.test.ts
@@ -86,12 +86,16 @@ describe("WebAudioTransport", () => {
       const startFn = vi.fn();
       const sourceNode = {
         buffer: null as AudioBuffer | null,
+        playbackRate: { value: 1 },
         start: startFn,
+        stop: vi.fn(),
+        disconnect: vi.fn(),
         connect: vi.fn(),
       };
       const gainNode = {
         gain: { value: 1 },
         connect: vi.fn(),
+        disconnect: vi.fn(),
       };
       const masterGain = {
         gain: { value: 1 },
@@ -161,4 +165,149 @@ describe("WebAudioTransport", () => {
       expect(mock.startFn).toHaveBeenCalledWith(0, 0);
     });
   });
+
+  describe("playback rate", () => {
+    function createMockAudioContext(currentTime = 100) {
+      const startFn = vi.fn();
+      const sourceNode = {
+        buffer: null as AudioBuffer | null,
+        playbackRate: { value: 1 },
+        start: startFn,
+        stop: vi.fn(),
+        disconnect: vi.fn(),
+        connect: vi.fn(),
+      };
+      const gainNode = {
+        gain: { value: 1 },
+        connect: vi.fn(),
+        disconnect: vi.fn(),
+      };
+      const masterGain = {
+        gain: { value: 1 },
+        connect: vi.fn(),
+      };
+      const ctx = {
+        currentTime,
+        state: "running",
+        resume: vi.fn(),
+        createBufferSource: vi.fn(() => sourceNode),
+        createGain: vi.fn(() => gainNode),
+        destination: {},
+        close: vi.fn(),
+      };
+      return { ctx, sourceNode, gainNode, masterGain, startFn };
+    }
+
+    function setupTransport(currentTime = 100) {
+      const transport = new WebAudioTransport();
+      const mock = createMockAudioContext(currentTime);
+      (transport as unknown as { _ctx: unknown })._ctx = mock.ctx;
+      (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain;
+      const gen = transport.startGeneration();
+      return { transport, mock, gen };
+    }
+
+    const mockBuffer = {} as AudioBuffer;
+    const mockEl = { muted: false } as HTMLMediaElement;
+
+    it("sets sourceNode.playbackRate.value when rate is provided", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+
+      expect(mock.sourceNode.playbackRate.value).toBe(2);
+    });
+
+    it("defaults rate to 1 when not provided", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen);
+
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+    });
+
+    it("scales delay by rate for future clips so they fire at the right wallclock", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      // compStart=10, compositionTime=2, rate=2 → 8s of comp time = 4s wallclock
+      await transport.schedulePlayback(mockEl, mockBuffer, 10, 0, 2, 1, gen, 2);
+
+      expect(mock.startFn).toHaveBeenCalledWith(104, 0);
+    });
+
+    it("keeps in-progress buffer offset at elapsed + mediaStart regardless of rate", async () => {
+      // Audio that has been playing for `elapsed` composition seconds at rate r:
+      //   wallclock elapsed = elapsed / r
+      //   buffer advanced  = (elapsed / r) × r = elapsed
+      // so the buffer offset is `elapsed + mediaStart` for any rate.
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+
+      expect(mock.startFn).toHaveBeenCalledWith(0, 3);
+    });
+
+    it("setRate updates active sources in place", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+
+      transport.setRate(2);
+
+      expect(mock.sourceNode.playbackRate.value).toBe(2);
+    });
+
+    it("setRate before any sources are scheduled does not throw", () => {
+      const transport = new WebAudioTransport();
+      expect(() => transport.setRate(2)).not.toThrow();
+    });
+
+    it("setRate clamps non-finite or non-positive values to 1", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+
+      transport.setRate(Number.NaN);
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+
+      transport.setRate(0);
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+
+      transport.setRate(-1);
+      expect(mock.sourceNode.playbackRate.value).toBe(1);
+    });
+
+    it("getTime advances at the configured rate", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+
+      // At schedule time, ctx.currentTime=100, compositionTime=8.
+      expect(transport.getTime()).toBeCloseTo(8, 10);
+
+      // Advance the audio-context clock by 0.5 wallclock seconds; at rate=2,
+      // composition time should have advanced 1s.
+      mock.ctx.currentTime = 100.5;
+      expect(transport.getTime()).toBeCloseTo(9, 10);
+    });
+
+    it("getTime tracks composition time after a mid-playback setRate", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+      expect(transport.getTime()).toBeCloseTo(8, 10);
+
+      // 0.5s passes at rate=1 → composition time = 8.5
+      mock.ctx.currentTime = 100.5;
+      expect(transport.getTime()).toBeCloseTo(8.5, 10);
+
+      // Bump rate to 2 — composition time should NOT jump.
+      transport.setRate(2);
+      expect(transport.getTime()).toBeCloseTo(8.5, 10);
+
+      // Another 0.5s wallclock at rate=2 → composition time = 9.5
+      mock.ctx.currentTime = 101;
+      expect(transport.getTime()).toBeCloseTo(9.5, 10);
+    });
+  });
 });
diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts
index 14c543063..932236f82 100644
--- a/packages/core/src/runtime/webAudioTransport.ts
+++ b/packages/core/src/runtime/webAudioTransport.ts
@@ -1,5 +1,10 @@
 import { swallow } from "./diagnostics";
 
+function normalizeRate(rate: number): number {
+  if (!Number.isFinite(rate) || rate <= 0) return 1;
+  return rate;
+}
+
 export type ScheduledSource = {
   el: HTMLMediaElement;
   sourceNode: AudioBufferSourceNode;
@@ -15,7 +20,12 @@ export class WebAudioTransport {
   private _bufferCache = new Map<string, AudioBuffer>();
   private _activeSources: ScheduledSource[] = [];
   private _masterGain: GainNode | null = null;
-  private _scheduleOffset = 0;
+  // Composition-time reference frame: at AudioContext time `_rateAnchorCtx`,
+  // composition time was `_rateAnchorComp`, and time has been advancing at
+  // `_rate` composition-seconds per wallclock-second since.
+  private _rateAnchorCtx = 0;
+  private _rateAnchorComp = 0;
+  private _rate = 1;
   private _paused = true;
   private _playGeneration = 0;
 
@@ -36,7 +46,7 @@ export class WebAudioTransport {
 
   getTime(): number {
     if (!this._ctx || this._paused) return -1;
-    return this._ctx.currentTime - this._scheduleOffset;
+    return this._rateAnchorComp + (this._ctx.currentTime - this._rateAnchorCtx) * this._rate;
   }
 
   async decodeAudioElement(el: HTMLMediaElement): Promise<AudioBuffer | null> {
@@ -73,6 +83,7 @@ export class WebAudioTransport {
     compositionTime: number,
     volume: number,
     generation: number,
+    rate = 1,
   ): Promise<ScheduledSource | null> {
     if (!this._ctx || !this._masterGain) return null;
     if (generation !== this._playGeneration) return null;
@@ -83,8 +94,11 @@ export class WebAudioTransport {
       }
       if (generation !== this._playGeneration) return null;
 
+      const safeRate = normalizeRate(rate);
+
       const sourceNode = this._ctx.createBufferSource();
       sourceNode.buffer = buffer;
+      sourceNode.playbackRate.value = safeRate;
 
       const gainNode = this._ctx.createGain();
       gainNode.gain.value = volume;
@@ -93,12 +107,20 @@ export class WebAudioTransport {
 
       const elapsed = compositionTime - compositionStart;
       const scheduledAt = this._ctx.currentTime;
-      this._scheduleOffset = scheduledAt - compositionTime;
+      this._rate = safeRate;
+      this._rateAnchorCtx = scheduledAt;
+      this._rateAnchorComp = compositionTime;
 
       if (elapsed >= 0) {
+        // Audio that has already been "playing" for `elapsed` composition
+        // seconds at rate r has advanced exactly `elapsed` buffer seconds —
+        // wallclock elapsed is `elapsed / r` and the source plays at rate r,
+        // so buffer advance = (elapsed / r) × r = elapsed.
         sourceNode.start(0, elapsed + mediaStart);
       } else {
-        const delay = -elapsed;
+        // Future clip: composition time will reach `compositionStart` after
+        // `-elapsed` composition seconds, which is `-elapsed / r` wallclock.
+        const delay = -elapsed / safeRate;
         sourceNode.start(scheduledAt + delay, mediaStart);
       }
 
@@ -123,6 +145,33 @@ export class WebAudioTransport {
     }
   }
 
+  /**
+   * Change the playback rate of every currently-active source in place.
+   *
+   * Also rebases the composition-time reference frame so `getTime()` keeps
+   * returning the same value across the rate change (i.e. the clock doesn't
+   * jump, it just advances at a different slope from here on).
+   *
+   * Note: sources scheduled to start in the future via `sourceNode.start(when)`
+   * keep their original wallclock start time. Callers that need rate-correct
+   * future-start times should `stopAll()` and reschedule.
+   */
+  setRate(rate: number): void {
+    const safeRate = normalizeRate(rate);
+    if (this._ctx && !this._paused) {
+      this._rateAnchorComp = this.getTime();
+      this._rateAnchorCtx = this._ctx.currentTime;
+    }
+    this._rate = safeRate;
+    for (const source of this._activeSources) {
+      try {
+        source.sourceNode.playbackRate.value = safeRate;
+      } catch (err) {
+        swallow("webAudioTransport.setRate", err);
+      }
+    }
+  }
+
   stopAll(): void {
     for (const source of this._activeSources) {
       try {

From 117029a719701dea742415f4984ff3eca43f3371 Mon Sep 17 00:00:00 2001
From: James <james.russo@heygen.com>
Date: Mon, 11 May 2026 14:11:54 +0000
Subject: [PATCH 2/2] refactor(core): tighten WebAudio rate fix per review

- Hoist duplicated test mock helpers (createMockAudioContext / setupTransport /
  mockBuffer / mockEl) from the two describe blocks to module scope.
- Drop redundant math-derivation comments in schedulePlayback; the dedicated
  rate-aware tests are the canonical proof.
- Tighten setRate JSDoc.
- Add no-op guard in setRate when the new rate equals the current rate, so a
  duplicate set-playback-rate postMessage doesn't re-anchor or walk active
  sources for nothing.
- Add a regression test for the no-op guard, and strengthen the clamp test
  to schedule at rate=2 first so the clamp-to-1 assertion is non-vacuous.
---
 .../src/runtime/webAudioTransport.test.ts     | 153 +++++++-----------
 .../core/src/runtime/webAudioTransport.ts     |  20 +--
 2 files changed, 67 insertions(+), 106 deletions(-)

diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts
index 6bccdf734..1dc6caf8f 100644
--- a/packages/core/src/runtime/webAudioTransport.test.ts
+++ b/packages/core/src/runtime/webAudioTransport.test.ts
@@ -1,6 +1,49 @@
 import { describe, it, expect, vi } from "vitest";
 import { WebAudioTransport } from "./webAudioTransport";
 
+function createMockAudioContext(currentTime = 100) {
+  const startFn = vi.fn();
+  const sourceNode = {
+    buffer: null as AudioBuffer | null,
+    playbackRate: { value: 1 },
+    start: startFn,
+    stop: vi.fn(),
+    disconnect: vi.fn(),
+    connect: vi.fn(),
+  };
+  const gainNode = {
+    gain: { value: 1 },
+    connect: vi.fn(),
+    disconnect: vi.fn(),
+  };
+  const masterGain = {
+    gain: { value: 1 },
+    connect: vi.fn(),
+  };
+  const ctx = {
+    currentTime,
+    state: "running",
+    resume: vi.fn(),
+    createBufferSource: vi.fn(() => sourceNode),
+    createGain: vi.fn(() => gainNode),
+    destination: {},
+    close: vi.fn(),
+  };
+  return { ctx, sourceNode, gainNode, masterGain, startFn };
+}
+
+function setupTransport(currentTime = 100) {
+  const transport = new WebAudioTransport();
+  const mock = createMockAudioContext(currentTime);
+  (transport as unknown as { _ctx: unknown })._ctx = mock.ctx;
+  (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain;
+  const gen = transport.startGeneration();
+  return { transport, mock, gen };
+}
+
+const mockBuffer = {} as AudioBuffer;
+const mockEl = { muted: false } as HTMLMediaElement;
+
 describe("WebAudioTransport", () => {
   it("tracks play generation for async race prevention", () => {
     const transport = new WebAudioTransport();
@@ -82,49 +125,6 @@ describe("WebAudioTransport", () => {
   });
 
   describe("schedulePlayback timing", () => {
-    function createMockAudioContext(currentTime = 100) {
-      const startFn = vi.fn();
-      const sourceNode = {
-        buffer: null as AudioBuffer | null,
-        playbackRate: { value: 1 },
-        start: startFn,
-        stop: vi.fn(),
-        disconnect: vi.fn(),
-        connect: vi.fn(),
-      };
-      const gainNode = {
-        gain: { value: 1 },
-        connect: vi.fn(),
-        disconnect: vi.fn(),
-      };
-      const masterGain = {
-        gain: { value: 1 },
-        connect: vi.fn(),
-      };
-      const ctx = {
-        currentTime,
-        state: "running",
-        resume: vi.fn(),
-        createBufferSource: vi.fn(() => sourceNode),
-        createGain: vi.fn(() => gainNode),
-        destination: {},
-        close: vi.fn(),
-      };
-      return { ctx, sourceNode, gainNode, masterGain, startFn };
-    }
-
-    function setupTransport(currentTime = 100) {
-      const transport = new WebAudioTransport();
-      const mock = createMockAudioContext(currentTime);
-      (transport as unknown as { _ctx: unknown })._ctx = mock.ctx;
-      (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain;
-      const gen = transport.startGeneration();
-      return { transport, mock, gen };
-    }
-
-    const mockBuffer = {} as AudioBuffer;
-    const mockEl = { muted: false } as HTMLMediaElement;
-
     it("starts in-progress clips immediately with correct buffer offset", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
@@ -167,49 +167,6 @@ describe("WebAudioTransport", () => {
   });
 
   describe("playback rate", () => {
-    function createMockAudioContext(currentTime = 100) {
-      const startFn = vi.fn();
-      const sourceNode = {
-        buffer: null as AudioBuffer | null,
-        playbackRate: { value: 1 },
-        start: startFn,
-        stop: vi.fn(),
-        disconnect: vi.fn(),
-        connect: vi.fn(),
-      };
-      const gainNode = {
-        gain: { value: 1 },
-        connect: vi.fn(),
-        disconnect: vi.fn(),
-      };
-      const masterGain = {
-        gain: { value: 1 },
-        connect: vi.fn(),
-      };
-      const ctx = {
-        currentTime,
-        state: "running",
-        resume: vi.fn(),
-        createBufferSource: vi.fn(() => sourceNode),
-        createGain: vi.fn(() => gainNode),
-        destination: {},
-        close: vi.fn(),
-      };
-      return { ctx, sourceNode, gainNode, masterGain, startFn };
-    }
-
-    function setupTransport(currentTime = 100) {
-      const transport = new WebAudioTransport();
-      const mock = createMockAudioContext(currentTime);
-      (transport as unknown as { _ctx: unknown })._ctx = mock.ctx;
-      (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain;
-      const gen = transport.startGeneration();
-      return { transport, mock, gen };
-    }
-
-    const mockBuffer = {} as AudioBuffer;
-    const mockEl = { muted: false } as HTMLMediaElement;
-
     it("sets sourceNode.playbackRate.value when rate is provided", async () => {
       const { transport, mock, gen } = setupTransport(100);
 
@@ -236,10 +193,6 @@ describe("WebAudioTransport", () => {
     });
 
     it("keeps in-progress buffer offset at elapsed + mediaStart regardless of rate", async () => {
-      // Audio that has been playing for `elapsed` composition seconds at rate r:
-      //   wallclock elapsed = elapsed / r
-      //   buffer advanced  = (elapsed / r) × r = elapsed
-      // so the buffer offset is `elapsed + mediaStart` for any rate.
       const { transport, mock, gen } = setupTransport(100);
 
       await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
@@ -263,16 +216,34 @@ describe("WebAudioTransport", () => {
       expect(() => transport.setRate(2)).not.toThrow();
     });
 
+    it("setRate is a no-op when the rate is unchanged", async () => {
+      const { transport, mock, gen } = setupTransport(100);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+
+      mock.ctx.currentTime = 100.5;
+      const timeBefore = transport.getTime();
+      transport.setRate(2);
+      const timeAfter = transport.getTime();
+
+      expect(timeAfter).toBe(timeBefore);
+      // No re-anchor, so the next 0.5s of wallclock still maps to 1s of comp time.
+      mock.ctx.currentTime = 101;
+      expect(transport.getTime()).toBeCloseTo(10, 10);
+    });
+
     it("setRate clamps non-finite or non-positive values to 1", async () => {
       const { transport, mock, gen } = setupTransport(100);
-      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1);
+      await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 2);
+      expect(mock.sourceNode.playbackRate.value).toBe(2);
 
       transport.setRate(Number.NaN);
       expect(mock.sourceNode.playbackRate.value).toBe(1);
 
+      transport.setRate(2);
       transport.setRate(0);
       expect(mock.sourceNode.playbackRate.value).toBe(1);
 
+      transport.setRate(2);
       transport.setRate(-1);
       expect(mock.sourceNode.playbackRate.value).toBe(1);
     });
diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts
index 932236f82..baa1113bc 100644
--- a/packages/core/src/runtime/webAudioTransport.ts
+++ b/packages/core/src/runtime/webAudioTransport.ts
@@ -112,14 +112,8 @@ export class WebAudioTransport {
       this._rateAnchorComp = compositionTime;
 
       if (elapsed >= 0) {
-        // Audio that has already been "playing" for `elapsed` composition
-        // seconds at rate r has advanced exactly `elapsed` buffer seconds —
-        // wallclock elapsed is `elapsed / r` and the source plays at rate r,
-        // so buffer advance = (elapsed / r) × r = elapsed.
         sourceNode.start(0, elapsed + mediaStart);
       } else {
-        // Future clip: composition time will reach `compositionStart` after
-        // `-elapsed` composition seconds, which is `-elapsed / r` wallclock.
         const delay = -elapsed / safeRate;
         sourceNode.start(scheduledAt + delay, mediaStart);
       }
@@ -146,18 +140,14 @@ export class WebAudioTransport {
   }
 
   /**
-   * Change the playback rate of every currently-active source in place.
-   *
-   * Also rebases the composition-time reference frame so `getTime()` keeps
-   * returning the same value across the rate change (i.e. the clock doesn't
-   * jump, it just advances at a different slope from here on).
-   *
-   * Note: sources scheduled to start in the future via `sourceNode.start(when)`
-   * keep their original wallclock start time. Callers that need rate-correct
-   * future-start times should `stopAll()` and reschedule.
+   * Rebases the composition-time reference frame before swapping rate so
+   * `getTime()` stays continuous across the change. Sources scheduled to
+   * start in the future keep their original wallclock start time — callers
+   * that need rate-correct future starts should `stopAll()` and reschedule.
    */
   setRate(rate: number): void {
     const safeRate = normalizeRate(rate);
+    if (safeRate === this._rate) return;
     if (this._ctx && !this._paused) {
       this._rateAnchorComp = this.getTime();
       this._rateAnchorCtx = this._ctx.currentTime;