Full overhawl of video & audio playback to make it more complete (#1432)

* Add support for capturing media attributes in rrweb-snapshot

* Add loop to mediaInteractionParam

* Add support for loop in RRMediaElement

* Add support for recording loop attribute on media elements

* Update video playback and fix bugs

* Update cross-origin iframe media attributes and player state

.changeset/cool-grapes-hug.md

@@ -0,0 +1,5 @@
+---
+'rrdom': patch
+---
+
+Support `loop` in `RRMediaElement`

.changeset/dirty-rules-dress.md

@@ -0,0 +1,5 @@
+---
+'rrweb-snapshot': minor
+---
+
+Video and Audio elements now also capture `playbackRate`, `muted`, `loop`, `volume`.

.changeset/mighty-ads-worry.md

@@ -0,0 +1,5 @@
+---
+'rrweb': minor
+---
+
+Full overhawl of `video` and `audio` element playback. More robust and fixes lots of bugs related to pausing/playing/skipping/muting/playbackRate etc.

.changeset/silver-pots-sit.md

@@ -0,0 +1,5 @@
+---
+'@rrweb/types': patch
+---
+
+Add `loop` to `mediaInteractionParam`

.changeset/smart-geckos-cover.md

@@ -0,0 +1,5 @@
+---
+'rrweb': patch
+---
+
+Record `loop` on `<audio>` & `<video>` elements.

packages/rrdom/src/diff.ts

@@ -250,6 +250,8 @@ function diffAfterUpdatingChildren(
             oldMediaElement.currentTime = newMediaRRElement.currentTime;
           if (newMediaRRElement.playbackRate !== undefined)
             oldMediaElement.playbackRate = newMediaRRElement.playbackRate;
+          if (newMediaRRElement.loop !== undefined)
+            oldMediaElement.loop = newMediaRRElement.loop;
           break;
         }
         case 'CANVAS': {

packages/rrdom/src/document.ts

@@ -563,6 +563,7 @@ export function BaseRRMediaElementImpl<
     public paused?: boolean;
     public muted?: boolean;
     public playbackRate?: number;
+    public loop?: boolean;
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     attachShadow(_init: ShadowRootInit): IRRElement {
       throw new Error(

packages/rrdom/test/diff.test.ts

@@ -280,13 +280,15 @@ describe('diff algorithm for rrdom', () => {
         rrMedia.muted = true;
         rrMedia.paused = false;
         rrMedia.playbackRate = 0.5;
+        rrMedia.loop = false;
 
         diff(element, rrMedia, replayer);
         expect(element.volume).toEqual(0.5);
         expect(element.currentTime).toEqual(100);
         expect(element.muted).toEqual(true);
         expect(element.paused).toEqual(false);
         expect(element.playbackRate).toEqual(0.5);
+        expect(element.loop).toEqual(false);
 
         rrMedia.paused = true;
         diff(element, rrMedia, replayer);

packages/rrdom/test/document.test.ts

@@ -1079,6 +1079,7 @@ describe('Basic RRDocument implementation', () => {
       expect(node.paused).toBeUndefined();
       expect(node.muted).toBeUndefined();
       expect(node.playbackRate).toBeUndefined();
+      expect(node.loop).toBeUndefined();
       expect(node.play).toBeDefined();
       expect(node.pause).toBeDefined();
       expect(node.toString()).toEqual('VIDEO ');

packages/rrweb-snapshot/src/rebuild.ts

@@ -342,6 +342,17 @@ function buildNode(
               break;
             default:
           }
+        } else if (
+          name === 'rr_mediaPlaybackRate' &&
+          typeof value === 'number'
+        ) {
+          (node as HTMLMediaElement).playbackRate = value;
+        } else if (name === 'rr_mediaMuted' && typeof value === 'boolean') {
+          (node as HTMLMediaElement).muted = value;
+        } else if (name === 'rr_mediaLoop' && typeof value === 'boolean') {
+          (node as HTMLMediaElement).loop = value;
+        } else if (name === 'rr_mediaVolume' && typeof value === 'number') {
+          (node as HTMLMediaElement).volume = value;
         }
       }
 

packages/rrweb-snapshot/src/snapshot.ts

@@ -12,6 +12,7 @@ import {
   ICanvas,
   elementNode,
   serializedElementNodeWithId,
+  type mediaAttributes,
 } from './types';
 import {
   Mirror,
@@ -761,10 +762,15 @@ function serializeElementNode(
   }
   // media elements
   if (tagName === 'audio' || tagName === 'video') {
-    attributes.rr_mediaState = (n as HTMLMediaElement).paused
+    const mediaAttributes = attributes as mediaAttributes;
+    mediaAttributes.rr_mediaState = (n as HTMLMediaElement).paused
       ? 'paused'
       : 'played';
-    attributes.rr_mediaCurrentTime = (n as HTMLMediaElement).currentTime;
+    mediaAttributes.rr_mediaCurrentTime = (n as HTMLMediaElement).currentTime;
+    mediaAttributes.rr_mediaPlaybackRate = (n as HTMLMediaElement).playbackRate;
+    mediaAttributes.rr_mediaMuted = (n as HTMLMediaElement).muted;
+    mediaAttributes.rr_mediaLoop = (n as HTMLMediaElement).loop;
+    mediaAttributes.rr_mediaVolume = (n as HTMLMediaElement).volume;
   }
   // Scroll
   if (!newlyAddedElement) {

packages/rrweb-snapshot/src/types.ts

@@ -82,6 +82,27 @@ export type tagMap = {
   [key: string]: string;
 };
 
+export type mediaAttributes = {
+  rr_mediaState: 'played' | 'paused';
+  rr_mediaCurrentTime: number;
+  /**
+   * for backwards compatibility this is optional but should always be set
+   */
+  rr_mediaPlaybackRate?: number;
+  /**
+   * for backwards compatibility this is optional but should always be set
+   */
+  rr_mediaMuted?: boolean;
+  /**
+   * for backwards compatibility this is optional but should always be set
+   */
+  rr_mediaLoop?: boolean;
+  /**
+   * for backwards compatibility this is optional but should always be set
+   */
+  rr_mediaVolume?: number;
+};
+
 // @deprecated
 export interface INode extends Node {
   __sn: serializedNodeWithId;

packages/rrweb/src/record/observer.ts

@@ -1042,7 +1042,7 @@ function initMediaInteractionObserver({
         ) {
           return;
         }
-        const { currentTime, volume, muted, playbackRate } =
+        const { currentTime, volume, muted, playbackRate, loop } =
           target as HTMLMediaElement;
         mediaInteractionCb({
           type,
@@ -1051,6 +1051,7 @@ function initMediaInteractionObserver({
           volume,
           muted,
           playbackRate,
+          loop,
         });
       }),
       sampling.media || 500,

packages/rrweb/src/replay/index.ts

@@ -47,7 +47,6 @@ import {
   ReplayerEvents,
   Handler,
   Emitter,
-  MediaInteractions,
   metaEvent,
   mutationData,
   scrollData,
@@ -81,6 +80,7 @@ import getInjectStyleRules from './styles/inject-style';
 import './styles/style.css';
 import canvasMutation from './canvas';
 import { deserializeArg } from './canvas/deserialize-args';
+import { MediaManager } from './media';
 
 const SKIP_TIME_INTERVAL = 5 * 1000;
 
@@ -142,6 +142,9 @@ export class Replayer {
   // Used to track StyleSheetObjects adopted on multiple document hosts.
   private styleMirror: StyleSheetMirror = new StyleSheetMirror();
 
+  // Used to track video & audio elements, and keep them in sync with general playback.
+  private mediaManager: MediaManager;
+
   private firstFullSnapshot: eventWithTime | true | null = null;
 
   private newDocumentQueue: addedNodeMutation[] = [];
@@ -324,6 +327,7 @@ export class Replayer {
       this.firstFullSnapshot = null;
       this.mirror.reset();
       this.styleMirror.reset();
+      this.mediaManager.reset();
     });
 
     const timer = new Timer([], {
@@ -366,6 +370,13 @@ export class Replayer {
         speed: state,
       });
     });
+    this.mediaManager = new MediaManager({
+      warn: this.warn.bind(this),
+      service: this.service,
+      speedService: this.speedService,
+      emitter: this.emitter,
+      getCurrentTime: this.getCurrentTime.bind(this),
+    });
 
     // rebuild first full snapshot as the poster of the player
     // maybe we can cache it for performance optimization
@@ -464,10 +475,16 @@ export class Replayer {
     };
   }
 
+  /**
+   * Get the actual time offset the player is at now compared to the first event.
+   */
   public getCurrentTime(): number {
     return this.timer.timeOffset + this.getTimeOffset();
   }
 
+  /**
+   * Get the time offset the player is at now compared to the first event, but without regard for the timer.
+   */
   public getTimeOffset(): number {
     const { baselineTime, events } = this.service.state.context;
     return baselineTime - events[0].timestamp;
@@ -527,6 +544,9 @@ export class Replayer {
    */
   public destroy() {
     this.pause();
+    this.mirror.reset();
+    this.styleMirror.reset();
+    this.mediaManager.reset();
     this.config.root.removeChild(this.wrapper);
     this.emitter.emit(ReplayerEvents.Destroy);
   }
@@ -667,9 +687,10 @@ export class Replayer {
             // Timer (requestAnimationFrame) can be faster than setTimeout(..., 1)
             this.firstFullSnapshot = true;
           }
+          this.mediaManager.reset();
+          this.styleMirror.reset();
           this.rebuildFullSnapshot(event, isSync);
           this.iframe.contentWindow?.scrollTo(event.data.initialOffset);
-          this.styleMirror.reset();
         };
         break;
       case EventType.IncrementalSnapshot:
@@ -778,6 +799,14 @@ export class Replayer {
     const collected: AppendedIframe[] = [];
     const afterAppend = (builtNode: Node, id: number) => {
       this.collectIframeAndAttachDocument(collected, builtNode);
+      if (this.mediaManager.isSupportedMediaElement(builtNode)) {
+        const { events } = this.service.state.context;
+        this.mediaManager.addMediaElements(
+          builtNode,
+          event.timestamp - events[0].timestamp,
+          this.mirror,
+        );
+      }
       for (const plugin of this.config.plugins || []) {
         if (plugin.onBuild)
           plugin.onBuild(builtNode, {
@@ -1261,35 +1290,14 @@ export class Replayer {
           return this.debugNodeNotFound(d, d.id);
         }
         const mediaEl = target as HTMLMediaElement | RRMediaElement;
-        try {
-          if (d.currentTime !== undefined) {
-            mediaEl.currentTime = d.currentTime;
-          }
-          if (d.volume !== undefined) {
-            mediaEl.volume = d.volume;
-          }
-          if (d.muted !== undefined) {
-            mediaEl.muted = d.muted;
-          }
-          if (d.type === MediaInteractions.Pause) {
-            mediaEl.pause();
-          }
-          if (d.type === MediaInteractions.Play) {
-            // remove listener for 'canplay' event because play() is async and returns a promise
-            // i.e. media will evntualy start to play when data is loaded
-            // 'canplay' event fires even when currentTime attribute changes which may lead to
-            // unexpeted behavior
-            void mediaEl.play();
-          }
-          if (d.type === MediaInteractions.RateChange) {
-            mediaEl.playbackRate = d.playbackRate;
-          }
-        } catch (error) {
-          this.warn(
-            // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/restrict-template-expressions
-            `Failed to replay media interactions: ${error.message || error}`,
-          );
-        }
+        const { events } = this.service.state.context;
+
+        this.mediaManager.mediaMutation({
+          target: mediaEl,
+          timeOffset: e.timestamp - events[0].timestamp,
+          mutation: d,
+        });
+
         break;
       }
       case IncrementalSource.StyleSheetRule:
@@ -1366,6 +1374,11 @@ export class Replayer {
     }
   }
 
+  /**
+   * Apply the mutation to the virtual dom or the real dom.
+   * @param d - The mutation data.
+   * @param isSync - Whether the mutation should be applied synchronously (while fast-forwarding).
+   */
   private applyMutation(d: mutationData, isSync: boolean) {
     // Only apply virtual dom optimization if the fast-forward process has node mutation. Because the cost of creating a virtual dom tree and executing the diff algorithm is usually higher than directly applying other kind of events.
     if (this.config.useVirtualDom && !this.usingVirtualDom && isSync) {