feat: Trigger an event with spatial video info (#6437)

This is a necessary step to implement VR in the UI

externs/shaka/codecs.js

@@ -42,3 +42,16 @@ shaka.extern.MPEG_PES;
  * @property {?number} time
  */
 shaka.extern.VideoNalu;
+
+
+/**
+ * @typedef {{
+ *   projection: ?string,
+ *   hfov: ?number
+ * }}
+ *
+ * @summary VideoNalu.
+ * @property {?string} projection
+ * @property {?number} hfov
+ */
+shaka.extern.SpatialVideoInfo;

lib/media/streaming_engine.js

@@ -128,6 +128,12 @@ shaka.media.StreamingEngine = class {
      * @private {!Map<shaka.extern.Stream, !shaka.media.SegmentPrefetch>}
      */
     this.audioPrefetchMap_ = new Map();
+
+    /** @private {!shaka.extern.SpatialVideoInfo} */
+    this.spatialVideoInfo_ = {
+      projection: null,
+      hfov: null,
+    };
   }
 
   /** @override */
@@ -1994,22 +2000,28 @@ shaka.media.StreamingEngine = class {
             let lastTimescale = null;
             const timescaleMap = new Map();
 
+            /** @type {!shaka.extern.SpatialVideoInfo} */
+            const spatialVideoInfo = {
+              projection: null,
+              hfov: null,
+            };
+
             const parser = new shaka.util.Mp4Parser();
             const Mp4Parser = shaka.util.Mp4Parser;
+            const Mp4BoxParsers = shaka.util.Mp4BoxParsers;
             parser.box('moov', Mp4Parser.children)
                 .box('trak', Mp4Parser.children)
                 .box('mdia', Mp4Parser.children)
                 .fullBox('mdhd', (box) => {
                   goog.asserts.assert(
                       box.version != null,
                       'MDHD is a full box and should have a valid version.');
-                  const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
+                  const parsedMDHDBox = Mp4BoxParsers.parseMDHD(
                       box.reader, box.version);
                   lastTimescale = parsedMDHDBox.timescale;
                 })
                 .box('hdlr', (box) => {
-                  const parsedHDLR = shaka.util.Mp4BoxParsers.parseHDLR(
-                      box.reader);
+                  const parsedHDLR = Mp4BoxParsers.parseHDLR(box.reader);
                   switch (parsedHDLR.handlerType) {
                     case 'soun':
                       timescaleMap.set(ContentType.AUDIO, lastTimescale);
@@ -2020,8 +2032,32 @@ shaka.media.StreamingEngine = class {
                   }
                   lastTimescale = null;
                 })
+                .box('minf', Mp4Parser.children)
+                .box('stbl', Mp4Parser.children)
+                .fullBox('stsd', Mp4Parser.sampleDescription)
+                .box('encv', Mp4Parser.visualSampleEntry)
+                .box('avc1', Mp4Parser.visualSampleEntry)
+                .box('avc3', Mp4Parser.visualSampleEntry)
+                .box('hev1', Mp4Parser.visualSampleEntry)
+                .box('hvc1', Mp4Parser.visualSampleEntry)
+                .box('dvav', Mp4Parser.visualSampleEntry)
+                .box('dva1', Mp4Parser.visualSampleEntry)
+                .box('dvh1', Mp4Parser.visualSampleEntry)
+                .box('dvhe', Mp4Parser.visualSampleEntry)
+                .box('vexu', Mp4Parser.children)
+                .box('proj', Mp4Parser.children)
+                .fullBox('prji', (box) => {
+                  const parsedPRJIBox = Mp4BoxParsers.parsePRJI(box.reader);
+                  spatialVideoInfo.projection = parsedPRJIBox.projection;
+                })
+                .box('hfov', (box) => {
+                  const parsedHFOVBox = Mp4BoxParsers.parseHFOV(box.reader);
+                  spatialVideoInfo.hfov = parsedHFOVBox.hfov;
+                })
                 .parse(initSegment);
 
+            this.updateSpatialVideoInfo_(spatialVideoInfo);
+
             if (timescaleMap.has(mediaState.type)) {
               reference.initSegmentReference.timescale =
                   timescaleMap.get(mediaState.type);
@@ -2697,6 +2733,31 @@ shaka.media.StreamingEngine = class {
     return true;
   }
 
+  /**
+   * Update the spatial video info and notify to the app.
+   *
+   * @param {shaka.extern.SpatialVideoInfo} info
+   * @private
+   */
+  updateSpatialVideoInfo_(info) {
+    if (this.spatialVideoInfo_.projection != info.projection ||
+        this.spatialVideoInfo_.hfov != info.hfov) {
+      const EventName = shaka.util.FakeEvent.EventName;
+      let event;
+      if (info.projection != null || info.hfov != null) {
+        const eventName = EventName.SpatialVideoInfoEvent;
+        const data = (new Map()).set('detail', info);
+        event = new shaka.util.FakeEvent(eventName, data);
+      } else {
+        const eventName = EventName.NoSpatialVideoInfoEvent;
+        event = new shaka.util.FakeEvent(eventName);
+      }
+      event.cancelable = true;
+      this.playerInterface_.onEvent(event);
+      this.spatialVideoInfo_ = info;
+    }
+  }
+
   /**
    * @param {shaka.media.StreamingEngine.MediaState_} mediaState
    * @return {string} A log prefix of the form ($CONTENT_TYPE:$STREAM_ID), e.g.,

lib/player.js

@@ -497,6 +497,29 @@ goog.requireType('shaka.media.PresentationTimeline');
  */
 
 
+/**
+ * @event shaka.Player.SpatialVideoInfoEvent
+ * @description Fired when the video has spatial video info. If a previous
+ *   event was fired, this include the new info.
+ * @property {string} type
+ *   'spatialvideoinfo'
+ * @property {shaka.extern.SpatialVideoInfo} detail
+ *   An object which contains the content of the emsg box.
+ * @exportDoc
+ */
+
+
+/**
+ * @event shaka.Player.NoSpatialVideoInfoEvent
+ * @description Fired when the video no longer has spatial video information.
+ *   For it to be fired, the shaka.Player.SpatialVideoInfoEvent event must
+ *   have been previously fired.
+ * @property {string} type
+ *   'nospatialvideoinfo'
+ * @exportDoc
+ */
+
+
 /**
  * @summary The main player object for Shaka Player.
  *

lib/util/fake_event.js

@@ -174,10 +174,12 @@ shaka.util.FakeEvent.EventName = {
   MediaQualityChanged: 'mediaqualitychanged',
   Metadata: 'metadata',
   Midpoint: 'midpoint',
+  NoSpatialVideoInfoEvent: 'nospatialvideoinfo',
   OnStateChange: 'onstatechange',
   RateChange: 'ratechange',
   SegmentAppended: 'segmentappended',
   SessionDataEvent: 'sessiondata',
+  SpatialVideoInfoEvent: 'spatialvideoinfo',
   StallDetected: 'stalldetected',
   Started: 'started',
   StateChanged: 'statechanged',

lib/util/mp4_box_parsers.js

@@ -522,16 +522,33 @@ shaka.util.Mp4BoxParsers = class {
   static parseHDLR(reader) {
     reader.skip(8); // Skip "pre_defined"
 
-    const data = reader.readBytes(4);
-    let handlerType = '';
-    handlerType += String.fromCharCode(data[0]);
-    handlerType += String.fromCharCode(data[1]);
-    handlerType += String.fromCharCode(data[2]);
-    handlerType += String.fromCharCode(data[3]);
-
+    const handlerType = reader.readTerminatedString();
     return {handlerType};
   }
 
+  /**
+   * Parses a PRJI box.
+   * @param {!shaka.util.DataViewReader} reader
+   * @return {!shaka.util.ParsedPRJIBox}
+   */
+  static parsePRJI(reader) {
+    const projection = reader.readTerminatedString();
+    return {projection};
+  }
+
+  /**
+   * Parses a HFOV box.
+   * @param {!shaka.util.DataViewReader} reader
+   * @return {!shaka.util.ParsedHFOVBox}
+   */
+  static parseHFOV(reader) {
+    const millidegrees = reader.readUint32();
+
+    return {
+      hfov: millidegrees / 1000,
+    };
+  }
+
   /**
    * Parses a COLR box.
    * @param {!shaka.util.DataViewReader} reader
@@ -866,6 +883,34 @@ shaka.util.ParsedAV1CBox;
  */
 shaka.util.ParsedHDLRBox;
 
+/**
+ * @typedef {{
+ *    projection: string
+ * }}
+ *
+ * @property {string} projection
+ *   A four-character code that identifies the type of the projection.
+ *   Possible values:
+ *    - Rectangular: ‘rect’
+ *    - Half equirectangular: ‘hequ’
+ *    - Equirectanguler: ?
+ *    - Fisheye: ‘fish’
+ *
+ * @exportDoc
+ */
+shaka.util.ParsedPRJIBox;
+
+/**
+ * @typedef {{
+ *    hfov: number
+ * }}
+ *
+ * @property {number} hfov
+ *
+ * @exportDoc
+ */
+shaka.util.ParsedHFOVBox;
+
 /**
  * @typedef {{
  *    videoRange: ?string