feat: Parse ID3 metadata (shaka-project#4409)

Co-authored-by: Alvaro Velad <alvaro.velad@mirada.tv>

build/types/core

@@ -76,6 +76,7 @@
 +../../lib/util/functional.js
 +../../lib/util/i_destroyable.js
 +../../lib/util/i_releasable.js
++../../lib/util/id3_utils.js
 +../../lib/util/iterables.js
 +../../lib/util/language_utils.js
 +../../lib/util/lazy.js
@@ -101,6 +102,7 @@
 +../../lib/util/switch_history.js
 +../../lib/util/text_parser.js
 +../../lib/util/timer.js
++../../lib/util/ts_parser.js
 +../../lib/util/uint8array_utils.js
 +../../lib/util/xml_utils.js
 

externs/mux.js

@@ -81,15 +81,13 @@ muxjs.mp4.Transmuxer = class {
  * @typedef {{
  *   initSegment: !Uint8Array,
  *   data: !Uint8Array,
- *   captions: !Array,
- *   metadata: !Array
+ *   captions: !Array
  * }}
  *
  * @description Transmuxed data from mux.js.
  * @property {!Uint8Array} initSegment
  * @property {!Uint8Array} data
  * @property {!Array} captions
- * @property {!Array} metadata
  * @exportDoc
  */
 muxjs.mp4.Transmuxer.Segment;
@@ -170,42 +168,3 @@ muxjs.mp4.ParsedClosedCaptions;
  */
 muxjs.mp4.ClosedCaption;
 
-
-/**
- * @typedef {{
- *   cueTime: number,
- *   data: !Uint8Array,
- *   dispatchType: string,
- *   dts: number,
- *   frames: !Array.<muxjs.mp4.MetadataFrame>,
- *   pts: number
- * }}
- *
- * @description metadata parsed from mux.js.
- * @property {number} cueTime
- * @property {number} data
- * @property {number} dispatchType
- * @property {number} dts
- * @property {string} frames
- * @property {string} pts
- */
-muxjs.mp4.Metadata;
-
-
-/**
- * @typedef {{
- *   data: string,
- *   description: string,
- *   id: string,
- *   key: string,
- *   value: string
- * }}
- *
- * @description metadata parsed from mux.js.
- * @property {number} data
- * @property {number} description
- * @property {number} id
- * @property {number} key
- * @property {string} value
- */
-muxjs.mp4.MetadataFrame;

externs/shaka/ads.js

@@ -120,7 +120,7 @@ shaka.extern.IAdManager = class extends EventTarget {
   onHlsTimedMetadata(metadata, timestampOffset) {}
 
   /**
-   * @param {shaka.extern.ID3Metadata} value
+   * @param {shaka.extern.MetadataFrame} value
    */
   onCueMetadataChange(value) {}
 };

externs/shaka/player.js

@@ -437,20 +437,63 @@ shaka.extern.DrmSupportType;
  */
 shaka.extern.SupportType;
 
-
 /**
- * @typedef {!Object.<string, ?>}
+ * @typedef {{
+ *   cueTime: ?number,
+ *   data: !Uint8Array,
+ *   frames: !Array.<shaka.extern.MetadataFrame>,
+ *   dts: ?number,
+ *   pts: ?number
+ * }}
  *
  * @description
  * ID3 metadata in format defined by
  * https://id3.org/id3v2.3.0#Declared_ID3v2_frames
  * The content of the field.
  *
+ * @property {?number} cueTime
+ * @property {!Uint8Array} data
+ * @property {!Array.<shaka.extern.MetadataFrame>} frames
+ * @property {?number} dts
+ * @property {?number} pts
+ *
  * @exportDoc
  */
 shaka.extern.ID3Metadata;
 
 
+/**
+ * @typedef {{
+ *   type: string,
+ *   size: number,
+ *   data: Uint8Array
+ * }}
+ *
+ * @description metadata raw frame.
+ * @property {string} type
+ * @property {number} size
+ * @property {Uint8Array} data
+ * @exportDoc
+ */
+shaka.extern.MetadataRawFrame;
+
+
+/**
+ * @typedef {{
+ *   key: string,
+ *   data: (ArrayBuffer|string),
+ *   description: string
+ * }}
+ *
+ * @description metadata frame parsed.
+ * @property {string} key
+ * @property {ArrayBuffer|string} data
+ * @property {string} description
+ * @exportDoc
+ */
+shaka.extern.MetadataFrame;
+
+
 /**
  * @typedef {{
  *   schemeIdUri: string,

lib/ads/server_side_ad_manager.js

@@ -237,7 +237,7 @@ shaka.ads.ServerSideAdManager = class {
   }
 
   /**
-   * @param {shaka.extern.ID3Metadata} value
+   * @param {shaka.extern.MetadataFrame} value
    */
   onCueMetadataChange(value) {
     // Native HLS over Safari/iOS/iPadOS
@@ -246,9 +246,9 @@ shaka.ads.ServerSideAdManager = class {
     // done through timed metadata. Timed metadata is carried as part of the
     // DAI stream content and carries ad break timing information used by the
     // SDK to track ad breaks.
-    if (value['key'] && value['data']) {
+    if (value.key && value.data) {
       const metadata = {};
-      metadata[value['key']] = value['data'];
+      metadata[value.key] = value.data;
       this.streamManager_.onTimedMetadata(metadata);
     }
   }

lib/media/media_source_engine.js

@@ -14,15 +14,18 @@ goog.require('shaka.media.SegmentReference');
 goog.require('shaka.media.TimeRangesUtils');
 goog.require('shaka.media.Transmuxer');
 goog.require('shaka.text.TextEngine');
+goog.require('shaka.util.BufferUtils');
 goog.require('shaka.util.Destroyer');
 goog.require('shaka.util.Error');
 goog.require('shaka.util.EventManager');
 goog.require('shaka.util.Functional');
 goog.require('shaka.util.IDestroyable');
+goog.require('shaka.util.Id3Utils');
 goog.require('shaka.util.ManifestParserUtils');
 goog.require('shaka.util.MimeUtils');
 goog.require('shaka.util.Platform');
 goog.require('shaka.util.PublicPromise');
+goog.require('shaka.util.TsParser');
 goog.require('shaka.lcevc.Dil');
 
 
@@ -568,6 +571,35 @@ shaka.media.MediaSourceEngine = class {
       return;
     }
 
+    const uint8ArrayData = shaka.util.BufferUtils.toUint8(data);
+    if (shaka.util.TsParser.probe(uint8ArrayData)) {
+      const metadata = new shaka.util.TsParser().parse(uint8ArrayData)
+          .getMetadata();
+      if (metadata.length) {
+        const timestampOffset =
+            this.sourceBuffers_[contentType].timestampOffset;
+        this.onMetadata_(metadata, timestampOffset,
+            reference ? reference.endTime : null);
+      }
+    } else {
+      const containerType = shaka.util.MimeUtils.getContainerType(
+          this.sourceBufferTypes_[contentType]);
+      if (containerType === 'aac') {
+        const frames = shaka.util.Id3Utils.getID3Frames(uint8ArrayData);
+        if (frames.length && reference) {
+          /** @private {shaka.extern.ID3Metadata} */
+          const metadata = {
+            cueTime: reference.startTime,
+            data: uint8ArrayData,
+            frames: frames,
+            dts: reference.startTime,
+            pts: reference.startTime,
+          };
+          this.onMetadata_([metadata], /* offset= */ 0, reference.endTime);
+        }
+      }
+    }
+
     if (this.transmuxers_[contentType]) {
       const transmuxedData =
           await this.transmuxers_[contentType].transmux(data);
@@ -576,15 +608,6 @@ shaka.media.MediaSourceEngine = class {
       if (!this.textEngine_) {
         this.reinitText('text/vtt', this.sequenceMode_);
       }
-
-      if (transmuxedData.metadata) {
-        const timestampOffset =
-            this.sourceBuffers_[contentType].timestampOffset;
-        this.onMetadata_(
-            transmuxedData.metadata,
-            timestampOffset,
-            reference ? reference.endTime : null);
-      }
       // This doesn't work for native TS support (ex. Edge/Chromecast),
       // since no transmuxing is needed for native TS.
       if (transmuxedData.captions && transmuxedData.captions.length) {

lib/media/streaming_engine.js

@@ -24,6 +24,7 @@ goog.require('shaka.util.Destroyer');
 goog.require('shaka.util.Error');
 goog.require('shaka.util.FakeEvent');
 goog.require('shaka.util.IDestroyable');
+goog.require('shaka.util.Id3Utils');
 goog.require('shaka.util.ManifestParserUtils');
 goog.require('shaka.util.MimeUtils');
 goog.require('shaka.util.Mp4Parser');
@@ -1814,6 +1815,21 @@ shaka.media.StreamingEngine = class {
       // A special scheme in DASH used to signal manifest updates.
       if (schemeId == 'urn:mpeg:dash:event:2012') {
         this.playerInterface_.onManifestUpdate();
+      } else if (schemeId == 'https://aomedia.org/emsg/ID3') {
+        // See https://aomediacodec.github.io/id3-emsg/
+        const frames = shaka.util.Id3Utils.getID3Frames(messageData);
+        if (frames.length && reference) {
+          /** @private {shaka.extern.ID3Metadata} */
+          const metadata = {
+            cueTime: reference.startTime,
+            data: messageData,
+            frames: frames,
+            dts: reference.startTime,
+            pts: reference.startTime,
+          };
+          this.playerInterface_.onMetadata(
+              [metadata], /* offset= */ 0, reference.endTime);
+        }
       } else {
         /** @type {shaka.extern.EmsgInfo} */
         const emsg = {
@@ -2191,7 +2207,8 @@ shaka.media.StreamingEngine = class {
  *     !shaka.util.ManifestParserUtils.ContentType),
  *   onInitSegmentAppended: function(!number,!shaka.media.InitSegmentReference),
  *   beforeAppendSegment: function(
- *     shaka.util.ManifestParserUtils.ContentType,!BufferSource):Promise
+ *     shaka.util.ManifestParserUtils.ContentType,!BufferSource):Promise,
+ *   onMetadata: !function(!Array.<shaka.extern.ID3Metadata>, number, ?number)
  * }}
  *
  * @property {function():number} getPresentationTime
@@ -2224,6 +2241,9 @@ shaka.media.StreamingEngine = class {
  * @property {!function(shaka.util.ManifestParserUtils.ContentType,
  *   !BufferSource):Promise} beforeAppendSegment
  *   A function called just before appending to the source buffer.
+ * @property
+ *  {!function(!Array.<shaka.extern.ID3Metadata>, number, ?number)} onMetadata
+ *   Called when an ID3 is found in a EMSG.
  */
 shaka.media.StreamingEngine.PlayerInterface;
 

lib/media/transmuxer.js

@@ -42,9 +42,6 @@ shaka.media.Transmuxer = class {
     /** @private {!Array.<muxjs.mp4.ClosedCaption>} */
     this.captions_ = [];
 
-    /** @private {!Array.<muxjs.mp4.Metadata>} */
-    this.metadata_ = [];
-
     /** @private {boolean} */
     this.isTransmuxing_ = false;
 
@@ -152,8 +149,7 @@ shaka.media.Transmuxer = class {
    * Transmux from Transport stream to MP4, using the mux.js library.
    * @param {BufferSource} data
    * @return {!Promise.<{data: !Uint8Array,
-   *                     captions: !Array.<!muxjs.mp4.ClosedCaption>,
-   *                     metadata: !Array.<!Object>}>}
+   *                     captions: !Array.<!muxjs.mp4.ClosedCaption>}>}
    */
   transmux(data) {
     goog.asserts.assert(!this.isTransmuxing_,
@@ -162,7 +158,6 @@ shaka.media.Transmuxer = class {
     this.transmuxPromise_ = new shaka.util.PublicPromise();
     this.transmuxedData_ = [];
     this.captions_ = [];
-    this.metadata_ = [];
 
     const dataArray = shaka.util.BufferUtils.toUint8(data);
     this.muxTransmuxer_.push(dataArray);
@@ -194,7 +189,6 @@ shaka.media.Transmuxer = class {
    */
   onTransmuxed_(segment) {
     this.captions_ = segment.captions;
-    this.metadata_ = segment.metadata;
     this.transmuxedData_.push(
         shaka.util.Uint8ArrayUtils.concat(segment.initSegment, segment.data));
   }
@@ -209,7 +203,6 @@ shaka.media.Transmuxer = class {
     const output = {
       data: shaka.util.Uint8ArrayUtils.concat(...this.transmuxedData_),
       captions: this.captions_,
-      metadata: this.metadata_,
     };
 
     this.transmuxPromise_.resolve(output);

lib/player.js

@@ -328,7 +328,7 @@ goog.requireType('shaka.routing.Payload');
  *   the cue applies.
  * @property {string} metadataType
  *   Type of metadata. Eg: org.id3 or org.mp4ra
- * @property {shaka.extern.ID3Metadata} payload
+ * @property {shaka.extern.MetadataFrame} payload
  *   The metadata itself
  * @exportDoc
  */
@@ -2706,11 +2706,11 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
    */
   processTimedMetadataMediaSrc_(metadata, offset, segmentEndTime) {
     for (const sample of metadata) {
-      if (sample['data'] && sample['cueTime'] && sample['frames']) {
-        const start = sample['cueTime'] + offset;
+      if (sample.data && sample.cueTime && sample.frames) {
+        const start = sample.cueTime + offset;
         const end = segmentEndTime;
-        const metadataType = 'ID3';
-        for (const frame of sample['frames']) {
+        const metadataType = 'org.id3';
+        for (const frame of sample.frames) {
           const payload = frame;
           this.dispatchMetadataEvent_(start, end, metadataType, payload);
         }
@@ -2729,7 +2729,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
    * @param {number} startTime
    * @param {?number} endTime
    * @param {string} metadataType
-   * @param {shaka.extern.ID3Metadata} payload
+   * @param {shaka.extern.MetadataFrame} payload
    * @private
    */
   dispatchMetadataEvent_(startTime, endTime, metadataType, payload) {
@@ -3101,6 +3101,9 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       beforeAppendSegment: (contentType, segment) => {
         return this.drmEngine_.parseInbandPssh(contentType, segment);
       },
+      onMetadata: (metadata, offset, endTime) => {
+        this.processTimedMetadataMediaSrc_(metadata, offset, endTime);
+      },
     };
 
     return new shaka.media.StreamingEngine(this.manifest_, playerInterface);