fix: Fix captions from MP4s with multiple trun boxes (shaka-project#5422

)

Closes shaka-project#5328

lib/cea/mp4_cea_parser.js

@@ -165,15 +165,17 @@ shaka.cea.Mp4CeaParser = class {
     // Fields that are found in MOOF boxes
     let defaultSampleDuration = this.defaultSampleDuration_;
     let defaultSampleSize = this.defaultSampleSize_;
-    let sampleData = [];
-    let moofOffset = null;
-    let trunOffset = null;
+    let moofOffset = 0;
+    /** @type {!Array<shaka.util.ParsedTRUNBox>} */
+    let parsedTRUNs = [];
     let baseMediaDecodeTime = null;
     let timescale = shaka.cea.CeaUtils.DEFAULT_TIMESCALE_VALUE;
 
     new Mp4Parser()
         .box('moof', (box) => {
           moofOffset = box.start;
+          // trun box parsing is reset on each moof.
+          parsedTRUNs = [];
           Mp4Parser.children(box);
         })
         .box('traf', Mp4Parser.children)
@@ -184,11 +186,8 @@ shaka.cea.Mp4CeaParser = class {
 
           const parsedTRUN = shaka.util.Mp4BoxParsers.parseTRUN(
               box.reader, box.version, box.flags);
-
-          sampleData = parsedTRUN.sampleData;
-          trunOffset = parsedTRUN.dataOffset;
+          parsedTRUNs.push(parsedTRUN);
         })
-
         .fullBox('tfhd', (box) => {
           goog.asserts.assert(
               box.flags != null,
@@ -212,7 +211,6 @@ shaka.cea.Mp4CeaParser = class {
             timescale = this.trackIdToTimescale_.get(trackId);
           }
         })
-
         .fullBox('tfdt', (box) => {
           goog.asserts.assert(
               box.version != null,
@@ -225,16 +223,19 @@ shaka.cea.Mp4CeaParser = class {
         })
         .box('mdat', (box) => {
           if (baseMediaDecodeTime === null) {
-            // This field should have been populated by
-            // the Base Media Decode time in the TFDT box
+            // This field should have been populated by the Base Media Decode
+            // Time in the tfdt box.
+            shaka.log.alwaysWarn(
+                'Unable to find base media decode time for CEA captions!');
             throw new shaka.util.Error(
                 shaka.util.Error.Severity.CRITICAL,
                 shaka.util.Error.Category.TEXT,
                 shaka.util.Error.Code.INVALID_MP4_CEA);
           }
-          const offset = (moofOffset || 0) + (trunOffset || 0) - box.start - 8;
+
+          const offset = moofOffset - box.start - 8;
           this.parseMdat_(box.reader, baseMediaDecodeTime, timescale,
-              defaultSampleDuration, defaultSampleSize, offset, sampleData,
+              defaultSampleDuration, defaultSampleSize, offset, parsedTRUNs,
               captionPackets);
         })
         .parse(mediaSegment, /* partialOkay= */ false);
@@ -250,12 +251,12 @@ shaka.cea.Mp4CeaParser = class {
    * @param {number} defaultSampleDuration
    * @param {number} defaultSampleSize
    * @param {number} offset
-   * @param {!Array<shaka.util.ParsedTRUNSample>} sampleData
+   * @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
    * @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
    * @private
    */
   parseMdat_(reader, time, timescale, defaultSampleDuration,
-      defaultSampleSize, offset, sampleData, captionPackets) {
+      defaultSampleSize, offset, parsedTRUNs, captionPackets) {
     const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
     const CeaUtils = shaka.cea.CeaUtils;
     let sampleIndex = 0;
@@ -266,11 +267,16 @@ shaka.cea.Mp4CeaParser = class {
     // composition time offset, we default to 0.
     let sampleSize = defaultSampleSize;
 
+    // Combine all sample data.  This assumes that the samples described across
+    // multiple trun boxes are still continuous in the mdat box.
+    const sampleDatas = parsedTRUNs.map((t) => t.sampleData);
+    const sampleData = sampleDatas.flat();
+
     if (sampleData.length) {
       sampleSize = sampleData[0].sampleSize || defaultSampleSize;
     }
 
-    reader.skip(offset);
+    reader.skip(offset + parsedTRUNs[0].dataOffset);
 
     while (reader.hasMoreData()) {
       const naluSize = reader.readUint32();
@@ -303,7 +309,7 @@ shaka.cea.Mp4CeaParser = class {
       if (isSeiMessage) {
         let timeOffset = 0;
 
-        if (sampleData.length > sampleIndex) {
+        if (sampleIndex < sampleData.length) {
           timeOffset = sampleData[sampleIndex].sampleCompositionTimeOffset || 0;
         }
 
@@ -326,7 +332,7 @@ shaka.cea.Mp4CeaParser = class {
       }
       sampleSize -= (naluSize + 4);
       if (sampleSize == 0) {
-        if (sampleData.length > sampleIndex) {
+        if (sampleIndex < sampleData.length) {
           time += sampleData[sampleIndex].sampleDuration ||
               defaultSampleDuration;
         } else {
@@ -335,7 +341,7 @@ shaka.cea.Mp4CeaParser = class {
 
         sampleIndex++;
 
-        if (sampleData.length > sampleIndex) {
+        if (sampleIndex < sampleData.length) {
           sampleSize = sampleData[sampleIndex].sampleSize || defaultSampleSize;
         } else {
           sampleSize = defaultSampleSize;

test/cea/mp4_cea_parser_unit.js

@@ -9,6 +9,10 @@ describe('Mp4CeaParser', () => {
   const ceaSegmentUri = '/base/test/test/assets/cea-segment.mp4';
   const h265ceaInitSegmentUri = '/base/test/test/assets/h265-cea-init.mp4';
   const h265ceaSegmentUri = '/base/test/test/assets/h265-cea-segment.mp4';
+  const multipleTrunInitSegmentUri =
+      '/base/test/test/assets/multiple-trun-init.mp4';
+  const multipleTrunSegmentUri =
+      '/base/test/test/assets/multiple-trun-segment.mp4';
   const Util = shaka.test.Util;
 
   /** @type {!ArrayBuffer} */
@@ -19,18 +23,27 @@ describe('Mp4CeaParser', () => {
   let h265ceaInitSegment;
   /** @type {!ArrayBuffer} */
   let h265ceaSegment;
+  /** @type {!ArrayBuffer} */
+  let multipleTrunInitSegment;
+  /** @type {!ArrayBuffer} */
+  let multipleTrunSegment;
 
   beforeAll(async () => {
-    const responses = await Promise.all([
+    [
+      ceaInitSegment,
+      ceaSegment,
+      h265ceaInitSegment,
+      h265ceaSegment,
+      multipleTrunInitSegment,
+      multipleTrunSegment,
+    ] = await Promise.all([
       shaka.test.Util.fetch(ceaInitSegmentUri),
       shaka.test.Util.fetch(ceaSegmentUri),
       shaka.test.Util.fetch(h265ceaInitSegmentUri),
       shaka.test.Util.fetch(h265ceaSegmentUri),
+      shaka.test.Util.fetch(multipleTrunInitSegmentUri),
+      shaka.test.Util.fetch(multipleTrunSegmentUri),
     ]);
-    ceaInitSegment = responses[0];
-    ceaSegment = responses[1];
-    h265ceaInitSegment = responses[2];
-    h265ceaSegment = responses[3];
   });
 
   /**
@@ -89,6 +102,16 @@ describe('Mp4CeaParser', () => {
     expect(ceaPackets.length).toBe(60);
   });
 
+  it('parses cea data from a segment with multiple trun boxes', () => {
+    const ceaParser = new shaka.cea.Mp4CeaParser();
+
+    ceaParser.init(multipleTrunInitSegment);
+    const ceaPackets = ceaParser.parse(multipleTrunSegment);
+    // The first trun box references samples with 48 CEA packets.
+    // The second trun box references samples with 132 more, for a total of 180.
+    expect(ceaPackets.length).toBe(180);
+  });
+
   it('parses an invalid init segment', () => {
     const cea708Parser = new shaka.cea.Mp4CeaParser();