fix(HLS): Fix external subtitles out of sync in HLS (#5491)

Fixes #5458 Fixes #5443 Backported to v4.3.x
shaka-project · Aug 21, 2023 · 38c8a88 · 38c8a88
1 parent cdf074b
commit 38c8a88
Show file tree

Hide file tree

Showing 18 changed files with 43 additions and 10 deletions.
diff --git a/externs/shaka/manifest.js b/externs/shaka/manifest.js
@@ -321,7 +321,8 @@ shaka.extern.FetchCryptoKeysFunction;
  *   tilesLayout: (string|undefined),
  *   matchedStreams:
  *      (!Array.<shaka.extern.Stream>|!Array.<shaka.extern.StreamDB>|
- *      undefined)
+ *      undefined),
+ *   external: boolean
  * }}
  *
  * @description
@@ -434,6 +435,9 @@ shaka.extern.FetchCryptoKeysFunction;
  * @property {(!Array.<shaka.extern.Stream>|!Array.<shaka.extern.StreamDB>|
  *   undefined)} matchedStreams
  *   The streams in all periods which match the stream. Used for Dash.
+ * @property {boolean} external
+ *   Indicate if the stream was added externally.
+ *   Eg: external text tracks.
  *
  * @exportDoc
  */

diff --git a/externs/shaka/offline.js b/externs/shaka/offline.js
@@ -132,7 +132,8 @@ shaka.extern.ManifestDB;
  *   audioSamplingRate: ?number,
  *   spatialAudio: boolean,
  *   closedCaptions: Map.<string, string>,
- *   tilesLayout: (string|undefined)
+ *   tilesLayout: (string|undefined),
+ *   external: boolean
  * }}
  *
  * @property {number} id
@@ -193,6 +194,9 @@ shaka.extern.ManifestDB;
  *   The value is a grid-item-dimension consisting of two positive decimal
  *   integers in the format: column-x-row ('4x3'). It describes the arrangement
  *   of Images in a Grid. The minimum valid LAYOUT is '1x1'.
+ * @property {boolean} external
+ *   Indicate if the stream was added externally.
+ *   Eg: external text tracks.
  */
 shaka.extern.StreamDB;
 

diff --git a/lib/dash/dash_parser.js b/lib/dash/dash_parser.js
@@ -1337,6 +1337,7 @@ shaka.dash.DashParser = class {
         hdr,
         tilesLayout,
         matchedStreams: [],
+        external: false,
       };
     }
 

diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js
@@ -2114,6 +2114,7 @@ shaka.hls.HlsParser = class {
       closedCaptions,
       hdr: undefined,
       tilesLayout: undefined,
+      external: false,
     };
   }
 

diff --git a/lib/media/media_source_engine.js b/lib/media/media_source_engine.js
@@ -369,7 +369,7 @@ shaka.media.MediaSourceEngine = class {
       let mimeType = shaka.util.MimeUtils.getFullType(
           stream.mimeType, stream.codecs);
       if (contentType == ContentType.TEXT) {
-        this.reinitText(mimeType, sequenceMode);
+        this.reinitText(mimeType, sequenceMode, stream.external);
       } else {
         if ((forceTransmux ||
             !shaka.media.Capabilities.isTypeSupported(mimeType)) &&
@@ -425,13 +425,14 @@ shaka.media.MediaSourceEngine = class {
    * Reinitialize the TextEngine for a new text type.
    * @param {string} mimeType
    * @param {boolean} sequenceMode
+   * @param {boolean} external
    */
-  reinitText(mimeType, sequenceMode) {
+  reinitText(mimeType, sequenceMode, external) {
     if (!this.textEngine_) {
       this.textEngine_ = new shaka.text.TextEngine(this.textDisplayer_);
     }
     this.textEngine_.initParser(mimeType, sequenceMode,
-        this.segmentRelativeVttTiming_);
+        external || this.segmentRelativeVttTiming_);
   }
 
   /**
@@ -651,7 +652,7 @@ shaka.media.MediaSourceEngine = class {
       // the video stream, so textEngine may not have been initialized.
       if (!this.textEngine_) {
         this.reinitText(shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE,
-            this.sequenceMode_);
+            this.sequenceMode_, /* external= */ false);
       }
       // This doesn't work for native TS support (ex. Edge/Chromecast),
       // since no transmuxing is needed for native TS.
@@ -671,7 +672,7 @@ shaka.media.MediaSourceEngine = class {
     } else if (hasClosedCaptions && contentType == ContentType.VIDEO) {
       if (!this.textEngine_) {
         this.reinitText(shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE,
-            this.sequenceMode_);
+            this.sequenceMode_, /* external= */ false);
       }
       if (!this.captionParser_) {
         this.captionParser_ = this.getCaptionParser(mimeType);

diff --git a/lib/media/streaming_engine.js b/lib/media/streaming_engine.js
@@ -244,7 +244,7 @@ shaka.media.StreamingEngine = class {
     const mimeType = shaka.util.MimeUtils.getFullType(
         stream.mimeType, stream.codecs);
     this.playerInterface_.mediaSourceEngine.reinitText(
-        mimeType, this.manifest_.sequenceMode);
+        mimeType, this.manifest_.sequenceMode, stream.external);
 
     const textDisplayer =
         this.playerInterface_.mediaSourceEngine.getTextDisplayer();
@@ -458,7 +458,7 @@ shaka.media.StreamingEngine = class {
       const fullMimeType = shaka.util.MimeUtils.getFullType(
           stream.mimeType, stream.codecs);
       this.playerInterface_.mediaSourceEngine.reinitText(
-          fullMimeType, this.manifest_.sequenceMode);
+          fullMimeType, this.manifest_.sequenceMode, stream.external);
     }
 
     // Releases the segmentIndex of the old stream.

diff --git a/lib/offline/indexeddb/v1_storage_cell.js b/lib/offline/indexeddb/v1_storage_cell.js
@@ -176,6 +176,7 @@ shaka.offline.indexeddb.V1StorageCell = class
       spatialAudio: false,
       closedCaptions: null,
       tilesLayout: undefined,
+      external: false,
     };
   }
 

diff --git a/lib/offline/indexeddb/v2_storage_cell.js b/lib/offline/indexeddb/v2_storage_cell.js
@@ -125,6 +125,7 @@ shaka.offline.indexeddb.V2StorageCell = class
       spatialAudio: false,
       closedCaptions: null,
       tilesLayout: undefined,
+      external: false,
     };
   }
 

diff --git a/lib/offline/manifest_converter.js b/lib/offline/manifest_converter.js
@@ -202,6 +202,7 @@ shaka.offline.ManifestConverter = class {
       spatialAudio: streamDB.spatialAudio,
       closedCaptions: streamDB.closedCaptions,
       tilesLayout: streamDB.tilesLayout,
+      external: streamDB.external,
     };
 
     return stream;

diff --git a/lib/offline/storage.js b/lib/offline/storage.js
@@ -1325,6 +1325,7 @@ shaka.offline.Storage = class {
       spatialAudio: stream.spatialAudio,
       closedCaptions: stream.closedCaptions,
       tilesLayout: stream.tilesLayout,
+      external: stream.external,
     };
 
     const startTime =

diff --git a/lib/player.js b/lib/player.js
@@ -2390,6 +2390,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
         audioSamplingRate: null,
         spatialAudio: false,
         closedCaptions: null,
+        external: false,
       },
       bandwidth: 100,
       allowedByApplication: true,
@@ -4823,6 +4824,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       audioSamplingRate: null,
       spatialAudio: false,
       closedCaptions: null,
+      external: true,
     };
 
     const fullMimeType = shaka.util.MimeUtils.getFullType(
@@ -4974,6 +4976,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
       spatialAudio: false,
       closedCaptions: null,
       tilesLayout: '1x1',
+      external: true,
     };
 
     this.manifest_.imageStreams.push(stream);
@@ -5393,6 +5396,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget {
               audioSamplingRate: null,
               spatialAudio: false,
               closedCaptions: null,
+              external: false,
             };
             manifest.textStreams.push(textStream);
             closedCaptionsSet.add(id);

diff --git a/lib/util/periods.js b/lib/util/periods.js
@@ -1491,6 +1491,7 @@ shaka.util.PeriodCombiner = class {
       audioSamplingRate: null,
       spatialAudio: false,
       closedCaptions: null,
+      external: false,
     };
   }
 
@@ -1527,6 +1528,7 @@ shaka.util.PeriodCombiner = class {
       audioSamplingRate: null,
       spatialAudio: false,
       closedCaptions: null,
+      external: false,
     };
   }
 

diff --git a/test/media/adaptation_set_unit.js b/test/media/adaptation_set_unit.js
@@ -218,6 +218,7 @@ describe('AdaptationSet', () => {
       forced: false,
       trickModeVideo: null,
       type: '',
+      external: false,
     };
   }
 });

diff --git a/test/media/media_source_engine_unit.js b/test/media/media_source_engine_unit.js
@@ -1262,7 +1262,7 @@ describe('MediaSourceEngine', () => {
     });
 
     it('destroys text engines', async () => {
-      mediaSourceEngine.reinitText('text/vtt', false);
+      mediaSourceEngine.reinitText('text/vtt', false, false);
 
       await mediaSourceEngine.destroy();
       expect(mockTextEngine).toBeTruthy();

diff --git a/test/offline/manifest_convert_unit.js b/test/offline/manifest_convert_unit.js
@@ -316,6 +316,7 @@ describe('ManifestConverter', () => {
       audioSamplingRate: null,
       spatialAudio: false,
       closedCaptions: null,
+      external: false,
     };
 
     return streamDB;
@@ -389,6 +390,7 @@ describe('ManifestConverter', () => {
       spatialAudio: false,
       closedCaptions: null,
       tilesLayout: undefined,
+      external: false,
     };
   }
 
@@ -438,6 +440,7 @@ describe('ManifestConverter', () => {
       spatialAudio: false,
       closedCaptions: null,
       tilesLayout: undefined,
+      external: false,
     };
   }
 
@@ -486,6 +489,7 @@ describe('ManifestConverter', () => {
       spatialAudio: false,
       closedCaptions: null,
       tilesLayout: undefined,
+      external: false,
     };
   }
 
@@ -531,6 +535,7 @@ describe('ManifestConverter', () => {
       spatialAudio: streamDb.spatialAudio,
       closedCaptions: streamDb.closedCaptions,
       tilesLayout: streamDb.tilesLayout,
+      external: streamDb.external,
     };
 
     expect(stream).toEqual(expectedStream);

diff --git a/test/test/util/manifest_generator.js b/test/test/util/manifest_generator.js
@@ -541,6 +541,8 @@ shaka.test.ManifestGenerator.Stream = class {
       this.hdr = undefined;
       /** @type {(string|undefined)} */
       this.tilesLayout = undefined;
+      /** @type {boolean} */
+      this.external = false;
     }
 
     /** @type {shaka.extern.Stream} */

diff --git a/test/test/util/offline_utils.js b/test/test/util/offline_utils.js
@@ -56,6 +56,7 @@ shaka.test.OfflineUtils = class {
       audioSamplingRate: null,
       spatialAudio: false,
       closedCaptions: null,
+      external: false,
     };
   }
 

diff --git a/test/test/util/streaming_engine_util.js b/test/test/util/streaming_engine_util.js
@@ -410,6 +410,7 @@ shaka.test.StreamingEngineUtil = class {
       roles: [],
       forced: false,
       spatialAudio: false,
+      external: false,
     };
   }
 
@@ -447,6 +448,7 @@ shaka.test.StreamingEngineUtil = class {
       roles: [],
       forced: false,
       spatialAudio: false,
+      external: false,
     };
   }
 
@@ -482,6 +484,7 @@ shaka.test.StreamingEngineUtil = class {
       roles: [],
       forced: false,
       spatialAudio: false,
+      external: false,
     };
   }
 };