Remove requirement for subtitles to be synced with media PTS when Web…

…VTT MPEGTS map is not present
video-dev · Jun 27, 2023 · a567ef5 · a567ef5
1 parent 206eb23
commit a567ef5
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 31 deletions.
diff --git a/src/controller/subtitle-stream-controller.ts b/src/controller/subtitle-stream-controller.ts
@@ -450,7 +450,7 @@ export class SubtitleStreamController
       }
       foundFrag = this.mapToInitFragWhenRequired(foundFrag) as Fragment;
       if (foundFrag.sn !== 'initSegment') {
-        // Load earlier fragment to make up for misaligned playlists
+        // Load earlier fragment to make up for misaligned playlists and cues that extend beyond end of segment
         const curSNIdx = foundFrag.sn - trackDetails.startSN;
         const prevFrag = fragments[curSNIdx - 1];
         if (

diff --git a/src/controller/timeline-controller.ts b/src/controller/timeline-controller.ts
@@ -304,7 +304,7 @@ export class TimelineController implements ComponentAPI {
     this.captionsTracks = {};
     this.nonNativeCaptionsTracks = {};
     this.textTracks = [];
-    this.unparsedVttFrags = this.unparsedVttFrags || [];
+    this.unparsedVttFrags = [];
     this.initPTS = [];
     if (this.cea608Parser1 && this.cea608Parser2) {
       this.cea608Parser1.reset();
@@ -477,24 +477,9 @@ export class TimelineController implements ComponentAPI {
     data: FragDecryptedData | FragLoadedData
   ) {
     const { frag, payload } = data;
-    const { initPTS, unparsedVttFrags } = this;
     if (frag.type === PlaylistLevelType.SUBTITLE) {
       // If fragment is subtitle type, parse as WebVTT.
       if (payload.byteLength) {
-        // We need an initial synchronisation PTS. Store fragments as long as none has arrived.
-        if (!initPTS[frag.cc]) {
-          unparsedVttFrags.push(data);
-          if (initPTS.length) {
-            // finish unsuccessfully, otherwise the subtitle-stream-controller could be blocked from loading new frags.
-            this.hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, {
-              success: false,
-              frag,
-              error: new Error('Missing initial subtitle PTS'),
-            });
-          }
-          return;
-        }
-
         const decryptData = frag.decryptdata;
         // fragment after decryption has a stats object
         const decrypted = 'stats' in data;
@@ -516,7 +501,7 @@ export class TimelineController implements ComponentAPI {
           ) {
             this._parseIMSC1(frag, payload);
           } else {
-            this._parseVTTs(frag, payload, vttCCs);
+            this._parseVTTs(data);
           }
         }
       } else {
@@ -553,7 +538,16 @@ export class TimelineController implements ComponentAPI {
     );
   }
 
-  private _parseVTTs(frag: Fragment, payload: ArrayBuffer, vttCCs: any) {
+  private _parseVTTs(data: FragDecryptedData | FragLoadedData) {
+    const { frag, payload } = data;
+    // We need an initial synchronisation PTS. Store fragments as long as none has arrived
+    const { initPTS, unparsedVttFrags } = this;
+    const maxAvCC = initPTS.length - 1;
+    if (!initPTS[frag.cc] && maxAvCC === -1) {
+      unparsedVttFrags.push(data);
+      return;
+    }
+
     const hls = this.hls;
     // Parse the WebVTT file contents.
     const payloadWebVTT = frag.initSegment?.data
@@ -562,7 +556,7 @@ export class TimelineController implements ComponentAPI {
     parseWebVTT(
       payloadWebVTT,
       this.initPTS[frag.cc],
-      vttCCs,
+      this.vttCCs,
       frag.cc,
       frag.start,
       (cues) => {
@@ -573,7 +567,13 @@ export class TimelineController implements ComponentAPI {
         });
       },
       (error) => {
-        this._fallbackToIMSC1(frag, payload);
+        const missingInitPTS =
+          error.message === 'Missing initPTS for VTT MPEGTS';
+        if (missingInitPTS) {
+          unparsedVttFrags.push(data);
+        } else {
+          this._fallbackToIMSC1(frag, payload);
+        }
         // Something went wrong while parsing. Trigger event with success false.
         logger.log(`Failed to parse VTT cue: ${error}`);
         hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, {
@@ -631,10 +631,6 @@ export class TimelineController implements ComponentAPI {
   ) {
     const { frag } = data;
     if (frag.type === PlaylistLevelType.SUBTITLE) {
-      if (!this.initPTS[frag.cc]) {
-        this.unparsedVttFrags.push(data as unknown as FragLoadedData);
-        return;
-      }
       this.onFragLoaded(Events.FRAG_LOADED, data as unknown as FragLoadedData);
     }
   }

diff --git a/src/utils/webvtt-parser.ts b/src/utils/webvtt-parser.ts
@@ -92,7 +92,7 @@ const calculateOffset = function (vttCCs: VTTCCs, cc, presentationTime) {
 
 export function parseWebVTT(
   vttByteArray: ArrayBuffer,
-  initPTS: RationalTimestamp,
+  initPTS: RationalTimestamp | undefined,
   vttCCs: VTTCCs,
   cc: number,
   timeOffset: number,
@@ -107,10 +107,9 @@ export function parseWebVTT(
     .replace(LINEBREAKS, '\n')
     .split('\n');
   const cues: VTTCue[] = [];
-  const init90kHz = toMpegTsClockFromTimescale(
-    initPTS.baseTime,
-    initPTS.timescale
-  );
+  const init90kHz = initPTS
+    ? toMpegTsClockFromTimescale(initPTS.baseTime, initPTS.timescale)
+    : 0;
   let cueTime = '00:00.000';
   let timestampMapMPEGTS = 0;
   let timestampMapLOCAL = 0;
@@ -134,8 +133,11 @@ export function parseWebVTT(
         calculateOffset(vttCCs, cc, webVttMpegTsMapOffset);
       }
     }
-
     if (webVttMpegTsMapOffset) {
+      if (!initPTS) {
+        parsingError = new Error('Missing initPTS for VTT MPEGTS');
+        return;
+      }
       // If we have MPEGTS, offset = presentation time + discontinuity offset
       cueOffset = webVttMpegTsMapOffset - vttCCs.presentationOffset;
     }