Plot Date Range attributes on metadata text track

Resolves #2218 Other changes: - Remove cues on buffer flush based on track type associated with metadata schema (video: emsg, audio: org.id3) - Handle Delta Playlist Update Date Range updates - Parse and validate DateRange tags - Add hls.playingDate to API: gets program date time at media playhead (video.currentTime) - Fix code formatting with prettier
video-dev · Jun 2, 2022 · 7a7acc5 · 7a7acc5
1 parent fcb0d8b
commit 7a7acc5
Show file tree

Hide file tree

Showing 16 changed files with 629 additions and 57 deletions.
diff --git a/README.md b/README.md
@@ -41,7 +41,7 @@ HLS.js is written in [ECMAScript6] (`*.js`) and [TypeScript] (`*.ts`) (strongly
   - Packetized metadata (ID3v2.3.0) Elementary Stream
 - AAC container (audio only streams)
 - MPEG Audio container (MPEG-1/2 Audio Layer III audio only streams)
-- Timed Metadata for HTTP Live Streaming (in ID3 format, carried in MPEG-2 TS and FMP4 Emsg)
+- Timed Metadata for HTTP Live Streaming (ID3 format carried in MPEG-2 TS, Emsg in CMAF/Fragmented MP4, and DATERANGE playlist tags)
 - AES-128 decryption
 - SAMPLE-AES decryption (only supported if using MPEG-2 TS container)
 - Encrypted media extensions (EME) support for DRM (digital rights management)
@@ -102,18 +102,17 @@ The following properties are added to their respective variants' attribute list
 - `#EXT-X-PRELOAD-HINT:<attribute-list>`
 - `#EXT-X-SKIP:<attribute-list>`
 - `#EXT-X-RENDITION-REPORT:<attribute-list>`
+- `#EXT-X-DATERANGE:<attribute-list>`
 
 The following tags are added to their respective fragment's attribute list but are not implemented in streaming and playback.
 
-- `#EXT-X-DATERANGE:<attribute-list>` (Not added to metadata TextTracks. See [#2218](https://github.com/video-dev/hls.js/issues/2218))
 - `#EXT-X-BITRATE` (Not used in ABR controller)
 - `#EXT-X-GAP` (Not implemented. See [#2940](https://github.com/video-dev/hls.js/issues/2940))
 
 ### Not Supported
 
 For a complete list of issues, see ["Top priorities" in the Release Planning and Backlog project tab](https://github.com/video-dev/hls.js/projects/6). Codec support is dependent on the runtime environment (for example, not all browsers on the same OS support HEVC).
 
-- `#EXT-X-DATERANGE` in "metadata" TextTracks [#2218](https://github.com/video-dev/hls.js/issues/2218)
 - `#EXT-X-GAP` filling [#2940](https://github.com/video-dev/hls.js/issues/2940)
 - `#EXT-X-I-FRAME-STREAM-INF` I-frame Media Playlist files
 - `SAMPLE-AES` with fmp4, aac, mp3, vtt... segments (MPEG-2 TS only)

diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md
@@ -282,6 +282,31 @@ export interface CuesParsedData {
     type: 'captions' | 'subtitles';
 }
 
+// Warning: (ae-missing-release-tag) "DateRange" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export class DateRange {
+    constructor(dateRangeAttr: AttrList, dateRangeWithSameId?: DateRange);
+    // (undocumented)
+    attr: AttrList;
+    // (undocumented)
+    get class(): string;
+    // (undocumented)
+    get duration(): number | null;
+    // (undocumented)
+    get endDate(): Date | null;
+    // (undocumented)
+    get endOnNext(): boolean;
+    // (undocumented)
+    get id(): string;
+    // (undocumented)
+    get isValid(): boolean;
+    // (undocumented)
+    get plannedDuration(): number | null;
+    // (undocumented)
+    get startDate(): Date;
+}
+
 // Warning: (ae-missing-release-tag) "DRMSystemOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -918,6 +943,7 @@ class Hls implements HlsEventEmitter {
     on<E extends keyof HlsListeners, Context = undefined>(event: E, listener: HlsListeners[E], context?: Context): void;
     // (undocumented)
     once<E extends keyof HlsListeners, Context = undefined>(event: E, listener: HlsListeners[E], context?: Context): void;
+    get playingDate(): Date | null;
     recoverMediaError(): void;
     // (undocumented)
     removeAllListeners<E extends keyof HlsListeners>(event?: E | undefined): void;
@@ -1342,6 +1368,8 @@ export class LevelDetails {
     // (undocumented)
     canSkipUntil: number;
     // (undocumented)
+    dateRanges: Record<string, DateRange>;
+    // (undocumented)
     deltaUpdateFailed?: boolean;
     // (undocumented)
     get drift(): number;
@@ -1845,6 +1873,20 @@ export interface MetadataSample {
     len?: number;
     // (undocumented)
     pts: number;
+    // (undocumented)
+    type: MetadataSchema;
+}
+
+// Warning: (ae-missing-release-tag) "MetadataSchema" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export enum MetadataSchema {
+    // (undocumented)
+    audioId3 = "org.id3",
+    // (undocumented)
+    dateRange = "com.apple.quicktime.HLS",
+    // (undocumented)
+    emsg = "https://aomedia.org/emsg/ID3"
 }
 
 // Warning: (ae-missing-release-tag) "MP4RemuxerConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)

diff --git a/docs/API.md b/docs/API.md
@@ -130,6 +130,8 @@
   - [`hls.latency`](#hlslatency)
   - [`hls.maxLatency`](#hlsmaxlatency)
   - [`hls.targetLatency`](#hlstargetlatency)
+  - [`hls.drift`](#hlsdrift)
+  - [`hls.playingDate`](#hlsplayingdate)
 - [Runtime Events](#runtime-events)
 - [Loader Composition](#loader-composition)
 - [Errors](#errors)
@@ -1386,6 +1388,10 @@ get : target distance from the edge as calculated by the latency controller
 
 get : the rate at which the edge of the current live playlist is advancing or 1 if there is none
 
+### `hls.playingDate`
+
+get: the datetime value relative to media.currentTime for the active level Program Date Time if present
+
 ## Runtime Events
 
 hls.js fires a bunch of events, that could be registered and unregistered as below:

diff --git a/src/controller/id3-track-controller.ts b/src/controller/id3-track-controller.ts
@@ -5,9 +5,12 @@ import {
   removeCuesInRange,
 } from '../utils/texttrack-utils';
 import * as ID3 from '../demux/id3';
+import { DateRange, DateRangeAttribute } from '../loader/date-range';
+import { MetadataSchema } from '../types/demuxer';
 import type {
   BufferFlushingData,
   FragParsingMetadataData,
+  LevelUpdatedData,
   MediaAttachedData,
 } from '../types/events';
 import type { ComponentAPI } from '../types/component-api';
@@ -19,12 +22,38 @@ declare global {
   }
 }
 
+type Cue = VTTCue | TextTrackCue;
+
 const MIN_CUE_DURATION = 0.25;
 
+function getCueClass() {
+  // Attempt to recreate Safari functionality by creating
+  // WebKitDataCue objects when available and store the decoded
+  // ID3 data in the value property of the cue
+  return (self.WebKitDataCue || self.VTTCue || self.TextTrackCue) as any;
+}
+
+function dateRangeDateToTimelineSeconds(date: Date, offset: number): number {
+  return date.getTime() / 1000 - offset;
+}
+
+function hexToArrayBuffer(str): ArrayBuffer {
+  return Uint8Array.from(
+    str
+      .replace(/^0x/, '')
+      .replace(/([\da-fA-F]{2}) ?/g, '0x$1 ')
+      .replace(/ +$/, '')
+      .split(' ')
+  ).buffer;
+}
 class ID3TrackController implements ComponentAPI {
   private hls: Hls;
   private id3Track: TextTrack | null = null;
   private media: HTMLMediaElement | null = null;
+  private dateRangeCuesAppended: Record<
+    string,
+    { cues: Record<string, Cue>; dateRange: DateRange; durationKnown: boolean }
+  > = {};
 
   constructor(hls) {
     this.hls = hls;
@@ -39,16 +68,20 @@ class ID3TrackController implements ComponentAPI {
     const { hls } = this;
     hls.on(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
     hls.on(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
+    hls.on(Events.MANIFEST_LOADING, this.onManifestLoading, this);
     hls.on(Events.FRAG_PARSING_METADATA, this.onFragParsingMetadata, this);
     hls.on(Events.BUFFER_FLUSHING, this.onBufferFlushing, this);
+    hls.on(Events.LEVEL_UPDATED, this.onLevelUpdated, this);
   }
 
   private _unregisterListeners() {
     const { hls } = this;
     hls.off(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
     hls.off(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
+    hls.off(Events.MANIFEST_LOADING, this.onManifestLoading, this);
     hls.off(Events.FRAG_PARSING_METADATA, this.onFragParsingMetadata, this);
     hls.off(Events.BUFFER_FLUSHING, this.onBufferFlushing, this);
+    hls.off(Events.LEVEL_UPDATED, this.onLevelUpdated, this);
   }
 
   // Add ID3 metatadata text track.
@@ -66,6 +99,17 @@ class ID3TrackController implements ComponentAPI {
     clearCurrentCues(this.id3Track);
     this.id3Track = null;
     this.media = null;
+    this.dateRangeCuesAppended = {};
+  }
+
+  private onManifestLoading() {
+    this.dateRangeCuesAppended = {};
+  }
+
+  createTrack(media: HTMLMediaElement): TextTrack {
+    const track = this.getID3Track(media.textTracks) as TextTrack;
+    track.mode = 'hidden';
+    return track;
   }
 
   getID3Track(textTracks: TextTrackList): TextTrack | void {
@@ -96,17 +140,12 @@ class ID3TrackController implements ComponentAPI {
 
     // create track dynamically
     if (!this.id3Track) {
-      this.id3Track = this.getID3Track(this.media.textTracks) as TextTrack;
-      this.id3Track.mode = 'hidden';
+      this.id3Track = this.createTrack(this.media);
     }
 
     // VTTCue end time must be finite, so use playlist edge or fragment end until next fragment with same frame type is found
     const maxCueTime = details.edge || fragment.end;
-
-    // Attempt to recreate Safari functionality by creating
-    // WebKitDataCue objects when available and store the decoded
-    // ID3 data in the value property of the cue
-    const Cue = (self.WebKitDataCue || self.VTTCue || self.TextTrackCue) as any;
+    const Cue = getCueClass();
     let updateCueRanges = false;
     const frameTypesAdded: Record<string, number | null> = {};
 
@@ -127,6 +166,10 @@ class ID3TrackController implements ComponentAPI {
           if (!ID3.isTimeStampFrame(frame)) {
             const cue = new Cue(startTime, endTime, '');
             cue.value = frame;
+            const type = samples[i].type;
+            if (type) {
+              cue.type = type;
+            }
             this.id3Track.addCue(cue);
             frameTypesAdded[frame.key] = null;
             updateCueRanges = true;
@@ -161,12 +204,133 @@ class ID3TrackController implements ComponentAPI {
     event: Events.BUFFER_FLUSHING,
     { startOffset, endOffset, type }: BufferFlushingData
   ) {
-    if (!type || type === 'audio') {
-      // id3 cues come from parsed audio only remove cues when audio buffer is cleared
-      const { id3Track } = this;
-      if (id3Track) {
-        removeCuesInRange(id3Track, startOffset, endOffset);
+    const { id3Track } = this;
+    if (id3Track) {
+      let predicate;
+      if (type === 'audio') {
+        predicate = (cue) => (cue as any).type === MetadataSchema.audioId3;
+      } else if (type === 'video') {
+        predicate = (cue) => (cue as any).type === MetadataSchema.emsg;
+      } else {
+        predicate = (cue) =>
+          (cue as any).type === MetadataSchema.audioId3 ||
+          (cue as any).type === MetadataSchema.emsg;
+      }
+      removeCuesInRange(id3Track, startOffset, endOffset, predicate);
+    }
+  }
+
+  onLevelUpdated(event: Events.LEVEL_UPDATED, { details }: LevelUpdatedData) {
+    if (!this.media || !details.hasProgramDateTime) {
+      return;
+    }
+    const { dateRangeCuesAppended, id3Track } = this;
+    const { dateRanges } = details;
+    const ids = Object.keys(dateRanges);
+    // Remove cues from track not found in details.dateRanges
+    if (id3Track) {
+      const idsToRemove = Object.keys(dateRangeCuesAppended).filter(
+        (id) => !ids.includes(id)
+      );
+      for (let i = idsToRemove.length; i--; ) {
+        const id = idsToRemove[i];
+        Object.keys(dateRangeCuesAppended[id].cues).forEach((key) => {
+          id3Track.removeCue(dateRangeCuesAppended[id].cues[key]);
+        });
+        delete dateRangeCuesAppended[id];
+      }
+    }
+    // Exit if the playlist does not have Date Ranges or does not have Program Date Time
+    const lastFragment = details.fragments[details.fragments.length - 1];
+    if (ids.length === 0 || !Number.isFinite(lastFragment?.programDateTime)) {
+      return;
+    }
+
+    if (!this.id3Track) {
+      this.id3Track = this.createTrack(this.media);
+    }
+
+    const dateTimeOffset =
+      (lastFragment.programDateTime as number) / 1000 - lastFragment.start;
+    const maxCueTime = details.edge || lastFragment.end;
+    const Cue = getCueClass();
+
+    for (let i = 0; i < ids.length; i++) {
+      const id = ids[i];
+      const dateRange = dateRanges[id];
+      const appendedDateRangeCues = dateRangeCuesAppended[id];
+      const cues = appendedDateRangeCues?.cues || {};
+      let durationKnown = appendedDateRangeCues?.durationKnown || false;
+      const startTime = dateRangeDateToTimelineSeconds(
+        dateRange.startDate,
+        dateTimeOffset
+      );
+      let endTime = maxCueTime;
+      const endDate = dateRange.endDate;
+      if (endDate) {
+        endTime = dateRangeDateToTimelineSeconds(endDate, dateTimeOffset);
+        durationKnown = true;
+      } else if (dateRange.endOnNext && !durationKnown) {
+        const nextDateRangeWithSameClass = ids
+          .reduce((filterMapArray, id) => {
+            const candidate = dateRanges[id];
+            if (
+              candidate.class === dateRange.class &&
+              candidate.id !== id &&
+              candidate.startDate > dateRange.startDate
+            ) {
+              filterMapArray.push(candidate);
+            }
+            return filterMapArray;
+          }, [] as DateRange[])
+          .sort((a, b) => a.startDate.getTime() - b.startDate.getTime())[0];
+        if (nextDateRangeWithSameClass) {
+          endTime = dateRangeDateToTimelineSeconds(
+            nextDateRangeWithSameClass.startDate,
+            dateTimeOffset
+          );
+          durationKnown = true;
+        }
+      }
+
+      const attributes = Object.keys(dateRange.attr);
+      for (let j = 0; j < attributes.length; j++) {
+        const key = attributes[j];
+        if (
+          key === DateRangeAttribute.ID ||
+          key === DateRangeAttribute.CLASS ||
+          key === DateRangeAttribute.START_DATE ||
+          key === DateRangeAttribute.DURATION ||
+          key === DateRangeAttribute.END_DATE ||
+          key === DateRangeAttribute.END_ON_NEXT
+        ) {
+          continue;
+        }
+        let cue = cues[key] as any;
+        if (cue) {
+          if (durationKnown && !appendedDateRangeCues.durationKnown) {
+            cue.endTime = endTime;
+          }
+        } else {
+          let data = dateRange.attr[key];
+          cue = new Cue(startTime, endTime, '');
+          if (
+            key === DateRangeAttribute.SCTE35_OUT ||
+            key === DateRangeAttribute.SCTE35_IN
+          ) {
+            data = hexToArrayBuffer(data);
+          }
+          cue.value = { key, data };
+          cue.type = MetadataSchema.dateRange;
+          this.id3Track.addCue(cue);
+          cues[key] = cue;
+        }
       }
+      dateRangeCuesAppended[id] = {
+        cues,
+        dateRange,
+        durationKnown,
+      };
     }
   }
 }