Content Steering (#5191)

* Load Steering Manifest * Make UA check lazy for legacy unsupported MP3 in MP4 container check * Add Redundant Streams tests * Add FRAME-RATE to Redundant Stream group key and level sorting Optimize MANIFEST_PARSED level filtering Optimize Media Playlist check * Do not group Variant levels with PATHWAY-ID into Fallback Stream URLs (unless building without Content Steering support) Store and return seperate LevelAttributes per Fallback Stream URL Align Level audio and text group ids with Fallback Stream URLs (Level._urlId) * Content Steering Pathway grouping and selection #5074 * Support removeLevel and improve capLevelController with level index changes * Pathway Clone Variants * Pathway Clone Rendition Groups * Fix audio track alignment with timescales that differ and smooth switch when attributes do not change * Always emit LEVEL_SWITCHING on Pathway change to ensure audio and subtitle tracks update as well * Fix unregister listeners in fps and latency controller * Unit test clean up * Update README for Content Steering
video-dev · Feb 18, 2023 · 766ec37 · 766ec37
1 parent a741b6e
commit 766ec37
Show file tree

Hide file tree

Showing 56 changed files with 3,064 additions and 745 deletions.
diff --git a/README.md b/README.md
@@ -2,6 +2,7 @@
 [![npm](https://img.shields.io/npm/v/hls.js/canary.svg?style=flat)](https://www.npmjs.com/package/hls.js/v/canary)
 [![](https://data.jsdelivr.com/v1/package/npm/hls.js/badge?style=rounded)](https://www.jsdelivr.com/package/npm/hls.js)
 [![Sauce Test Status](https://saucelabs.com/buildstatus/robwalch)](https://app.saucelabs.com/u/robwalch)
+[![jsDeliver](https://data.jsdelivr.com/v1/package/npm/hls.js/badge)](https://www.jsdelivr.com/package/npm/hls.js)
 
 [comment]: <> ([![Sauce Test Status]&#40;https://saucelabs.com/browser-matrix/robwalch.svg&#41;]&#40;https://saucelabs.com/u/robwalch&#41;)
 
@@ -46,6 +47,7 @@ HLS.js is written in [ECMAScript6] (`*.js`) and [TypeScript] (`*.ts`) (strongly
 - SAMPLE-AES decryption (only supported if using MPEG-2 TS container)
 - Encrypted media extensions (EME) support for DRM (digital rights management)
   - FairPlay, PlayReady, Widevine CDMs with fmp4 segments
+- Level capping based on HTMLMediaElement resolution, dropped-frames, and HDCP-Level
 - CEA-608/708 captions
 - WebVTT subtitles
 - Alternate Audio Track Rendition (Master Playlist with Alternative Audio) for VoD and Live playlists
@@ -71,18 +73,20 @@ HLS.js is written in [ECMAScript6] (`*.js`) and [TypeScript] (`*.ts`) (strongly
 
 For details on the HLS format and these tags' meanings, see https://datatracker.ietf.org/doc/html/draft-pantos-hls-rfc8216bis
 
-#### Multi-Variant Playlist tags
+#### Multivariant Playlist tags
 
 - `#EXT-X-STREAM-INF:<attribute-list>`
   `<URI>`
 - `#EXT-X-MEDIA:<attribute-list>`
 - `#EXT-X-SESSION-DATA:<attribute-list>`
+- `#EXT-X-SESSION-KEY:<attribute-list>` EME Key-System selection and preloading
 - `#EXT-X-START:TIME-OFFSET=<n>`
-- `#EXT-X-DEFINE` Variable Substitution
+- `#EXT-X-CONTENT-STEERING:<attribute-list>` Content Steering
+- `#EXT-X-DEFINE:<attribute-list>` Variable Substitution
 
 The following properties are added to their respective variants' attribute list but are not implemented in their selection and playback.
 
-- `VIDEO-RANGE` and `HDCP-LEVEL` (See [#2489](https://github.com/video-dev/hls.js/issues/2489))
+- `VIDEO-RANGE` (See [#2489](https://github.com/video-dev/hls.js/issues/2489))
 
 #### Media Playlist tags
 
@@ -105,7 +109,7 @@ The following properties are added to their respective variants' attribute list
 - `#EXT-X-SKIP:<attribute-list>` Delta Playlists
 - `#EXT-X-RENDITION-REPORT:<attribute-list>`
 - `#EXT-X-DATERANGE:<attribute-list>` Metadata
-- `#EXT-X-DEFINE` Variable Substitution
+- `#EXT-X-DEFINE:<attribute-list>` Variable Import and Substitution
 
 The following tags are added to their respective fragment's attribute list but are not implemented in streaming and playback.
 
@@ -114,8 +118,6 @@ The following tags are added to their respective fragment's attribute list but a
 
 Parsed but missing feature support
 
-- `#EXT-X-CONTENT-STEERING:<attribute-list>` (See [#3988](https://github.com/video-dev/hls.js/issues/3988))
-  - #3988
 - `#EXT-X-PRELOAD-HINT:<attribute-list>` (See [#5074](https://github.com/video-dev/hls.js/issues/3988))
   - #5074
 

diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md
@@ -72,25 +72,13 @@ export interface AudioTracksUpdatedData {
 // Warning: (ae-missing-release-tag) "AudioTrackSwitchedData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
-export interface AudioTrackSwitchedData {
-    // (undocumented)
-    id: number;
+export interface AudioTrackSwitchedData extends MediaPlaylist {
 }
 
 // Warning: (ae-missing-release-tag) "AudioTrackSwitchingData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
-export interface AudioTrackSwitchingData {
-    // (undocumented)
-    groupId: string;
-    // (undocumented)
-    id: number;
-    // (undocumented)
-    name: string;
-    // (undocumented)
-    type: MediaPlaylistType | 'main';
-    // (undocumented)
-    url: string;
+export interface AudioTrackSwitchingData extends MediaPlaylist {
 }
 
 // Warning: (ae-missing-release-tag) "BackBufferData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -270,6 +258,14 @@ export type CMCDControllerConfig = {
     useHeaders?: boolean;
 };
 
+// Warning: (ae-missing-release-tag) "ContentSteeringOptions" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export type ContentSteeringOptions = {
+    uri: string;
+    pathwayId: string;
+};
+
 // Warning: (ae-missing-release-tag) "CuesInterface" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -1055,6 +1051,7 @@ export type HlsConfig = {
     emeController?: typeof EMEController;
     cmcd?: CMCDControllerConfig;
     cmcdController?: typeof CMCDController;
+    contentSteeringController?: typeof ContentSteeringController;
     abrController: typeof AbrController;
     bufferController: typeof BufferController;
     capLevelController: typeof CapLevelController;
@@ -1321,11 +1318,15 @@ export type LatencyControllerConfig = {
 export class Level {
     constructor(data: LevelParsed);
     // (undocumented)
-    readonly attrs: LevelAttributes;
+    addFallback(data: LevelParsed): void;
+    // (undocumented)
+    get attrs(): LevelAttributes;
+    // (undocumented)
+    readonly _attrs: LevelAttributes[];
     // (undocumented)
     readonly audioCodec: string | undefined;
     // (undocumented)
-    audioGroupIds?: string[];
+    audioGroupIds?: (string | undefined)[];
     // (undocumented)
     readonly bitrate: number;
     // (undocumented)
@@ -1350,9 +1351,11 @@ export class Level {
     // (undocumented)
     readonly name: string | undefined;
     // (undocumented)
+    get pathwayId(): string;
+    // (undocumented)
     realBitrate: number;
     // (undocumented)
-    textGroupIds?: string[];
+    textGroupIds?: (string | undefined)[];
     // (undocumented)
     readonly unknownCodecs: string[] | undefined;
     // (undocumented)
@@ -1381,43 +1384,29 @@ export interface LevelAttributes extends AttrList {
     // (undocumented)
     'FRAME-RATE'?: string;
     // (undocumented)
-    'HDCP-LEVEL'?: string;
+    'HDCP-LEVEL'?: 'TYPE-0' | 'TYPE-1' | 'NONE';
     // (undocumented)
     'PATHWAY-ID'?: string;
     // (undocumented)
-    'PROGRAM-ID'?: string;
+    'STABLE-VARIANT-ID'?: string;
     // (undocumented)
-    'VIDEO-RANGE'?: string;
+    'SUPPLEMENTAL-CODECS'?: string;
     // (undocumented)
-    AUDIO?: string;
+    'VIDEO-RANGE'?: 'SDR' | 'HLG' | 'PQ';
     // (undocumented)
-    AUTOSELECT?: string;
+    AUDIO?: string;
     // (undocumented)
     BANDWIDTH?: string;
     // (undocumented)
-    BYTERANGE?: string;
-    // (undocumented)
-    CHARACTERISTICS?: string;
-    // (undocumented)
     CODECS?: string;
     // (undocumented)
-    DEFAULT?: string;
-    // (undocumented)
-    FORCED?: string;
-    // (undocumented)
-    LANGUAGE?: string;
-    // (undocumented)
-    NAME?: string;
-    // (undocumented)
     RESOLUTION?: string;
     // (undocumented)
     SCORE?: string;
     // (undocumented)
     SUBTITLES?: string;
     // (undocumented)
-    TYPE?: string;
-    // (undocumented)
-    URI?: string;
+    VIDEO?: string;
 }
 
 // Warning: (ae-missing-release-tag) "LevelControllerConfig" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -1714,6 +1703,8 @@ export interface Loader<T extends LoaderContext> {
     destroy(): void;
     getCacheAge?: () => number | null;
     // (undocumented)
+    getResponseHeader?: (name: string) => string | null;
+    // (undocumented)
     load(context: LoaderContext, config: LoaderConfiguration, callbacks: LoaderCallbacks<T>): void;
     // (undocumented)
     stats: LoaderStats;
@@ -1740,7 +1731,7 @@ export interface LoaderCallbacks<T extends LoaderContext> {
 // @public (undocumented)
 export interface LoaderConfiguration {
     // (undocumented)
-    highWaterMark: number;
+    highWaterMark?: number;
     // (undocumented)
     maxRetry: number;
     // (undocumented)
@@ -1802,7 +1793,7 @@ export type LoaderOnTimeout<T extends LoaderContext> = (stats: LoaderStats, cont
 // @public (undocumented)
 export interface LoaderResponse {
     // (undocumented)
-    data: string | ArrayBuffer;
+    data: string | ArrayBuffer | Object;
     // (undocumented)
     url: string;
 }
@@ -1869,7 +1860,7 @@ export interface ManifestLoadedData {
     // (undocumented)
     captions?: MediaPlaylist[];
     // (undocumented)
-    contentSteering: Object | null;
+    contentSteering: ContentSteeringOptions | null;
     // (undocumented)
     levels: LevelParsed[];
     // (undocumented)
@@ -1940,6 +1931,40 @@ export interface MediaAttachingData {
     media: HTMLMediaElement;
 }
 
+// Warning: (ae-missing-release-tag) "MediaAttributes" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public (undocumented)
+export interface MediaAttributes extends AttrList {
+    // (undocumented)
+    'ASSOC-LANGUAGE'?: string;
+    // (undocumented)
+    'GROUP-ID': string;
+    // (undocumented)
+    'INSTREAM-ID'?: string;
+    // (undocumented)
+    'PATHWAY-ID'?: string;
+    // (undocumented)
+    'STABLE-RENDITION-ID'?: string;
+    // (undocumented)
+    AUTOSELECT?: 'YES' | 'NO';
+    // (undocumented)
+    CHANNELS?: string;
+    // (undocumented)
+    CHARACTERISTICS?: string;
+    // (undocumented)
+    DEFAULT?: 'YES' | 'NO';
+    // (undocumented)
+    FORCED?: 'YES' | 'NO';
+    // (undocumented)
+    LANGUAGE?: string;
+    // (undocumented)
+    NAME: string;
+    // (undocumented)
+    TYPE?: 'AUDIO' | 'VIDEO' | 'SUBTITLES' | 'CLOSED-CAPTIONS';
+    // (undocumented)
+    URI?: string;
+}
+
 // Warning: (ae-missing-release-tag) "MediaKeyFunc" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
@@ -1948,15 +1973,17 @@ export type MediaKeyFunc = (keySystem: KeySystems, supportedConfigurations: Medi
 // Warning: (ae-missing-release-tag) "MediaPlaylist" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
-export interface MediaPlaylist extends LevelParsed {
+export interface MediaPlaylist extends Omit<LevelParsed, 'attrs'> {
+    // (undocumented)
+    attrs: MediaAttributes;
     // (undocumented)
     autoselect: boolean;
     // (undocumented)
     default: boolean;
     // (undocumented)
     forced: boolean;
     // (undocumented)
-    groupId?: string;
+    groupId: string;
     // (undocumented)
     id: number;
     // (undocumented)
@@ -2334,20 +2361,21 @@ export type VariableMap = Record<string, string>;
 
 // Warnings were encountered during analysis:
 //
-// src/config.ts:93:3 - (ae-forgotten-export) The symbol "MediaKeySessionContext" needs to be exported by the entry point hls.d.ts
-// src/config.ts:108:3 - (ae-forgotten-export) The symbol "DRMSystemsConfiguration" needs to be exported by the entry point hls.d.ts
-// src/config.ts:211:3 - (ae-forgotten-export) The symbol "ILogger" needs to be exported by the entry point hls.d.ts
-// src/config.ts:221:3 - (ae-forgotten-export) The symbol "AudioStreamController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:222:3 - (ae-forgotten-export) The symbol "AudioTrackController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:224:3 - (ae-forgotten-export) The symbol "SubtitleStreamController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:225:3 - (ae-forgotten-export) The symbol "SubtitleTrackController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:226:3 - (ae-forgotten-export) The symbol "TimelineController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:228:3 - (ae-forgotten-export) The symbol "EMEController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:231:3 - (ae-forgotten-export) The symbol "CMCDController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:233:3 - (ae-forgotten-export) The symbol "AbrController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:234:3 - (ae-forgotten-export) The symbol "BufferController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:235:3 - (ae-forgotten-export) The symbol "CapLevelController" needs to be exported by the entry point hls.d.ts
-// src/config.ts:236:3 - (ae-forgotten-export) The symbol "FPSController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:97:3 - (ae-forgotten-export) The symbol "MediaKeySessionContext" needs to be exported by the entry point hls.d.ts
+// src/config.ts:112:3 - (ae-forgotten-export) The symbol "DRMSystemsConfiguration" needs to be exported by the entry point hls.d.ts
+// src/config.ts:215:3 - (ae-forgotten-export) The symbol "ILogger" needs to be exported by the entry point hls.d.ts
+// src/config.ts:225:3 - (ae-forgotten-export) The symbol "AudioStreamController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:226:3 - (ae-forgotten-export) The symbol "AudioTrackController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:228:3 - (ae-forgotten-export) The symbol "SubtitleStreamController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:229:3 - (ae-forgotten-export) The symbol "SubtitleTrackController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:230:3 - (ae-forgotten-export) The symbol "TimelineController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:232:3 - (ae-forgotten-export) The symbol "EMEController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:235:3 - (ae-forgotten-export) The symbol "CMCDController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:237:3 - (ae-forgotten-export) The symbol "ContentSteeringController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:239:3 - (ae-forgotten-export) The symbol "AbrController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:240:3 - (ae-forgotten-export) The symbol "BufferController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:241:3 - (ae-forgotten-export) The symbol "CapLevelController" needs to be exported by the entry point hls.d.ts
+// src/config.ts:242:3 - (ae-forgotten-export) The symbol "FPSController" needs to be exported by the entry point hls.d.ts
 
 // (No @packageDocumentation comment for this package)
 

diff --git a/docs/API.md b/docs/API.md
@@ -1160,8 +1160,6 @@ parameter should be a float greater than [abrEwmaFastVoD](#abrewmafastvod)
 
 Default bandwidth estimate in bits/s prior to collecting fragment bandwidth samples.
 
-parameter should be a float
-
 ### `abrBandWidthFactor`
 
 (default: `0.95`)

diff --git a/src/config.ts b/src/config.ts
@@ -11,6 +11,7 @@ import EMEController, {
   MediaKeySessionContext,
 } from './controller/eme-controller';
 import CMCDController from './controller/cmcd-controller';
+import ContentSteeringController from './controller/content-steering-controller';
 import XhrLoader from './utils/xhr-loader';
 import FetchLoader, { fetchSupported } from './utils/fetch-loader';
 import Cues from './utils/cues';
@@ -32,6 +33,9 @@ export type ABRControllerConfig = {
   abrEwmaSlowLive: number;
   abrEwmaFastVoD: number;
   abrEwmaSlowVoD: number;
+  /**
+   * Default bandwidth estimate in bits/s prior to collecting fragment bandwidth samples
+   */
   abrEwmaDefaultEstimate: number;
   abrBandWidthFactor: number;
   abrBandWidthUpFactor: number;
@@ -229,6 +233,8 @@ export type HlsConfig = {
   // CMCD
   cmcd?: CMCDControllerConfig;
   cmcdController?: typeof CMCDController;
+  // Content Steering
+  contentSteeringController?: typeof ContentSteeringController;
 
   abrController: typeof AbrController;
   bufferController: typeof BufferController;
@@ -357,6 +363,9 @@ export const hlsDefaultConfig: HlsConfig = {
   audioTrackController: __USE_ALT_AUDIO__ ? AudioTrackController : undefined,
   emeController: __USE_EME_DRM__ ? EMEController : undefined,
   cmcdController: __USE_CMCD__ ? CMCDController : undefined,
+  contentSteeringController: __USE_CONTENT_STEERING__
+    ? ContentSteeringController
+    : undefined,
 };
 
 function timelineConfig(): TimelineControllerConfig {

diff --git a/src/controller/abr-controller.ts b/src/controller/abr-controller.ts
@@ -465,15 +465,26 @@ class AbrController implements AbrComponentAPI {
       : fragCurrent
       ? fragCurrent.duration
       : 0;
+    let levelSkippedMin = minAutoLevel;
+    let levelSkippedMax = -1;
     for (let i = maxAutoLevel; i >= minAutoLevel; i--) {
       const levelInfo = levels[i];
 
       if (
         !levelInfo ||
         (currentCodecSet && levelInfo.codecSet !== currentCodecSet)
       ) {
+        if (levelInfo) {
+          levelSkippedMin = Math.min(i, levelSkippedMin);
+          levelSkippedMax = Math.max(i, levelSkippedMax);
+        }
         continue;
       }
+      if (levelSkippedMax !== -1) {
+        logger.trace(
+          `[abr] Skipped level(s) ${levelSkippedMin}-${levelSkippedMax} with CODECS:"${levels[levelSkippedMax].attrs.CODECS}"; not compatible with "${level.attrs.CODECS}"`
+        );
+      }
 
       const levelDetails = levelInfo.details;
       const avgDuration =