Improve bandwidth estimation and adaptive switching with smaller segm…

…ents and higher TTFB

Fixes video-dev#3578 (special thanks to @Oleksandr0xB for submitting video-dev#4283)
Fixes video-dev#3563 and Closes video-dev#3595 (special thanks to @kanongil)

api-extractor/report/hls.js.api.md

@@ -219,7 +219,7 @@ export interface BufferFlushingData {
     type: SourceBufferName | null;
 }
 
-// Warning: (ae-missing-release-tag) "BufferInfo" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+// Warning: (ae-missing-release-tag) "BufferInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
 export type BufferInfo = {
@@ -229,7 +229,7 @@ export type BufferInfo = {
     nextStart?: number;
 };
 
-// Warning: (ae-missing-release-tag) "CapLevelControllerConfig" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+// Warning: (ae-missing-release-tag) "CapLevelControllerConfig" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)
 export type CapLevelControllerConfig = {

src/controller/abr-controller.ts

@@ -156,17 +156,13 @@ class AbrController implements ComponentAPI {
     const expectedLen =
       stats.total ||
       Math.max(stats.loaded, Math.round((duration * level.maxBitrate) / 8));
-    let timeStreaming = timeLoading - ttfb;
-    if (timeStreaming < 1 && loadedFirstByte) {
-      timeStreaming = Math.min(timeLoading, (stats.loaded * 8) / bwEstimate);
-    }
     const loadRate = loadedFirstByte
-      ? (stats.loaded * 1000) / timeStreaming
+      ? (stats.loaded * 1000) / (timeLoading - ttfb)
       : 0;
     // fragLoadDelay is an estimate of the time (in seconds) it will take to buffer the remainder of the fragment
     const fragLoadedDelay = loadRate
       ? (expectedLen - stats.loaded) / loadRate
-      : (expectedLen * 8) / bwEstimate + ttfbEstimate / 1000;
+      : expectedLen / bwEstimate + ttfbEstimate;
     // Only downswitch if the time to finish loading the current fragment is greater than the amount of buffer left
     if (fragLoadedDelay <= bufferStarvationDelay) {
       return;
@@ -183,9 +179,10 @@ class AbrController implements ComponentAPI {
       // compute time to load next fragment at lower level
       // 8 = bits per byte (bps/Bps)
       const levelNextBitrate = levels[nextLoadLevel].maxBitrate;
-      const bwe = loadRate ? loadRate * 8 : bwEstimate;
-      fragLevelNextLoadedDelay =
-        (duration * levelNextBitrate) / bwe + ttfbEstimate / 1000;
+      fragLevelNextLoadedDelay = loadRate
+        ? (duration * levelNextBitrate) / (8 * 0.8 * loadRate)
+        : duration / bwEstimate + ttfbEstimate;
+
       if (fragLevelNextLoadedDelay < bufferStarvationDelay) {
         break;
       }
@@ -195,7 +192,6 @@ class AbrController implements ComponentAPI {
     if (fragLevelNextLoadedDelay >= fragLoadedDelay) {
       return;
     }
-
     // if estimated load time of new segment is completely unreasonable, ignore and do not emergency switch down
     if (fragLevelNextLoadedDelay > duration * 10) {
       return;
@@ -240,23 +236,22 @@ class AbrController implements ComponentAPI {
     event: Events.FRAG_LOADED,
     { frag, part }: FragLoadedData
   ) {
-    const stats = part ? part.stats : frag.stats;
-    if (frag.type === PlaylistLevelType.MAIN) {
-      this.bwEstimator.sampleTTFB(stats.loading.first - stats.loading.start);
-    }
     if (this.ignoreFragment(frag)) {
       return;
     }
+    const stats = part ? part.stats : frag.stats;
+    const duration = part ? part.duration : frag.duration;
     // stop monitoring bw once frag loaded
     this.clearTimer();
     // store level id after successful fragment load
     this.lastLoadedFragLevel = frag.level;
     // reset forced auto level value so that next level will be selected
     this._nextAutoLevel = -1;
 
+    this.bwEstimator.sampleTTFB(stats.loading.first - stats.loading.start);
+
     // compute level average bitrate
     if (this.hls.config.abrMaxWithRealBitrate) {
-      const duration = part ? part.duration : frag.duration;
       const level = this.hls.levels[frag.level];
       const loadedBytes =
         (level.loaded ? level.loaded.bytes : 0) + stats.loaded;

src/controller/audio-stream-controller.ts

@@ -363,7 +363,11 @@ class AudioStreamController
       return;
     }
 
-    this.loadFragment(frag, levelInfo, targetBufferTime);
+    if (frag.decryptdata?.keyFormat === 'identity' && !frag.decryptdata?.key) {
+      this.loadKey(frag, trackDetails);
+    } else {
+      this.loadFragment(frag, levelInfo, targetBufferTime);
+    }
   }
 
   protected getMaxBufferLength(mainBufferLength?: number): number {
@@ -823,7 +827,7 @@ class AudioStreamController
       fragState === FragmentState.PARTIAL
     ) {
       if (frag.sn === 'initSegment') {
-        this._loadInitSegment(frag, track);
+        this._loadInitSegment(frag);
       } else if (
         track.details?.live &&
         !Number.isFinite(this.initPTS[frag.cc])

src/controller/base-stream-controller.ts

@@ -266,11 +266,19 @@ export default class BaseStreamController
     this.startPosition = this.lastCurrentTime = 0;
   }
 
-  protected onLevelSwitching(
-    event: Events.LEVEL_SWITCHING,
-    data: LevelSwitchingData
-  ): void {
-    this.fragLoadError = 0;
+  onKeyLoaded(event: Events.KEY_LOADED, data: KeyLoadedData) {
+    if (
+      this.state !== State.KEY_LOADING ||
+      data.frag !== this.fragCurrent ||
+      !this.levels
+    ) {
+      return;
+    }
+    this.state = State.IDLE;
+    const level = this.levels[data.frag.level];
+    if (level.details) {
+      this.loadFragment(data.frag, level, data.frag.start);
+    }
   }
 
   protected onHandlerDestroying() {
@@ -535,7 +543,7 @@ export default class BaseStreamController
 
   protected _doFragLoad(
     frag: Fragment,
-    level: Level,
+    level?: Level,
     targetBufferTime: number | null = null,
     progressCallback?: FragmentLoadProgressCallback
   ): Promise<PartsLoadedData | FragLoadedData | null> {
@@ -563,6 +571,7 @@ export default class BaseStreamController
     }
 
     targetBufferTime = Math.max(frag.start, targetBufferTime || 0);
+    const details = level?.details;
     if (this.config.lowLatencyMode && details) {
       const partList = details.partList;
       if (partList && progressCallback) {

src/controller/stream-controller.ts

@@ -320,7 +320,13 @@ export default class StreamController
       frag = frag.initSegment;
     }
 
-    this.loadFragment(frag, levelInfo, targetBufferTime);
+    // We want to load the key if we're dealing with an identity key, because we will decrypt
+    // this content using the key we fetch. Other keys will be handled by the DRM CDM via EME.
+    if (frag.decryptdata?.keyFormat === 'identity' && !frag.decryptdata?.key) {
+      this.loadKey(frag, levelDetails);
+    } else {
+      this.loadFragment(frag, levelInfo, targetBufferTime);
+    }
   }
 
   protected loadFragment(

src/controller/subtitle-stream-controller.ts

@@ -413,11 +413,7 @@ export class SubtitleStreamController
     targetBufferTime: number
   ) {
     this.fragCurrent = frag;
-    if (frag.sn === 'initSegment') {
-      this._loadInitSegment(frag, level);
-    } else {
-      super.loadFragment(frag, level, targetBufferTime);
-    }
+    super.loadFragment(frag, level, targetBufferTime);
   }
 
   get mediaBufferTimeRanges(): Bufferable {