abr-controller.ts

import EwmaBandWidthEstimator from '../utils/ewma-bandwidth-estimator';
import { Events } from '../events';
import { BufferHelper } from '../utils/buffer-helper';
import { ErrorDetails } from '../errors';
import { PlaylistLevelType } from '../types/loader';
import { logger } from '../utils/logger';
import type { Bufferable } from '../utils/buffer-helper';
import type { Fragment } from '../loader/fragment';
import type { Part } from '../loader/fragment';
import type { LoaderStats } from '../types/loader';
import type Hls from '../hls';
import type {
  FragLoadingData,
  FragLoadedData,
  FragBufferedData,
  ErrorData,
  LevelLoadedData,
} from '../types/events';
import type { ComponentAPI } from '../types/component-api';

class AbrController implements ComponentAPI {
  protected hls: Hls;
  private lastLoadedFragLevel: number = 0;
  private _nextAutoLevel: number = -1;
  private timer?: number;
  private onCheck: Function = this._abandonRulesCheck.bind(this);
  private fragCurrent: Fragment | null = null;
  private partCurrent: Part | null = null;
  private bitrateTestDelay: number = 0;

  public readonly bwEstimator: EwmaBandWidthEstimator;

  constructor(hls: Hls) {
    this.hls = hls;

    const config = hls.config;
    this.bwEstimator = new EwmaBandWidthEstimator(
      config.abrEwmaSlowVoD,
      config.abrEwmaFastVoD,
      config.abrEwmaDefaultEstimate
    );

    this.registerListeners();
  }

  protected registerListeners() {
    const { hls } = this;
    hls.on(Events.FRAG_LOADING, this.onFragLoading, this);
    hls.on(Events.FRAG_LOADED, this.onFragLoaded, this);
    hls.on(Events.FRAG_BUFFERED, this.onFragBuffered, this);
    hls.on(Events.LEVEL_LOADED, this.onLevelLoaded, this);
    hls.on(Events.ERROR, this.onError, this);
  }

  protected unregisterListeners() {
    const { hls } = this;
    hls.off(Events.FRAG_LOADING, this.onFragLoading, this);
    hls.off(Events.FRAG_LOADED, this.onFragLoaded, this);
    hls.off(Events.FRAG_BUFFERED, this.onFragBuffered, this);
    hls.off(Events.LEVEL_LOADED, this.onLevelLoaded, this);
    hls.off(Events.ERROR, this.onError, this);
  }

  public destroy() {
    this.unregisterListeners();
    this.clearTimer();
  }

  protected onFragLoading(event: Events.FRAG_LOADING, data: FragLoadingData) {
    const frag = data.frag;
    if (frag.type === PlaylistLevelType.MAIN) {
      if (!this.timer) {
        this.fragCurrent = frag;
        this.partCurrent = data.part ?? null;
        this.timer = self.setInterval(this.onCheck, 100);
      }
    }
  }

  protected onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) {
    const config = this.hls.config;
    if (data.details.live) {
      this.bwEstimator.update(config.abrEwmaSlowLive, config.abrEwmaFastLive);
    } else {
      this.bwEstimator.update(config.abrEwmaSlowVoD, config.abrEwmaFastVoD);
    }
  }

  /*
      This method monitors the download rate of the current fragment, and will downswitch if that fragment will not load
      quickly enough to prevent underbuffering
    */
  private _abandonRulesCheck() {
    const { fragCurrent: frag, partCurrent: part, hls } = this;
    const { autoLevelEnabled, config, media } = hls;
    if (!frag || !media) {
      return;
    }

    const stats: LoaderStats = part ? part.stats : frag.stats;
    const duration = part ? part.duration : frag.duration;
    // If loading has been aborted and not in lowLatencyMode, stop timer and return
    if (stats.aborted) {
      logger.warn('frag loader destroy or aborted, disarm abandonRules');
      this.clearTimer();
      // reset forced auto level value so that next level will be selected
      this._nextAutoLevel = -1;
      return;
    }

    // This check only runs if we're in ABR mode and actually playing
    if (
      !autoLevelEnabled ||
      media.paused ||
      !media.playbackRate ||
      !media.readyState
    ) {
      return;
    }

    const requestDelay = performance.now() - stats.loading.start;
    const playbackRate = Math.abs(media.playbackRate);
    // In order to work with a stable bandwidth, only begin monitoring bandwidth after half of the fragment has been loaded
    if (requestDelay <= (500 * duration) / playbackRate) {
      return;
    }

    const { levels, minAutoLevel } = hls;
    const level = levels[frag.level];
    const expectedLen =
      stats.total ||
      Math.max(stats.loaded, Math.round((duration * level.maxBitrate) / 8));
    const loadRate = Math.max(
      1,
      stats.bwEstimate
        ? stats.bwEstimate / 8
        : (stats.loaded * 1000) / requestDelay
    );
    // fragLoadDelay is an estimate of the time (in seconds) it will take to buffer the entire fragment
    const fragLoadedDelay = (expectedLen - stats.loaded) / loadRate;

    const pos = media.currentTime;
    // bufferStarvationDelay is an estimate of the amount time (in seconds) it will take to exhaust the buffer
    const bufferStarvationDelay =
      (BufferHelper.bufferInfo(media, pos, config.maxBufferHole).end - pos) /
      playbackRate;

    // Attempt an emergency downswitch only if less than 2 fragment lengths are buffered, and the time to finish loading
    // the current fragment is greater than the amount of buffer we have left
    if (
      bufferStarvationDelay >= (2 * duration) / playbackRate ||
      fragLoadedDelay <= bufferStarvationDelay
    ) {
      return;
    }

    let fragLevelNextLoadedDelay: number = Number.POSITIVE_INFINITY;
    let nextLoadLevel: number;
    // Iterate through lower level and try to find the largest one that avoids rebuffering
    for (
      nextLoadLevel = frag.level - 1;
      nextLoadLevel > minAutoLevel;
      nextLoadLevel--
    ) {
      // compute time to load next fragment at lower level
      // 0.8 : consider only 80% of current bw to be conservative
      // 8 = bits per byte (bps/Bps)
      const levelNextBitrate = levels[nextLoadLevel].maxBitrate;
      fragLevelNextLoadedDelay =
        (duration * levelNextBitrate) / (8 * 0.8 * loadRate);

      if (fragLevelNextLoadedDelay < bufferStarvationDelay) {
        break;
      }
    }
    // Only emergency switch down if it takes less time to load a new fragment at lowest level instead of continuing
    // to load the current one
    if (fragLevelNextLoadedDelay >= fragLoadedDelay) {
      return;
    }
    const bwEstimate: number = this.bwEstimator.getEstimate();
    logger.warn(`Fragment ${frag.sn}${
      part ? ' part ' + part.index : ''
    } of level ${
      frag.level
    } is loading too slowly and will cause an underbuffer; aborting and switching to level ${nextLoadLevel}
      Current BW estimate: ${
        Number.isFinite(bwEstimate) ? (bwEstimate / 1024).toFixed(3) : 'Unknown'
      } Kb/s
      Estimated load time for current fragment: ${fragLoadedDelay.toFixed(3)} s
      Estimated load time for the next fragment: ${fragLevelNextLoadedDelay.toFixed(
        3
      )} s
      Time to underbuffer: ${bufferStarvationDelay.toFixed(3)} s`);
    hls.nextLoadLevel = nextLoadLevel;
    this.bwEstimator.sample(requestDelay, stats.loaded);
    this.clearTimer();
    if (frag.loader) {
      this.fragCurrent = this.partCurrent = null;
      frag.loader.abort();
    }
    hls.trigger(Events.FRAG_LOAD_EMERGENCY_ABORTED, { frag, part, stats });
  }

  protected onFragLoaded(
    event: Events.FRAG_LOADED,
    { frag, part }: FragLoadedData
  ) {
    if (
      frag.type === PlaylistLevelType.MAIN &&
      Number.isFinite(frag.sn as number)
    ) {
      const stats = part ? part.stats : frag.stats;
      const duration = part ? part.duration : frag.duration;
      // stop monitoring bw once frag loaded
      this.clearTimer();
      // store level id after successful fragment load
      this.lastLoadedFragLevel = frag.level;
      // reset forced auto level value so that next level will be selected
      this._nextAutoLevel = -1;

      // compute level average bitrate
      if (this.hls.config.abrMaxWithRealBitrate) {
        const level = this.hls.levels[frag.level];
        const loadedBytes =
          (level.loaded ? level.loaded.bytes : 0) + stats.loaded;
        const loadedDuration =
          (level.loaded ? level.loaded.duration : 0) + duration;
        level.loaded = { bytes: loadedBytes, duration: loadedDuration };
        level.realBitrate = Math.round((8 * loadedBytes) / loadedDuration);
      }
      if (frag.bitrateTest) {
        const fragBufferedData: FragBufferedData = {
          stats,
          frag,
          part,
          id: frag.type,
        };
        this.onFragBuffered(Events.FRAG_BUFFERED, fragBufferedData);
        frag.bitrateTest = false;
      }
    }
  }

  protected onFragBuffered(
    event: Events.FRAG_BUFFERED,
    data: FragBufferedData
  ) {
    const { frag, part } = data;
    const stats = part ? part.stats : frag.stats;

    if (stats.aborted) {
      return;
    }
    // Only count non-alt-audio frags which were actually buffered in our BW calculations
    if (frag.type !== PlaylistLevelType.MAIN || frag.sn === 'initSegment') {
      return;
    }
    // Use the difference between parsing and request instead of buffering and request to compute fragLoadingProcessing;
    // rationale is that buffer appending only happens once media is attached. This can happen when config.startFragPrefetch
    // is used. If we used buffering in that case, our BW estimate sample will be very large.
    const processingMs = stats.parsing.end - stats.loading.start;
    this.bwEstimator.sample(processingMs, stats.loaded);
    stats.bwEstimate = this.bwEstimator.getEstimate();
    if (frag.bitrateTest) {
      this.bitrateTestDelay = processingMs / 1000;
    } else {
      this.bitrateTestDelay = 0;
    }
  }

  protected onError(event: Events.ERROR, data: ErrorData) {
    // stop timer in case of frag loading error
    switch (data.details) {
      case ErrorDetails.FRAG_LOAD_ERROR:
      case ErrorDetails.FRAG_LOAD_TIMEOUT:
        this.clearTimer();
        break;
      default:
        break;
    }
  }

  clearTimer() {
    self.clearInterval(this.timer);
    this.timer = undefined;
  }

  // return next auto level
  get nextAutoLevel() {
    const forcedAutoLevel = this._nextAutoLevel;
    const bwEstimator = this.bwEstimator;
    // in case next auto level has been forced, and bw not available or not reliable, return forced value
    if (
      forcedAutoLevel !== -1 &&
      (!bwEstimator || !bwEstimator.canEstimate())
    ) {
      return forcedAutoLevel;
    }

    // compute next level using ABR logic
    let nextABRAutoLevel = this.getNextABRAutoLevel();
    // if forced auto level has been defined, use it to cap ABR computed quality level
    if (forcedAutoLevel !== -1) {
      nextABRAutoLevel = Math.min(forcedAutoLevel, nextABRAutoLevel);
    }

    return nextABRAutoLevel;
  }

  private getNextABRAutoLevel() {
    const { fragCurrent, partCurrent, hls } = this;
    const { maxAutoLevel, config, minAutoLevel, media } = hls;
    const currentFragDuration = partCurrent
      ? partCurrent.duration
      : fragCurrent
      ? fragCurrent.duration
      : 0;
    const pos = media ? media.currentTime : 0;

    // playbackRate is the absolute value of the playback rate; if media.playbackRate is 0, we use 1 to load as
    // if we're playing back at the normal rate.
    const playbackRate =
      media && media.playbackRate !== 0 ? Math.abs(media.playbackRate) : 1.0;
    const avgbw = this.bwEstimator
      ? this.bwEstimator.getEstimate()
      : config.abrEwmaDefaultEstimate;
    // bufferStarvationDelay is the wall-clock time left until the playback buffer is exhausted.
    const bufferStarvationDelay =
      (BufferHelper.bufferInfo(media as Bufferable, pos, config.maxBufferHole)
        .end -
        pos) /
      playbackRate;

    // First, look to see if we can find a level matching with our avg bandwidth AND that could also guarantee no rebuffering at all
    let bestLevel = this.findBestLevel(
      avgbw,
      minAutoLevel,
      maxAutoLevel,
      bufferStarvationDelay,
      config.abrBandWidthFactor,
      config.abrBandWidthUpFactor
    );
    if (bestLevel >= 0) {
      return bestLevel;
    }
    logger.trace(
      `${
        bufferStarvationDelay ? 'rebuffering expected' : 'buffer is empty'
      }, finding optimal quality level`
    );
    // not possible to get rid of rebuffering ... let's try to find level that will guarantee less than maxStarvationDelay of rebuffering
    // if no matching level found, logic will return 0
    let maxStarvationDelay = currentFragDuration
      ? Math.min(currentFragDuration, config.maxStarvationDelay)
      : config.maxStarvationDelay;
    let bwFactor = config.abrBandWidthFactor;
    let bwUpFactor = config.abrBandWidthUpFactor;

    if (!bufferStarvationDelay) {
      // in case buffer is empty, let's check if previous fragment was loaded to perform a bitrate test
      const bitrateTestDelay = this.bitrateTestDelay;
      if (bitrateTestDelay) {
        // if it is the case, then we need to adjust our max starvation delay using maxLoadingDelay config value
        // max video loading delay used in  automatic start level selection :
        // in that mode ABR controller will ensure that video loading time (ie the time to fetch the first fragment at lowest quality level +
        // the time to fetch the fragment at the appropriate quality level is less than ```maxLoadingDelay``` )
        // cap maxLoadingDelay and ensure it is not bigger 'than bitrate test' frag duration
        const maxLoadingDelay = currentFragDuration
          ? Math.min(currentFragDuration, config.maxLoadingDelay)
          : config.maxLoadingDelay;
        maxStarvationDelay = maxLoadingDelay - bitrateTestDelay;
        logger.trace(
          `bitrate test took ${Math.round(
            1000 * bitrateTestDelay
          )}ms, set first fragment max fetchDuration to ${Math.round(
            1000 * maxStarvationDelay
          )} ms`
        );
        // don't use conservative factor on bitrate test
        bwFactor = bwUpFactor = 1;
      }
    }
    bestLevel = this.findBestLevel(
      avgbw,
      minAutoLevel,
      maxAutoLevel,
      bufferStarvationDelay + maxStarvationDelay,
      bwFactor,
      bwUpFactor
    );
    return Math.max(bestLevel, 0);
  }

  private findBestLevel(
    currentBw: number,
    minAutoLevel: number,
    maxAutoLevel: number,
    maxFetchDuration: number,
    bwFactor: number,
    bwUpFactor: number
  ): number {
    const {
      fragCurrent,
      partCurrent,
      lastLoadedFragLevel: currentLevel,
    } = this;
    const { levels } = this.hls;
    const level = levels[currentLevel];
    const live = !!level?.details?.live;
    const currentCodecSet = level?.codecSet;

    const currentFragDuration = partCurrent
      ? partCurrent.duration
      : fragCurrent
      ? fragCurrent.duration
      : 0;
    for (let i = maxAutoLevel; i >= minAutoLevel; i--) {
      const levelInfo = levels[i];

      if (
        !levelInfo ||
        (currentCodecSet && levelInfo.codecSet !== currentCodecSet)
      ) {
        continue;
      }

      const levelDetails = levelInfo.details;
      const avgDuration =
        (partCurrent
          ? levelDetails?.partTarget
          : levelDetails?.averagetargetduration) || currentFragDuration;

      let adjustedbw: number;
      // follow algorithm captured from stagefright :
      // https://android.googlesource.com/platform/frameworks/av/+/master/media/libstagefright/httplive/LiveSession.cpp
      // Pick the highest bandwidth stream below or equal to estimated bandwidth.
      // consider only 80% of the available bandwidth, but if we are switching up,
      // be even more conservative (70%) to avoid overestimating and immediately
      // switching back.
      if (i <= currentLevel) {
        adjustedbw = bwFactor * currentBw;
      } else {
        adjustedbw = bwUpFactor * currentBw;
      }

      const bitrate: number = levels[i].maxBitrate;
      const fetchDuration: number = (bitrate * avgDuration) / adjustedbw;

      logger.trace(
        `level/adjustedbw/bitrate/avgDuration/maxFetchDuration/fetchDuration: ${i}/${Math.round(
          adjustedbw
        )}/${bitrate}/${avgDuration}/${maxFetchDuration}/${fetchDuration}`
      );
      // if adjusted bw is greater than level bitrate AND
      if (
        adjustedbw > bitrate &&
        // fragment fetchDuration unknown OR live stream OR fragment fetchDuration less than max allowed fetch duration, then this level matches
        // we don't account for max Fetch Duration for live streams, this is to avoid switching down when near the edge of live sliding window ...
        // special case to support startLevel = -1 (bitrateTest) on live streams : in that case we should not exit loop so that findBestLevel will return -1
        (!fetchDuration ||
          (live && !this.bitrateTestDelay) ||
          fetchDuration < maxFetchDuration)
      ) {
        // as we are looping from highest to lowest, this will return the best achievable quality level
        return i;
      }
    }
    // not enough time budget even with quality level 0 ... rebuffering might happen
    return -1;
  }

  set nextAutoLevel(nextLevel) {
    this._nextAutoLevel = nextLevel;
  }
}

export default AbrController;