Skip to content

Commit

Permalink
Improve bandwidth estimation and adaptive switching (#4825)
Browse files Browse the repository at this point in the history 
* Improve bandwidth estimation and adaptive switching with smaller segments and higher TTFB
Fixes #3578 (special thanks to @Oleksandr0xB for submitting #4283)
Fixes #3563 and Closes #3595 (special thanks to @kanongil)

* Load rate and loaded delay calculation fixes

* Convert ttfbEstimate to seconds

* Include main variant init segments in TTFB sampling

* Use ttfb estimate in abandon rules down-switch timing
  • Loading branch information
@robwalch
robwalch committed Jan 26, 2023
1 parent 10375a2 commit ffd67eb
Show file tree
Hide file tree
Showing 11 changed files with 259 additions and 151 deletions.
13 changes: 11 additions & 2 deletions api-extractor/report/hls.js.api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,16 @@ export interface BufferFlushingData {
type: SourceBufferName | null;
}

// Warning: (ae-missing-release-tag) "BufferInfo" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export type BufferInfo = {
len: number;
start: number;
end: number;
nextStart?: number;
};

// Warning: (ae-missing-release-tag) "CapLevelControllerConfig" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
Expand Down Expand Up @@ -961,8 +971,6 @@ class Hls implements HlsEventEmitter {
get lowLatencyMode(): boolean;
// Warning: (ae-setter-with-docs) The doc comment for the property "lowLatencyMode" must appear on the getter, not the setter.
set lowLatencyMode(mode: boolean);
// Warning: (ae-forgotten-export) The symbol "BufferInfo" needs to be exported by the entry point hls.d.ts
//
// (undocumented)
get mainForwardBufferInfo(): BufferInfo | null;
get manualLevel(): number;
Expand Down Expand Up @@ -1011,6 +1019,7 @@ class Hls implements HlsEventEmitter {
get targetLatency(): number | null;
// (undocumented)
trigger<E extends keyof HlsListeners>(event: E, eventObject: Parameters<HlsListeners[E]>[1]): boolean;
get ttfbEstimate(): number;
// (undocumented)
readonly userConfig: Partial<HlsConfig>;
// (undocumented)
Expand Down
1 change: 1 addition & 0 deletions demo/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1212,6 +1212,7 @@ function checkBuffer() {
log += `Dropped frames: ${video.webkitDroppedFrameCount}\n`;
}
}
log += `TTFB Estimate: ${hls.ttfbEstimate.toFixed(3)}\n`;
log += `Bandwidth Estimate: ${hls.bandwidthEstimate.toFixed(3)}\n`;
if (events.isLive) {
log +=
Expand Down
210 changes: 128 additions & 82 deletions src/controller/abr-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class AbrController implements AbrComponentAPI {
protected hls: Hls;
private lastLoadedFragLevel: number = 0;
private _nextAutoLevel: number = -1;
private timer?: number;
private timer: number = -1;
private onCheck: Function = this._abandonRulesCheck.bind(this);
private fragCurrent: Fragment | null = null;
private partCurrent: Part | null = null;
Expand Down Expand Up @@ -69,13 +69,13 @@ class AbrController implements AbrComponentAPI {

protected onFragLoading(event: Events.FRAG_LOADING, data: FragLoadingData) {
const frag = data.frag;
if (frag.type === PlaylistLevelType.MAIN) {
if (!this.timer) {
this.fragCurrent = frag;
this.partCurrent = data.part ?? null;
this.timer = self.setInterval(this.onCheck, 100);
}
if (this.ignoreFragment(frag)) {
return;
}
this.fragCurrent = frag;
this.partCurrent = data.part ?? null;
this.clearTimer();
this.timer = self.setInterval(this.onCheck, 100);
}

protected onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) {
Expand All @@ -98,8 +98,10 @@ class AbrController implements AbrComponentAPI {
return;
}

const now = performance.now();
const stats: LoaderStats = part ? part.stats : frag.stats;
const duration = part ? part.duration : frag.duration;
const timeLoading = now - stats.loading.start;
// If frag loading is aborted, complete, or from lowest level, stop timer and return
if (
stats.aborted ||
Expand Down Expand Up @@ -127,30 +129,44 @@ class AbrController implements AbrComponentAPI {
return;
}

const requestDelay = performance.now() - stats.loading.start;
const ttfbEstimate = this.bwEstimator.getEstimateTTFB();
const playbackRate = Math.abs(media.playbackRate);
// In order to work with a stable bandwidth, only begin monitoring bandwidth after half of the fragment has been loaded
if (requestDelay <= (500 * duration) / playbackRate) {
// To maintain stable adaptive playback, only begin monitoring frag loading after half or more of its playback duration has passed
if (
timeLoading <=
Math.max(ttfbEstimate, 1000 * (duration / (playbackRate * 2)))
) {
return;
}

// bufferStarvationDelay is an estimate of the amount time (in seconds) it will take to exhaust the buffer
const bufferStarvationDelay = bufferInfo.len / playbackRate;
// Only downswitch if less than 2 fragment lengths are buffered
if (bufferStarvationDelay >= (2 * duration) / playbackRate) {
return;
}

const loadedFirstByte = stats.loaded && stats.loading.first;
const ttfb = stats.loading.first
? stats.loading.first - stats.loading.start
: -1;
const loadedFirstByte = stats.loaded && ttfb > -1;
const bwEstimate: number = this.bwEstimator.getEstimate();
const { levels, minAutoLevel } = hls;
const level = levels[frag.level];
const expectedLen =
stats.total ||
Math.max(stats.loaded, Math.round((duration * level.maxBitrate) / 8));
const loadRate = loadedFirstByte ? (stats.loaded * 1000) / requestDelay : 0;

let timeStreaming = timeLoading - ttfb;
if (timeStreaming < 1 && loadedFirstByte) {
timeStreaming = Math.min(timeLoading, (stats.loaded * 8) / bwEstimate);
}
const loadRate = loadedFirstByte
? (stats.loaded * 1000) / timeStreaming
: 0;
// fragLoadDelay is an estimate of the time (in seconds) it will take to buffer the remainder of the fragment
const fragLoadedDelay = loadRate
? (expectedLen - stats.loaded) / loadRate
: (expectedLen * 8) / bwEstimate;

// bufferStarvationDelay is an estimate of the amount time (in seconds) it will take to exhaust the buffer
const bufferStarvationDelay = bufferInfo.len / playbackRate;

: (expectedLen * 8) / bwEstimate + ttfbEstimate / 1000;
// Only downswitch if the time to finish loading the current fragment is greater than the amount of buffer left
if (fragLoadedDelay <= bufferStarvationDelay) {
return;
Expand All @@ -165,13 +181,11 @@ class AbrController implements AbrComponentAPI {
nextLoadLevel--
) {
// compute time to load next fragment at lower level
// 0.8 : consider only 80% of current bw to be conservative
// 8 = bits per byte (bps/Bps)
const levelNextBitrate = levels[nextLoadLevel].maxBitrate;
fragLevelNextLoadedDelay = loadRate
? (duration * levelNextBitrate) / (8 * 0.8 * loadRate)
: (duration * levelNextBitrate) / bwEstimate;

const bwe = loadRate ? loadRate * 8 : bwEstimate;
fragLevelNextLoadedDelay =
(duration * levelNextBitrate) / bwe + ttfbEstimate / 1000;
if (fragLevelNextLoadedDelay < bufferStarvationDelay) {
break;
}
Expand All @@ -181,26 +195,41 @@ class AbrController implements AbrComponentAPI {
if (fragLevelNextLoadedDelay >= fragLoadedDelay) {
return;
}
logger.warn(`Fragment ${frag.sn}${

// if estimated load time of new segment is completely unreasonable, ignore and do not emergency switch down
if (fragLevelNextLoadedDelay > duration * 10) {
return;
}
hls.nextLoadLevel = nextLoadLevel;
if (loadedFirstByte) {
// If there has been loading progress, sample bandwidth using loading time offset by minimum TTFB time
this.bwEstimator.sample(
timeLoading - Math.min(ttfbEstimate, ttfb),
stats.loaded
);
} else {
// If there has been no loading progress, sample TTFB
this.bwEstimator.sampleTTFB(timeLoading);
}

this.clearTimer();
logger.warn(`[abr] Fragment ${frag.sn}${
part ? ' part ' + part.index : ''
} of level ${
frag.level
} is loading too slowly and will cause an underbuffer; aborting and switching to level ${nextLoadLevel}
} of level ${frag.level} is loading too slowly;
Time to underbuffer: ${bufferStarvationDelay.toFixed(3)} s
Estimated load time for current fragment: ${fragLoadedDelay.toFixed(3)} s
Estimated load time for down switch fragment: ${fragLevelNextLoadedDelay.toFixed(
3
)} s
TTFB estimate: ${ttfb}
Current BW estimate: ${
Number.isFinite(bwEstimate) ? (bwEstimate / 1024).toFixed(3) : 'Unknown'
} Kb/s
Estimated load time for current fragment: ${fragLoadedDelay.toFixed(3)} s
Estimated load time for the next fragment: ${fragLevelNextLoadedDelay.toFixed(
New BW estimate: ${(this.bwEstimator.getEstimate() / 1024).toFixed(
3
)} s
Time to underbuffer: ${bufferStarvationDelay.toFixed(3)} s`);
hls.nextLoadLevel = nextLoadLevel;
if (loadedFirstByte) {
// If there has been loading progress, sample bandwidth
this.bwEstimator.sample(requestDelay, stats.loaded);
}
this.clearTimer();
if (frag.loader || frag.keyLoader) {
)} Kb/s
Aborting and switching to level ${nextLoadLevel}`);
if (frag.loader) {
this.fragCurrent = this.partCurrent = null;
frag.abortRequests();
}
Expand All @@ -211,38 +240,40 @@ class AbrController implements AbrComponentAPI {
event: Events.FRAG_LOADED,
{ frag, part }: FragLoadedData
) {
if (
frag.type === PlaylistLevelType.MAIN &&
Number.isFinite(frag.sn as number)
) {
const stats = part ? part.stats : frag.stats;
const duration = part ? part.duration : frag.duration;
// stop monitoring bw once frag loaded
this.clearTimer();
// store level id after successful fragment load
this.lastLoadedFragLevel = frag.level;
// reset forced auto level value so that next level will be selected
this._nextAutoLevel = -1;
const stats = part ? part.stats : frag.stats;
if (frag.type === PlaylistLevelType.MAIN) {
this.bwEstimator.sampleTTFB(stats.loading.first - stats.loading.start);
}
if (this.ignoreFragment(frag)) {
return;
}
// stop monitoring bw once frag loaded
this.clearTimer();
// store level id after successful fragment load
this.lastLoadedFragLevel = frag.level;
// reset forced auto level value so that next level will be selected
this._nextAutoLevel = -1;

// compute level average bitrate
if (this.hls.config.abrMaxWithRealBitrate) {
const level = this.hls.levels[frag.level];
const loadedBytes =
(level.loaded ? level.loaded.bytes : 0) + stats.loaded;
const loadedDuration =
(level.loaded ? level.loaded.duration : 0) + duration;
level.loaded = { bytes: loadedBytes, duration: loadedDuration };
level.realBitrate = Math.round((8 * loadedBytes) / loadedDuration);
}
if (frag.bitrateTest) {
const fragBufferedData: FragBufferedData = {
stats,
frag,
part,
id: frag.type,
};
this.onFragBuffered(Events.FRAG_BUFFERED, fragBufferedData);
}
// compute level average bitrate
if (this.hls.config.abrMaxWithRealBitrate) {
const duration = part ? part.duration : frag.duration;
const level = this.hls.levels[frag.level];
const loadedBytes =
(level.loaded ? level.loaded.bytes : 0) + stats.loaded;
const loadedDuration =
(level.loaded ? level.loaded.duration : 0) + duration;
level.loaded = { bytes: loadedBytes, duration: loadedDuration };
level.realBitrate = Math.round((8 * loadedBytes) / loadedDuration);
}
if (frag.bitrateTest) {
const fragBufferedData: FragBufferedData = {
stats,
frag,
part,
id: frag.type,
};
this.onFragBuffered(Events.FRAG_BUFFERED, fragBufferedData);
frag.bitrateTest = false;
}
}

Expand All @@ -251,19 +282,24 @@ class AbrController implements AbrComponentAPI {
data: FragBufferedData
) {
const { frag, part } = data;
const stats = part ? part.stats : frag.stats;
const stats = part?.stats.loaded ? part.stats : frag.stats;

if (stats.aborted) {
return;
}
// Only count non-alt-audio frags which were actually buffered in our BW calculations
if (frag.type !== PlaylistLevelType.MAIN || frag.sn === 'initSegment') {
if (this.ignoreFragment(frag)) {
return;
}
// Use the difference between parsing and request instead of buffering and request to compute fragLoadingProcessing;
// rationale is that buffer appending only happens once media is attached. This can happen when config.startFragPrefetch
// is used. If we used buffering in that case, our BW estimate sample will be very large.
const processingMs = stats.parsing.end - stats.loading.start;
const processingMs =
stats.parsing.end -
stats.loading.start -
Math.min(
stats.loading.first - stats.loading.start,
this.bwEstimator.getEstimateTTFB()
);
this.bwEstimator.sample(processingMs, stats.loaded);
stats.bwEstimate = this.bwEstimator.getEstimate();
if (frag.bitrateTest) {
Expand Down Expand Up @@ -293,9 +329,13 @@ class AbrController implements AbrComponentAPI {
}
}

clearTimer() {
private ignoreFragment(frag: Fragment): boolean {
// Only count non-alt-audio frags which were actually buffered in our BW calculations
return frag.type !== PlaylistLevelType.MAIN || frag.sn === 'initSegment';
}

public clearTimer() {
self.clearInterval(this.timer);
this.timer = undefined;
}

// return next auto level
Expand All @@ -321,7 +361,7 @@ class AbrController implements AbrComponentAPI {
return nextABRAutoLevel;
}

private getNextABRAutoLevel() {
private getNextABRAutoLevel(): number {
const { fragCurrent, partCurrent, hls } = this;
const { maxAutoLevel, config, minAutoLevel, media } = hls;
const currentFragDuration = partCurrent
Expand Down Expand Up @@ -355,7 +395,7 @@ class AbrController implements AbrComponentAPI {
return bestLevel;
}
logger.trace(
`${
`[abr] ${
bufferStarvationDelay ? 'rebuffering expected' : 'buffer is empty'
}, finding optimal quality level`
);
Expand All @@ -381,7 +421,7 @@ class AbrController implements AbrComponentAPI {
: config.maxLoadingDelay;
maxStarvationDelay = maxLoadingDelay - bitrateTestDelay;
logger.trace(
`bitrate test took ${Math.round(
`[abr] bitrate test took ${Math.round(
1000 * bitrateTestDelay
)}ms, set first fragment max fetchDuration to ${Math.round(
1000 * maxStarvationDelay
Expand Down Expand Up @@ -454,13 +494,19 @@ class AbrController implements AbrComponentAPI {
adjustedbw = bwUpFactor * currentBw;
}

const ttfbEstimate = this.bwEstimator.getEstimateTTFB();
const bitrate: number = levels[i].maxBitrate;
const fetchDuration: number = (bitrate * avgDuration) / adjustedbw;
const fetchDuration: number =
(bitrate * avgDuration) / adjustedbw + ttfbEstimate / 1000;

logger.trace(
`level/adjustedbw/bitrate/avgDuration/maxFetchDuration/fetchDuration: ${i}/${Math.round(
adjustedbw
)}/${bitrate}/${avgDuration}/${maxFetchDuration}/${fetchDuration}`
`[abr] level:${i} adjustedbw-bitrate:${Math.round(
adjustedbw - bitrate
)} avgDuration:${avgDuration.toFixed(
1
)} maxFetchDuration:${maxFetchDuration.toFixed(
1
)} fetchDuration:${fetchDuration.toFixed(1)}`
);
// if adjusted bw is greater than level bitrate AND
if (
Expand Down
Loading

0 comments on commit ffd67eb

Please sign in to comment.