diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 7688af69b51..e7af4df8f01 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -41,6 +41,8 @@ export class AbrController implements AbrComponentAPI { // (undocumented) protected onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData): void; // (undocumented) + protected onLevelSwitching(event: Events.LEVEL_SWITCHING, data: LevelSwitchingData): void; + // (undocumented) protected registerListeners(): void; // (undocumented) protected unregisterListeners(): void; @@ -232,8 +234,6 @@ export class BasePlaylistController implements NetworkComponentAPI { // (undocumented) protected requestScheduled: number; // (undocumented) - protected retryCount: number; - // (undocumented) protected shouldLoadPlaylist(playlist: Level | MediaPlaylist): boolean; // (undocumented) startLoad(): void; @@ -879,15 +879,35 @@ export type EMEControllerConfig = { requestMediaKeySystemAccessFunc: MediaKeyFunc | null; }; +// Warning: (ae-missing-release-tag) "ErrorActionFlags" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export enum ErrorActionFlags { + // (undocumented) + MoveAllAlternatesMatchingHDCP = 2, + // (undocumented) + MoveAllAlternatesMatchingHost = 1, + // (undocumented) + None = 0, + // (undocumented) + SwitchToSDR = 4 +} + // Warning: (ae-missing-release-tag) "ErrorController" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class ErrorController { +export class ErrorController implements NetworkComponentAPI { constructor(hls: Hls); // (undocumented) destroy(): void; // (undocumented) onErrorOut(event: Events.ERROR, data: ErrorData): void; + // (undocumented) + sendAlternateToPenaltyBox(data: ErrorData): void; + // (undocumented) + startLoad(startPosition: number): void; + // (undocumented) + stopLoad(): void; } // Warning: (ae-missing-release-tag) "ErrorData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -904,13 +924,15 @@ export interface ErrorData { context?: PlaylistLoaderContext; // (undocumented) details: ErrorDetails; - // (undocumented) + // @deprecated (undocumented) err?: { message: string; }; // (undocumented) error: Error; // (undocumented) + errorAction?: IErrorAction; + // (undocumented) event?: keyof HlsListeners | 'demuxerWorker'; // (undocumented) fatal: boolean; @@ -1446,7 +1468,7 @@ export type HdcpLevel = (typeof HdcpLevels)[number]; // Warning: (ae-missing-release-tag) "HdcpLevels" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export const HdcpLevels: readonly ["NONE", "TYPE-0", "TYPE-1", "TYPE-2", null]; +export const HdcpLevels: readonly ["NONE", "TYPE-0", "TYPE-1", null]; // @public class Hls implements HlsEventEmitter { @@ -1793,6 +1815,19 @@ export class HlsUrlParameters { skip?: HlsSkip; } +// Warning: (ae-missing-release-tag) "IErrorAction" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type IErrorAction = { + action: NetworkErrorAction; + flags: ErrorActionFlags; + retryCount?: number; + retryConfig?: RetryConfig; + hdcpLevel?: HdcpLevel; + nextAutoLevel?: number; + resolved?: boolean; +}; + // Warning: (ae-missing-release-tag) "ILogger" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) @@ -2261,7 +2296,7 @@ export interface LevelUpdatedData { // Warning: (ae-missing-release-tag) "LiveBackBufferData" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // -// @public +// @public @deprecated (undocumented) export interface LiveBackBufferData extends BackBufferData { } @@ -2381,7 +2416,7 @@ export interface LoaderResponse { // (undocumented) code?: number; // (undocumented) - data: string | ArrayBuffer | Object; + data?: string | ArrayBuffer | Object; // (undocumented) text?: string; // (undocumented) @@ -2673,6 +2708,24 @@ export interface NetworkComponentAPI extends ComponentAPI { stopLoad(): void; } +// Warning: (ae-missing-release-tag) "NetworkErrorAction" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export enum NetworkErrorAction { + // (undocumented) + DoNothing = 0, + // (undocumented) + InsertDiscontinuity = 4, + // (undocumented) + RemoveAlternatePermanently = 3, + // (undocumented) + RetryRequest = 5, + // (undocumented) + SendAlternateToPenaltyBox = 2, + // (undocumented) + SendEndCallback = 1 +} + // Warning: (ae-missing-release-tag) "NonNativeTextTrack" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/src/config.ts b/src/config.ts index 6969f6a7cd3..092239d5d02 100644 --- a/src/config.ts +++ b/src/config.ts @@ -291,6 +291,13 @@ export type HlsConfig = { FragmentLoaderConfig & PlaylistLoaderConfig; +const defaultLoadPolicy: LoaderConfig = { + maxTimeToFirstByteMs: 8000, + maxLoadTimeMs: 20000, + timeoutRetry: null, + errorRetry: null, +}; + /** * @ignore * If possible, keep hlsDefaultConfig shallow @@ -374,12 +381,7 @@ export const hlsDefaultConfig: HlsConfig = { enableID3MetadataCues: true, certLoadPolicy: { - default: { - maxTimeToFirstByteMs: 8000, - maxLoadTimeMs: 20000, - timeoutRetry: null, - errorRetry: null, - }, + default: defaultLoadPolicy, }, keyLoadPolicy: { default: { @@ -448,20 +450,22 @@ export const hlsDefaultConfig: HlsConfig = { }, }, steeringManifestLoadPolicy: { - default: { - maxTimeToFirstByteMs: 10000, - maxLoadTimeMs: 20000, - timeoutRetry: { - maxNumRetry: 2, - retryDelayMs: 0, - maxRetryDelayMs: 0, - }, - errorRetry: { - maxNumRetry: 1, - retryDelayMs: 1000, - maxRetryDelayMs: 8000, - }, - }, + default: __USE_CONTENT_STEERING__ + ? { + maxTimeToFirstByteMs: 10000, + maxLoadTimeMs: 20000, + timeoutRetry: { + maxNumRetry: 2, + retryDelayMs: 0, + maxRetryDelayMs: 0, + }, + errorRetry: { + maxNumRetry: 1, + retryDelayMs: 1000, + maxRetryDelayMs: 8000, + }, + } + : defaultLoadPolicy, }, // These default settings are deprecated in favor of the above policies diff --git a/src/controller/abr-controller.ts b/src/controller/abr-controller.ts index d383ad228ba..60512a5242e 100644 --- a/src/controller/abr-controller.ts +++ b/src/controller/abr-controller.ts @@ -11,6 +11,7 @@ import type { FragLoadedData, FragBufferedData, LevelLoadedData, + LevelSwitchingData, } from '../types/events'; import type { AbrComponentAPI } from '../types/component-api'; @@ -44,6 +45,7 @@ class AbrController implements AbrComponentAPI { hls.on(Events.FRAG_LOADING, this.onFragLoading, this); hls.on(Events.FRAG_LOADED, this.onFragLoaded, this); hls.on(Events.FRAG_BUFFERED, this.onFragBuffered, this); + hls.on(Events.LEVEL_SWITCHING, this.onLevelSwitching, this); hls.on(Events.LEVEL_LOADED, this.onLevelLoaded, this); } @@ -52,6 +54,7 @@ class AbrController implements AbrComponentAPI { hls.off(Events.FRAG_LOADING, this.onFragLoading, this); hls.off(Events.FRAG_LOADED, this.onFragLoaded, this); hls.off(Events.FRAG_BUFFERED, this.onFragBuffered, this); + hls.off(Events.LEVEL_SWITCHING, this.onLevelSwitching, this); hls.off(Events.LEVEL_LOADED, this.onLevelLoaded, this); } @@ -74,6 +77,13 @@ class AbrController implements AbrComponentAPI { this.timer = self.setInterval(this.onCheck, 100); } + protected onLevelSwitching( + event: Events.LEVEL_SWITCHING, + data: LevelSwitchingData + ): void { + this.clearTimer(); + } + protected onLevelLoaded(event: Events.LEVEL_LOADED, data: LevelLoadedData) { const config = this.hls.config; if (data.details.live) { diff --git a/src/controller/audio-stream-controller.ts b/src/controller/audio-stream-controller.ts index 4431b7e12ee..aca8033f8cf 100644 --- a/src/controller/audio-stream-controller.ts +++ b/src/controller/audio-stream-controller.ts @@ -400,9 +400,9 @@ class AudioStreamController if (fragCurrent) { fragCurrent.abortRequests(); + this.fragmentTracker.removeFragment(fragCurrent); } - this.fragCurrent = null; - this.clearWaitingFragment(); + this.resetLoadingState(); // destroy useless transmuxer when switching audio to main if (!altAudio) { this.resetTransmuxer(); diff --git a/src/controller/audio-track-controller.ts b/src/controller/audio-track-controller.ts index 20ff2cfd22f..dd149fb77b8 100644 --- a/src/controller/audio-track-controller.ts +++ b/src/controller/audio-track-controller.ts @@ -92,7 +92,6 @@ class AudioTrackController extends BasePlaylistController { ); if (id === this.trackId) { - this.retryCount = 0; this.playlistLoaded(id, data, curDetails); } } diff --git a/src/controller/base-playlist-controller.ts b/src/controller/base-playlist-controller.ts index e666011f36c..e69a2d1f211 100644 --- a/src/controller/base-playlist-controller.ts +++ b/src/controller/base-playlist-controller.ts @@ -11,14 +11,14 @@ import type { TrackLoadedData, } from '../types/events'; import { ErrorData } from '../types/events'; -import { ErrorDetails } from '../errors'; +import { NetworkErrorAction } from '../errors'; +import { getRetryDelay, isTimeoutError } from '../utils/error-helper'; export default class BasePlaylistController implements NetworkComponentAPI { protected hls: Hls; protected timer: number = -1; protected requestScheduled: number = -1; protected canLoad: boolean = false; - protected retryCount: number = 0; protected log: (msg: any) => void; protected warn: (msg: any) => void; @@ -41,7 +41,6 @@ export default class BasePlaylistController implements NetworkComponentAPI { public startLoad(): void { this.canLoad = true; - this.retryCount = 0; this.requestScheduled = -1; this.loadPlaylist(); } @@ -290,44 +289,37 @@ export default class BasePlaylistController implements NetworkComponentAPI { } protected checkRetry(errorEvent: ErrorData): boolean { - const { playlistLoadPolicy } = this.hls.config; const errorDetails = errorEvent.details; - const isTimeout = - errorDetails === ErrorDetails.LEVEL_LOAD_TIMEOUT || - errorDetails === ErrorDetails.AUDIO_TRACK_LOAD_TIMEOUT || - errorDetails === ErrorDetails.SUBTITLE_TRACK_LOAD_TIMEOUT; - const httpStatus = errorEvent.response?.code; - const retryConfig = - playlistLoadPolicy.default[`${isTimeout ? 'timeout' : 'error'}Retry`]; + const isTimeout = isTimeoutError(errorEvent); + const errorAction = errorEvent.errorAction; + const { action, retryCount = 0, retryConfig } = errorAction || {}; const retry = - !!retryConfig && - this.retryCount < retryConfig.maxNumRetry && - httpStatus !== 0; + action === NetworkErrorAction.RetryRequest && + !!errorAction && + !!retryConfig; if (retry) { this.requestScheduled = -1; - const retryCount = ++this.retryCount; if (isTimeout && errorEvent.context?.deliveryDirectives) { // The LL-HLS request already timed out so retry immediately this.warn( - `Retrying playlist loading ${retryCount}/${retryConfig.maxNumRetry} after "${errorDetails}" without delivery-directives` + `Retrying playlist loading ${retryCount + 1}/${ + retryConfig.maxNumRetry + } after "${errorDetails}" without delivery-directives` ); this.loadPlaylist(); } else { - // exponential backoff capped to max retry delay - const backoffFactor = - retryConfig.backoff === 'linear' ? 1 : Math.pow(2, retryCount - 1); - const delay = Math.min( - backoffFactor * retryConfig.retryDelayMs, - retryConfig.maxRetryDelayMs - ); + const delay = getRetryDelay(retryConfig, retryCount); // Schedule level/track reload this.timer = self.setTimeout(() => this.loadPlaylist(), delay); this.warn( - `Retrying playlist loading ${retryCount}/${retryConfig.maxNumRetry} after "${errorDetails}" in ${delay}ms` + `Retrying playlist loading ${retryCount + 1}/${ + retryConfig.maxNumRetry + } after "${errorDetails}" in ${delay}ms` ); } // `levelRetry = true` used to inform other controllers that a retry is happening errorEvent.levelRetry = true; + errorAction.resolved = true; } return retry; } diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index 3413f1f5270..f95ee7b7ac8 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -3,7 +3,7 @@ import { FragmentState } from './fragment-tracker'; import { Bufferable, BufferHelper, BufferInfo } from '../utils/buffer-helper'; import { logger } from '../utils/logger'; import { Events } from '../events'; -import { ErrorDetails, ErrorTypes } from '../errors'; +import { ErrorDetails, ErrorTypes, NetworkErrorAction } from '../errors'; import { ChunkMetadata } from '../types/transmuxer'; import { appendUint8Array } from '../utils/mp4-tools'; import { alignStream } from '../utils/discontinuities'; @@ -47,6 +47,7 @@ import type { HlsConfig } from '../config'; import type { NetworkComponentAPI } from '../types/component-api'; import type { SourceBufferName } from '../types/buffer'; import type { RationalTimestamp } from '../utils/timescale-conversion'; +import { getRetryDelay } from '../utils/error-helper'; type ResolveFragLoaded = (FragLoadedEndData) => void; type RejectFragLoaded = (LoadError) => void; @@ -133,7 +134,7 @@ export default class BaseStreamController this.fragmentLoader.abort(); this.keyLoader.abort(); const frag = this.fragCurrent; - if (frag) { + if (frag?.loader) { frag.abortRequests(); this.fragmentTracker.removeFragment(frag); } @@ -1354,24 +1355,17 @@ export default class BaseStreamController filterType: PlaylistLevelType, data: ErrorData ) { - if (data.chunkMeta) { - // Parsing Error: no retries + if (data.chunkMeta && !data.frag) { const context = this.getCurrentContext(data.chunkMeta); - if ( - context && - !this.fragContextChanged(context.frag) && - context.frag.type === filterType - ) { - this.resetFragmentErrors(filterType); + if (context) { + data.frag = context.frag; } - return; } const frag = data.frag; // Handle frag error related to caller's filterType if (!frag || frag.type !== filterType || !this.levels) { return; } - const level = this.levels[frag.level]; if (this.fragContextChanged(frag)) { this.warn( `Frag load error must match current frag to retry ${frag.url} > ${this.fragCurrent?.url}` @@ -1379,43 +1373,39 @@ export default class BaseStreamController return; } // keep retrying until the limit will be reached - const fragmentErrors = this.levels.reduce( - (acc, level) => acc + level.fragmentError, - 0 - ); - const retryCount = level ? fragmentErrors + 1 : Infinity; - const { fragLoadPolicy, keyLoadPolicy } = this.config; - const isTimeout = - data.details === ErrorDetails.FRAG_LOAD_TIMEOUT || - data.details === ErrorDetails.KEY_LOAD_TIMEOUT; - const retryConfig = ( - data.details.startsWith('key') ? keyLoadPolicy : fragLoadPolicy - ).default[`${isTimeout ? 'timeout' : 'error'}Retry`]; - const retry = !!retryConfig && retryCount <= retryConfig.maxNumRetry; - if (retry) { - level.fragmentError++; + const errorAction = data.errorAction; + const { action, retryCount = 0, retryConfig } = errorAction || {}; + if ( + errorAction && + action === NetworkErrorAction.RetryRequest && + retryConfig + ) { if (!this.loadedmetadata) { this.startFragRequested = false; this.nextLoadPosition = this.startPosition; } - // exponential backoff capped to max retry delay - const backoffFactor = - retryConfig.backoff === 'linear' ? 1 : Math.pow(2, fragmentErrors); - const delay = Math.min( - backoffFactor * retryConfig.retryDelayMs, - retryConfig.maxRetryDelayMs - ); + const delay = getRetryDelay(retryConfig, retryCount); this.warn( - `Fragment ${frag.sn} of ${filterType} ${frag.level} errored with ${data.details}, retrying loading ${retryCount}/${retryConfig.maxNumRetry} in ${delay}ms` + `Fragment ${frag.sn} of ${filterType} ${frag.level} errored with ${ + data.details + }, retrying loading ${retryCount + 1}/${ + retryConfig.maxNumRetry + } in ${delay}ms` ); + errorAction.resolved = true; this.retryDate = self.performance.now() + delay; this.state = State.FRAG_LOADING_WAITING_RETRY; - } else if (data.levelRetry) { + } else if (retryConfig && errorAction) { this.resetFragmentErrors(filterType); + if (retryCount < retryConfig.maxNumRetry) { + // Network retry is skipped for when level switch is preferred + errorAction.resolved = true; + } else { + logger.warn( + `${data.details} reached or exceeded max retry (${retryCount})` + ); + } } else { - logger.warn(`${data.details} reached max retry (${fragmentErrors})`); - // `levelRetry = false` used to inform other controllers that if a level switch is not possible, error should escalated to fatal - data.levelRetry = false; this.state = State.ERROR; } } @@ -1545,6 +1535,9 @@ export default class BaseStreamController frag, reason: `Found no media in msn ${frag.sn} of level "${level.url}"`, }); + if (!this.hls) { + return; + } this.resetTransmuxer(); // For this error fallthrough. Marking parsed will allow advancing to next fragment. } diff --git a/src/controller/content-steering-controller.ts b/src/controller/content-steering-controller.ts index 95af8f9ab00..7703578c340 100644 --- a/src/controller/content-steering-controller.ts +++ b/src/controller/content-steering-controller.ts @@ -1,10 +1,17 @@ import { Events } from '../events'; import { Level } from '../types/level'; import { AttrList } from '../utils/attr-list'; +import { addGroupId } from './level-controller'; +import { ErrorActionFlags, NetworkErrorAction } from '../errors'; import { logger } from '../utils/logger'; import type Hls from '../hls'; import type { NetworkComponentAPI } from '../types/component-api'; -import type { ManifestLoadedData, ManifestParsedData } from '../types/events'; +import type { + ErrorData, + ManifestLoadedData, + ManifestParsedData, +} from '../types/events'; +import type { RetryConfig } from '../config'; import type { Loader, LoaderCallbacks, @@ -15,8 +22,6 @@ import type { } from '../types/loader'; import type { LevelParsed } from '../types/level'; import type { MediaAttributes, MediaPlaylist } from '../types/media-playlist'; -import { addGroupId } from './level-controller'; -import { RetryConfig } from '../config'; export type SteeringManifest = { VERSION: 1; @@ -39,6 +44,8 @@ type UriReplacement = { 'PER-RENDITION-URIS'?: { [stableRenditionId: string]: string }; }; +const PATHWAY_PENALTY_DURATION_MS = 300000; + export default class ContentSteeringController implements NetworkComponentAPI { private readonly hls: Hls; private log: (msg: any) => void; @@ -49,10 +56,12 @@ export default class ContentSteeringController implements NetworkComponentAPI { private timeToLoad: number = 300; private reloadTimer: number = -1; private updated: number = 0; + private started: boolean = false; private enabled: boolean = true; private levels: Level[] | null = null; private audioTracks: MediaPlaylist[] | null = null; private subtitleTracks: MediaPlaylist[] | null = null; + private penalizedPathways: { [pathwayId: string]: number } = {}; constructor(hls: Hls) { this.hls = hls; @@ -65,6 +74,7 @@ export default class ContentSteeringController implements NetworkComponentAPI { hls.on(Events.MANIFEST_LOADING, this.onManifestLoading, this); hls.on(Events.MANIFEST_LOADED, this.onManifestLoaded, this); hls.on(Events.MANIFEST_PARSED, this.onManifestParsed, this); + hls.on(Events.ERROR, this.onError, this); } private unregisterListeners() { @@ -75,14 +85,16 @@ export default class ContentSteeringController implements NetworkComponentAPI { hls.off(Events.MANIFEST_LOADING, this.onManifestLoading, this); hls.off(Events.MANIFEST_LOADED, this.onManifestLoaded, this); hls.off(Events.MANIFEST_PARSED, this.onManifestParsed, this); + hls.off(Events.ERROR, this.onError, this); } startLoad(): void { + this.started = true; self.clearTimeout(this.reloadTimer); if (this.enabled && this.uri) { if (this.updated) { const ttl = Math.max( - this.timeToLoad * 1000 - (Date.now() - this.updated), + this.timeToLoad * 1000 - (performance.now() - this.updated), 0 ); this.scheduleRefresh(this.uri, ttl); @@ -93,6 +105,7 @@ export default class ContentSteeringController implements NetworkComponentAPI { } stopLoad(): void { + this.started = false; if (this.loader) { this.loader.destroy(); this.loader = null; @@ -135,7 +148,9 @@ export default class ContentSteeringController implements NetworkComponentAPI { } this.pathwayId = contentSteering.pathwayId; this.uri = contentSteering.uri; - this.startLoad(); + if (this.started) { + this.startLoad(); + } } private onManifestParsed( @@ -146,6 +161,33 @@ export default class ContentSteeringController implements NetworkComponentAPI { this.subtitleTracks = data.subtitleTracks; } + private onError(event: Events.ERROR, data: ErrorData) { + const { errorAction } = data; + if ( + errorAction?.action === NetworkErrorAction.SendAlternateToPenaltyBox && + errorAction.flags === ErrorActionFlags.MoveAllAlternatesMatchingHost + ) { + let pathwayPriority = this.pathwayPriority; + const pathwayId = this.pathwayId; + if (!this.penalizedPathways[pathwayId]) { + this.penalizedPathways[pathwayId] = performance.now(); + } + if (!pathwayPriority && this.levels) { + // If PATHWAY-PRIORITY was not provided, list pathways for error handling + pathwayPriority = this.levels.reduce((pathways, level) => { + if (pathways.indexOf(level.pathwayId) === -1) { + pathways.push(level.pathwayId); + } + return pathways; + }, [] as string[]); + } + if (pathwayPriority && pathwayPriority.length > 1) { + this.updatePathwayPriority(pathwayPriority); + errorAction.resolved = this.pathwayId !== pathwayId; + } + } + } + public filterParsedLevels(levels: Level[]): Level[] { // Filter levels to only include those that are in the initial pathway this.levels = levels; @@ -177,8 +219,20 @@ export default class ContentSteeringController implements NetworkComponentAPI { private updatePathwayPriority(pathwayPriority: string[]) { this.pathwayPriority = pathwayPriority; let levels: Level[] | undefined; + + // Evaluate if we should remove the pathway from the penalized list + const penalizedPathways = this.penalizedPathways; + const now = performance.now(); + Object.keys(penalizedPathways).forEach((pathwayId) => { + if (now - penalizedPathways[pathwayId] > PATHWAY_PENALTY_DURATION_MS) { + delete penalizedPathways[pathwayId]; + } + }); for (let i = 0; i < pathwayPriority.length; i++) { const pathwayId = pathwayPriority[i]; + if (penalizedPathways[pathwayId]) { + continue; + } if (pathwayId === this.pathwayId) { return; } @@ -322,7 +376,7 @@ export default class ContentSteeringController implements NetworkComponentAPI { this.log(`Steering VERSION ${steeringData.VERSION} not supported!`); return; } - this.updated = Date.now(); + this.updated = performance.now(); this.timeToLoad = steeringData.TTL; const { 'RELOAD-URI': reloadUri, diff --git a/src/controller/eme-controller.ts b/src/controller/eme-controller.ts index dec0831d26a..61b1ba34cf7 100644 --- a/src/controller/eme-controller.ts +++ b/src/controller/eme-controller.ts @@ -33,10 +33,15 @@ import type { ErrorData, ManifestLoadedData, } from '../types/events'; -import type { EMEControllerConfig } from '../config'; +import type { EMEControllerConfig, HlsConfig, LoadPolicy } from '../config'; import type { Fragment } from '../loader/fragment'; +import type { + Loader, + LoaderCallbacks, + LoaderConfiguration, + LoaderContext, +} from '../types/loader'; -const MAX_LICENSE_REQUEST_FAILURES = 3; const LOGGER_PREFIX = '[eme]'; interface KeySystemAccessPromises { @@ -65,7 +70,11 @@ class EMEController implements ComponentAPI { public static CDMCleanupPromise: Promise | void; private readonly hls: Hls; - private readonly config: EMEControllerConfig; + private readonly config: EMEControllerConfig & { + loader: { new (confg: HlsConfig): Loader }; + certLoadPolicy: LoadPolicy; + keyLoadPolicy: LoadPolicy; + }; private media: HTMLMediaElement | null = null; private keyFormatPromise: Promise | null = null; private keySystemAccessPromises: { @@ -832,36 +841,73 @@ class EMEController implements ComponentAPI { private fetchServerCertificate( keySystem: KeySystems ): Promise { + const config = this.config; + const Loader = config.loader; + const certLoader = new Loader(config as HlsConfig) as Loader; + const url = this.getServerCertificateUrl(keySystem); + if (!url) { + return Promise.resolve(); + } + this.log(`Fetching serverCertificate for "${keySystem}"`); return new Promise((resolve, reject) => { - const url = this.getServerCertificateUrl(keySystem); - if (!url) { - return resolve(); - } - this.log(`Fetching serverCertificate for "${keySystem}"`); - const xhr = new XMLHttpRequest(); - xhr.open('GET', url, true); - xhr.responseType = 'arraybuffer'; - xhr.onreadystatechange = () => { - if (xhr.readyState === XMLHttpRequest.DONE) { - if (xhr.status === 200) { - resolve(xhr.response); - } else { - reject( - new EMEKeyError( - { - type: ErrorTypes.KEY_SYSTEM_ERROR, - details: - ErrorDetails.KEY_SYSTEM_SERVER_CERTIFICATE_REQUEST_FAILED, - fatal: true, - networkDetails: xhr, + const loaderContext: LoaderContext = { + responseType: 'arraybuffer', + url, + }; + const loadPolicy = config.certLoadPolicy.default; + const loaderConfig: LoaderConfiguration = { + loadPolicy, + timeout: loadPolicy.maxLoadTimeMs, + maxRetry: 0, + retryDelay: 0, + maxRetryDelay: 0, + }; + const loaderCallbacks: LoaderCallbacks = { + onSuccess: (response, stats, context, networkDetails) => { + resolve(response.data as ArrayBuffer); + }, + onError: (response, contex, networkDetails, stats) => { + reject( + new EMEKeyError( + { + type: ErrorTypes.KEY_SYSTEM_ERROR, + details: + ErrorDetails.KEY_SYSTEM_SERVER_CERTIFICATE_REQUEST_FAILED, + fatal: true, + networkDetails, + response: { + url: loaderContext.url, + data: undefined, + ...response, }, - `"${keySystem}" certificate request XHR failed (${url}). Status: ${xhr.status} (${xhr.statusText})` - ) - ); - } - } + }, + `"${keySystem}" certificate request failed (${url}). Status: ${response.code} (${response.text})` + ) + ); + }, + onTimeout: (stats, context, networkDetails) => { + reject( + new EMEKeyError( + { + type: ErrorTypes.KEY_SYSTEM_ERROR, + details: + ErrorDetails.KEY_SYSTEM_SERVER_CERTIFICATE_REQUEST_FAILED, + fatal: true, + networkDetails, + response: { + url: loaderContext.url, + data: undefined, + }, + }, + `"${keySystem}" certificate request timed out (${url})` + ) + ); + }, + onAbort: (stats, context, networkDetails) => { + reject(new Error('aborted')); + }, }; - xhr.send(); + certLoader.load(loaderContext, loaderConfig, loaderCallbacks); }); } @@ -980,6 +1026,7 @@ class EMEController implements ComponentAPI { keySessionContext: MediaKeySessionContext, licenseChallenge: Uint8Array ): Promise { + const keyLoadPolicy = this.config.keyLoadPolicy.default; return new Promise((resolve, reject) => { const url = this.getLicenseServerUrl(keySessionContext.keySystem); this.log(`Sending license request to URL: ${url}`); @@ -1013,9 +1060,11 @@ class EMEController implements ComponentAPI { } resolve(data); } else { + const retryConfig = keyLoadPolicy.errorRetry; + const maxNumRetry = retryConfig ? retryConfig.maxNumRetry : 0; this._requestLicenseFailureCount++; if ( - this._requestLicenseFailureCount > MAX_LICENSE_REQUEST_FAILURES || + this._requestLicenseFailureCount > maxNumRetry || (xhr.status >= 400 && xhr.status < 500) ) { reject( @@ -1025,15 +1074,19 @@ class EMEController implements ComponentAPI { details: ErrorDetails.KEY_SYSTEM_LICENSE_REQUEST_FAILED, fatal: true, networkDetails: xhr, + response: { + url, + data: undefined as any, + code: xhr.status, + text: xhr.statusText, + }, }, `License Request XHR failed (${url}). Status: ${xhr.status} (${xhr.statusText})` ) ); } else { const attemptsLeft = - MAX_LICENSE_REQUEST_FAILURES - - this._requestLicenseFailureCount + - 1; + maxNumRetry - this._requestLicenseFailureCount + 1; this.warn( `Retrying license request, ${attemptsLeft} attempts left` ); diff --git a/src/controller/error-controller.ts b/src/controller/error-controller.ts index 41709a06c61..db12de0eb5e 100644 --- a/src/controller/error-controller.ts +++ b/src/controller/error-controller.ts @@ -1,14 +1,23 @@ import { Events } from '../events'; -import { ErrorDetails, ErrorTypes } from '../errors'; +import { + ErrorActionFlags, + ErrorDetails, + ErrorTypes, + IErrorAction, + NetworkErrorAction, +} from '../errors'; import { PlaylistContextType, PlaylistLevelType } from '../types/loader'; +import { getRetryConfig, shouldRetry } from '../utils/error-helper'; +import { HdcpLevels } from '../types/level'; import { logger } from '../utils/logger'; import type Hls from '../hls'; +import type { NetworkComponentAPI } from '../types/component-api'; import type { ErrorData } from '../types/events'; import type { Fragment } from '../loader/fragment'; -import { HdcpLevel, HdcpLevels } from '../types/level'; -export default class ErrorController { +export default class ErrorController implements NetworkComponentAPI { private readonly hls: Hls; + private playlistError: number = 0; private log: (msg: any) => void; private warn: (msg: any) => void; private error: (msg: any) => void; @@ -36,89 +45,128 @@ export default class ErrorController { this.hls = null; } + startLoad(startPosition: number): void { + this.playlistError = 0; + } + + stopLoad(): void {} + + private getVariantLevelIndex(frag: Fragment | undefined): number { + return frag?.type === PlaylistLevelType.MAIN + ? frag.level + : this.hls.loadLevel; + } + private onError(event: Events.ERROR, data: ErrorData) { if (data.fatal) { return; } const hls = this.hls; const context = data.context; - const level = hls.levels[hls.loadLevel]; switch (data.details) { case ErrorDetails.FRAG_LOAD_ERROR: case ErrorDetails.FRAG_LOAD_TIMEOUT: case ErrorDetails.KEY_LOAD_ERROR: case ErrorDetails.KEY_LOAD_TIMEOUT: + data.errorAction = this.getFragRetryOrSwitchAction(data); + return; + case ErrorDetails.FRAG_PARSING_ERROR: + case ErrorDetails.FRAG_DECRYPT_ERROR: { + // Switch level if possible, otherwise allow retry count to reach max error retries + data.errorAction = this.getFragRetryOrSwitchAction(data); + data.errorAction.action = NetworkErrorAction.SendAlternateToPenaltyBox; + return; + } + case ErrorDetails.LEVEL_EMPTY_ERROR: + case ErrorDetails.LEVEL_PARSING_ERROR: { - // Share fragment error count accross media options (main, audio, subs) - // This allows for level based rendition switching when media option assets fail - const variantLevelIndex = this.getVariantLevelIndex(data.frag); - const level = hls.levels[variantLevelIndex]; - // Switch levels when out of retried or level index out of bounds - if (level) { - const { fragLoadPolicy, keyLoadPolicy } = hls.config; - const isTimeout = - data.details === ErrorDetails.FRAG_LOAD_TIMEOUT || - data.details === ErrorDetails.KEY_LOAD_TIMEOUT; - const retryConfig = ( - data.details.startsWith('key') ? keyLoadPolicy : fragLoadPolicy - ).default[`${isTimeout ? 'timeout' : 'error'}Retry`]; - const fragmentErrors = hls.levels.reduce( - (acc, level) => acc + level.fragmentError, - 0 + // Only retry when empty and live + const levelIndex = + data.parent === PlaylistLevelType.MAIN + ? (data.level as number) + : hls.loadLevel; + if ( + data.details === ErrorDetails.LEVEL_EMPTY_ERROR && + !!data.context?.levelDetails?.live + ) { + data.errorAction = this.getPlaylistRetryOrSwitchAction( + data, + levelIndex ); - const retry = - !!retryConfig && fragmentErrors < retryConfig.maxNumRetry; - if (!retry) { - this.levelSwitch(data, variantLevelIndex); - } } else { - this.levelSwitch(data, variantLevelIndex); + // Escalate to fatal if not retrying or switching + data.levelRetry = false; + data.errorAction = this.getLevelSwitchAction(data, levelIndex); + } + } + return; + case ErrorDetails.LEVEL_LOAD_ERROR: + case ErrorDetails.LEVEL_LOAD_TIMEOUT: + if (typeof context?.level === 'number') { + data.errorAction = this.getPlaylistRetryOrSwitchAction( + data, + context.level + ); + } + return; + case ErrorDetails.AUDIO_TRACK_LOAD_ERROR: + case ErrorDetails.AUDIO_TRACK_LOAD_TIMEOUT: + case ErrorDetails.SUBTITLE_LOAD_ERROR: + case ErrorDetails.SUBTITLE_TRACK_LOAD_TIMEOUT: + // Switch to redundant level when track fails to load + if (context) { + const level = hls.levels[hls.loadLevel]; + if ( + level && + ((context.type === PlaylistContextType.AUDIO_TRACK && + level.audioGroupIds && + context.groupId === level.audioGroupIds[level.urlId]) || + (context.type === PlaylistContextType.SUBTITLE_TRACK && + level.textGroupIds && + context.groupId === level.textGroupIds[level.urlId])) + ) { + // redundant failover + data.errorAction = { + action: NetworkErrorAction.SendAlternateToPenaltyBox, + flags: ErrorActionFlags.MoveAllAlternatesMatchingHost, + }; + return; } } return; case ErrorDetails.KEY_SYSTEM_STATUS_OUTPUT_RESTRICTED: { + const level = hls.levels[hls.loadLevel]; const restrictedHdcpLevel = level?.attrs['HDCP-LEVEL']; if (restrictedHdcpLevel) { - hls.maxHdcpLevel = - HdcpLevels[ - HdcpLevels.indexOf(restrictedHdcpLevel as HdcpLevel) - 1 - ]; - this.warn( - `Restricting playback to HDCP-LEVEL of "${hls.maxHdcpLevel}" or lower` - ); + data.errorAction = { + action: NetworkErrorAction.SendAlternateToPenaltyBox, + flags: ErrorActionFlags.MoveAllAlternatesMatchingHDCP, + hdcpLevel: restrictedHdcpLevel, + }; } } - break; - case ErrorDetails.FRAG_PARSING_ERROR: - case ErrorDetails.FRAG_DECRYPT_ERROR: { - const levelIndex = this.getVariantLevelIndex(data.frag); - // Switch level if possible, otherwise allow retry count to reach max error retries - this.levelSwitch(data, levelIndex); return; - } + case ErrorDetails.BUFFER_ADD_CODEC_ERROR: case ErrorDetails.REMUX_ALLOC_ERROR: - this.levelSwitch(data, data.level ?? hls.loadLevel); + data.errorAction = this.getLevelSwitchAction( + data, + data.level ?? hls.loadLevel + ); return; - case ErrorDetails.AUDIO_TRACK_LOAD_ERROR: - case ErrorDetails.AUDIO_TRACK_LOAD_TIMEOUT: - case ErrorDetails.SUBTITLE_LOAD_ERROR: - case ErrorDetails.SUBTITLE_TRACK_LOAD_TIMEOUT: - // Switch to redundant level when track fails to load - if ( - context && - level && - ((context.type === PlaylistContextType.AUDIO_TRACK && - level.audioGroupIds && - context.groupId === level.audioGroupIds[level.urlId]) || - (context.type === PlaylistContextType.SUBTITLE_TRACK && - level.textGroupIds && - context.groupId === level.textGroupIds[level.urlId])) - ) { - this.redundantFailover(hls.loadLevel); - return; - } + case ErrorDetails.INTERNAL_EXCEPTION: + case ErrorDetails.BUFFER_APPENDING_ERROR: + case ErrorDetails.BUFFER_APPEND_ERROR: + case ErrorDetails.BUFFER_FULL_ERROR: + case ErrorDetails.LEVEL_SWITCH_ERROR: + case ErrorDetails.BUFFER_STALLED_ERROR: + case ErrorDetails.BUFFER_SEEK_OVER_HOLE: + case ErrorDetails.BUFFER_NUDGE_ON_STALL: + data.errorAction = { + action: NetworkErrorAction.DoNothing, + flags: ErrorActionFlags.None, + }; return; } @@ -126,62 +174,89 @@ export default class ErrorController { const levelIndex = this.getVariantLevelIndex(data.frag); // Do not retry level. Escalate to fatal if switching levels fails. data.levelRetry = false; - this.levelSwitch(data, levelIndex); + data.errorAction = this.getLevelSwitchAction(data, levelIndex); return; } } - public onErrorOut(event: Events.ERROR, data: ErrorData) { + private getPlaylistRetryOrSwitchAction( + data: ErrorData, + levelIndex: number | null | undefined + ): IErrorAction { const hls = this.hls; + const retryConfig = getRetryConfig(hls.config.playlistLoadPolicy, data); + const retryCount = this.playlistError++; + const httpStatus = data.response?.code; + const retry = shouldRetry(retryConfig, retryCount, httpStatus); + if (retry) { + return { + action: NetworkErrorAction.RetryRequest, + flags: ErrorActionFlags.None, + retryConfig, + retryCount, + }; + } + // Do not perform level switch if an error occurred using delivery directives + // Allow reload without directives (handled in playlist-loader) + if (data.context?.deliveryDirectives) { + return { + action: NetworkErrorAction.DoNothing, + flags: ErrorActionFlags.None, + retryConfig: retryConfig || { + maxNumRetry: 0, + retryDelayMs: 0, + maxRetryDelayMs: 0, + }, + retryCount, + }; + } + return this.getLevelSwitchAction(data, levelIndex); + } - if (!data.fatal) { - const context = data.context; - - switch (data.details) { - case ErrorDetails.LEVEL_EMPTY_ERROR: - case ErrorDetails.LEVEL_PARSING_ERROR: - if (!data.levelRetry) { - const levelIndex = - data.parent === PlaylistLevelType.MAIN - ? (data.level as number) - : hls.loadLevel; - this.levelSwitch(data, levelIndex); - } - break; - case ErrorDetails.LEVEL_LOAD_ERROR: - case ErrorDetails.LEVEL_LOAD_TIMEOUT: - if (!data.levelRetry) { - // Do not perform level switch if an error occurred using delivery directives - // Attempt to reload level without directives first - if (context && !context.deliveryDirectives) { - this.levelSwitch(data, context.level); - } - } - break; - case ErrorDetails.FRAG_PARSING_ERROR: - case ErrorDetails.FRAG_DECRYPT_ERROR: - case ErrorDetails.FRAG_LOAD_ERROR: - case ErrorDetails.FRAG_LOAD_TIMEOUT: - case ErrorDetails.KEY_LOAD_ERROR: - case ErrorDetails.KEY_LOAD_TIMEOUT: - if (data.levelRetry === false) { - data.fatal = true; - } + private getFragRetryOrSwitchAction(data: ErrorData): IErrorAction { + const hls = this.hls; + // Share fragment error count accross media options (main, audio, subs) + // This allows for level based rendition switching when media option assets fail + const variantLevelIndex = this.getVariantLevelIndex(data.frag); + const level = hls.levels[variantLevelIndex]; + const { fragLoadPolicy, keyLoadPolicy } = hls.config; + const retryConfig = getRetryConfig( + data.details.startsWith('key') ? keyLoadPolicy : fragLoadPolicy, + data + ); + const fragmentErrors = hls.levels.reduce( + (acc, level) => acc + level.fragmentError, + 0 + ); + // Switch levels when out of retried or level index out of bounds + if (level) { + level.fragmentError++; + const httpStatus = data.response?.code; + const retry = shouldRetry(retryConfig, fragmentErrors, httpStatus); + if (retry) { + return { + action: NetworkErrorAction.RetryRequest, + flags: ErrorActionFlags.None, + retryConfig, + retryCount: fragmentErrors, + }; } } - - if (data.fatal) { - hls.stopLoad(); - return; + // Reach max retry count, or Missing level reference + // Switch to valid index + const errorAction = this.getLevelSwitchAction(data, variantLevelIndex); + // Add retry details to allow skipping of FRAG_PARSING_ERROR + if (retryConfig) { + errorAction.retryConfig = retryConfig; + errorAction.retryCount = fragmentErrors; } - - this.hls.nextLoadLevel = this.hls.nextAutoLevel; + return errorAction; } - private levelSwitch( - errorEvent: ErrorData, + private getLevelSwitchAction( + data: ErrorData, levelIndex: number | null | undefined - ): void { + ): IErrorAction { const hls = this.hls; if (levelIndex === null || levelIndex === undefined) { levelIndex = hls.loadLevel; @@ -192,12 +267,8 @@ export default class ErrorController { const redundantLevels = level.url.length; // Try redundant fail-over until level.loadError reaches redundantLevels if (redundantLevels > 1 && level.loadError < redundantLevels) { - errorEvent.levelRetry = true; - this.redundantFailover(levelIndex); - return; - } - - if (hls.autoLevelEnabled) { + data.levelRetry = true; + } else if (hls.autoLevelEnabled) { // Search for next level to retry let nextLevel = -1; const levels = hls.levels; @@ -212,21 +283,87 @@ export default class ErrorController { } } if (nextLevel > -1 && hls.loadLevel !== nextLevel) { - this.warn(`${errorEvent.details}: switching to level ${nextLevel}`); - errorEvent.levelRetry = true; - this.hls.nextAutoLevel = nextLevel; - } else if (errorEvent.levelRetry === false) { - // No levels to switch to and no more retries - // TODO: switch pathways first - errorEvent.fatal = true; + data.levelRetry = true; + return { + action: NetworkErrorAction.SendAlternateToPenaltyBox, + flags: ErrorActionFlags.None, + nextAutoLevel: nextLevel, + }; } - } else { - // TODO: switch pathways in manual level mode } } + // No levels to switch / Manual level selection / Level not found + // Resolve with Pathway switch, Redundant fail-over, or stay on lowest Level + return { + action: NetworkErrorAction.SendAlternateToPenaltyBox, + flags: ErrorActionFlags.MoveAllAlternatesMatchingHost, + }; } - private redundantFailover(levelIndex: number) { + public onErrorOut(event: Events.ERROR, data: ErrorData) { + switch (data.errorAction?.action) { + case NetworkErrorAction.DoNothing: + break; + case NetworkErrorAction.SendAlternateToPenaltyBox: + this.sendAlternateToPenaltyBox(data); + if (!data.errorAction.resolved) { + data.fatal = true; + } + break; + case NetworkErrorAction.RetryRequest: + // handled by stream and playlist/level controllers + break; + } + + if (data.fatal) { + this.hls.stopLoad(); + return; + } + } + + sendAlternateToPenaltyBox(data: ErrorData) { + const hls = this.hls; + const errorAction = data.errorAction; + if (!errorAction) { + return; + } + const { flags, hdcpLevel, nextAutoLevel } = errorAction; + + switch (flags) { + case ErrorActionFlags.None: + this.warn(`${data.details}: switching to level ${nextAutoLevel}`); + if (nextAutoLevel !== undefined) { + this.hls.nextAutoLevel = nextAutoLevel; + errorAction.resolved = true; + } + // Stream controller is responsible for this but won't swicth on false start + this.hls.nextLoadLevel = this.hls.nextAutoLevel; + break; + case ErrorActionFlags.MoveAllAlternatesMatchingHost: + { + const levelIndex = + data.parent === PlaylistLevelType.MAIN + ? (data.level as number) + : hls.loadLevel; + // Handle Redundant Levels here. Patway switching is handled by content-steering-controller + if (!errorAction.resolved) { + errorAction.resolved = this.redundantFailover(levelIndex); + } + } + break; + case ErrorActionFlags.MoveAllAlternatesMatchingHDCP: + if (hdcpLevel) { + hls.maxHdcpLevel = HdcpLevels[HdcpLevels.indexOf(hdcpLevel) - 1]; + errorAction.resolved = true; + } + this.warn( + `Restricting playback to HDCP-LEVEL of "${hls.maxHdcpLevel}" or lower` + ); + break; + } + } + + private redundantFailover(levelIndex: number): boolean { const hls = this.hls; const level = hls.levels[levelIndex]; const redundantLevels = level.url.length; @@ -238,16 +375,13 @@ export default class ErrorController { level.url[newUrlId] }"` ); + this.playlistError = 0; hls.levels.forEach((lv) => { lv.urlId = newUrlId; }); hls.nextLoadLevel = levelIndex; + return true; } - } - - private getVariantLevelIndex(frag: Fragment | undefined): number { - return frag?.type === PlaylistLevelType.MAIN - ? frag.level - : this.hls.loadLevel; + return false; } } diff --git a/src/controller/level-controller.ts b/src/controller/level-controller.ts index 1bdc052c183..8a2f3f704b4 100644 --- a/src/controller/level-controller.ts +++ b/src/controller/level-controller.ts @@ -18,7 +18,7 @@ import { Events } from '../events'; import { ErrorTypes, ErrorDetails } from '../errors'; import { isCodecSupportedInMp4 } from '../utils/codecs'; import BasePlaylistController from './base-playlist-controller'; -import { PlaylistLevelType } from '../types/loader'; +import { PlaylistContextType, PlaylistLevelType } from '../types/loader'; import type Hls from '../hls'; import type { HlsUrlParameters, LevelParsed } from '../types/level'; import type { MediaPlaylist } from '../types/media-playlist'; @@ -170,7 +170,7 @@ export default class LevelController extends BasePlaylistController { videoCodecFound ||= !!videoCodec; audioCodecFound ||= !!audioCodec; return ( - (!unknownCodecs || !unknownCodecs.length) && + !unknownCodecs?.length && (!audioCodec || isCodecSupportedInMp4(audioCodec, 'audio')) && (!videoCodec || isCodecSupportedInMp4(videoCodec, 'video')) ); @@ -185,16 +185,21 @@ export default class LevelController extends BasePlaylistController { } if (levels.length === 0) { - const error = new Error( - 'no level with compatible codecs found in manifest' - ); - this.hls.trigger(Events.ERROR, { - type: ErrorTypes.MEDIA_ERROR, - details: ErrorDetails.MANIFEST_INCOMPATIBLE_CODECS_ERROR, - fatal: true, - url: data.url, - error, - reason: error.message, + // Dispatch error after MANIFEST_LOADED is done propagating + Promise.resolve().then(() => { + if (this.hls) { + const error = new Error( + 'no level with compatible codecs found in manifest' + ); + this.hls.trigger(Events.ERROR, { + type: ErrorTypes.MEDIA_ERROR, + details: ErrorDetails.MANIFEST_INCOMPATIBLE_CODECS_ERROR, + fatal: true, + url: data.url, + error, + reason: error.message, + }); + } }); return; } @@ -415,28 +420,15 @@ export default class LevelController extends BasePlaylistController { } protected onError(event: Events.ERROR, data: ErrorData) { - if (data.fatal) { + if (data.fatal || !data.context) { return; } - // try to recover not fatal errors - switch (data.details) { - case ErrorDetails.LEVEL_EMPTY_ERROR: - case ErrorDetails.LEVEL_PARSING_ERROR: - // Only retry when empty and live - if ( - data.details === ErrorDetails.LEVEL_EMPTY_ERROR && - !!data.context?.levelDetails?.live - ) { - this.checkRetry(data); - } else { - // Escalate to fatal if not retrying or switching - data.levelRetry = false; - } - break; - case ErrorDetails.LEVEL_LOAD_ERROR: - case ErrorDetails.LEVEL_LOAD_TIMEOUT: - this.checkRetry(data); - break; + + if ( + data.context.type === PlaylistContextType.LEVEL && + data.context.level === this.level + ) { + this.checkRetry(data); } } @@ -467,7 +459,6 @@ export default class LevelController extends BasePlaylistController { // reset level load error counter on successful level loaded only if there is no issues with fragments if (curLevel.fragmentError === 0) { curLevel.loadError = 0; - this.retryCount = 0; } this.playlistLoaded(level, data, curLevel.details); } else if (data.deliveryDirectives?.skip) { diff --git a/src/controller/stream-controller.ts b/src/controller/stream-controller.ts index ac4fb7516af..f427cf317d3 100644 --- a/src/controller/stream-controller.ts +++ b/src/controller/stream-controller.ts @@ -743,6 +743,7 @@ export default class StreamController if (fragCurrent) { this.log('Switching to main audio track, cancel main fragment load'); fragCurrent.abortRequests(); + this.fragmentTracker.removeFragment(fragCurrent); } // destroy transmuxer to force init segment generation (following audio switch) this.resetTransmuxer(); diff --git a/src/controller/subtitle-track-controller.ts b/src/controller/subtitle-track-controller.ts index 458d9999289..8d07f8c8e71 100644 --- a/src/controller/subtitle-track-controller.ts +++ b/src/controller/subtitle-track-controller.ts @@ -175,7 +175,6 @@ class SubtitleTrackController extends BasePlaylistController { ); if (id === this.trackId) { - this.retryCount = 0; this.playlistLoaded(id, data, curDetails); } } diff --git a/src/demux/tsdemuxer.ts b/src/demux/tsdemuxer.ts index d8e37ba4e54..3332228aa64 100644 --- a/src/demux/tsdemuxer.ts +++ b/src/demux/tsdemuxer.ts @@ -626,15 +626,15 @@ class TSDemuxer implements Demuxer { } if (!track.sps) { - const expGolombDecoder = new ExpGolomb(unit.data); + const sps = unit.data; + const expGolombDecoder = new ExpGolomb(sps); const config = expGolombDecoder.readSPS(); track.width = config.width; track.height = config.height; track.pixelRatio = config.pixelRatio; - // TODO: `track.sps` is defined as a `number[]`, but we're setting it to a `Uint8Array[]`. - track.sps = [unit.data] as any; + track.sps = [sps]; track.duration = this._duration; - const codecarray = unit.data.subarray(1, 4); + const codecarray = sps.subarray(1, 4); let codecstring = 'avc1.'; for (let i = 0; i < 3; i++) { let h = codecarray[i].toString(16); @@ -655,8 +655,7 @@ class TSDemuxer implements Demuxer { } if (!track.pps) { - // TODO: `track.pss` is defined as a `number[]`, but we're setting it to a `Uint8Array[]`. - track.pps = [unit.data] as any; + track.pps = [unit.data]; } break; diff --git a/src/errors.ts b/src/errors.ts index 5687c34e77e..b230d4b456f 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,3 +1,6 @@ +import { RetryConfig } from './config'; +import { HdcpLevel } from './types/level'; + export enum ErrorTypes { // Identifier for a network error (loading error / timeout ...) NETWORK_ERROR = 'networkError', @@ -86,3 +89,29 @@ export enum ErrorDetails { // Uncategorized error UNKNOWN = 'unknown', } + +export enum NetworkErrorAction { + DoNothing = 0, + SendEndCallback = 1, // Reserved for future use + SendAlternateToPenaltyBox = 2, + RemoveAlternatePermanently = 3, // Reserved for future use + InsertDiscontinuity = 4, // Reserved for future use + RetryRequest = 5, +} + +export enum ErrorActionFlags { + None = 0, + MoveAllAlternatesMatchingHost = 1, + MoveAllAlternatesMatchingHDCP = 1 << 1, + SwitchToSDR = 1 << 2, // Reserved for future use +} + +export type IErrorAction = { + action: NetworkErrorAction; + flags: ErrorActionFlags; + retryCount?: number; + retryConfig?: RetryConfig; + hdcpLevel?: HdcpLevel; + nextAutoLevel?: number; + resolved?: boolean; +}; diff --git a/src/hls.ts b/src/hls.ts index eaf582e8538..2610d4a885b 100644 --- a/src/hls.ts +++ b/src/hls.ts @@ -186,7 +186,6 @@ export default class Hls implements HlsEventEmitter { fpsController, id3TrackController, fragmentTracker, - errorController, ]; this.audioTrackController = this.createController( @@ -228,6 +227,7 @@ export default class Hls implements HlsEventEmitter { // Error controller handles errors before and after all other controllers // This listener will be invoked after all other controllers error listeners + networkControllers.push(errorController); const onErrorOut = errorController.onErrorOut; if (typeof onErrorOut === 'function') { this.on(Events.ERROR, onErrorOut, errorController); @@ -1036,4 +1036,9 @@ export type { SubtitleTracksUpdatedData, SubtitleTrackSwitchData, } from './types/events'; +export type { + IErrorAction, + NetworkErrorAction, + ErrorActionFlags, +} from './errors'; export type { AttrList } from './utils/attr-list'; diff --git a/src/loader/playlist-loader.ts b/src/loader/playlist-loader.ts index 2391972960c..9c52a2c6c0f 100644 --- a/src/loader/playlist-loader.ts +++ b/src/loader/playlist-loader.ts @@ -535,7 +535,9 @@ class PlaylistLoader implements NetworkComponentAPI { stats: LoaderStats ): void { let message = `A network ${ - timeout ? 'timeout' : 'error' + timeout + ? 'timeout' + : 'error' + (response ? ' (status ' + response.code + ')' : '') } occurred while loading ${context.type}`; if (context.type === PlaylistContextType.LEVEL) { message += `: ${context.level} id: ${context.id}`; diff --git a/src/types/demuxer.ts b/src/types/demuxer.ts index 92794c3c759..4fa4c6d81c8 100644 --- a/src/types/demuxer.ts +++ b/src/types/demuxer.ts @@ -72,8 +72,8 @@ export interface DemuxedVideoTrack extends DemuxedTrack { height?: number; pixelRatio?: [number, number]; audFound?: boolean; - pps?: number[]; - sps?: number[]; + pps?: Uint8Array[]; + sps?: Uint8Array[]; naluState?: number; samples: AvcSample[] | Uint8Array; } diff --git a/src/types/events.ts b/src/types/events.ts index db4fa4c0bf0..318d197d14e 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -22,12 +22,12 @@ import type { Track, TrackSet } from './track'; import type { SourceBufferName } from './buffer'; import type { ChunkMetadata } from './transmuxer'; import type { LoadStats } from '../loader/load-stats'; -import type { ErrorDetails, ErrorTypes } from '../errors'; +import type { ErrorDetails, ErrorTypes, IErrorAction } from '../errors'; import type { MetadataSample, UserdataSample } from './demuxer'; import type { AttrList } from '../utils/attr-list'; import type { HlsListeners } from '../events'; -import { KeyLoaderInfo } from '../loader/key-loader'; -import { LevelKey } from '../loader/level-key'; +import type { KeyLoaderInfo } from '../loader/key-loader'; +import type { LevelKey } from '../loader/level-key'; export interface MediaAttachingData { media: HTMLMediaElement; @@ -222,12 +222,13 @@ export interface FPSDropLevelCappingData { export interface ErrorData { type: ErrorTypes; details: ErrorDetails; + error: Error; fatal: boolean; + errorAction?: IErrorAction; buffer?: number; bytes?: number; chunkMeta?: ChunkMetadata; context?: PlaylistLoaderContext; - error: Error; event?: keyof HlsListeners | 'demuxerWorker'; frag?: Fragment; level?: number | undefined; @@ -240,6 +241,9 @@ export interface ErrorData { response?: LoaderResponse; url?: string; parent?: PlaylistLevelType; + /** + * @deprecated Use ErrorData.error + */ err?: { // comes from transmuxer interface message: string; @@ -358,6 +362,6 @@ export interface BackBufferData { } /** - * Deprecated; please use BackBufferData + * @deprecated Use BackBufferData */ export interface LiveBackBufferData extends BackBufferData {} diff --git a/src/types/level.ts b/src/types/level.ts index e9a063c49f8..0ced190b105 100644 --- a/src/types/level.ts +++ b/src/types/level.ts @@ -36,7 +36,7 @@ export interface LevelAttributes extends AttrList { 'VIDEO-RANGE'?: 'SDR' | 'HLG' | 'PQ'; } -export const HdcpLevels = ['NONE', 'TYPE-0', 'TYPE-1', 'TYPE-2', null] as const; +export const HdcpLevels = ['NONE', 'TYPE-0', 'TYPE-1', null] as const; export type HdcpLevel = (typeof HdcpLevels)[number]; export type VariableMap = Record; diff --git a/src/types/loader.ts b/src/types/loader.ts index 99b20bb53b3..236822e74aa 100644 --- a/src/types/loader.ts +++ b/src/types/loader.ts @@ -62,7 +62,7 @@ export interface LoaderConfiguration { export interface LoaderResponse { url: string; - data: string | ArrayBuffer | Object; + data?: string | ArrayBuffer | Object; // Errors can include HTTP status code and error message code?: number; text?: string; diff --git a/src/utils/error-helper.ts b/src/utils/error-helper.ts new file mode 100644 index 00000000000..846a86cc410 --- /dev/null +++ b/src/utils/error-helper.ts @@ -0,0 +1,54 @@ +import { LoadPolicy, RetryConfig } from '../config'; +import { ErrorDetails } from '../errors'; +import { ErrorData } from '../types/events'; + +export function isTimeoutError(error: ErrorData): boolean { + switch (error.details) { + case ErrorDetails.FRAG_LOAD_TIMEOUT: + case ErrorDetails.KEY_LOAD_TIMEOUT: + case ErrorDetails.LEVEL_LOAD_TIMEOUT: + case ErrorDetails.MANIFEST_LOAD_TIMEOUT: + return true; + } + return false; +} + +export function getRetryConfig( + loadPolicy: LoadPolicy, + error: ErrorData +): RetryConfig | null { + const isTimeout = isTimeoutError(error); + return loadPolicy.default[`${isTimeout ? 'timeout' : 'error'}Retry`]; +} + +export function getRetryDelay( + retryConfig: RetryConfig, + retryCount: number +): number { + // exponential backoff capped to max retry delay + const backoffFactor = + retryConfig.backoff === 'linear' ? 1 : Math.pow(2, retryCount); + return Math.min( + backoffFactor * retryConfig.retryDelayMs, + retryConfig.maxRetryDelayMs + ); +} + +export function shouldRetry( + retryConfig: RetryConfig | null, + retryCount: number, + httpStatus: number | undefined +): retryConfig is RetryConfig & boolean { + return ( + !!retryConfig && + retryCount < retryConfig.maxNumRetry && + retryForHttpStatus(httpStatus) + ); +} + +export function retryForHttpStatus(httpStatus: number | undefined) { + return ( + httpStatus === undefined || + (httpStatus !== 0 && (httpStatus < 400 || httpStatus > 499)) + ); +} diff --git a/src/utils/xhr-loader.ts b/src/utils/xhr-loader.ts index 25f9d35e767..b59e6494bc1 100644 --- a/src/utils/xhr-loader.ts +++ b/src/utils/xhr-loader.ts @@ -8,6 +8,7 @@ import type { } from '../types/loader'; import { LoadStats } from '../loader/load-stats'; import { RetryConfig } from '../config'; +import { getRetryDelay, shouldRetry } from './error-helper'; const AGE_HEADER_LINE_REGEX = /^age:\s*[\d.]+\s*$/im; @@ -204,14 +205,11 @@ class XhrLoader implements Loader { this.callbacks.onSuccess(response, stats, context, xhr); } else { const retryConfig = config.loadPolicy.errorRetry; + const retryCount = stats.retry; // if max nb of retries reached or if http status between 400 and 499 (such error cannot be recovered, retrying is useless), return error - if ( - (status >= 400 && status <= 500) || - (status >= 502 && status <= 504) || - status === 0 || - !retryConfig || - stats.retry >= retryConfig.maxNumRetry - ) { + if (shouldRetry(retryConfig, retryCount, status)) { + this.retry(retryConfig); + } else { logger.error(`${status} while loading ${context.url}`); this.callbacks!.onError( { code: status, text: xhr.statusText }, @@ -219,8 +217,6 @@ class XhrLoader implements Loader { xhr, stats ); - } else if (retryConfig) { - this.retry(retryConfig); } } } @@ -229,7 +225,8 @@ class XhrLoader implements Loader { loadtimeout(): void { const retryConfig = this.config?.loadPolicy.timeoutRetry; - if (retryConfig && this.stats.retry < retryConfig.maxNumRetry) { + const retryCount = this.stats.retry; + if (retryConfig && retryCount < retryConfig.maxNumRetry) { this.retry(retryConfig); } else { logger.warn(`timeout while loading ${this.context.url}`); @@ -243,13 +240,14 @@ class XhrLoader implements Loader { retry(retryConfig: RetryConfig) { const { context, stats } = this; - if (this.retryDelay === 0) { - this.retryDelay = retryConfig.retryDelayMs; - } + this.retryDelay = getRetryDelay(retryConfig, stats.retry); + stats.retry++; logger.warn( `${status ? 'HTTP Status ' + status : 'Timeout'} while loading ${ context.url - }, retrying in ${this.retryDelay}ms` + }, retrying ${stats.retry}/${retryConfig.maxNumRetry} in ${ + this.retryDelay + }ms` ); // abort and reset internal state this.abortInternal(); @@ -260,12 +258,6 @@ class XhrLoader implements Loader { this.loadInternal.bind(this), this.retryDelay ); - const backoffFactor = retryConfig.backoff === 'exponential' ? 2 : 1; - this.retryDelay = Math.min( - backoffFactor * this.retryDelay, - retryConfig.maxRetryDelayMs - ); - stats.retry++; } loadprogress(event: ProgressEvent): void { diff --git a/tests/unit/controller/buffer-controller-operations.ts b/tests/unit/controller/buffer-controller-operations.ts index 213714ba251..f7eec8a0b9a 100644 --- a/tests/unit/controller/buffer-controller-operations.ts +++ b/tests/unit/controller/buffer-controller-operations.ts @@ -267,8 +267,6 @@ describe('BufferController', function () { data.id, 'The id of the event should be equal to the frag type' ).to.equal(frag.type); - // TODO: remove stats from event & place onto frag - // expect(data.stats).to.equal({}); } catch (e) { reject(e); } diff --git a/tests/unit/controller/content-steering-controller.ts b/tests/unit/controller/content-steering-controller.ts index 7c93a254a00..d5d83fc7820 100644 --- a/tests/unit/controller/content-steering-controller.ts +++ b/tests/unit/controller/content-steering-controller.ts @@ -142,6 +142,7 @@ describe('ContentSteeringController', function () { describe('Steering Manifest', function () { it('loads the steering manifest', function () { + contentSteeringController.startLoad(); contentSteeringController.onManifestLoaded(Events.MANIFEST_LOADED, { contentSteering: { uri: 'http://example.com/manifest.json', @@ -776,6 +777,7 @@ function loadSteeringManifest( partialManifest: Partial, steering: ConentSteeringControllerTestable ) { + steering.startLoad(); const response: LoaderResponse = { url: '', data: { diff --git a/tests/unit/controller/error-controller.ts b/tests/unit/controller/error-controller.ts index cf32aae9d6b..58aa4a31beb 100644 --- a/tests/unit/controller/error-controller.ts +++ b/tests/unit/controller/error-controller.ts @@ -148,7 +148,7 @@ describe('ErrorController Integration Tests', function () { expectFatalErrorEventToStopPlayer( hls, ErrorDetails.MANIFEST_LOAD_ERROR, - 'A network error occurred while loading manifest' + 'A network error (status 400) occurred while loading manifest' ) ); }); @@ -186,7 +186,7 @@ describe('ErrorController Integration Tests', function () { expectFatalErrorEventToStopPlayer( hls, ErrorDetails.MANIFEST_LOAD_ERROR, - 'A network error occurred while loading manifest' + 'A network error (status 501) occurred while loading manifest' ) ); }); @@ -322,11 +322,7 @@ describe('ErrorController Integration Tests', function () { errorIndex = data.level; server.respondWith(data.url, testResponses['noTargetDuration.m3u8']); }); - hls.on(Events.LEVEL_LOADING, (event, data) => { - Promise.resolve().then(() => { - server.respond(); - }); - }); + hls.on(Events.LEVEL_LOADING, loadingEventCallback(server, timers)); return new Promise((resolve) => { hls.on(Events.ERROR, (event, data) => resolve(data)); server.respond(); @@ -358,9 +354,7 @@ describe('ErrorController Integration Tests', function () { errorIndex = data.level; server.respondWith(data.url, [400, {}, '']); }); - hls.on(Events.LEVEL_LOADING, (event, data) => { - server.respond(); - }); + hls.on(Events.LEVEL_LOADING, loadingEventCallback(server, timers)); return new Promise((resolve) => { hls.on(Events.ERROR, (event, data) => resolve(data)); server.respond(); @@ -368,7 +362,7 @@ describe('ErrorController Integration Tests', function () { expect(data.details).to.equal(ErrorDetails.LEVEL_LOAD_ERROR); expect(data.fatal).to.equal(false, 'Error should not be fatal'); expect(data.error.message).to.equal( - 'A network error occurred while loading level: 1 id: 0', + 'A network error (status 400) occurred while loading level: 1 id: 0', data.error.message ); hls.stopLoad.should.have.been.calledOnce; @@ -411,23 +405,21 @@ describe('ErrorController Integration Tests', function () { describe('Segment Error Handling', function () { it('Fragment HTTP Load Errors retry fragLoadPolicy `errorRetry.maxNumRetry` times before switching down and continues until no lower levels are available', function () { server.respondWith('multivariantPlaylist.m3u8/segment.mp4', [ - 400, + 500, {}, new ArrayBuffer(0), ]); hls.loadSource('multivariantPlaylist.m3u8'); - hls.on(Events.LEVEL_LOADING, (event, data) => { - server.respond(); - }); - hls.on(Events.FRAG_LOADING, (event, data) => { - server.respond(); - }); + hls.on(Events.LEVEL_LOADING, loadingEventCallback(server, timers)); + hls.on(Events.FRAG_LOADING, loadingEventCallback(server, timers)); return new Promise((resolve, reject) => { hls.on(Events.ERROR, (event, data) => { if (data.fatal) { resolve(data); } else { - timers.tick(hls.config.fragLoadPolicy.default.maxTimeToFirstByteMs); + timers.tick( + hls.config.fragLoadPolicy.default.errorRetry!.maxRetryDelayMs + ); } }); hls.on(Events.FRAG_LOADED, (event, data) => @@ -443,7 +435,7 @@ describe('ErrorController Integration Tests', function () { const finalAssertion = expectFatalErrorEventToStopPlayer( hls, ErrorDetails.FRAG_LOAD_ERROR, - 'HTTP Error 400 Bad Request' + 'HTTP Error 500 Internal Server Error' ); finalAssertion(errorData); }); @@ -558,10 +550,18 @@ segment.mp4 hls.loadSource('oneSegmentVod-mp2ts.m3u8'); hls.on(Events.LEVEL_LOADING, loadingEventCallback(server, timers)); hls.on(Events.FRAG_LOADING, loadingEventCallback(server, timers)); + let errorCount = 0; return new Promise((resolve, reject) => { hls.on(Events.ERROR, (event, data) => { - if (data.fatal) { + errorCount++; + if (errorCount === 3 && data.fatal) { resolve(data); + } else if (data.fatal) { + reject( + new Error( + `Error fatal before retries exhausted: "${data.error.message}"` + ) + ); } else { timers.tick(8000); } @@ -709,7 +709,7 @@ segment.mp4 hls.on(Events.FRAG_LOADING, loadingEventCallback(server, timers)); hls.on(Events.ERROR, (event, data) => { errors.push(data); - timers.tick(2000); + Promise.resolve().then(() => timers.tick(2000)); }); return new Promise((resolve, reject) => { hls.on(Events.LEVEL_SWITCHING, (event, data) => { @@ -744,7 +744,9 @@ segment.mp4 resolve(data); }); hls.on(Events.ERROR, (event, data) => { - reject(new Error('Unexpected error after fallback')); + reject( + new Error(`Unexpected error after fallback: ${data.error}`) + ); }); }); }) @@ -754,7 +756,7 @@ segment.mp4 'Error should not be fatal' ); expect(data.frag.url).to.equal( - 'http://www.baz.com/video-segment.mp4' + 'http://www.baz.com/audio-segment.mp4' ); }); }); @@ -817,7 +819,7 @@ segment.mp4 hls.on(Events.FRAG_LOADING, loadingEventCallback(server, timers)); hls.on(Events.ERROR, (event, data) => { errors.push(data); - timers.tick(2000); + Promise.resolve().then(() => timers.tick(2000)); }); return new Promise((resolve, reject) => { hls.on(Events.LEVEL_SWITCHING, (event, data) => { @@ -862,7 +864,7 @@ segment.mp4 'Error should not be fatal' ); expect(data.frag.url).to.equal( - 'http://www.baz.com/video-segment.mp4' + 'http://www.baz.com/audio-segment.mp4' ); }); }); diff --git a/tests/unit/controller/level-controller.ts b/tests/unit/controller/level-controller.ts index eee1bf01503..810d324bc52 100644 --- a/tests/unit/controller/level-controller.ts +++ b/tests/unit/controller/level-controller.ts @@ -186,13 +186,15 @@ describe('LevelController', function () { url: 'https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8', }); - expect(hls.trigger).to.have.been.calledWith(Events.ERROR, { - type: ErrorTypes.MEDIA_ERROR, - details: ErrorDetails.MANIFEST_INCOMPATIBLE_CODECS_ERROR, - fatal: true, - url: 'https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8', - error: hls.trigger.getCall(0).lastArg.error, - reason: 'no level with compatible codecs found in manifest', + return Promise.resolve().then(() => { + expect(hls.trigger).to.have.been.calledWith(Events.ERROR, { + type: ErrorTypes.MEDIA_ERROR, + details: ErrorDetails.MANIFEST_INCOMPATIBLE_CODECS_ERROR, + fatal: true, + url: 'https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8', + error: hls.trigger.getCall(0).lastArg.error, + reason: 'no level with compatible codecs found in manifest', + }); }); }); diff --git a/tests/unit/controller/stream-controller.ts b/tests/unit/controller/stream-controller.ts index 9a6f3a39bac..501b89bdff2 100644 --- a/tests/unit/controller/stream-controller.ts +++ b/tests/unit/controller/stream-controller.ts @@ -33,7 +33,12 @@ describe('StreamController', function () { beforeEach(function () { fake = sinon.useFakeXMLHttpRequest(); - hls = new Hls({}); + hls = new Hls({ + // Enable debug to catch callback errors and enable logging in these tests: + // debug: true, + startFragPrefetch: true, + enableWorker: false, + }); streamController = hls['streamController']; fragmentTracker = streamController['fragmentTracker']; streamController['startFragRequested'] = true; diff --git a/tests/unit/loader/playlist-loader.ts b/tests/unit/loader/playlist-loader.ts index e8639661a3a..3f7190a60cb 100644 --- a/tests/unit/loader/playlist-loader.ts +++ b/tests/unit/loader/playlist-loader.ts @@ -2381,11 +2381,6 @@ a{$bar}.mp4 }); }); -describe('#EXT-X-CONTENT-STEERING', function () { - // TODO: CONTENT-STEERING - it('', function () {}); -}); - function expectWithJSONMessage(value: any, msg?: string) { return expect(value, `${msg || 'actual:'} ${JSON.stringify(value, null, 2)}`); }