From cfe8af5ff928fe7466b103429a6325917579ce70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Velad=20Galv=C3=A1n?= Date: Fri, 12 Aug 2022 22:01:17 +0200 Subject: [PATCH] feat: Automatic ABR quality restrictions based on size (#4404) Closes #2333 --- demo/common/message_ids.js | 2 + demo/config.js | 6 ++- demo/locales/en.json | 2 + demo/locales/source.json | 8 ++++ externs/shaka/abr_manager.js | 8 ++++ externs/shaka/player.js | 15 ++++++- lib/abr/simple_abr_manager.js | 76 ++++++++++++++++++++++++++++++-- lib/player.js | 7 +++ lib/util/player_configuration.js | 2 + 9 files changed, 119 insertions(+), 7 deletions(-) diff --git a/demo/common/message_ids.js b/demo/common/message_ids.js index cffd96756b..a067989618 100644 --- a/demo/common/message_ids.js +++ b/demo/common/message_ids.js @@ -188,6 +188,7 @@ shakaDemo.MessageIds = { IGNORE_DASH_DRM: 'DEMO_IGNORE_DASH_DRM', IGNORE_DASH_MAX_SEGMENT_DURATION: 'DEMO_IGNORE_DASH_MAX_SEGMENT_DURATION', IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY: 'DEMO_IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY', + IGNORE_DEVICE_PIXEL_RATIO: 'DEMO_IGNORE_DEVICE_PIXEL_RATIO', IGNORE_HLS_IMAGE_FAILURES: 'DEMO_IGNORE_HLS_IMAGE_FAILURES', IGNORE_HLS_TEXT_FAILURES: 'DEMO_IGNORE_HLS_TEXT_FAILURES', IGNORE_MANIFEST_PROGRAM_DATE_TIME: 'DEMO_IGNORE_MANIFEST_PROGRAM_DATE_TIME', @@ -229,6 +230,7 @@ shakaDemo.MessageIds = { PREFER_FORCED_SUBS: 'DEMO_PREFER_FORCED_SUBS', PREFER_NATIVE_HLS: 'DEMO_PREFER_NATIVE_HLS', REBUFFERING_GOAL: 'DEMO_REBUFFERING_GOAL', + RESTRICT_TO_ELEMENT_SIZE: 'DEMO_RESTRICT_TO_ELEMENT_SIZE', RESTRICTIONS_SECTION_HEADER: 'DEMO_RESTRICTIONS_SECTION_HEADER', SAFE_SEEK_OFFSET: 'DEMO_SAFE_SEEK_OFFSET', SAFE_SKIP_DISTANCE: 'DEMO_SAFE_SKIP_DISTANCE', diff --git a/demo/config.js b/demo/config.js index c54ab7f271..62d46fa7f9 100644 --- a/demo/config.js +++ b/demo/config.js @@ -271,7 +271,11 @@ shakaDemo.Config = class { /* canBeDecimal= */ true) .addNumberInput_(MessageIds.SLOW_HALF_LIFE, 'abr.advanced.slowHalfLife', - /* canBeDecimal= */ true); + /* canBeDecimal= */ true) + .addBoolInput_(MessageIds.RESTRICT_TO_ELEMENT_SIZE, + 'abr.restrictToElementSize') + .addBoolInput_(MessageIds.IGNORE_DEVICE_PIXEL_RATIO, + 'abr.ignoreDevicePixelRatio'); this.addRetrictionsSection_('abr', MessageIds.ADAPTATION_RESTRICTIONS_SECTION_HEADER); } diff --git a/demo/locales/en.json b/demo/locales/en.json index 63da8909e5..be7907470c 100644 --- a/demo/locales/en.json +++ b/demo/locales/en.json @@ -95,6 +95,7 @@ "DEMO_IGNORE_DASH_EMPTY_ADAPTATION_SET": "Ignore empty DASH AdaptationSets", "DEMO_IGNORE_DASH_MAX_SEGMENT_DURATION": "Ignore DASH maxSegmentDuration", "DEMO_IGNORE_DASH_SUGGESTED_PRESENTATION_DELAY": "Ignore DASH suggestedPresentationDelay", + "DEMO_IGNORE_DEVICE_PIXEL_RATIO": "Ignore device pixel ratio", "DEMO_IGNORE_HLS_IMAGE_FAILURES": "Ignore HLS Image Stream Failures", "DEMO_IGNORE_HLS_TEXT_FAILURES": "Ignore HLS Text Stream Failures", "DEMO_IMA_ASSET_KEY": "Asset key (for LIVE DAI Content)", @@ -177,6 +178,7 @@ "DEMO_PROMPT_YES": "Yes", "DEMO_REBUFFERING_GOAL": "Rebuffering Goal", "DEMO_REPORT_BUG": "REPORT BUG", + "DEMO_RESTRICT_TO_ELEMENT_SIZE": "Restrict to element size", "DEMO_RESTRICTIONS_SECTION_HEADER": "Restrictions", "DEMO_SAFE_SEEK_OFFSET": "Safe Seek Offset", "DEMO_SAFE_SKIP_DISTANCE": "Safe Skip Distance", diff --git a/demo/locales/source.json b/demo/locales/source.json index ec29388ecb..de9940b1aa 100644 --- a/demo/locales/source.json +++ b/demo/locales/source.json @@ -383,6 +383,10 @@ "description": "The name of a configuration value.", "message": "Ignore [PROPER_NAME:DASH] [JARGON:suggestedPresentationDelay]" }, + "DEMO_IGNORE_DEVICE_PIXEL_RATIO": { + "description": "The name of a configuration value.", + "message": "Ignore device pixel ratio" + }, "DEMO_IGNORE_HLS_TEXT_FAILURES": { "description": "The name of a configuration value.", "message": "Ignore [PROPER_NAME:HLS] Text Stream Failures" @@ -711,6 +715,10 @@ "description": "A link in the header, that files a bug report.", "message": "REPORT BUG" }, + "DEMO_RESTRICT_TO_ELEMENT_SIZE": { + "description": "The name of a configuration value.", + "message": "Restrict to element size" + }, "DEMO_RESTRICTIONS_SECTION_HEADER": { "description": "The header for a section of configuration values.", "message": "Restrictions" diff --git a/externs/shaka/abr_manager.js b/externs/shaka/abr_manager.js index 56a7444d9e..f421ac7f41 100644 --- a/externs/shaka/abr_manager.js +++ b/externs/shaka/abr_manager.js @@ -103,6 +103,14 @@ shaka.extern.AbrManager = class { */ playbackRateChanged(rate) {} + /** + * Set media element. + * + * @param {HTMLMediaElement} mediaElement + * @exportDoc + */ + setMediaElement(mediaElement) {} + /** * Sets the ABR configuration. * diff --git a/externs/shaka/player.js b/externs/shaka/player.js index f5bd8180cc..8713625888 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -1000,7 +1000,9 @@ shaka.extern.StreamingConfiguration; * switchInterval: number, * bandwidthUpgradeTarget: number, * bandwidthDowngradeTarget: number, - * advanced: shaka.extern.AdvancedAbrConfiguration + * advanced: shaka.extern.AdvancedAbrConfiguration, + * restrictToElementSize: boolean, + * ignoreDevicePixelRatio: boolean * }} * * @property {boolean} enabled @@ -1029,7 +1031,16 @@ shaka.extern.StreamingConfiguration; * The largest fraction of the estimated bandwidth we should use. We should * downgrade to avoid this. * @property {shaka.extern.AdvancedAbrConfiguration} advanced - * Advanced ABR configuration. + * Advanced ABR configuration + * @property {boolean} restrictToElementSize + * If true, restrict the quality to media element size. + * Note: The use of ResizeObserver is required for it to work properly. If + * true without ResizeObserver, it behaves as false. + * Defaults false. + * @property {boolean} ignoreDevicePixelRatio + * If true,device pixel ratio is ignored when restricting the quality to + * media element size. + * Defaults false. * @exportDoc */ shaka.extern.AbrConfiguration; diff --git a/lib/abr/simple_abr_manager.js b/lib/abr/simple_abr_manager.js index 1d8bf09f5a..d799dbf06b 100644 --- a/lib/abr/simple_abr_manager.js +++ b/lib/abr/simple_abr_manager.js @@ -10,6 +10,7 @@ goog.require('goog.asserts'); goog.require('shaka.abr.EwmaBandwidthEstimator'); goog.require('shaka.log'); goog.require('shaka.util.StreamUtils'); +goog.require('shaka.util.Timer'); /** @@ -85,6 +86,22 @@ shaka.abr.SimpleAbrManager = class { /** @private {?shaka.extern.AbrConfiguration} */ this.config_ = null; + + /** @private {HTMLMediaElement} */ + this.mediaElement_ = null; + + /** @private {ResizeObserver} */ + this.resizeObserver_ = null; + + /** @private {shaka.util.Timer} */ + this.resizeObserverTimer_ = new shaka.util.Timer(() => { + if (this.config_.restrictToElementSize) { + const chosenVariant = this.chooseVariant(); + if (chosenVariant) { + this.switch_(chosenVariant); + } + } + }); } @@ -98,6 +115,14 @@ shaka.abr.SimpleAbrManager = class { this.variants_ = []; this.playbackRate_ = 1; this.lastTimeChosenMs_ = null; + this.mediaElement_ = null; + + if (this.resizeObserver_) { + this.resizeObserver_.disconnect(); + this.resizeObserver_ = null; + } + + this.resizeObserverTimer_.stop(); // Don't reset |startupComplete_|: if we've left the startup interval, we // can start using bandwidth estimates right away after init() is called. @@ -120,9 +145,19 @@ shaka.abr.SimpleAbrManager = class { chooseVariant() { const SimpleAbrManager = shaka.abr.SimpleAbrManager; + let maxHeight = Infinity; + let maxWidth = Infinity; + + if (this.resizeObserver_ && this.config_.restrictToElementSize) { + const devicePixelRatio = + this.config_.ignoreDevicePixelRatio ? 1 : window.devicePixelRatio; + maxHeight = this.mediaElement_.clientWidth * devicePixelRatio; + maxWidth = this.mediaElement_.clientHeight * devicePixelRatio; + } + // Get sorted Variants. let sortedVariants = SimpleAbrManager.filterAndSortVariants_( - this.config_.restrictions, this.variants_); + this.config_.restrictions, this.variants_, maxHeight, maxWidth); const defaultBandwidthEstimate = this.getDefaultBandwidth_(); const currentBandwidth = this.bandwidthEstimator_.getBandwidthEstimate( @@ -137,7 +172,8 @@ shaka.abr.SimpleAbrManager = class { shaka.log.warning('No variants met the ABR restrictions. ' + 'Choosing a variant by lowest bandwidth.'); sortedVariants = SimpleAbrManager.filterAndSortVariants_( - /* restrictions= */ null, this.variants_); + /* restrictions= */ null, this.variants_, + /* maxHeight= */ Infinity, /* maxWidth= */ Infinity); sortedVariants = [sortedVariants[0]]; } @@ -243,6 +279,28 @@ shaka.abr.SimpleAbrManager = class { } + /** + * @override + * @export + */ + setMediaElement(mediaElement) { + this.mediaElement_ = mediaElement; + if (this.resizeObserver_) { + this.resizeObserver_.disconnect(); + this.resizeObserver_ = null; + } + if (this.mediaElement_ && 'ResizeObserver' in window) { + this.resizeObserver_ = new ResizeObserver(() => { + const SimpleAbrManager = shaka.abr.SimpleAbrManager; + // Batch up resize changes before checking them. + this.resizeObserverTimer_.tickAfter( + /* seconds= */ SimpleAbrManager.RESIZE_OBSERVER_BATCH_TIME); + }); + this.resizeObserver_.observe(this.mediaElement_); + } + } + + /** * @override * @export @@ -320,11 +378,13 @@ shaka.abr.SimpleAbrManager = class { /** * @param {?shaka.extern.Restrictions} restrictions * @param {!Array.} variants + * @param {!number} maxHeight + * @param {!number} maxWidth * @return {!Array.} variants filtered according to * |restrictions| and sorted in ascending order of bandwidth. * @private */ - static filterAndSortVariants_(restrictions, variants) { + static filterAndSortVariants_(restrictions, variants, maxHeight, maxWidth) { if (restrictions) { variants = variants.filter((variant) => { // This was already checked in another scope, but the compiler doesn't @@ -333,7 +393,7 @@ shaka.abr.SimpleAbrManager = class { return shaka.util.StreamUtils.meetsRestrictions( variant, restrictions, - /* maxHwRes= */ {width: Infinity, height: Infinity}); + /* maxHwRes= */ {width: maxWidth, height: maxHeight}); }); } @@ -342,3 +402,11 @@ shaka.abr.SimpleAbrManager = class { }); } }; + + +/** + * The amount of time, in seconds, we wait to batch up rapid resize changes. + * This allows us to avoid multiple resize events in most cases. + * @type {number} + */ +shaka.abr.SimpleAbrManager.RESIZE_OBSERVER_BATCH_TIME = 1; diff --git a/lib/player.js b/lib/player.js index 94918e62f1..ba71bedfac 100644 --- a/lib/player.js +++ b/lib/player.js @@ -1948,6 +1948,12 @@ shaka.Player = class extends shaka.util.FakeEventTarget { if (!this.abrManager_ || this.abrManagerFactory_ != abrFactory) { this.abrManagerFactory_ = abrFactory; this.abrManager_ = abrFactory(); + if (typeof this.abrManager_.setMediaElement != 'function') { + shaka.Deprecate.deprecateFeature(5, + 'AbrManager', + 'Please use an AbrManager with setMediaElement function.'); + this.abrManager_.setMediaElement = () => {}; + } this.abrManager_.configure(this.config_.abr); } @@ -1970,6 +1976,7 @@ shaka.Player = class extends shaka.util.FakeEventTarget { this.abrManager_.init((variant, clearBuffer, safeMargin) => { return this.switch_(variant, clearBuffer, safeMargin); }); + this.abrManager_.setMediaElement(mediaElement); this.playhead_ = this.createPlayhead(has.startTime); this.playheadObservers_ = this.createPlayheadObserversForMSE_(); diff --git a/lib/util/player_configuration.js b/lib/util/player_configuration.js index f737530bab..b75e8e9fce 100644 --- a/lib/util/player_configuration.js +++ b/lib/util/player_configuration.js @@ -244,6 +244,8 @@ shaka.util.PlayerConfiguration = class { fastHalfLife: 2, slowHalfLife: 5, }, + restrictToElementSize: false, + ignoreDevicePixelRatio: false, }; const cmcd = {