From 79d5fd8ca0a7c5f3738d539aec226395dc10b83e Mon Sep 17 00:00:00 2001 From: Michelle Zhuo Date: Tue, 6 Apr 2021 12:12:20 -0700 Subject: [PATCH] feat(MediaCap): Add preferredDecodingAttributes config We'll allow users to configure the decoding attributes they prefer when choosing codecs through the configuration. The attributes include 'smooth', 'powerEfficient' and 'bandwidth'. For example, if the user configures the field as ['smooth', 'powerEfficient'], we'll filter the variants and keep the smooth ones first, and if we have more than one available variants, we'll filter and keep the power efficient variants. After that, we choose the codecs with lowest bandwidth if we have multiple codecs available. Issue #1391 Change-Id: Ief3f6d8ff98fabff5ec99bb0365cdc6a36d2ab2d --- externs/shaka/player.js | 4 ++ lib/offline/storage.js | 4 +- lib/player.js | 3 +- lib/util/multi_map.js | 8 +++ lib/util/player_configuration.js | 1 + lib/util/stream_utils.js | 98 +++++++++++++++++++++++++++++--- test/util/stream_utils_unit.js | 45 ++++++++++++++- 7 files changed, 151 insertions(+), 12 deletions(-) diff --git a/externs/shaka/player.js b/externs/shaka/player.js index a0ea3f28ac..e68c970304 100644 --- a/externs/shaka/player.js +++ b/externs/shaka/player.js @@ -981,6 +981,7 @@ shaka.extern.OfflineConfiguration; * preferredVariantRole: string, * preferredTextRole: string, * preferredAudioChannelCount: number, + * preferredDecodingAttributes: !Array., * preferForcedSubs: boolean, * restrictions: shaka.extern.Restrictions, * playRangeStart: number, @@ -1016,6 +1017,9 @@ shaka.extern.OfflineConfiguration; * The preferred role to use for text tracks. * @property {number} preferredAudioChannelCount * The preferred number of audio channels. + * @property {!Array.} preferredDecodingAttributes + * The list of preferred attributes of decodingInfo, in the order of their + * priorities. * @property {boolean} preferForcedSubs * If true, a forced text track is preferred. Defaults to false. * If the content has no forced captions and the value is true, diff --git a/lib/offline/storage.js b/lib/offline/storage.js index 5b813134c6..15c91e01ff 100644 --- a/lib/offline/storage.js +++ b/lib/offline/storage.js @@ -501,8 +501,10 @@ shaka.offline.Storage = class { // Choose the codec that has the lowest average bandwidth. const preferredAudioChannelCount = config.preferredAudioChannelCount; + const preferredDecodingAttributes = config.preferredDecodingAttributes; + StreamUtils.chooseCodecsAndFilterManifest( - manifest, preferredAudioChannelCount); + manifest, preferredAudioChannelCount, preferredDecodingAttributes); for (const variant of manifest.variants) { goog.asserts.assert( diff --git a/lib/player.js b/lib/player.js index 768931749f..7c7ece86a3 100644 --- a/lib/player.js +++ b/lib/player.js @@ -1849,7 +1849,8 @@ shaka.Player = class extends shaka.util.FakeEventTarget { // If the content is multi-codec and the browser can play more than one of // them, choose codecs now before we initialize streaming. shaka.util.StreamUtils.chooseCodecsAndFilterManifest( - this.manifest_, this.config_.preferredAudioChannelCount); + this.manifest_, this.config_.preferredAudioChannelCount, + this.config_.preferredDecodingAttributes); this.streamingEngine_ = this.createStreamingEngine(); this.streamingEngine_.configure(this.config_.streaming); diff --git a/lib/util/multi_map.js b/lib/util/multi_map.js index a26c033c7d..96363dc3e8 100644 --- a/lib/util/multi_map.js +++ b/lib/util/multi_map.js @@ -102,4 +102,12 @@ shaka.util.MultiMap = class { size() { return Object.keys(this.map_).length; } + + /** + * Get a list of all the keys. + * @return {!Array.} + */ + keys() { + return Object.keys(this.map_); + } }; diff --git a/lib/util/player_configuration.js b/lib/util/player_configuration.js index 03fe4e2c54..01fbe22d8f 100644 --- a/lib/util/player_configuration.js +++ b/lib/util/player_configuration.js @@ -243,6 +243,7 @@ shaka.util.PlayerConfiguration = class { preferredTextRole: '', preferredAudioChannelCount: 2, preferForcedSubs: false, + preferredDecodingAttributes: [], restrictions: { minWidth: 0, maxWidth: Infinity, diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index 3c54229902..3333d155fc 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -29,8 +29,10 @@ shaka.util.StreamUtils = class { * Also filters out variants that have too many audio channels. * @param {!shaka.extern.Manifest} manifest * @param {number} preferredAudioChannelCount + * @param {!Array.} preferredDecodingAttributes */ - static chooseCodecsAndFilterManifest(manifest, preferredAudioChannelCount) { + static chooseCodecsAndFilterManifest(manifest, preferredAudioChannelCount, + preferredDecodingAttributes) { const StreamUtils = shaka.util.StreamUtils; // To start, consider a subset of variants based on audio channel @@ -46,12 +48,13 @@ shaka.util.StreamUtils = class { let variantsByCodecs = StreamUtils.getVariantsByCodecs_(variants); variantsByCodecs = StreamUtils.filterVariantsByDensity_(variantsByCodecs); - const bestCodecs = StreamUtils.findBestCodecs_(variantsByCodecs); + const bestCodecs = StreamUtils.chooseCodecs_(variantsByCodecs, + preferredDecodingAttributes); // Filter out any variants that don't match, forcing AbrManager to choose - // from the most efficient variants possible. + // from a single video codec and a single audio codec possible. manifest.variants = manifest.variants.filter((variant) => { - const codecs = StreamUtils.getGroupVariantCodecs_(variant); + const codecs = StreamUtils.getVariantCodecs_(variant); if (codecs == bestCodecs) { return true; } @@ -71,8 +74,8 @@ shaka.util.StreamUtils = class { static getVariantsByCodecs_(variants) { const variantsByCodecs = new shaka.util.MultiMap(); for (const variant of variants) { - const group = shaka.util.StreamUtils.getGroupVariantCodecs_(variant); - variantsByCodecs.push(group, variant); + const variantCodecs = shaka.util.StreamUtils.getVariantCodecs_(variant); + variantsByCodecs.push(variantCodecs, variant); } return variantsByCodecs; @@ -80,6 +83,8 @@ shaka.util.StreamUtils = class { /** * Filters variants by density. + * Get variants by codecs map with the max density where all codecs are + * present. * * @param {!shaka.util.MultiMap.} variantsByCodecs * @return {!shaka.util.MultiMap.} @@ -120,6 +125,74 @@ shaka.util.StreamUtils = class { return maxDensity ? codecGroupsByDensity.get(maxDensity) : variantsByCodecs; } + /** + * Choose the codecs by configured preferred decoding attributes. + * + * @param {!shaka.util.MultiMap.} variantsByCodecs + * @param {!Array.} attributes + * @return {string} + * @private + */ + static chooseCodecs_(variantsByCodecs, attributes) { + const StreamUtils = shaka.util.StreamUtils; + + for (const attribute of attributes) { + if (attribute == StreamUtils.DecodingAttributes.SMOOTH || + attribute == StreamUtils.DecodingAttributes.POWER) { + variantsByCodecs = StreamUtils.chooseCodecsByMediaCapabilitiesInfo_( + variantsByCodecs, attribute); + // If we only have one smooth or powerEfficient codecs, choose it as the + // best codecs. + if (variantsByCodecs.size() == 1) { + return variantsByCodecs.keys()[0]; + } + } else if (attribute == StreamUtils.DecodingAttributes.BANDWIDTH) { + return StreamUtils.findCodecsByLowestBandwidth_(variantsByCodecs); + } + } + // If there's no configured decoding preferences, or we have multiple codecs + // that meets the configured decoding preferences, choose the one with + // the lowest bandwidth. + return StreamUtils.findCodecsByLowestBandwidth_(variantsByCodecs); + } + + /** + * Choose the best codecs by configured preferred MediaCapabilitiesInfo + * attributes. + * + * @param {!shaka.util.MultiMap.} variantsByCodecs + * @param {string} attribute + * @return {!shaka.util.MultiMap.} + * @private + */ + static chooseCodecsByMediaCapabilitiesInfo_(variantsByCodecs, attribute) { + let highestScore = 0; + const bestVariantsByCodecs = new shaka.util.MultiMap(); + variantsByCodecs.forEach((codecs, variants) => { + let sum = 0; + let num = 0; + + for (const variant of variants) { + if (variant.decodingInfos.length) { + sum += variant.decodingInfos[0][attribute] ? 1 : 0; + num++; + } + } + + const averageScore = sum / num; + shaka.log.info('codecs', codecs, 'avg', attribute, averageScore); + + if (averageScore > highestScore) { + bestVariantsByCodecs.clear(); + bestVariantsByCodecs.push(codecs, variants); + highestScore = averageScore; + } else if (averageScore == highestScore) { + bestVariantsByCodecs.push(codecs, variants); + } + }); + return bestVariantsByCodecs; + } + /** * Find the lowest-bandwidth (best) codecs. * Compute the average bandwidth for each group of variants. @@ -128,7 +201,7 @@ shaka.util.StreamUtils = class { * @return {string} * @private */ - static findBestCodecs_(variantsByCodecs) { + static findCodecsByLowestBandwidth_(variantsByCodecs) { let bestCodecs = ''; let lowestAverageBandwidth = Infinity; @@ -163,7 +236,7 @@ shaka.util.StreamUtils = class { * @return {string} * @private */ - static getGroupVariantCodecs_(variant) { + static getVariantCodecs_(variant) { // Only consider the base of the codec string. For example, these should // both be considered the same codec: avc1.42c01e, avc1.4d401f let baseVideoCodec = ''; @@ -1332,3 +1405,12 @@ shaka.util.StreamUtils = class { /** @private {number} */ shaka.util.StreamUtils.nextTrackId_ = 0; + +/** + * @enum {string} + */ +shaka.util.StreamUtils.DecodingAttributes = { + SMOOTH: 'smooth', + POWER: 'powerEfficient', + BANDWIDTH: 'bandwidth', +}; diff --git a/test/util/stream_utils_unit.js b/test/util/stream_utils_unit.js index da7534722d..1bbec0edc5 100644 --- a/test/util/stream_utils_unit.js +++ b/test/util/stream_utils_unit.js @@ -707,7 +707,7 @@ describe('StreamUtils', () => { addVariant2160Vp9(manifest); }); - shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2); + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2, []); expect(manifest.variants.length).toBe(2); expect(manifest.variants[0].video.codecs).toBe(vp09Codecs); @@ -720,10 +720,51 @@ describe('StreamUtils', () => { addVariant1080Vp9(manifest); }); - shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2); + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2, []); expect(manifest.variants.length).toBe(1); expect(manifest.variants[0].video.codecs).toBe(vp09Codecs); }); + + it('chooses variants by decoding attributes', async () => { + manifest = shaka.test.ManifestGenerator.generate((manifest) => { + manifest.addVariant(0, (variant) => { + variant.bandwidth = 4058558; + variant.addVideo(1, (stream) => { + stream.mime('video', 'notsmooth'); + }); + }); + manifest.addVariant(1, (variant) => { + variant.bandwidth = 4781002; + variant.addVideo(2, (stream) => { + stream.mime('video', 'smooth'); + }); + }); + manifest.addVariant(3, (variant) => { + variant.addVideo(4, (stream) => { + variant.bandwidth = 5058558; + stream.mime('video', 'smooth-2'); + }); + }); + }); + navigator.mediaCapabilities.decodingInfo = + shaka.test.Util.spyFunc(decodingInfoSpy); + decodingInfoSpy.and.callFake((config) => { + const res = config.video.contentType.includes('notsmooth') ? + {supported: true, smooth: false} : + {supported: true, smooth: true}; + return Promise.resolve(res); + }); + + await StreamUtils.getDecodingInfosForVariants(manifest.variants, + /* usePersistentLicenses= */false); + + shaka.util.StreamUtils.chooseCodecsAndFilterManifest(manifest, 2, + [shaka.util.StreamUtils.DecodingAttributes.SMOOTH]); + // 2 video codecs are smooth. Choose the one with the lowest bandwidth. + expect(manifest.variants.length).toBe(1); + expect(manifest.variants[0].id).toBe(1); + expect(manifest.variants[0].video.id).toBe(2); + }); }); });