From 198a6d42f6efb29bcf763273d66a17834b3acc6f Mon Sep 17 00:00:00 2001 From: Michelle Zhuo Date: Sun, 28 Mar 2021 21:39:12 -0700 Subject: [PATCH] feat(MediaCap): Guess the codecs of multiplexd stream for MediaCap If we have a multiplexd stream with audio and video, the video codecs are a combined string containing both audio and video codecs. Before sending the request to decodingInfo(), we should check which codec is audio and which one is video. This is a follow-up of commit 966a756. Issue #1391 Change-Id: Ic9b1c5972a99f63a715c74ae068b85f43efac447 --- lib/hls/hls_parser.js | 130 +++--------------------------- lib/util/manifest_parser_utils.js | 114 ++++++++++++++++++++++++++ lib/util/stream_utils.js | 15 ++-- 3 files changed, 134 insertions(+), 125 deletions(-) diff --git a/lib/hls/hls_parser.js b/lib/hls/hls_parser.js index bb0047f3f9..833d5e8162 100644 --- a/lib/hls/hls_parser.js +++ b/lib/hls/hls_parser.js @@ -596,17 +596,20 @@ shaka.hls.HlsParser = class { const allCodecs = this.getCodecsForVariantTag_(variantTag); if (subGroupId) { - const textCodecs = this.guessCodecsSafe_(ContentType.TEXT, allCodecs); + const textCodecs = shaka.util.ManifestParserUtils.guessCodecsSafe( + ContentType.TEXT, allCodecs); goog.asserts.assert(textCodecs != null, 'Text codecs should be valid.'); this.groupIdToCodecsMap_.set(subGroupId, textCodecs); shaka.util.ArrayUtils.remove(allCodecs, textCodecs); } if (audioGroupId) { - const codecs = this.guessCodecs_(ContentType.AUDIO, allCodecs); + const codecs = shaka.util.ManifestParserUtils.guessCodecs( + ContentType.AUDIO, allCodecs); this.groupIdToCodecsMap_.set(audioGroupId, codecs); } if (videoGroupId) { - const codecs = this.guessCodecs_(ContentType.VIDEO, allCodecs); + const codecs = shaka.util.ManifestParserUtils.guessCodecs( + ContentType.VIDEO, allCodecs); this.groupIdToCodecsMap_.set(videoGroupId, codecs); } } @@ -774,8 +777,10 @@ shaka.hls.HlsParser = class { return audio && audio.verbatimMediaPlaylistUri == streamURI; }); - const videoCodecs = this.guessCodecsSafe_(ContentType.VIDEO, allCodecs); - const audioCodecs = this.guessCodecsSafe_(ContentType.AUDIO, allCodecs); + const videoCodecs = shaka.util.ManifestParserUtils.guessCodecsSafe( + ContentType.VIDEO, allCodecs); + const audioCodecs = shaka.util.ManifestParserUtils.guessCodecsSafe( + ContentType.AUDIO, allCodecs); if (audioCodecs && !videoCodecs) { // There are no associated media tags, and there's only audio codec, @@ -1196,7 +1201,7 @@ shaka.hls.HlsParser = class { } const closedCaptions = this.getClosedCaptions_(tag, type); - const codecs = this.guessCodecs_(type, allCodecs); + const codecs = shaka.util.ManifestParserUtils.guessCodecs(type, allCodecs); const streamInfo = await this.createStreamInfo_(verbatimMediaPlaylistUri, codecs, type, /* language= */ 'und', /* primary= */ false, /* name= */ null, /* channelcount= */ null, closedCaptions, @@ -2291,65 +2296,6 @@ shaka.hls.HlsParser = class { } } - /** - * Attempts to guess which codecs from the codecs list belong to a given - * content type. - * Assumes that at least one codec is correct, and throws if none are. - * - * @param {string} contentType - * @param {!Array.} codecs - * @return {string} - * @private - */ - guessCodecs_(contentType, codecs) { - if (codecs.length == 1) { - return codecs[0]; - } - - const match = this.guessCodecsSafe_(contentType, codecs); - // A failure is specifically denoted by null; an empty string represents a - // valid match of no codec. - if (match != null) { - return match; - } - - // Unable to guess codecs. - throw new shaka.util.Error( - shaka.util.Error.Severity.CRITICAL, - shaka.util.Error.Category.MANIFEST, - shaka.util.Error.Code.HLS_COULD_NOT_GUESS_CODECS, - codecs); - } - - /** - * Attempts to guess which codecs from the codecs list belong to a given - * content type. Does not assume a single codec is anything special, and does - * not throw if it fails to match. - * - * @param {string} contentType - * @param {!Array.} codecs - * @return {?string} or null if no match is found - * @private - */ - guessCodecsSafe_(contentType, codecs) { - const formats = - shaka.hls.HlsParser.CODEC_REGEXPS_BY_CONTENT_TYPE_[contentType]; - for (const format of formats) { - for (const codec of codecs) { - if (format.test(codec.trim())) { - return codec.trim(); - } - } - } - - // Text does not require a codec string. - if (contentType == shaka.util.ManifestParserUtils.ContentType.TEXT) { - return ''; - } - - return null; - } - /** * Replaces the variables of a given URI. * @@ -2719,60 +2665,6 @@ shaka.hls.HlsParser.StreamInfo; shaka.hls.HlsParser.StreamInfos; -/** - * A list of regexps to detect well-known video codecs. - * - * @const {!Array.} - * @private - */ -shaka.hls.HlsParser.VIDEO_CODEC_REGEXPS_ = [ - /^avc/, - /^hev/, - /^hvc/, - /^vp0?[89]/, - /^av1$/, -]; - - -/** - * A list of regexps to detect well-known audio codecs. - * - * @const {!Array.} - * @private - */ -shaka.hls.HlsParser.AUDIO_CODEC_REGEXPS_ = [ - /^vorbis$/, - /^opus$/, - /^flac$/, - /^mp4a/, - /^[ae]c-3$/, -]; - - -/** - * A list of regexps to detect well-known text codecs. - * - * @const {!Array.} - * @private - */ -shaka.hls.HlsParser.TEXT_CODEC_REGEXPS_ = [ - /^vtt$/, - /^wvtt/, - /^stpp/, -]; - - -/** - * @const {!Object.>} - * @private - */ -shaka.hls.HlsParser.CODEC_REGEXPS_BY_CONTENT_TYPE_ = { - 'audio': shaka.hls.HlsParser.AUDIO_CODEC_REGEXPS_, - 'video': shaka.hls.HlsParser.VIDEO_CODEC_REGEXPS_, - 'text': shaka.hls.HlsParser.TEXT_CODEC_REGEXPS_, -}; - - /** * @const {!Object.} * @private diff --git a/lib/util/manifest_parser_utils.js b/lib/util/manifest_parser_utils.js index 47d365614d..c4aaee5bd7 100644 --- a/lib/util/manifest_parser_utils.js +++ b/lib/util/manifest_parser_utils.js @@ -7,6 +7,7 @@ goog.provide('shaka.util.ManifestParserUtils'); goog.require('goog.Uri'); +goog.require('shaka.util.Error'); goog.require('shaka.util.Functional'); @@ -58,6 +59,66 @@ shaka.util.ManifestParserUtils = class { keyIds: new Set(), }; } + + + /** + * Attempts to guess which codecs from the codecs list belong to a given + * content type. + * Assumes that at least one codec is correct, and throws if none are. + * + * @param {string} contentType + * @param {!Array.} codecs + * @return {string} + */ + static guessCodecs(contentType, codecs) { + if (codecs.length == 1) { + return codecs[0]; + } + + const match = shaka.util.ManifestParserUtils.guessCodecsSafe( + contentType, codecs); + // A failure is specifically denoted by null; an empty string represents a + // valid match of no codec. + if (match != null) { + return match; + } + + // Unable to guess codecs. + throw new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.MANIFEST, + shaka.util.Error.Code.HLS_COULD_NOT_GUESS_CODECS, + codecs); + } + + + /** + * Attempts to guess which codecs from the codecs list belong to a given + * content type. Does not assume a single codec is anything special, and does + * not throw if it fails to match. + * + * @param {string} contentType + * @param {!Array.} codecs + * @return {?string} or null if no match is found + */ + static guessCodecsSafe(contentType, codecs) { + const formats = shaka.util.ManifestParserUtils + .CODEC_REGEXPS_BY_CONTENT_TYPE_[contentType]; + for (const format of formats) { + for (const codec of codecs) { + if (format.test(codec.trim())) { + return codec.trim(); + } + } + } + + // Text does not require a codec string. + if (contentType == shaka.util.ManifestParserUtils.ContentType.TEXT) { + return ''; + } + + return null; + } }; @@ -91,3 +152,56 @@ shaka.util.ManifestParserUtils.TextStreamKind = { * @const {number} */ shaka.util.ManifestParserUtils.GAP_OVERLAP_TOLERANCE_SECONDS = 1 / 15; + + +/** + * A list of regexps to detect well-known video codecs. + * + * @const {!Array.} + * @private + */ +shaka.util.ManifestParserUtils.VIDEO_CODEC_REGEXPS_ = [ + /^avc/, + /^hev/, + /^hvc/, + /^vp0?[89]/, + /^av1$/, +]; + + +/** + * A list of regexps to detect well-known audio codecs. + * + * @const {!Array.} + * @private + */ +shaka.util.ManifestParserUtils.AUDIO_CODEC_REGEXPS_ = [ + /^vorbis$/, + /^opus$/, + /^flac$/, + /^mp4a/, + /^[ae]c-3$/, +]; + + +/** + * A list of regexps to detect well-known text codecs. + * + * @const {!Array.} + * @private + */ +shaka.util.ManifestParserUtils.TEXT_CODEC_REGEXPS_ = [ + /^vtt$/, + /^wvtt/, + /^stpp/, +]; + + +/** + * @const {!Object.>} + */ +shaka.util.ManifestParserUtils.CODEC_REGEXPS_BY_CONTENT_TYPE_ = { + 'audio': shaka.util.ManifestParserUtils.AUDIO_CODEC_REGEXPS_, + 'video': shaka.util.ManifestParserUtils.VIDEO_CODEC_REGEXPS_, + 'text': shaka.util.ManifestParserUtils.TEXT_CODEC_REGEXPS_, +}; diff --git a/lib/util/stream_utils.js b/lib/util/stream_utils.js index f709caf1cf..2d7b940ac4 100644 --- a/lib/util/stream_utils.js +++ b/lib/util/stream_utils.js @@ -433,15 +433,18 @@ shaka.util.StreamUtils = class { }; if (video) { - let audioCodec; - let videoCodec = video.codecs; - + let videoCodecs = video.codecs; // For multiplexed streams with audio+video codecs, the config should have // AudioConfiguration and VideoConfiguration. if (video.codecs.includes(',')) { - [videoCodec, audioCodec] = video.codecs.split(','); + const allCodecs = video.codecs.split(','); + videoCodecs = shaka.util.ManifestParserUtils.guessCodecs( + ContentType.VIDEO, allCodecs); + const audioCodecs = shaka.util.ManifestParserUtils.guessCodecs( + ContentType.AUDIO, allCodecs); + const audioFullType = shaka.util.MimeUtils.getFullOrConvertedType( - video.mimeType, audioCodec, ContentType.AUDIO); + video.mimeType, audioCodecs, ContentType.AUDIO); mediaDecodingConfig.audio = { contentType: audioFullType, channels: 2, @@ -451,7 +454,7 @@ shaka.util.StreamUtils = class { }; } const fullType = shaka.util.MimeUtils.getFullOrConvertedType( - video.mimeType, videoCodec, ContentType.VIDEO); + video.mimeType, videoCodecs, ContentType.VIDEO); // VideoConfiguration mediaDecodingConfig.video = { contentType: fullType,