From 6f8399791352b6ccb6f3803c5163be4999c075f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Velad=20Galv=C3=A1n?= Date: Sat, 24 Jun 2023 12:59:21 +0200 Subject: [PATCH] feat: Add AC3 transmuxer (#5297) --- build/types/transmuxer | 2 + lib/transmuxer/ac3.js | 151 ++++++++++++++++ lib/transmuxer/ac3_transmuxer.js | 208 ++++++++++++++++++++++ lib/util/mp4_generator.js | 10 +- lib/util/platform.js | 6 +- shaka-player.uncompiled.js | 2 + test/transmuxer/transmuxer_integration.js | 39 ++++ 7 files changed, 411 insertions(+), 7 deletions(-) create mode 100644 lib/transmuxer/ac3.js create mode 100644 lib/transmuxer/ac3_transmuxer.js diff --git a/build/types/transmuxer b/build/types/transmuxer index 7b0e7584ac..83b14994fd 100644 --- a/build/types/transmuxer +++ b/build/types/transmuxer @@ -1,6 +1,8 @@ # Optional plugins related to transmuxer. +../../lib/transmuxer/aac_transmuxer.js ++../../lib/transmuxer/ac3.js ++../../lib/transmuxer/ac3_transmuxer.js +../../lib/transmuxer/adts.js +../../lib/transmuxer/mp3_transmuxer.js +../../lib/transmuxer/mpeg_audio.js diff --git a/lib/transmuxer/ac3.js b/lib/transmuxer/ac3.js new file mode 100644 index 0000000000..4416ad99c6 --- /dev/null +++ b/lib/transmuxer/ac3.js @@ -0,0 +1,151 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.transmuxer.Ac3'); + + +/** + * AC3 utils + */ +shaka.transmuxer.Ac3 = class { + /** + * @param {!Uint8Array} data + * @param {!number} offset + * @return {?{sampleRate: number, channelCount: number, + * audioConfig: !Uint8Array, frameLength: number}} + */ + static parseFrame(data, offset) { + if (offset + 8 > data.length) { + // not enough bytes left + return null; + } + + if (data[offset] !== 0x0b || data[offset + 1] !== 0x77) { + // invalid magic + return null; + } + + // get sample rate + const samplingRateCode = data[offset + 4] >> 6; + if (samplingRateCode >= 3) { + // invalid sampling rate + return null; + } + + const samplingRateMap = [48000, 44100, 32000]; + + // get frame size + const frameSizeCode = data[offset + 4] & 0x3f; + const frameSizeMap = [ + 64, 69, 96, 64, 70, 96, 80, 87, 120, 80, 88, 120, 96, 104, 144, 96, 105, + 144, 112, 121, 168, 112, 122, 168, 128, 139, 192, 128, 140, 192, 160, + 174, 240, 160, 175, 240, 192, 208, 288, 192, 209, 288, 224, 243, 336, + 224, 244, 336, 256, 278, 384, 256, 279, 384, 320, 348, 480, 320, 349, + 480, 384, 417, 576, 384, 418, 576, 448, 487, 672, 448, 488, 672, 512, + 557, 768, 512, 558, 768, 640, 696, 960, 640, 697, 960, 768, 835, 1152, + 768, 836, 1152, 896, 975, 1344, 896, 976, 1344, 1024, 1114, 1536, 1024, + 1115, 1536, 1152, 1253, 1728, 1152, 1254, 1728, 1280, 1393, 1920, 1280, + 1394, 1920, + ]; + + const frameLength = frameSizeMap[frameSizeCode * 3 + samplingRateCode] * 2; + if (offset + frameLength > data.length) { + return null; + } + + // get channel count + const channelMode = data[offset + 6] >> 5; + let skipCount = 0; + if (channelMode === 2) { + skipCount += 2; + } else { + if ((channelMode & 1) && channelMode !== 1) { + skipCount += 2; + } + if (channelMode & 4) { + skipCount += 2; + } + } + + const lowFrequencyEffectsChannelOn = + (((data[offset + 6] << 8) | data[offset + 7]) >> (12 - skipCount)) & 1; + + const channelsMap = [2, 1, 2, 3, 3, 4, 4, 5]; + + // Audio config for DAC3 box + const bitStreamIdentification = data[offset + 5] >> 3; + const bitStreamMode = data[offset + 5] & 7; + + const config = new Uint8Array([ + (samplingRateCode << 6) | + (bitStreamIdentification << 1) | + (bitStreamMode >> 2), + ((bitStreamMode & 3) << 6) | + (channelMode << 3) | + (lowFrequencyEffectsChannelOn << 2) | + (frameSizeCode >> 4), + (frameSizeCode << 4) & 0xe0, + ]); + + return { + sampleRate: samplingRateMap[samplingRateCode], + channelCount: channelsMap[channelMode] + lowFrequencyEffectsChannelOn, + audioConfig: config, + frameLength: frameLength, + }; + } + + /** + * @param {!Uint8Array} data + * @param {!number} offset + * @return {boolean} + */ + static canParse(data, offset) { + return offset + 64 < data.length; + } + + /** + * @param {!Uint8Array} data + * @param {!number} offset + * @return {boolean} + */ + static probe(data, offset) { + // look for the ac-3 sync bytes + if (data[offset] === 0x0b && + data[offset + 1] === 0x77) { + // check the bsid (bitStreamIdentification) to confirm ac-3 + let bsid = 0; + let numBits = 5; + offset += numBits; + /** @type {?number} */ + let temp = null; + /** @type {?number} */ + let mask = null; + /** @type {?number} */ + let byte = null; + while (numBits > 0) { + byte = data[offset]; + // read remaining bits, upto 8 bits at a time + const bits = Math.min(numBits, 8); + const shift = 8 - bits; + mask = (0xff000000 >>> (24 + shift)) << shift; + temp = (byte & mask) >> shift; + bsid = !bsid ? temp : (bsid << bits) | temp; + offset += 1; + numBits -= bits; + } + if (bsid < 16) { + return true; + } + } + return false; + } +}; + +/** + * @const {number} + */ +shaka.transmuxer.Ac3.AC3_SAMPLES_PER_FRAME = 1536; diff --git a/lib/transmuxer/ac3_transmuxer.js b/lib/transmuxer/ac3_transmuxer.js new file mode 100644 index 0000000000..e8bc106eb4 --- /dev/null +++ b/lib/transmuxer/ac3_transmuxer.js @@ -0,0 +1,208 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.transmuxer.Ac3Transmuxer'); + +goog.require('shaka.media.Capabilities'); +goog.require('shaka.transmuxer.Ac3'); +goog.require('shaka.transmuxer.TransmuxerEngine'); +goog.require('shaka.util.BufferUtils'); +goog.require('shaka.util.Error'); +goog.require('shaka.util.Id3Utils'); +goog.require('shaka.util.ManifestParserUtils'); +goog.require('shaka.util.Mp4Generator'); +goog.require('shaka.util.Uint8ArrayUtils'); + + +/** + * @implements {shaka.extern.Transmuxer} + * @export + */ +shaka.transmuxer.Ac3Transmuxer = class { + /** + * @param {string} mimeType + */ + constructor(mimeType) { + /** @private {string} */ + this.originalMimeType_ = mimeType; + + /** @private {number} */ + this.frameIndex_ = 0; + + /** @private {!Map.} */ + this.initSegments = new Map(); + } + + + /** + * @override + * @export + */ + destroy() { + this.initSegments.clear(); + } + + + /** + * Check if the mime type and the content type is supported. + * @param {string} mimeType + * @param {string=} contentType + * @return {boolean} + * @override + * @export + */ + isSupported(mimeType, contentType) { + const Capabilities = shaka.media.Capabilities; + + if (!this.isAc3Container_(mimeType)) { + return false; + } + const ContentType = shaka.util.ManifestParserUtils.ContentType; + return Capabilities.isTypeSupported( + this.convertCodecs(ContentType.AUDIO, mimeType)); + } + + + /** + * Check if the mimetype is 'audio/ac3'. + * @param {string} mimeType + * @return {boolean} + * @private + */ + isAc3Container_(mimeType) { + return mimeType.toLowerCase().split(';')[0] == 'audio/ac3'; + } + + + /** + * @override + * @export + */ + convertCodecs(contentType, mimeType) { + if (this.isAc3Container_(mimeType)) { + return 'audio/mp4; codecs="ac-3"'; + } + return mimeType; + } + + + /** + * @override + * @export + */ + getOrginalMimeType() { + return this.originalMimeType_; + } + + + /** + * @override + * @export + */ + transmux(data, stream, reference, duration) { + const Ac3 = shaka.transmuxer.Ac3; + const Uint8ArrayUtils = shaka.util.Uint8ArrayUtils; + + const uint8ArrayData = shaka.util.BufferUtils.toUint8(data); + + const id3Data = shaka.util.Id3Utils.getID3Data(uint8ArrayData); + let offset = id3Data.length; + for (; offset < uint8ArrayData.length; offset++) { + if (Ac3.probe(uint8ArrayData, offset)) { + break; + } + } + + let timestamp = reference.endTime * 1000; + + const frames = shaka.util.Id3Utils.getID3Frames(id3Data); + if (frames.length && reference) { + const metadataTimestamp = frames.find((frame) => { + return frame.description === + 'com.apple.streaming.transportStreamTimestamp'; + }); + if (metadataTimestamp) { + timestamp = /** @type {!number} */(metadataTimestamp.data); + } + } + + /** @type {number} */ + let sampleRate = 0; + + /** @type {!Uint8Array} */ + let audioConfig = new Uint8Array([]); + + /** @type {!Array.} */ + const samples = []; + + while (offset < uint8ArrayData.length) { + const frame = Ac3.parseFrame(uint8ArrayData, offset); + if (!frame) { + return Promise.reject(new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.MEDIA, + shaka.util.Error.Code.TRANSMUXING_FAILED)); + } + stream.audioSamplingRate = frame.sampleRate; + stream.channelsCount = frame.channelCount; + sampleRate = frame.sampleRate; + audioConfig = frame.audioConfig; + + const frameData = uint8ArrayData.subarray( + offset, offset + frame.frameLength); + + samples.push({ + data: frameData, + size: frame.frameLength, + duration: Ac3.AC3_SAMPLES_PER_FRAME, + cts: 0, + flags: { + isLeading: 0, + isDependedOn: 0, + hasRedundancy: 0, + degradPrio: 0, + dependsOn: 2, + isNonSync: 0, + }, + }); + offset += frame.frameLength; + } + /** @type {number} */ + const baseMediaDecodeTime = Math.floor(timestamp * sampleRate / 1000); + + /** @type {shaka.util.Mp4Generator.StreamInfo} */ + const streamInfo = { + timescale: sampleRate, + duration: duration, + videoNalus: [], + audioConfig: audioConfig, + data: { + sequenceNumber: this.frameIndex_, + baseMediaDecodeTime: baseMediaDecodeTime, + samples: samples, + }, + stream: stream, + }; + const mp4Generator = new shaka.util.Mp4Generator(streamInfo); + let initSegment; + if (!this.initSegments.has(stream.id)) { + initSegment = mp4Generator.initSegment(); + this.initSegments.set(stream.id, initSegment); + } else { + initSegment = this.initSegments.get(stream.id); + } + const segmentData = mp4Generator.segmentData(); + + this.frameIndex_++; + const transmuxData = Uint8ArrayUtils.concat(initSegment, segmentData); + return Promise.resolve(transmuxData); + } +}; + +shaka.transmuxer.TransmuxerEngine.registerTransmuxer( + 'audio/ac3', + () => new shaka.transmuxer.Ac3Transmuxer('audio/ac3'), + shaka.transmuxer.TransmuxerEngine.PluginPriority.FALLBACK); diff --git a/lib/util/mp4_generator.js b/lib/util/mp4_generator.js index 7bfc375806..b441558cdc 100644 --- a/lib/util/mp4_generator.js +++ b/lib/util/mp4_generator.js @@ -310,12 +310,14 @@ shaka.util.Mp4Generator = class { break; case ContentType.AUDIO: if (this.stream_.mimeType === 'audio/mpeg' || - this.stream_.codecs.includes('mp3') || - this.stream_.codecs.includes('mp4a.40.34')) { + this.stream_.codecs.includes('mp3') || + this.stream_.codecs.includes('mp4a.40.34')) { bytes = this.mp3_(); - } else if (this.stream_.codecs.includes('ac-3')) { + } else if (this.stream_.mimeType === 'audio/ac3' || + this.stream_.codecs.includes('ac-3')) { bytes = this.ac3_(); - } else if (this.stream_.codecs.includes('ec-3')) { + } else if (this.stream_.mimeType === 'audio/ec3' || + this.stream_.codecs.includes('ec-3')) { bytes = this.ec3_(); } else { bytes = this.mp4a_(); diff --git a/lib/util/platform.js b/lib/util/platform.js index 4eb5f6e47e..3adf21ed54 100644 --- a/lib/util/platform.js +++ b/lib/util/platform.js @@ -193,10 +193,10 @@ shaka.util.Platform = class { * @return {boolean} */ static isChrome() { - // The Edge user agent will also contain the "Chrome" keyword, so we need - // to make sure this is not Edge. + // The Edge Legacy user agent will also contain the "Chrome" keyword, so we + // need to make sure this is not Edge Legacy. return shaka.util.Platform.userAgentContains_('Chrome') && - !shaka.util.Platform.isEdge(); + !shaka.util.Platform.isLegacyEdge(); } /** diff --git a/shaka-player.uncompiled.js b/shaka-player.uncompiled.js index 3d002e79c5..f3fba3735c 100644 --- a/shaka-player.uncompiled.js +++ b/shaka-player.uncompiled.js @@ -66,6 +66,8 @@ goog.require('shaka.cea.CeaDecoder'); goog.require('shaka.cea.Mp4CeaParser'); goog.require('shaka.cea.TsCeaParser'); goog.require('shaka.transmuxer.AacTransmuxer'); +goog.require('shaka.transmuxer.Ac3'); +goog.require('shaka.transmuxer.Ac3Transmuxer'); goog.require('shaka.transmuxer.ADTS'); goog.require('shaka.transmuxer.Mp3Transmuxer'); goog.require('shaka.transmuxer.MpegAudio'); diff --git a/test/transmuxer/transmuxer_integration.js b/test/transmuxer/transmuxer_integration.js index 98a686aefe..ffc784a972 100644 --- a/test/transmuxer/transmuxer_integration.js +++ b/test/transmuxer/transmuxer_integration.js @@ -43,6 +43,7 @@ describe('Transmuxer Player', () => { // soft restrictions cannot be met. player.configure('abr.restrictions.maxHeight', 1); player.configure('mediaSource.forceTransmux', true); + player.configure('streaming.useNativeHlsOnSafari', false); // Grab event manager from the uncompiled library: eventManager = new shaka.util.EventManager(); @@ -106,6 +107,44 @@ describe('Transmuxer Player', () => { await player.unload(); }); + it('raw AC3', async () => { + if (!MediaSource.isTypeSupported('audio/mp4; codecs="ac-3"')) { + return; + } + // This tests is flaky in some Tizen devices, so we need omit it for now. + if (shaka.util.Platform.isTizen()) { + return; + } + // It seems that AC3 on Edge Windows from github actions is not working + // (in the lab AC3 is working). The AC3 detection is currently hard-coded + // to true, which leads to a failure in GitHub's environment. + // We must enable this, once it is resolved: + // https://bugs.chromium.org/p/chromium/issues/detail?id=1450313 + const chromeVersion = shaka.util.Platform.chromeVersion(); + if (shaka.util.Platform.isEdge() && + chromeVersion && chromeVersion <= 116) { + return; + } + + // eslint-disable-next-line max-len + const url = 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_ts/a2/prog_index.m3u8'; + + await player.load(url, /* startTime= */ null, + /* mimeType= */ undefined); + video.play(); + expect(player.isLive()).toBe(false); + + // Wait for the video to start playback. If it takes longer than 10 + // seconds, fail the test. + await waiter.waitForMovementOrFailOnTimeout(video, 10); + + // Play for 15 seconds, but stop early if the video ends. If it takes + // longer than 45 seconds, fail the test. + await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 15, 45); + + await player.unload(); + }); + it('muxed H.264+AAC in TS', async () => { // eslint-disable-next-line max-len const url = 'https://cf-sf-video.wmspanel.com/local/raw/BigBuckBunny_320x180.mp4/playlist.m3u8';