diff --git a/README.md b/README.md index 2ba12155c4..c6254a2224 100644 --- a/README.md +++ b/README.md @@ -280,6 +280,7 @@ Shaka Player supports: - EC-3 in MPEG-2 TS to EC-3 in MP4 - MP3 in MPEG-2 TS to MP3 in MP4 - MP3 in MPEG-2 TS to raw MP3 + - Opus in MPEG-2 TS to MP3 in MP4 - H.264 in MPEG-2 TS to H.264 in MP4 - H.265 in MPEG-2 TS to H.265 in MP4 - Muxed content in MPEG-2 TS with the previous codecs diff --git a/build/types/transmuxer b/build/types/transmuxer index c249dfc39b..ccd2dabab8 100644 --- a/build/types/transmuxer +++ b/build/types/transmuxer @@ -11,4 +11,5 @@ +../../lib/transmuxer/mp3_transmuxer.js +../../lib/transmuxer/mpeg_audio.js +../../lib/transmuxer/mpeg_ts_transmuxer.js ++../../lib/transmuxer/opus.js +../../lib/transmuxer/ts_transmuxer.js diff --git a/karma.conf.js b/karma.conf.js index fea092f86d..b747afa817 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -255,6 +255,7 @@ module.exports = (config) => { {pattern: 'test/test/assets/hls-ts-muxed-ac3-h264/*', included: false}, {pattern: 'test/test/assets/hls-ts-muxed-mp3-h264/*', included: false}, {pattern: 'test/test/assets/hls-ts-muxed-ec3-h264/*', included: false}, + {pattern: 'test/test/assets/hls-ts-muxed-opus-h264/*', included: false}, {pattern: 'test/test/assets/hls-ts-raw-aac/*', included: false}, {pattern: 'test/test/assets/hls-ts-rollover/*', included: false}, {pattern: 'dist/shaka-player.ui.js', included: false}, diff --git a/lib/transmuxer/opus.js b/lib/transmuxer/opus.js new file mode 100644 index 0000000000..82bbdaf113 --- /dev/null +++ b/lib/transmuxer/opus.js @@ -0,0 +1,91 @@ +/*! @license + * Shaka Player + * Copyright 2016 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +goog.provide('shaka.transmuxer.Opus'); + +goog.requireType('shaka.util.TsParser'); + + +/** + * Opus utils + */ +shaka.transmuxer.Opus = class { + /** + * @param {!shaka.util.TsParser.OpusMetadata} metadata + * @return {!Uint8Array} + */ + static getAudioConfig(metadata) { + let mapping = []; + switch (metadata.channelConfigCode) { + case 0x01: + case 0x02: + mapping = [0x0]; + break; + case 0x00: // dualmono + mapping = [0xFF, 1, 1, 0, 1]; + break; + case 0x80: // dualmono + mapping = [0xFF, 2, 0, 0, 1]; + break; + case 0x03: + mapping = [0x01, 2, 1, 0, 2, 1]; + break; + case 0x04: + mapping = [0x01, 2, 2, 0, 1, 2, 3]; + break; + case 0x05: + mapping = [0x01, 3, 2, 0, 4, 1, 2, 3]; + break; + case 0x06: + mapping = [0x01, 4, 2, 0, 4, 1, 2, 3, 5]; + break; + case 0x07: + mapping = [0x01, 4, 2, 0, 4, 1, 2, 3, 5, 6]; + break; + case 0x08: + mapping = [0x01, 5, 3, 0, 6, 1, 2, 3, 4, 5, 7]; + break; + case 0x82: + mapping = [0x01, 1, 2, 0, 1]; + break; + case 0x83: + mapping = [0x01, 1, 3, 0, 1, 2]; + break; + case 0x84: + mapping = [0x01, 1, 4, 0, 1, 2, 3]; + break; + case 0x85: + mapping = [0x01, 1, 5, 0, 1, 2, 3, 4]; + break; + case 0x86: + mapping = [0x01, 1, 6, 0, 1, 2, 3, 4, 5]; + break; + case 0x87: + mapping = [0x01, 1, 7, 0, 1, 2, 3, 4, 5, 6]; + break; + case 0x88: + mapping = [0x01, 1, 8, 0, 1, 2, 3, 4, 5, 6, 7]; + break; + } + + return new Uint8Array([ + 0x00, // Version (1) + metadata.channelCount, // OutputChannelCount: 2 + 0x00, 0x00, // PreSkip: 2 + (metadata.sampleRate >>> 24) & 0xFF, // Audio sample rate: 4 + (metadata.sampleRate >>> 17) & 0xFF, + (metadata.sampleRate >>> 8) & 0xFF, + (metadata.sampleRate >>> 0) & 0xFF, + 0x00, 0x00, // Global Gain : 2 + ...mapping, + ]); + } +}; + +/** + * @const {number} + */ +shaka.transmuxer.Opus.OPUS_AUDIO_SAMPLE_PER_FRAME = 960; diff --git a/lib/transmuxer/ts_transmuxer.js b/lib/transmuxer/ts_transmuxer.js index 6407ae3b79..93ea592d1f 100644 --- a/lib/transmuxer/ts_transmuxer.js +++ b/lib/transmuxer/ts_transmuxer.js @@ -14,6 +14,7 @@ goog.require('shaka.transmuxer.Ec3'); goog.require('shaka.transmuxer.H264'); goog.require('shaka.transmuxer.H265'); goog.require('shaka.transmuxer.MpegAudio'); +goog.require('shaka.transmuxer.Opus'); goog.require('shaka.transmuxer.TransmuxerEngine'); goog.require('shaka.util.BufferUtils'); goog.require('shaka.util.Error'); @@ -274,6 +275,10 @@ shaka.transmuxer.TsTransmuxer = class { streamInfo = this.getMp3StreamInfo_(tsParser, stream, duration); break; + case 'opus': + streamInfo = + this.getOpusStreamInfo_(tsParser, stream, duration); + break; } if (streamInfo) { streamInfos.push(streamInfo); @@ -734,6 +739,115 @@ shaka.transmuxer.TsTransmuxer = class { } + /** + * @param {shaka.util.TsParser} tsParser + * @param {shaka.extern.Stream} stream + * @param {number} duration + * @return {shaka.util.Mp4Generator.StreamInfo} + * @private + */ + getOpusStreamInfo_(tsParser, stream, duration) { + const Opus = shaka.transmuxer.Opus; + const timescale = shaka.util.TsParser.Timescale; + + /** @type {!Array.} */ + const samples = []; + + let firstPts = null; + + /** @type {?shaka.util.TsParser.OpusMetadata} */ + const opusMetadata = tsParser.getOpusMetadata(); + + if (!opusMetadata) { + throw new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.MEDIA, + shaka.util.Error.Code.TRANSMUXING_FAILED); + } + + /** @type {!Uint8Array} */ + const audioConfig = Opus.getAudioConfig(opusMetadata); + + /** @type {number} */ + const sampleRate = opusMetadata.sampleRate; + + for (const audioData of tsParser.getAudioData()) { + const data = audioData.data; + if (firstPts == null && audioData.pts !== null) { + firstPts = audioData.pts; + } + let offset = 0; + while (offset < data.length) { + const opusPendingTrimStart = (data[offset + 1] & 0x10) !== 0; + const trimEnd = (data[offset + 1] & 0x08) !== 0; + let index = offset + 2; + let size = 0; + + while (data[index] === 0xFF) { + size += 255; + index += 1; + } + size += data[index]; + index += 1; + index += opusPendingTrimStart ? 2 : 0; + index += trimEnd ? 2 : 0; + + const sample = data.slice(index, index + size); + + samples.push({ + data: sample, + size: sample.byteLength, + duration: Opus.OPUS_AUDIO_SAMPLE_PER_FRAME, + cts: 0, + flags: { + isLeading: 0, + isDependedOn: 0, + hasRedundancy: 0, + degradPrio: 0, + dependsOn: 2, + isNonSync: 0, + }, + }); + offset = index + size; + } + } + + if (audioConfig.byteLength == 0 || firstPts == null) { + throw new shaka.util.Error( + shaka.util.Error.Severity.CRITICAL, + shaka.util.Error.Category.MEDIA, + shaka.util.Error.Code.TRANSMUXING_FAILED); + } + + stream.audioSamplingRate = opusMetadata.sampleRate; + stream.channelsCount = opusMetadata.channelCount; + + + /** @type {number} */ + const baseMediaDecodeTime = firstPts / timescale * sampleRate; + + return { + id: stream.id, + type: shaka.util.ManifestParserUtils.ContentType.AUDIO, + codecs: 'opus', + encrypted: stream.encrypted && stream.drmInfos.length > 0, + timescale: sampleRate, + duration: duration, + videoNalus: [], + audioConfig: audioConfig, + videoConfig: new Uint8Array([]), + hSpacing: 0, + vSpacing: 0, + data: { + sequenceNumber: this.frameIndex_, + baseMediaDecodeTime: baseMediaDecodeTime, + samples: samples, + }, + stream: stream, + }; + } + + /** * @param {shaka.util.TsParser} tsParser * @param {shaka.extern.Stream} stream @@ -933,6 +1047,7 @@ shaka.transmuxer.TsTransmuxer.SUPPORTED_AUDIO_CODECS_ = [ 'ac-3', 'ec-3', 'mp3', + 'opus', ]; /** diff --git a/lib/util/mp4_generator.js b/lib/util/mp4_generator.js index e151989a17..81a82d044e 100644 --- a/lib/util/mp4_generator.js +++ b/lib/util/mp4_generator.js @@ -314,6 +314,8 @@ shaka.util.Mp4Generator = class { bytes = this.ac3_(streamInfo); } else if (streamInfo.codecs.includes('ec-3')) { bytes = this.ec3_(streamInfo); + } else if (streamInfo.codecs.includes('opus')) { + bytes = this.opus_(streamInfo); } else { bytes = this.mp4a_(streamInfo); } @@ -611,6 +613,27 @@ shaka.util.Mp4Generator = class { this.audioStsd_(streamInfo), dec3Box, sinfBox); } + /** + * Generate a Opus box + * + * @param {shaka.util.Mp4Generator.StreamInfo} streamInfo + * @return {!Uint8Array} + * @private + */ + opus_(streamInfo) { + const Mp4Generator = shaka.util.Mp4Generator; + const dopsBox = Mp4Generator.box('dOps', streamInfo.audioConfig); + + let boxName = 'Opus'; + let sinfBox = new Uint8Array([]); + if (streamInfo.encrypted) { + sinfBox = this.sinf_(streamInfo); + boxName = 'enca'; + } + return Mp4Generator.box(boxName, + this.audioStsd_(streamInfo), dopsBox, sinfBox); + } + /** * Generate a MP4A box * diff --git a/lib/util/ts_parser.js b/lib/util/ts_parser.js index b588eabfe5..640e502869 100644 --- a/lib/util/ts_parser.js +++ b/lib/util/ts_parser.js @@ -62,6 +62,9 @@ shaka.util.TsParser = class { /** @private {?number} */ this.referenceDts_ = null; + + /** @private {?shaka.util.TsParser.OpusMetadata} */ + this.opusMetadata_ = null; } /** @@ -302,6 +305,29 @@ shaka.util.TsParser = class { result.audioCodec = 'aac'; } break; + // DVB extension descriptor + case 0x7f: + if (result.audioCodec == 'opus') { + const extensionDescriptorId = data[parsePos + 2]; + let channelConfigCode = null; + // User defined (provisional Opus) + if (extensionDescriptorId === 0x80) { + channelConfigCode = data[parsePos + 3]; + } + + if (channelConfigCode == null) { + // Not Supported Opus channel count. + break; + } + const channelCount = (channelConfigCode & 0x0F) === 0 ? + 2 : (channelConfigCode & 0x0F); + this.opusMetadata_ = { + channelCount, + channelConfigCode, + sampleRate: 48000, + }; + } + break; } parsePos += descriptorLen; remaining -= descriptorLen; @@ -1113,6 +1139,15 @@ shaka.util.TsParser = class { return videoInfo; } + /** + * Return the Opus metadata + * + * @return {?shaka.util.TsParser.OpusMetadata} + */ + getOpusMetadata() { + return this.opusMetadata_; + } + /** * Convert a byte to 2 digits of hex. (Only handles values 0-255.) * @@ -1272,3 +1307,18 @@ shaka.util.TsParser.PROFILES_WITH_OPTIONAL_SPS_DATA_ = */ shaka.util.TsParser.PMT; + +/** + * @typedef {{ + * channelCount: number, + * channelConfigCode: number, + * sampleRate: number + * }} + * + * @summary PMT. + * @property {number} channelCount + * @property {number} channelConfigCode + * @property {number} sampleRate + */ +shaka.util.TsParser.OpusMetadata; + diff --git a/shaka-player.uncompiled.js b/shaka-player.uncompiled.js index c5e656e128..3450848302 100644 --- a/shaka-player.uncompiled.js +++ b/shaka-player.uncompiled.js @@ -75,6 +75,7 @@ goog.require('shaka.transmuxer.H265'); goog.require('shaka.transmuxer.Mp3Transmuxer'); goog.require('shaka.transmuxer.MpegAudio'); goog.require('shaka.transmuxer.MpegTsTransmuxer'); +goog.require('shaka.transmuxer.Opus'); goog.require('shaka.transmuxer.TransmuxerEngine'); goog.require('shaka.transmuxer.MssTransmuxer'); goog.require('shaka.transmuxer.TsTransmuxer'); diff --git a/test/test/assets/hls-ts-muxed-opus-h264/file1.ts b/test/test/assets/hls-ts-muxed-opus-h264/file1.ts new file mode 100644 index 0000000000..ff4e6f60ca Binary files /dev/null and b/test/test/assets/hls-ts-muxed-opus-h264/file1.ts differ diff --git a/test/test/assets/hls-ts-muxed-opus-h264/file2.ts b/test/test/assets/hls-ts-muxed-opus-h264/file2.ts new file mode 100644 index 0000000000..a67a1d4eed Binary files /dev/null and b/test/test/assets/hls-ts-muxed-opus-h264/file2.ts differ diff --git a/test/test/assets/hls-ts-muxed-opus-h264/file3.ts b/test/test/assets/hls-ts-muxed-opus-h264/file3.ts new file mode 100644 index 0000000000..512b433176 Binary files /dev/null and b/test/test/assets/hls-ts-muxed-opus-h264/file3.ts differ diff --git a/test/test/assets/hls-ts-muxed-opus-h264/playlist.m3u8 b/test/test/assets/hls-ts-muxed-opus-h264/playlist.m3u8 new file mode 100644 index 0000000000..9a0663bb26 --- /dev/null +++ b/test/test/assets/hls-ts-muxed-opus-h264/playlist.m3u8 @@ -0,0 +1,11 @@ +#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:8 +#EXT-X-PLAYLIST-TYPE:VOD +#EXTINF:9.138255, +file1.ts +#EXTINF:8.789333, +file2.ts +#EXTINF:1.208332, +file3.ts +#EXT-X-ENDLIST diff --git a/test/transmuxer/transmuxer_integration.js b/test/transmuxer/transmuxer_integration.js index 624aa55c3c..ccce76472b 100644 --- a/test/transmuxer/transmuxer_integration.js +++ b/test/transmuxer/transmuxer_integration.js @@ -445,5 +445,26 @@ describe('Transmuxer Player', () => { await player.unload(); }); + + it('H.264+Opus in TS', async () => { + if (!MediaSource.isTypeSupported('audio/mp4; codecs="opus"')) { + pending('Codec opus is not supported by the platform.'); + } + + // eslint-disable-next-line max-len + await player.load('/base/test/test/assets/hls-ts-muxed-opus-h264/playlist.m3u8'); + await video.play(); + expect(player.isLive()).toBe(false); + + // Wait for the video to start playback. If it takes longer than 10 + // seconds, fail the test. + await waiter.waitForMovementOrFailOnTimeout(video, 10); + + // Play for 15 seconds, but stop early if the video ends. If it takes + // longer than 45 seconds, fail the test. + await waiter.waitUntilPlayheadReachesOrFailOnTimeout(video, 15, 45); + + await player.unload(); + }); }); });