feat: Add Opus TS transmuxer (#6387)

README.md

@@ -280,6 +280,7 @@ Shaka Player supports:
   - EC-3 in MPEG-2 TS to EC-3 in MP4
   - MP3 in MPEG-2 TS to MP3 in MP4
   - MP3 in MPEG-2 TS to raw MP3
+  - Opus in MPEG-2 TS to MP3 in MP4
   - H.264 in MPEG-2 TS to H.264 in MP4
   - H.265 in MPEG-2 TS to H.265 in MP4
   - Muxed content in MPEG-2 TS with the previous codecs

build/types/transmuxer

@@ -11,4 +11,5 @@
 +../../lib/transmuxer/mp3_transmuxer.js
 +../../lib/transmuxer/mpeg_audio.js
 +../../lib/transmuxer/mpeg_ts_transmuxer.js
++../../lib/transmuxer/opus.js
 +../../lib/transmuxer/ts_transmuxer.js

karma.conf.js

@@ -255,6 +255,7 @@ module.exports = (config) => {
       {pattern: 'test/test/assets/hls-ts-muxed-ac3-h264/*', included: false},
       {pattern: 'test/test/assets/hls-ts-muxed-mp3-h264/*', included: false},
       {pattern: 'test/test/assets/hls-ts-muxed-ec3-h264/*', included: false},
+      {pattern: 'test/test/assets/hls-ts-muxed-opus-h264/*', included: false},
       {pattern: 'test/test/assets/hls-ts-raw-aac/*', included: false},
       {pattern: 'test/test/assets/hls-ts-rollover/*', included: false},
       {pattern: 'dist/shaka-player.ui.js', included: false},

lib/transmuxer/opus.js

@@ -0,0 +1,91 @@
+/*! @license
+ * Shaka Player
+ * Copyright 2016 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+goog.provide('shaka.transmuxer.Opus');
+
+goog.requireType('shaka.util.TsParser');
+
+
+/**
+ * Opus utils
+ */
+shaka.transmuxer.Opus = class {
+  /**
+   * @param {!shaka.util.TsParser.OpusMetadata} metadata
+   * @return {!Uint8Array}
+   */
+  static getAudioConfig(metadata) {
+    let mapping = [];
+    switch (metadata.channelConfigCode) {
+      case 0x01:
+      case 0x02:
+        mapping = [0x0];
+        break;
+      case 0x00: // dualmono
+        mapping = [0xFF, 1, 1, 0, 1];
+        break;
+      case 0x80: // dualmono
+        mapping = [0xFF, 2, 0, 0, 1];
+        break;
+      case 0x03:
+        mapping = [0x01, 2, 1, 0, 2, 1];
+        break;
+      case 0x04:
+        mapping = [0x01, 2, 2, 0, 1, 2, 3];
+        break;
+      case 0x05:
+        mapping = [0x01, 3, 2, 0, 4, 1, 2, 3];
+        break;
+      case 0x06:
+        mapping = [0x01, 4, 2, 0, 4, 1, 2, 3, 5];
+        break;
+      case 0x07:
+        mapping = [0x01, 4, 2, 0, 4, 1, 2, 3, 5, 6];
+        break;
+      case 0x08:
+        mapping = [0x01, 5, 3, 0, 6, 1, 2, 3, 4, 5, 7];
+        break;
+      case 0x82:
+        mapping = [0x01, 1, 2, 0, 1];
+        break;
+      case 0x83:
+        mapping = [0x01, 1, 3, 0, 1, 2];
+        break;
+      case 0x84:
+        mapping = [0x01, 1, 4, 0, 1, 2, 3];
+        break;
+      case 0x85:
+        mapping = [0x01, 1, 5, 0, 1, 2, 3, 4];
+        break;
+      case 0x86:
+        mapping = [0x01, 1, 6, 0, 1, 2, 3, 4, 5];
+        break;
+      case 0x87:
+        mapping = [0x01, 1, 7, 0, 1, 2, 3, 4, 5, 6];
+        break;
+      case 0x88:
+        mapping = [0x01, 1, 8, 0, 1, 2, 3, 4, 5, 6, 7];
+        break;
+    }
+
+    return new Uint8Array([
+      0x00,         // Version (1)
+      metadata.channelCount, // OutputChannelCount: 2
+      0x00, 0x00,   // PreSkip: 2
+      (metadata.sampleRate >>> 24) & 0xFF,  // Audio sample rate: 4
+      (metadata.sampleRate >>> 17) & 0xFF,
+      (metadata.sampleRate >>> 8) & 0xFF,
+      (metadata.sampleRate >>> 0) & 0xFF,
+      0x00, 0x00,  // Global Gain : 2
+      ...mapping,
+    ]);
+  }
+};
+
+/**
+ * @const {number}
+ */
+shaka.transmuxer.Opus.OPUS_AUDIO_SAMPLE_PER_FRAME = 960;

lib/transmuxer/ts_transmuxer.js

@@ -14,6 +14,7 @@ goog.require('shaka.transmuxer.Ec3');
 goog.require('shaka.transmuxer.H264');
 goog.require('shaka.transmuxer.H265');
 goog.require('shaka.transmuxer.MpegAudio');
+goog.require('shaka.transmuxer.Opus');
 goog.require('shaka.transmuxer.TransmuxerEngine');
 goog.require('shaka.util.BufferUtils');
 goog.require('shaka.util.Error');
@@ -274,6 +275,10 @@ shaka.transmuxer.TsTransmuxer = class {
             streamInfo =
               this.getMp3StreamInfo_(tsParser, stream, duration);
             break;
+          case 'opus':
+            streamInfo =
+              this.getOpusStreamInfo_(tsParser, stream, duration);
+            break;
         }
         if (streamInfo) {
           streamInfos.push(streamInfo);
@@ -734,6 +739,115 @@ shaka.transmuxer.TsTransmuxer = class {
   }
 
 
+  /**
+   * @param {shaka.util.TsParser} tsParser
+   * @param {shaka.extern.Stream} stream
+   * @param {number} duration
+   * @return {shaka.util.Mp4Generator.StreamInfo}
+   * @private
+   */
+  getOpusStreamInfo_(tsParser, stream, duration) {
+    const Opus = shaka.transmuxer.Opus;
+    const timescale = shaka.util.TsParser.Timescale;
+
+    /** @type {!Array.<shaka.util.Mp4Generator.Mp4Sample>} */
+    const samples = [];
+
+    let firstPts = null;
+
+    /** @type {?shaka.util.TsParser.OpusMetadata} */
+    const opusMetadata = tsParser.getOpusMetadata();
+
+    if (!opusMetadata) {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.CRITICAL,
+          shaka.util.Error.Category.MEDIA,
+          shaka.util.Error.Code.TRANSMUXING_FAILED);
+    }
+
+    /** @type {!Uint8Array} */
+    const audioConfig = Opus.getAudioConfig(opusMetadata);
+
+    /** @type {number} */
+    const sampleRate = opusMetadata.sampleRate;
+
+    for (const audioData of tsParser.getAudioData()) {
+      const data = audioData.data;
+      if (firstPts == null && audioData.pts !== null) {
+        firstPts = audioData.pts;
+      }
+      let offset = 0;
+      while (offset < data.length) {
+        const opusPendingTrimStart = (data[offset + 1] & 0x10) !== 0;
+        const trimEnd = (data[offset + 1] & 0x08) !== 0;
+        let index = offset + 2;
+        let size = 0;
+
+        while (data[index] === 0xFF) {
+          size += 255;
+          index += 1;
+        }
+        size += data[index];
+        index += 1;
+        index += opusPendingTrimStart ? 2 : 0;
+        index += trimEnd ? 2 : 0;
+
+        const sample = data.slice(index, index + size);
+
+        samples.push({
+          data: sample,
+          size: sample.byteLength,
+          duration: Opus.OPUS_AUDIO_SAMPLE_PER_FRAME,
+          cts: 0,
+          flags: {
+            isLeading: 0,
+            isDependedOn: 0,
+            hasRedundancy: 0,
+            degradPrio: 0,
+            dependsOn: 2,
+            isNonSync: 0,
+          },
+        });
+        offset = index + size;
+      }
+    }
+
+    if (audioConfig.byteLength == 0 || firstPts == null) {
+      throw new shaka.util.Error(
+          shaka.util.Error.Severity.CRITICAL,
+          shaka.util.Error.Category.MEDIA,
+          shaka.util.Error.Code.TRANSMUXING_FAILED);
+    }
+
+    stream.audioSamplingRate = opusMetadata.sampleRate;
+    stream.channelsCount = opusMetadata.channelCount;
+
+
+    /** @type {number} */
+    const baseMediaDecodeTime = firstPts / timescale * sampleRate;
+
+    return {
+      id: stream.id,
+      type: shaka.util.ManifestParserUtils.ContentType.AUDIO,
+      codecs: 'opus',
+      encrypted: stream.encrypted && stream.drmInfos.length > 0,
+      timescale: sampleRate,
+      duration: duration,
+      videoNalus: [],
+      audioConfig: audioConfig,
+      videoConfig: new Uint8Array([]),
+      hSpacing: 0,
+      vSpacing: 0,
+      data: {
+        sequenceNumber: this.frameIndex_,
+        baseMediaDecodeTime: baseMediaDecodeTime,
+        samples: samples,
+      },
+      stream: stream,
+    };
+  }
+
+
   /**
    * @param {shaka.util.TsParser} tsParser
    * @param {shaka.extern.Stream} stream
@@ -933,6 +1047,7 @@ shaka.transmuxer.TsTransmuxer.SUPPORTED_AUDIO_CODECS_ = [
   'ac-3',
   'ec-3',
   'mp3',
+  'opus',
 ];
 
 /**

lib/util/mp4_generator.js

@@ -314,6 +314,8 @@ shaka.util.Mp4Generator = class {
           bytes = this.ac3_(streamInfo);
         } else if (streamInfo.codecs.includes('ec-3')) {
           bytes = this.ec3_(streamInfo);
+        } else if (streamInfo.codecs.includes('opus')) {
+          bytes = this.opus_(streamInfo);
         } else {
           bytes = this.mp4a_(streamInfo);
         }
@@ -611,6 +613,27 @@ shaka.util.Mp4Generator = class {
         this.audioStsd_(streamInfo), dec3Box, sinfBox);
   }
 
+  /**
+   * Generate a Opus box
+   *
+   * @param {shaka.util.Mp4Generator.StreamInfo} streamInfo
+   * @return {!Uint8Array}
+   * @private
+   */
+  opus_(streamInfo) {
+    const Mp4Generator = shaka.util.Mp4Generator;
+    const dopsBox = Mp4Generator.box('dOps', streamInfo.audioConfig);
+
+    let boxName = 'Opus';
+    let sinfBox = new Uint8Array([]);
+    if (streamInfo.encrypted) {
+      sinfBox = this.sinf_(streamInfo);
+      boxName = 'enca';
+    }
+    return Mp4Generator.box(boxName,
+        this.audioStsd_(streamInfo), dopsBox, sinfBox);
+  }
+
   /**
    * Generate a MP4A box
    *

lib/util/ts_parser.js

@@ -62,6 +62,9 @@ shaka.util.TsParser = class {
 
     /** @private {?number} */
     this.referenceDts_ = null;
+
+    /** @private {?shaka.util.TsParser.OpusMetadata} */
+    this.opusMetadata_ = null;
   }
 
   /**
@@ -302,6 +305,29 @@ shaka.util.TsParser = class {
                     result.audioCodec = 'aac';
                   }
                   break;
+                // DVB extension descriptor
+                case 0x7f:
+                  if (result.audioCodec == 'opus') {
+                    const extensionDescriptorId = data[parsePos + 2];
+                    let channelConfigCode = null;
+                    // User defined (provisional Opus)
+                    if (extensionDescriptorId === 0x80) {
+                      channelConfigCode = data[parsePos + 3];
+                    }
+
+                    if (channelConfigCode == null) {
+                      // Not Supported Opus channel count.
+                      break;
+                    }
+                    const channelCount = (channelConfigCode & 0x0F) === 0 ?
+                        2 : (channelConfigCode & 0x0F);
+                    this.opusMetadata_ = {
+                      channelCount,
+                      channelConfigCode,
+                      sampleRate: 48000,
+                    };
+                  }
+                  break;
               }
               parsePos += descriptorLen;
               remaining -= descriptorLen;
@@ -1113,6 +1139,15 @@ shaka.util.TsParser = class {
     return videoInfo;
   }
 
+  /**
+   * Return the Opus metadata
+   *
+   * @return {?shaka.util.TsParser.OpusMetadata}
+   */
+  getOpusMetadata() {
+    return this.opusMetadata_;
+  }
+
   /**
    * Convert a byte to 2 digits of hex.  (Only handles values 0-255.)
    *
@@ -1272,3 +1307,18 @@ shaka.util.TsParser.PROFILES_WITH_OPTIONAL_SPS_DATA_ =
  */
 shaka.util.TsParser.PMT;
 
+
+/**
+ * @typedef {{
+ *   channelCount: number,
+ *   channelConfigCode: number,
+ *   sampleRate: number
+ * }}
+ *
+ * @summary PMT.
+ * @property {number} channelCount
+ * @property {number} channelConfigCode
+ * @property {number} sampleRate
+ */
+shaka.util.TsParser.OpusMetadata;
+

shaka-player.uncompiled.js

@@ -75,6 +75,7 @@ goog.require('shaka.transmuxer.H265');
 goog.require('shaka.transmuxer.Mp3Transmuxer');
 goog.require('shaka.transmuxer.MpegAudio');
 goog.require('shaka.transmuxer.MpegTsTransmuxer');
+goog.require('shaka.transmuxer.Opus');
 goog.require('shaka.transmuxer.TransmuxerEngine');
 goog.require('shaka.transmuxer.MssTransmuxer');
 goog.require('shaka.transmuxer.TsTransmuxer');

test/test/assets/hls-ts-muxed-opus-h264/playlist.m3u8

@@ -0,0 +1,11 @@
+#EXTM3U
+#EXT-X-VERSION:3
+#EXT-X-TARGETDURATION:8
+#EXT-X-PLAYLIST-TYPE:VOD
+#EXTINF:9.138255,
+file1.ts
+#EXTINF:8.789333,
+file2.ts
+#EXTINF:1.208332,
+file3.ts
+#EXT-X-ENDLIST