Apply DTS-HD and DTS Express detection to MP4

libraries/extractor/src/main/java/androidx/media3/extractor/DtsUtil.java

@@ -44,6 +44,15 @@
 @UnstableApi
 public final class DtsUtil {
 
+  /**
+   * Returns whether {@code mimeType} is {@link MimeTypes#AUDIO_DTS} or {@link
+   * MimeTypes#AUDIO_DTS_HD}.
+   */
+  public static boolean isDtsBaseAudioMimeType(@Nullable String mimeType) {
+    return Objects.equals(mimeType, MimeTypes.AUDIO_DTS)
+        || Objects.equals(mimeType, MimeTypes.AUDIO_DTS_HD);
+  }
+
   /** Information parsed from a DTS frame header. */
   public static final class DtsHeader {
     /** The mime type of the DTS bitstream. */

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/FragmentedMp4Extractor.java

@@ -49,6 +49,7 @@
 import androidx.media3.extractor.CeaUtil;
 import androidx.media3.extractor.ChunkIndex;
 import androidx.media3.extractor.ChunkIndexMerger;
+import androidx.media3.extractor.DtsUtil;
 import androidx.media3.extractor.Extractor;
 import androidx.media3.extractor.ExtractorInput;
 import androidx.media3.extractor.ExtractorOutput;
@@ -1851,6 +1852,15 @@ private boolean readSample(ExtractorInput input) throws IOException {
         }
       }
     } else {
+      Format pendingFormat = trackBundle.pendingFormat;
+      if (pendingFormat != null && DtsUtil.isDtsBaseAudioMimeType(track.format.sampleMimeType)) {
+        trackBundle.baseFormat =
+            DtsUtil.updateFormatWithDtsHdInfo(input, sampleSize, trackBundle.baseFormat);
+        Format outputFormat =
+            trackBundle.baseFormat.buildUpon().setDrmInitData(pendingFormat.drmInitData).build();
+        trackBundle.output.format(outputFormat);
+        trackBundle.pendingFormat = null;
+      }
       while (sampleBytesWritten < sampleSize) {
         int writtenBytes = output.sampleData(input, sampleSize - sampleBytesWritten, false);
         sampleBytesWritten += writtenBytes;
@@ -2302,7 +2312,13 @@ private static final class TrackBundle {
     public int currentTrackRunIndex;
     public int firstSampleToOutputIndex;
 
-    private final Format baseFormat;
+    /**
+     * A {@link Format} that needs to be passed to {@link #output}, after being possibly modified
+     * based on sample data, before {@link TrackOutput#sampleMetadata} is called.
+     */
+    @Nullable private Format pendingFormat;
+
+    private Format baseFormat;
     private final ParsableByteArray encryptionSignalByte;
     private final ParsableByteArray defaultInitializationVector;
 
@@ -2321,13 +2337,18 @@ public TrackBundle(
       scratch = new ParsableByteArray();
       encryptionSignalByte = new ParsableByteArray(1);
       defaultInitializationVector = new ParsableByteArray();
+      if (DtsUtil.isDtsBaseAudioMimeType(baseFormat.sampleMimeType)) {
+        pendingFormat = baseFormat;
+      }
       reset(moovSampleTable, defaultSampleValues);
     }
 
     public void reset(TrackSampleTable moovSampleTable, DefaultSampleValues defaultSampleValues) {
       this.moovSampleTable = moovSampleTable;
       this.defaultSampleValues = defaultSampleValues;
-      output.format(baseFormat);
+      if (pendingFormat == null) {
+        output.format(baseFormat);
+      }
       resetFragmentInfo();
     }
 
@@ -2339,7 +2360,11 @@ public void updateDrmInitData(DrmInitData drmInitData) {
       @Nullable String schemeType = encryptionBox != null ? encryptionBox.schemeType : null;
       DrmInitData updatedDrmInitData = drmInitData.copyWithSchemeType(schemeType);
       Format format = baseFormat.buildUpon().setDrmInitData(updatedDrmInitData).build();
-      output.format(format);
+      if (pendingFormat != null) {
+        pendingFormat = format;
+      } else {
+        output.format(format);
+      }
     }
 
     /** Resets the current fragment, sample indices and {@link #currentlyInFragment} boolean. */

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java

@@ -46,6 +46,7 @@
 import androidx.media3.container.NalUnitUtil;
 import androidx.media3.extractor.Ac3Util;
 import androidx.media3.extractor.Ac4Util;
+import androidx.media3.extractor.DtsUtil;
 import androidx.media3.extractor.Extractor;
 import androidx.media3.extractor.ExtractorInput;
 import androidx.media3.extractor.ExtractorOutput;
@@ -744,10 +745,13 @@ private void processMoovAtom(ContainerBox moov) throws ParserException {
       // The moov and esds boxes don't contain enough information to distinguish between MPEG
       // audio layers 1, 2 and 3, but the distinction is important to select the right MIME type
       // for MediaCodec decoders (and other decoders that handle the same audio/mpeg-L1 and
-      // audio/mpeg-L2 MIME types). So we store the format with audio/mpeg for now, and then
-      // update the MIME type and pass it to TrackOutput.format(...) based on the layer info in
-      // the first sample.
-      boolean isMpegAudio = Objects.equals(track.format.sampleMimeType, MimeTypes.AUDIO_MPEG);
+      // audio/mpeg-L2 MIME types). DTS has a similar problem where we can't distinguish DTS,
+      // DTS-HD and DTS Express. So we store the format with a placeholder MIME for now, and then
+      // update the MIME type and pass it to TrackOutput.format(...) based on the info in the first
+      // sample.
+      boolean needsSamplesForMimeType =
+          Objects.equals(track.format.sampleMimeType, MimeTypes.AUDIO_MPEG)
+              || DtsUtil.isDtsBaseAudioMimeType(track.format.sampleMimeType);
       boolean needsChapterMetadata = false;
       if (!omitTrackSampleTable && track.chapterTrackId != C.INDEX_UNSET) {
         for (TrackSampleTable chapterSampleTable : chapterSampleTables) {
@@ -757,7 +761,7 @@ private void processMoovAtom(ContainerBox moov) throws ParserException {
           }
         }
       }
-      if (isMpegAudio || needsChapterMetadata) {
+      if (needsSamplesForMimeType || needsChapterMetadata) {
         mp4Track.pendingFormat = format;
       } else {
         mp4Track.trackOutput.format(format);
@@ -1013,8 +1017,16 @@ private int readSample(ExtractorInput input, PositionHolder positionHolder) thro
                     .build()
                 : pendingFormat);
         track.pendingFormat = null;
-      } else if (trueHdSampleRechunker != null) {
-        trueHdSampleRechunker.startSample(input);
+      } else {
+        Format pendingFormat = track.pendingFormat;
+        if (pendingFormat != null
+            && DtsUtil.isDtsBaseAudioMimeType(track.track.format.sampleMimeType)) {
+          track.trackOutput.format(
+              DtsUtil.updateFormatWithDtsHdInfo(input, sampleSize, pendingFormat));
+          track.pendingFormat = null;
+        } else if (trueHdSampleRechunker != null) {
+          trueHdSampleRechunker.startSample(input);
+        }
       }
 
       while (sampleBytesWritten < sampleSize) {
@@ -1103,10 +1115,11 @@ private int readQuickTimeChapters(ExtractorInput input, PositionHolder seekPosit
             currentFormat.buildUpon().setMetadata(new Metadata(filteredEntries)).build();
 
         // The format was kept pending in processMoovAtom either because it was waiting for chapter
-        // metadata, or because it is MPEG audio (which needs to wait for the first sample to
+        // metadata, or because it is MPEG or DTS audio (which needs to wait for the first sample to
         // determine the exact MIME type). We have now applied the chapter metadata, so we can
-        // output the format, unless it is also MPEG audio.
-        if (Objects.equals(updatedFormat.sampleMimeType, MimeTypes.AUDIO_MPEG)) {
+        // output the format, unless it is also MPEG or DTS audio.
+        if (Objects.equals(updatedFormat.sampleMimeType, MimeTypes.AUDIO_MPEG)
+            || DtsUtil.isDtsBaseAudioMimeType(updatedFormat.sampleMimeType)) {
           track.pendingFormat = updatedFormat;
         } else {
           track.trackOutput.format(updatedFormat);
@@ -1370,7 +1383,7 @@ private static final class Mp4Track {
      * A {@link Format} that needs to be passed to {@link #trackOutput}, after being possibly
      * modified based on sample data, before {@link TrackOutput#sampleMetadata} is called.
      */
-    @Nullable public Format pendingFormat;
+    @Nullable private Format pendingFormat;
 
     public Mp4Track(Track track, TrackSampleTable sampleTable, TrackOutput trackOutput) {
       this.track = track;

libraries/extractor/src/test/java/androidx/media3/extractor/mp4/FragmentedMp4ExtractorParameterizedTest.java

@@ -188,6 +188,22 @@ public void sampleWithOpusTrack() throws Exception {
         /* peekLimit= */ 540);
   }
 
+  @Test
+  public void sampleWithDtsExpress() throws Exception {
+    assertExtractorBehavior(
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        "media/mp4/sample_fragmented_dts_express.mp4",
+        /* peekLimit= */ 4096);
+  }
+
+  @Test
+  public void sampleWithDtsHdMa() throws Exception {
+    assertExtractorBehavior(
+        /* closedCaptionFormats= */ ImmutableList.of(),
+        "media/mp4/sample_fragmented_dts_hd_ma.mp4",
+        /* peekLimit= */ 4096);
+  }
+
   @Test
   public void samplePartiallyFragmented() throws Exception {
     assertExtractorBehavior(

libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorParameterizedTest.java

@@ -143,6 +143,16 @@ public void mp4SampleWithDolbyTrueHDTrack() throws Exception {
     assertExtractorBehavior("media/mp4/sample_dthd.mp4", /* peekLimit= */ 3450);
   }
 
+  @Test
+  public void mp4SampleWithDtsExpress() throws Exception {
+    assertExtractorBehavior("media/mp4/sample_dts_express.mp4", /* peekLimit= */ 4096);
+  }
+
+  @Test
+  public void mp4SampleWithDtsHdMa() throws Exception {
+    assertExtractorBehavior("media/mp4/sample_dts_hd_ma.mp4", /* peekLimit= */ 4096);
+  }
+
   @Test
   public void mp4SampleWithColrMdcvAndClli() throws Exception {
     assertExtractorBehavior("media/mp4/sample_with_colr_mdcv_and_clli.mp4", /* peekLimit= */ 50);

libraries/test_data/src/test/assets/extractordumps/mp4/sample_dts_express.mp4.0.dump

@@ -0,0 +1,67 @@
+seekMap:
+  isSeekable = true
+  duration = 939000
+  getPosition(0) = [[timeUs=0, position=44]]
+  getPosition(1) = [[timeUs=1, position=44]]
+  getPosition(469500) = [[timeUs=469500, position=20524]]
+  getPosition(939000) = [[timeUs=939000, position=41004]]
+numberOfTracks = 1
+track 0:
+  total output bytes = 45056
+  sample count = 11
+  track duration = 939000
+  format 0:
+    averageBitrate = 384000
+    id = 1
+    containerMimeType = audio/mp4
+    sampleMimeType = audio/vnd.dts.hd;profile=lbr
+    maxInputSize = 4126
+    channelCount = 6
+    sampleRate = 48000
+    language = und
+    metadata = entries=[Mp4AlternateGroup: 1, TSSE: description=null: values=[Lavf62.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
+  sample 0:
+    time = 0
+    flags = 1
+    data = length 4096, hash 1F0B79C5
+  sample 1:
+    time = 85333
+    flags = 1
+    data = length 4096, hash 2EC282A1
+  sample 2:
+    time = 170666
+    flags = 1
+    data = length 4096, hash 6B7902F0
+  sample 3:
+    time = 256000
+    flags = 1
+    data = length 4096, hash 8FC4EE2C
+  sample 4:
+    time = 341333
+    flags = 1
+    data = length 4096, hash 67899547
+  sample 5:
+    time = 426666
+    flags = 1
+    data = length 4096, hash 1BE4CF1C
+  sample 6:
+    time = 512000
+    flags = 1
+    data = length 4096, hash 620F5E51
+  sample 7:
+    time = 597333
+    flags = 1
+    data = length 4096, hash 4D3E0644
+  sample 8:
+    time = 682666
+    flags = 1
+    data = length 4096, hash F69B5FED
+  sample 9:
+    time = 768000
+    flags = 1
+    data = length 4096, hash 93D31EA7
+  sample 10:
+    time = 853333
+    flags = 536870913
+    data = length 4096, hash 41F1D921
+tracksEnded = true

libraries/test_data/src/test/assets/extractordumps/mp4/sample_dts_express.mp4.1.dump

@@ -0,0 +1,55 @@
+seekMap:
+  isSeekable = true
+  duration = 939000
+  getPosition(0) = [[timeUs=0, position=44]]
+  getPosition(1) = [[timeUs=1, position=44]]
+  getPosition(469500) = [[timeUs=469500, position=20524]]
+  getPosition(939000) = [[timeUs=939000, position=41004]]
+numberOfTracks = 1
+track 0:
+  total output bytes = 32768
+  sample count = 8
+  track duration = 939000
+  format 0:
+    averageBitrate = 384000
+    id = 1
+    containerMimeType = audio/mp4
+    sampleMimeType = audio/vnd.dts.hd;profile=lbr
+    maxInputSize = 4126
+    channelCount = 6
+    sampleRate = 48000
+    language = und
+    metadata = entries=[Mp4AlternateGroup: 1, TSSE: description=null: values=[Lavf62.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
+  sample 0:
+    time = 256000
+    flags = 1
+    data = length 4096, hash 8FC4EE2C
+  sample 1:
+    time = 341333
+    flags = 1
+    data = length 4096, hash 67899547
+  sample 2:
+    time = 426666
+    flags = 1
+    data = length 4096, hash 1BE4CF1C
+  sample 3:
+    time = 512000
+    flags = 1
+    data = length 4096, hash 620F5E51
+  sample 4:
+    time = 597333
+    flags = 1
+    data = length 4096, hash 4D3E0644
+  sample 5:
+    time = 682666
+    flags = 1
+    data = length 4096, hash F69B5FED
+  sample 6:
+    time = 768000
+    flags = 1
+    data = length 4096, hash 93D31EA7
+  sample 7:
+    time = 853333
+    flags = 536870913
+    data = length 4096, hash 41F1D921
+tracksEnded = true

libraries/test_data/src/test/assets/extractordumps/mp4/sample_dts_express.mp4.2.dump

@@ -0,0 +1,39 @@
+seekMap:
+  isSeekable = true
+  duration = 939000
+  getPosition(0) = [[timeUs=0, position=44]]
+  getPosition(1) = [[timeUs=1, position=44]]
+  getPosition(469500) = [[timeUs=469500, position=20524]]
+  getPosition(939000) = [[timeUs=939000, position=41004]]
+numberOfTracks = 1
+track 0:
+  total output bytes = 16384
+  sample count = 4
+  track duration = 939000
+  format 0:
+    averageBitrate = 384000
+    id = 1
+    containerMimeType = audio/mp4
+    sampleMimeType = audio/vnd.dts.hd;profile=lbr
+    maxInputSize = 4126
+    channelCount = 6
+    sampleRate = 48000
+    language = und
+    metadata = entries=[Mp4AlternateGroup: 1, TSSE: description=null: values=[Lavf62.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
+  sample 0:
+    time = 597333
+    flags = 1
+    data = length 4096, hash 4D3E0644
+  sample 1:
+    time = 682666
+    flags = 1
+    data = length 4096, hash F69B5FED
+  sample 2:
+    time = 768000
+    flags = 1
+    data = length 4096, hash 93D31EA7
+  sample 3:
+    time = 853333
+    flags = 536870913
+    data = length 4096, hash 41F1D921
+tracksEnded = true

libraries/test_data/src/test/assets/extractordumps/mp4/sample_dts_express.mp4.3.dump

@@ -0,0 +1,27 @@
+seekMap:
+  isSeekable = true
+  duration = 939000
+  getPosition(0) = [[timeUs=0, position=44]]
+  getPosition(1) = [[timeUs=1, position=44]]
+  getPosition(469500) = [[timeUs=469500, position=20524]]
+  getPosition(939000) = [[timeUs=939000, position=41004]]
+numberOfTracks = 1
+track 0:
+  total output bytes = 4096
+  sample count = 1
+  track duration = 939000
+  format 0:
+    averageBitrate = 384000
+    id = 1
+    containerMimeType = audio/mp4
+    sampleMimeType = audio/vnd.dts.hd;profile=lbr
+    maxInputSize = 4126
+    channelCount = 6
+    sampleRate = 48000
+    language = und
+    metadata = entries=[Mp4AlternateGroup: 1, TSSE: description=null: values=[Lavf62.3.100], Mp4Timestamp: creation time=0, modification time=0, timescale=1000]
+  sample 0:
+    time = 853333
+    flags = 536870913
+    data = length 4096, hash 41F1D921
+tracksEnded = true