From fea4376779ff9b2167363ed970bc1672e1bae01f Mon Sep 17 00:00:00 2001 From: olly Date: Sun, 19 Apr 2020 17:12:26 +0100 Subject: [PATCH] Merge trick play tracks into main track groups Issue: #6054 PiperOrigin-RevId: 307285068 --- RELEASENOTES.md | 5 + .../source/dash/DashMediaPeriod.java | 122 +++++++--- .../source/dash/DashMediaPeriodTest.java | 222 +++++++++++++++--- .../testutil/MediaPeriodAsserts.java | 17 +- 4 files changed, 291 insertions(+), 75 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 9daedd60665..36fd99ef420 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -97,6 +97,11 @@ * Remove generics from DRM components. * Downloads: Merge downloads in `SegmentDownloader` to improve overall download speed ([#5978](https://github.com/google/ExoPlayer/issues/5978)). +* DASH: + * Merge trick play adaptation sets (i.e., adaptation sets marked with + `http://dashif.org/guidelines/trickmode`) into the same `TrackGroup` as + the main adaptation sets to which they refer. Trick play tracks are + marked with the `C.ROLE_FLAG_TRICK_PLAY` flag. * MP3: Add `IndexSeeker` for accurate seeks in VBR streams ([#6787](https://github.com/google/ExoPlayer/issues/6787)). This seeker is enabled by passing `FLAG_ENABLE_INDEX_SEEKING` to the `Mp3Extractor`. It may diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java index 2f2cc26623c..e1a441f36fa 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java @@ -16,6 +16,7 @@ package com.google.android.exoplayer2.source.dash; import android.util.Pair; +import android.util.SparseArray; import android.util.SparseIntArray; import androidx.annotation.IntDef; import androidx.annotation.Nullable; @@ -516,51 +517,94 @@ private static Pair buildTrackGroups( return Pair.create(new TrackGroupArray(trackGroups), trackGroupInfos); } + /** + * Groups adaptation sets. Two adaptations sets belong to the same group if either: + * + * + * + * @param adaptationSets The adaptation sets to merge. + * @return An array of groups, where each group is an array of adaptation set indices. + */ private static int[][] getGroupedAdaptationSetIndices(List adaptationSets) { int adaptationSetCount = adaptationSets.size(); - SparseIntArray idToIndexMap = new SparseIntArray(adaptationSetCount); + SparseIntArray adaptationSetIdToIndex = new SparseIntArray(adaptationSetCount); + List> adaptationSetGroupedIndices = new ArrayList<>(adaptationSetCount); + SparseArray> adaptationSetIndexToGroupedIndices = + new SparseArray<>(adaptationSetCount); + + // Initially make each adaptation set belong to its own group. Also build the + // adaptationSetIdToIndex map. for (int i = 0; i < adaptationSetCount; i++) { - idToIndexMap.put(adaptationSets.get(i).id, i); + adaptationSetIdToIndex.put(adaptationSets.get(i).id, i); + List initialGroup = new ArrayList<>(); + initialGroup.add(i); + adaptationSetGroupedIndices.add(initialGroup); + adaptationSetIndexToGroupedIndices.put(i, initialGroup); } - int[][] groupedAdaptationSetIndices = new int[adaptationSetCount][]; - boolean[] adaptationSetUsedFlags = new boolean[adaptationSetCount]; - - int groupCount = 0; + // Merge adaptation set groups. for (int i = 0; i < adaptationSetCount; i++) { - if (adaptationSetUsedFlags[i]) { - // This adaptation set has already been included in a group. - continue; - } - adaptationSetUsedFlags[i] = true; + int mergedGroupIndex = i; + AdaptationSet adaptationSet = adaptationSets.get(i); + + // Trick-play adaptation sets are merged with their corresponding main adaptation sets. @Nullable - Descriptor adaptationSetSwitchingProperty = - findAdaptationSetSwitchingProperty(adaptationSets.get(i).supplementalProperties); - if (adaptationSetSwitchingProperty == null) { - groupedAdaptationSetIndices[groupCount++] = new int[] {i}; - } else { - String[] extraAdaptationSetIds = Util.split(adaptationSetSwitchingProperty.value, ","); - int[] adaptationSetIndices = new int[1 + extraAdaptationSetIds.length]; - adaptationSetIndices[0] = i; - int outputIndex = 1; - for (String adaptationSetId : extraAdaptationSetIds) { - int extraIndex = - idToIndexMap.get(Integer.parseInt(adaptationSetId), /* valueIfKeyNotFound= */ -1); - if (extraIndex != -1) { - adaptationSetUsedFlags[extraIndex] = true; - adaptationSetIndices[outputIndex] = extraIndex; - outputIndex++; - } + Descriptor trickPlayProperty = findTrickPlayProperty(adaptationSet.essentialProperties); + if (trickPlayProperty == null) { + // Trick-play can also be specified using a supplemental property. + trickPlayProperty = findTrickPlayProperty(adaptationSet.supplementalProperties); + } + if (trickPlayProperty != null) { + int mainAdaptationSetId = Integer.parseInt(trickPlayProperty.value); + int mainAdaptationSetIndex = + adaptationSetIdToIndex.get(mainAdaptationSetId, /* valueIfKeyNotFound= */ -1); + if (mainAdaptationSetIndex != -1) { + mergedGroupIndex = mainAdaptationSetIndex; } - if (outputIndex < adaptationSetIndices.length) { - adaptationSetIndices = Arrays.copyOf(adaptationSetIndices, outputIndex); + } + + // Adaptation sets that are safe for switching are merged, using the smallest index for the + // merged group. + if (mergedGroupIndex == i) { + @Nullable + Descriptor adaptationSetSwitchingProperty = + findAdaptationSetSwitchingProperty(adaptationSet.supplementalProperties); + if (adaptationSetSwitchingProperty != null) { + String[] otherAdaptationSetIds = Util.split(adaptationSetSwitchingProperty.value, ","); + for (String adaptationSetId : otherAdaptationSetIds) { + int otherAdaptationSetId = + adaptationSetIdToIndex.get( + Integer.parseInt(adaptationSetId), /* valueIfKeyNotFound= */ -1); + if (otherAdaptationSetId != -1) { + mergedGroupIndex = Math.min(mergedGroupIndex, otherAdaptationSetId); + } + } } - groupedAdaptationSetIndices[groupCount++] = adaptationSetIndices; + } + + // Merge the groups if necessary. + if (mergedGroupIndex != i) { + List thisGroup = adaptationSetIndexToGroupedIndices.get(i); + List mergedGroup = adaptationSetIndexToGroupedIndices.get(mergedGroupIndex); + mergedGroup.addAll(thisGroup); + adaptationSetIndexToGroupedIndices.put(i, mergedGroup); + adaptationSetGroupedIndices.remove(thisGroup); } } - return groupCount < adaptationSetCount - ? Arrays.copyOf(groupedAdaptationSetIndices, groupCount) : groupedAdaptationSetIndices; + int[][] groupedAdaptationSetIndices = new int[adaptationSetGroupedIndices.size()][]; + for (int i = 0; i < groupedAdaptationSetIndices.length; i++) { + groupedAdaptationSetIndices[i] = Util.toArray(adaptationSetGroupedIndices.get(i)); + // Restore the original adaptation set order within each group. + Arrays.sort(groupedAdaptationSetIndices[i]); + } + return groupedAdaptationSetIndices; } /** @@ -747,9 +791,19 @@ private ChunkSampleStream buildSampleStream(TrackGroupInfo trac @Nullable private static Descriptor findAdaptationSetSwitchingProperty(List descriptors) { + return findDescriptor(descriptors, "urn:mpeg:dash:adaptation-set-switching:2016"); + } + + @Nullable + private static Descriptor findTrickPlayProperty(List descriptors) { + return findDescriptor(descriptors, "http://dashif.org/guidelines/trickmode"); + } + + @Nullable + private static Descriptor findDescriptor(List descriptors, String schemeIdUri) { for (int i = 0; i < descriptors.size(); i++) { Descriptor descriptor = descriptors.get(i); - if ("urn:mpeg:dash:adaptation-set-switching:2016".equals(descriptor.schemeIdUri)) { + if (schemeIdUri.equals(descriptor.schemeIdUri)) { return descriptor; } } diff --git a/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/DashMediaPeriodTest.java b/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/DashMediaPeriodTest.java index e9e5f3030c6..5a5318c6704 100644 --- a/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/DashMediaPeriodTest.java +++ b/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/DashMediaPeriodTest.java @@ -26,6 +26,8 @@ import com.google.android.exoplayer2.source.CompositeSequenceableLoaderFactory; import com.google.android.exoplayer2.source.MediaSource.MediaPeriodId; import com.google.android.exoplayer2.source.MediaSourceEventListener.EventDispatcher; +import com.google.android.exoplayer2.source.TrackGroup; +import com.google.android.exoplayer2.source.TrackGroupArray; import com.google.android.exoplayer2.source.dash.PlayerEmsgHandler.PlayerEmsgCallback; import com.google.android.exoplayer2.source.dash.manifest.AdaptationSet; import com.google.android.exoplayer2.source.dash.manifest.DashManifest; @@ -35,7 +37,6 @@ import com.google.android.exoplayer2.source.dash.manifest.SegmentBase.SingleSegmentBase; import com.google.android.exoplayer2.source.dash.manifest.UtcTimingElement; import com.google.android.exoplayer2.testutil.MediaPeriodAsserts; -import com.google.android.exoplayer2.testutil.MediaPeriodAsserts.FilterableManifestMediaPeriodFactory; import com.google.android.exoplayer2.upstream.Allocator; import com.google.android.exoplayer2.upstream.LoadErrorHandlingPolicy; import com.google.android.exoplayer2.upstream.LoaderErrorThrower; @@ -43,6 +44,7 @@ import com.google.android.exoplayer2.util.MimeTypes; import java.util.Arrays; import java.util.Collections; +import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.annotation.LooperMode; @@ -53,7 +55,7 @@ public final class DashMediaPeriodTest { @Test - public void getSteamKeys_isCompatibleWithDashManifestFilter() { + public void getStreamKeys_isCompatibleWithDashManifestFilter() { // Test manifest which covers various edge cases: // - Multiple periods. // - Single and multiple representations per adaptation set. @@ -61,83 +63,220 @@ public void getSteamKeys_isCompatibleWithDashManifestFilter() { // - Embedded track groups. // All cases are deliberately combined in one test to catch potential indexing problems which // only occur in combination. - DashManifest testManifest = + DashManifest manifest = createDashManifest( createPeriod( createAdaptationSet( /* id= */ 0, - /* trackType= */ C.TRACK_TYPE_VIDEO, + C.TRACK_TYPE_VIDEO, /* descriptor= */ null, createVideoRepresentation(/* bitrate= */ 1000000))), createPeriod( createAdaptationSet( /* id= */ 100, - /* trackType= */ C.TRACK_TYPE_VIDEO, - /* descriptor= */ createSwitchDescriptor(/* ids...= */ 103, 104), + C.TRACK_TYPE_VIDEO, + createSwitchDescriptor(/* ids...= */ 103, 104), createVideoRepresentationWithInbandEventStream(/* bitrate= */ 200000), createVideoRepresentationWithInbandEventStream(/* bitrate= */ 400000), createVideoRepresentationWithInbandEventStream(/* bitrate= */ 600000)), createAdaptationSet( /* id= */ 101, - /* trackType= */ C.TRACK_TYPE_AUDIO, - /* descriptor= */ createSwitchDescriptor(/* ids...= */ 102), + C.TRACK_TYPE_AUDIO, + createSwitchDescriptor(/* ids...= */ 102), createAudioRepresentation(/* bitrate= */ 48000), createAudioRepresentation(/* bitrate= */ 96000)), createAdaptationSet( /* id= */ 102, - /* trackType= */ C.TRACK_TYPE_AUDIO, - /* descriptor= */ createSwitchDescriptor(/* ids...= */ 101), + C.TRACK_TYPE_AUDIO, + createSwitchDescriptor(/* ids...= */ 101), createAudioRepresentation(/* bitrate= */ 256000)), createAdaptationSet( /* id= */ 103, - /* trackType= */ C.TRACK_TYPE_VIDEO, - /* descriptor= */ createSwitchDescriptor(/* ids...= */ 100, 104), + C.TRACK_TYPE_VIDEO, + createSwitchDescriptor(/* ids...= */ 100, 104), createVideoRepresentationWithInbandEventStream(/* bitrate= */ 800000), createVideoRepresentationWithInbandEventStream(/* bitrate= */ 1000000)), createAdaptationSet( /* id= */ 104, - /* trackType= */ C.TRACK_TYPE_VIDEO, - /* descriptor= */ createSwitchDescriptor(/* ids...= */ 100, 103), + C.TRACK_TYPE_VIDEO, + createSwitchDescriptor(/* ids...= */ 100, 103), createVideoRepresentationWithInbandEventStream(/* bitrate= */ 2000000)), createAdaptationSet( /* id= */ 105, - /* trackType= */ C.TRACK_TYPE_TEXT, + C.TRACK_TYPE_TEXT, /* descriptor= */ null, createTextRepresentation(/* language= */ "eng")), createAdaptationSet( /* id= */ 105, - /* trackType= */ C.TRACK_TYPE_TEXT, + C.TRACK_TYPE_TEXT, /* descriptor= */ null, createTextRepresentation(/* language= */ "ger")))); - FilterableManifestMediaPeriodFactory mediaPeriodFactory = - (manifest, periodIndex) -> - new DashMediaPeriod( - /* id= */ periodIndex, - manifest, - periodIndex, - mock(DashChunkSource.Factory.class), - mock(TransferListener.class), - DrmSessionManager.getDummyDrmSessionManager(), - mock(LoadErrorHandlingPolicy.class), - new EventDispatcher() - .withParameters( - /* windowIndex= */ 0, - /* mediaPeriodId= */ new MediaPeriodId(/* periodUid= */ new Object()), - /* mediaTimeOffsetMs= */ 0), - /* elapsedRealtimeOffsetMs= */ 0, - mock(LoaderErrorThrower.class), - mock(Allocator.class), - mock(CompositeSequenceableLoaderFactory.class), - mock(PlayerEmsgCallback.class)); // Ignore embedded metadata as we don't want to select primary group just to get embedded track. MediaPeriodAsserts.assertGetStreamKeysAndManifestFilterIntegration( - mediaPeriodFactory, - testManifest, + DashMediaPeriodTest::createDashMediaPeriod, + manifest, /* periodIndex= */ 1, /* ignoredMimeType= */ "application/x-emsg"); } + @Test + public void adaptationSetSwitchingProperty_mergesTrackGroups() { + DashManifest manifest = + createDashManifest( + createPeriod( + createAdaptationSet( + /* id= */ 0, + C.TRACK_TYPE_VIDEO, + createSwitchDescriptor(/* ids...= */ 1, 2), + createVideoRepresentation(/* bitrate= */ 0), + createVideoRepresentation(/* bitrate= */ 1)), + createAdaptationSet( + /* id= */ 3, + C.TRACK_TYPE_VIDEO, + /* descriptor= */ null, + createVideoRepresentation(/* bitrate= */ 300)), + createAdaptationSet( + /* id= */ 2, + C.TRACK_TYPE_VIDEO, + createSwitchDescriptor(/* ids...= */ 0, 1), + createVideoRepresentation(/* bitrate= */ 200), + createVideoRepresentation(/* bitrate= */ 201)), + createAdaptationSet( + /* id= */ 1, + C.TRACK_TYPE_VIDEO, + createSwitchDescriptor(/* ids...= */ 0, 2), + createVideoRepresentation(/* bitrate= */ 100)))); + DashMediaPeriod dashMediaPeriod = createDashMediaPeriod(manifest, 0); + List adaptationSets = manifest.getPeriod(0).adaptationSets; + + // We expect the three adaptation sets with the switch descriptor to be merged, retaining the + // representations in their original order. + TrackGroupArray expectedTrackGroups = + new TrackGroupArray( + new TrackGroup( + adaptationSets.get(0).representations.get(0).format, + adaptationSets.get(0).representations.get(1).format, + adaptationSets.get(2).representations.get(0).format, + adaptationSets.get(2).representations.get(1).format, + adaptationSets.get(3).representations.get(0).format), + new TrackGroup(adaptationSets.get(1).representations.get(0).format)); + + MediaPeriodAsserts.assertTrackGroups(dashMediaPeriod, expectedTrackGroups); + } + + @Test + public void trickPlayProperty_mergesTrackGroups() { + DashManifest manifest = + createDashManifest( + createPeriod( + createAdaptationSet( + /* id= */ 0, + C.TRACK_TYPE_VIDEO, + createTrickPlayDescriptor(/* mainAdaptationSetId= */ 1), + createVideoRepresentation(/* bitrate= */ 0), + createVideoRepresentation(/* bitrate= */ 1)), + createAdaptationSet( + /* id= */ 1, + C.TRACK_TYPE_VIDEO, + /* descriptor= */ null, + createVideoRepresentation(/* bitrate= */ 100)), + createAdaptationSet( + /* id= */ 2, + C.TRACK_TYPE_VIDEO, + /* descriptor= */ null, + createVideoRepresentation(/* bitrate= */ 200), + createVideoRepresentation(/* bitrate= */ 201)), + createAdaptationSet( + /* id= */ 3, + C.TRACK_TYPE_VIDEO, + createTrickPlayDescriptor(/* mainAdaptationSetId= */ 2), + createVideoRepresentation(/* bitrate= */ 300)))); + DashMediaPeriod dashMediaPeriod = createDashMediaPeriod(manifest, 0); + List adaptationSets = manifest.getPeriod(0).adaptationSets; + + // We expect the trick play adaptation sets to be merged with the ones to which they refer, + // retaining representations in their original order. + TrackGroupArray expectedTrackGroups = + new TrackGroupArray( + new TrackGroup( + adaptationSets.get(0).representations.get(0).format, + adaptationSets.get(0).representations.get(1).format, + adaptationSets.get(1).representations.get(0).format), + new TrackGroup( + adaptationSets.get(2).representations.get(0).format, + adaptationSets.get(2).representations.get(1).format, + adaptationSets.get(3).representations.get(0).format)); + + MediaPeriodAsserts.assertTrackGroups(dashMediaPeriod, expectedTrackGroups); + } + + @Test + public void adaptationSetSwitchingProperty_andTrickPlayProperty_mergesTrackGroups() { + DashManifest manifest = + createDashManifest( + createPeriod( + createAdaptationSet( + /* id= */ 0, + C.TRACK_TYPE_VIDEO, + createTrickPlayDescriptor(/* mainAdaptationSetId= */ 1), + createVideoRepresentation(/* bitrate= */ 0), + createVideoRepresentation(/* bitrate= */ 1)), + createAdaptationSet( + /* id= */ 1, + C.TRACK_TYPE_VIDEO, + createSwitchDescriptor(/* ids...= */ 2), + createVideoRepresentation(/* bitrate= */ 100)), + createAdaptationSet( + /* id= */ 2, + C.TRACK_TYPE_VIDEO, + createSwitchDescriptor(/* ids...= */ 1), + createVideoRepresentation(/* bitrate= */ 200), + createVideoRepresentation(/* bitrate= */ 201)), + createAdaptationSet( + /* id= */ 3, + C.TRACK_TYPE_VIDEO, + createTrickPlayDescriptor(/* mainAdaptationSetId= */ 2), + createVideoRepresentation(/* bitrate= */ 300)))); + DashMediaPeriod dashMediaPeriod = createDashMediaPeriod(manifest, 0); + List adaptationSets = manifest.getPeriod(0).adaptationSets; + + // We expect all adaptation sets to be merged into one group, retaining representations in their + // original order. + TrackGroupArray expectedTrackGroups = + new TrackGroupArray( + new TrackGroup( + adaptationSets.get(0).representations.get(0).format, + adaptationSets.get(0).representations.get(1).format, + adaptationSets.get(1).representations.get(0).format, + adaptationSets.get(2).representations.get(0).format, + adaptationSets.get(2).representations.get(1).format, + adaptationSets.get(3).representations.get(0).format)); + + MediaPeriodAsserts.assertTrackGroups(dashMediaPeriod, expectedTrackGroups); + } + + private static DashMediaPeriod createDashMediaPeriod(DashManifest manifest, int periodIndex) { + return new DashMediaPeriod( + /* id= */ periodIndex, + manifest, + periodIndex, + mock(DashChunkSource.Factory.class), + mock(TransferListener.class), + DrmSessionManager.getDummyDrmSessionManager(), + mock(LoadErrorHandlingPolicy.class), + new EventDispatcher() + .withParameters( + /* windowIndex= */ 0, + /* mediaPeriodId= */ new MediaPeriodId(/* periodUid= */ new Object()), + /* mediaTimeOffsetMs= */ 0), + /* elapsedRealtimeOffsetMs= */ 0, + mock(LoaderErrorThrower.class), + mock(Allocator.class), + mock(CompositeSequenceableLoaderFactory.class), + mock(PlayerEmsgCallback.class)); + } + private static DashManifest createDashManifest(Period... periods) { return new DashManifest( /* availabilityStartTimeMs= */ 0, @@ -228,6 +367,13 @@ private static Descriptor createSwitchDescriptor(int... ids) { /* id= */ null); } + private static Descriptor createTrickPlayDescriptor(int mainAdaptationSetId) { + return new Descriptor( + /* schemeIdUri= */ "http://dashif.org/guidelines/trickmode", + /* value= */ Integer.toString(mainAdaptationSetId), + /* id= */ null); + } + private static Descriptor getInbandEventDescriptor() { return new Descriptor( /* schemeIdUri= */ "inBandSchemeIdUri", /* value= */ "inBandValue", /* id= */ "inBandId"); diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/MediaPeriodAsserts.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/MediaPeriodAsserts.java index 27d6b086dcd..197280159d9 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/MediaPeriodAsserts.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/MediaPeriodAsserts.java @@ -54,6 +54,17 @@ public interface FilterableManifestMediaPeriodFactory trackGroupArray = new AtomicReference<>(); DummyMainThread dummyMainThread = new DummyMainThread(); ConditionVariable preparedCondition = new ConditionVariable();