Skip to content

Commit

Permalink
Make ExoPlayer.setVideoEffects() timestamp start from 0
Browse files Browse the repository at this point in the history 
This is consistent with `Transformer` and `CompositionPlayer`

Issue: #1098
PiperOrigin-RevId: 646446824
  • Loading branch information
@claincly
claincly authored and Copybara-Service committed Jun 25, 2024
1 parent 867410f commit 73bf852
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 13 deletions.
3 changes: 3 additions & 0 deletions RELEASENOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
* Fix potential `IndexOutOfBoundsException` caused by extractors reporting
additional tracks after the initial preparation step
([#1476](https://github.com/androidx/media/issues/1476)).
* `Effects` in `ExoPlayer.setVideoEffect()` will receive the timestamps
with the renderer offset removed
([#1098](https://github.com/androidx/media/issues/1098)).
* Transformer:
* Track Selection:
* Extractors:
Expand Down
20 changes: 18 additions & 2 deletions ...ries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException;
import androidx.media3.exoplayer.source.MediaSource;
import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatcher;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
Expand Down Expand Up @@ -177,6 +178,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
private int tunnelingAudioSessionId;
/* package */ @Nullable OnFrameRenderedListenerV23 tunnelingOnFrameRenderedListener;
@Nullable private VideoFrameMetadataListener frameMetadataListener;
private long startPositionUs;

/**
* @param context A context.
Expand Down Expand Up @@ -414,6 +416,7 @@ public MediaCodecVideoRenderer(
tunnelingAudioSessionId = C.AUDIO_SESSION_ID_UNSET;
reportedVideoSize = null;
rendererPriority = C.PRIORITY_PLAYBACK;
startPositionUs = C.TIME_UNSET;
}

// FrameTimingEvaluator methods
Expand Down Expand Up @@ -714,6 +717,19 @@ public void enableMayRenderStartOfStream() {
}
}

@Override
protected void onStreamChanged(
Format[] formats,
long startPositionUs,
long offsetUs,
MediaSource.MediaPeriodId mediaPeriodId)
throws ExoPlaybackException {
super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId);
if (this.startPositionUs == C.TIME_UNSET) {
this.startPositionUs = startPositionUs;
}
}

@Override
protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException {
if (videoSink != null) {
Expand Down Expand Up @@ -814,6 +830,7 @@ protected void onReset() {
super.onReset();
} finally {
hasSetVideoSink = false;
startPositionUs = C.TIME_UNSET;
if (placeholderSurface != null) {
releasePlaceholderSurface();
}
Expand Down Expand Up @@ -1446,8 +1463,7 @@ protected boolean processOutputBuffer(
* position) to the frame presentation time, in microseconds.
*/
protected long getBufferTimestampAdjustmentUs() {
// TODO - b/333514379: Make effect-enabled effect timestamp start from zero.
return 0;
return -startPositionUs;
}

private boolean maybeReleaseFrame(
Expand Down
77 changes: 67 additions & 10 deletions ...former/VideoTimestampConsistencyTest.java → ...mer/mh/VideoTimestampConsistencyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

package androidx.media3.transformer;
package androidx.media3.transformer.mh;

import static androidx.media3.common.util.Util.usToMs;
import static androidx.media3.transformer.AndroidTestUtil.JPG_SINGLE_PIXEL_ASSET;
Expand All @@ -27,6 +27,18 @@
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
import androidx.media3.effect.GlEffect;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.transformer.Composition;
import androidx.media3.transformer.CompositionPlayer;
import androidx.media3.transformer.EditedMediaItem;
import androidx.media3.transformer.EditedMediaItemSequence;
import androidx.media3.transformer.Effects;
import androidx.media3.transformer.ExportTestResult;
import androidx.media3.transformer.InputTimestampRecordingShaderProgram;
import androidx.media3.transformer.PlayerTestListener;
import androidx.media3.transformer.SurfaceTestActivity;
import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner;
import androidx.test.ext.junit.rules.ActivityScenarioRule;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.platform.app.InstrumentationRegistry;
Expand Down Expand Up @@ -68,6 +80,7 @@ public class VideoTimestampConsistencyTest {
private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
private final Context applicationContext = instrumentation.getContext().getApplicationContext();

private ExoPlayer exoplayer;
private CompositionPlayer compositionPlayer;
private SurfaceView surfaceView;

Expand Down Expand Up @@ -95,7 +108,8 @@ public void oneImageComposition_timestampsAreConsistent() throws Exception {
.setFrameRate(30)
.build();

compareTimestamps(ImmutableList.of(image), IMAGE_TIMESTAMPS_US_500_MS_30_FPS);
compareTimestamps(
ImmutableList.of(image), IMAGE_TIMESTAMPS_US_500_MS_30_FPS, /* containsImage= */ true);
}

@Test
Expand All @@ -105,7 +119,8 @@ public void oneVideoComposition_timestampsAreConsistent() throws Exception {
.setDurationUs(MP4_ASSET.videoDurationUs)
.build();

compareTimestamps(ImmutableList.of(video), MP4_ASSET_FRAME_TIMESTAMPS_US);
compareTimestamps(
ImmutableList.of(video), MP4_ASSET_FRAME_TIMESTAMPS_US, /* containsImage= */ false);
}

@Test
Expand Down Expand Up @@ -138,7 +153,8 @@ public void twoVideosComposition_clippingTheFirst_timestampsAreConsistent() thro
timestampUs -> ((MP4_ASSET.videoDurationUs - clippedStartUs) + timestampUs)))
.build();

compareTimestamps(ImmutableList.of(video1, video2), expectedTimestamps);
compareTimestamps(
ImmutableList.of(video1, video2), expectedTimestamps, /* containsImage= */ false);
}

@Test
Expand All @@ -162,7 +178,8 @@ public void twoVideosComposition_timestampsAreConsistent() throws Exception {
timestampUs -> (MP4_ASSET.videoDurationUs + timestampUs)))
.build();

compareTimestamps(ImmutableList.of(video1, video2), expectedTimestamps);
compareTimestamps(
ImmutableList.of(video1, video2), expectedTimestamps, /* containsImage= */ false);
}

@Test
Expand Down Expand Up @@ -198,7 +215,8 @@ public void twoImagesComposition_timestampsAreConsistent() throws Exception {
timestampUs -> (timestampUs + imageDurationUs)))
.build();

compareTimestamps(ImmutableList.of(image1, image2), expectedTimestamps);
compareTimestamps(
ImmutableList.of(image1, image2), expectedTimestamps, /* containsImage= */ true);
}

@Test
Expand Down Expand Up @@ -227,7 +245,8 @@ public void imageThenVideoComposition_timestampsAreConsistent() throws Exception
MP4_ASSET_FRAME_TIMESTAMPS_US, timestampUs -> (timestampUs + imageDurationUs)))
.build();

compareTimestamps(ImmutableList.of(image, video), expectedTimestamps);
compareTimestamps(
ImmutableList.of(image, video), expectedTimestamps, /* containsImage= */ true);
}

@Test
Expand Down Expand Up @@ -257,7 +276,8 @@ public void videoThenImageComposition_timestampsAreConsistent() throws Exception
timestampUs -> (MP4_ASSET.videoDurationUs + timestampUs)))
.build();

compareTimestamps(ImmutableList.of(video, image), expectedTimestamps);
compareTimestamps(
ImmutableList.of(video, image), expectedTimestamps, /* containsImage= */ true);
}

@Test
Expand Down Expand Up @@ -295,16 +315,26 @@ public void videoThenImageComposition_clippingVideo_timestampsAreConsistent() th
timestampUs -> ((MP4_ASSET.videoDurationUs - clippedStartUs) + timestampUs)))
.build();

compareTimestamps(ImmutableList.of(video, image), expectedTimestamps);
compareTimestamps(
ImmutableList.of(video, image), expectedTimestamps, /* containsImage= */ true);
}

private void compareTimestamps(List<EditedMediaItem> mediaItems, List<Long> expectedTimestamps)
private void compareTimestamps(
List<EditedMediaItem> mediaItems, List<Long> expectedTimestamps, boolean containsImage)
throws Exception {
ImmutableList<Long> timestampsFromCompositionPlayer =
getTimestampsFromCompositionPlayer(mediaItems);
ImmutableList<Long> timestampsFromTransformer = getTimestampsFromTransformer(mediaItems);

assertThat(timestampsFromCompositionPlayer).isEqualTo(timestampsFromTransformer);

if (!containsImage) {
// ExoPlayer doesn't support image playback with effects.
ImmutableList<Long> timestampsFromExoPlayer =
getTimestampsFromExoPlayer(
Lists.transform(mediaItems, editedMediaItem -> editedMediaItem.mediaItem));
assertThat(timestampsFromCompositionPlayer).isEqualTo(timestampsFromExoPlayer);
}
assertThat(timestampsFromTransformer).isEqualTo(expectedTimestamps);
}

Expand All @@ -318,6 +348,7 @@ private ImmutableList<Long> getTimestampsFromTransformer(List<EditedMediaItem> e
/* effects= */ ImmutableList.of(
(GlEffect) (context, useHdr) -> timestampRecordingShaderProgram));

@SuppressWarnings("unused")
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(
applicationContext, new Transformer.Builder(applicationContext).build())
Expand Down Expand Up @@ -365,6 +396,32 @@ private ImmutableList<Long> getTimestampsFromCompositionPlayer(
return timestampRecordingShaderProgram.getInputTimestampsUs();
}

private ImmutableList<Long> getTimestampsFromExoPlayer(List<MediaItem> mediaItems)
throws Exception {
PlayerTestListener playerListener = new PlayerTestListener(TEST_TIMEOUT_MS);
InputTimestampRecordingShaderProgram timestampRecordingShaderProgram =
new InputTimestampRecordingShaderProgram();

instrumentation.runOnMainSync(
() -> {
exoplayer = new ExoPlayer.Builder(applicationContext).build();
// Set a surface on the player even though there is no UI on this test. We need a surface
// otherwise the player will skip/drop video frames.
exoplayer.setVideoSurfaceView(surfaceView);
exoplayer.addListener(playerListener);
exoplayer.setMediaItems(mediaItems);
exoplayer.setVideoEffects(
ImmutableList.of((GlEffect) (context, useHdr) -> timestampRecordingShaderProgram));
exoplayer.prepare();
exoplayer.play();
});

playerListener.waitUntilPlayerEnded();
instrumentation.runOnMainSync(() -> exoplayer.release());

return timestampRecordingShaderProgram.getInputTimestampsUs();
}

private static ImmutableList<EditedMediaItem> prependVideoEffects(
List<EditedMediaItem> editedMediaItems, List<Effect> effects) {
ImmutableList.Builder<EditedMediaItem> prependedItems = new ImmutableList.Builder<>();
Expand Down
2 changes: 1 addition & 1 deletion libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ private EditedMediaItem(
}

/** Returns a {@link Builder} initialized with the values of this instance. */
/* package */ Builder buildUpon() {
public Builder buildUpon() {
return new Builder(this);
}

Expand Down

0 comments on commit 73bf852

Please sign in to comment.