From 4c73241058041fd978e9748b2158168d4e6e702d Mon Sep 17 00:00:00 2001 From: tianyifeng Date: Thu, 6 Oct 2022 09:58:37 +0000 Subject: [PATCH] Provide access to original media timestamps in AudioSink. * Add `setOutputStreamOffsetUs(long)` method in `AudioSink`. * Add private methods `setOutputStreamOffsetUs(long)` method in `MediaCodecRenderer` and `DecoderAudioRenderer`. * Add protected method `onOutputStreamOffsetUs(long)` method in `MediaCodecRenderer`, in which: * `MediaCodecRenderer` itself will be no-op for this method. * `MediaCodecAudioRenderer` will propagate this value to its `audioSink`. * Add logics in `DecoderAudioRenderer` to calculate `outputStreamOffsetUs`. PiperOrigin-RevId: 479265429 --- .../android/exoplayer2/audio/AudioSink.java | 8 +++ .../audio/DecoderAudioRenderer.java | 47 ++++++++++++++- .../exoplayer2/audio/ForwardingAudioSink.java | 5 ++ .../audio/MediaCodecAudioRenderer.java | 5 ++ .../mediacodec/MediaCodecRenderer.java | 28 +++++++-- .../audio/DecoderAudioRendererTest.java | 10 +++- .../audio/MediaCodecAudioRendererTest.java | 57 +++++++++++++++++++ 7 files changed, 152 insertions(+), 8 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java index 192f2dfa92d..c51a3ddd8eb 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java @@ -427,6 +427,14 @@ boolean handleBuffer(ByteBuffer buffer, long presentationTimeUs, int encodedAcce @RequiresApi(23) default void setPreferredDevice(@Nullable AudioDeviceInfo audioDeviceInfo) {} + /** + * Sets the offset that is added to the media timestamp before it is passed as {@code + * presentationTimeUs} in {@link #handleBuffer(ByteBuffer, long, int)}. + * + * @param outputStreamOffsetUs The output stream offset in microseconds. + */ + default void setOutputStreamOffsetUs(long outputStreamOffsetUs) {} + /** * Enables tunneling, if possible. The sink is reset if tunneling was previously disabled. * Enabling tunneling is only possible if the sink is based on a platform {@link AudioTrack}, and diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java index 5e8960cca3f..546ca63d96a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java @@ -118,6 +118,11 @@ public abstract class DecoderAudioRenderer< * end of stream signal to indicate that it has output any remaining buffers before we release it. */ private static final int REINITIALIZATION_STATE_WAIT_END_OF_STREAM = 2; + /** + * Generally there is zero or one pending output stream offset. We track more offsets to allow for + * pending output streams that have fewer frames than the codec latency. + */ + private static final int MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT = 10; private final EventDispatcher eventDispatcher; private final AudioSink audioSink; @@ -147,6 +152,9 @@ public abstract class DecoderAudioRenderer< private boolean allowPositionDiscontinuity; private boolean inputStreamEnded; private boolean outputStreamEnded; + private long outputStreamOffsetUs; + private final long[] pendingOutputStreamOffsetsUs; + private int pendingOutputStreamOffsetCount; public DecoderAudioRenderer() { this(/* eventHandler= */ null, /* eventListener= */ null); @@ -206,6 +214,8 @@ public DecoderAudioRenderer( flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance(); decoderReinitializationState = REINITIALIZATION_STATE_NONE; audioTrackNeedsConfigure = true; + setOutputStreamOffsetUs(C.TIME_UNSET); + pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; } /** @@ -390,7 +400,7 @@ private boolean drainOutputBuffer() audioSink.handleDiscontinuity(); } if (outputBuffer.isFirstSample()) { - audioSink.handleDiscontinuity(); + processFirstSampleOfStream(); } } @@ -436,6 +446,27 @@ private boolean drainOutputBuffer() return false; } + private void processFirstSampleOfStream() { + audioSink.handleDiscontinuity(); + if (pendingOutputStreamOffsetCount != 0) { + setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]); + pendingOutputStreamOffsetCount--; + System.arraycopy( + pendingOutputStreamOffsetsUs, + /* srcPos= */ 1, + pendingOutputStreamOffsetsUs, + /* destPos= */ 0, + pendingOutputStreamOffsetCount); + } + } + + private void setOutputStreamOffsetUs(long outputStreamOffsetUs) { + this.outputStreamOffsetUs = outputStreamOffsetUs; + if (outputStreamOffsetUs != C.TIME_UNSET) { + audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs); + } + } + private boolean feedInputBuffer() throws DecoderException, ExoPlaybackException { if (decoder == null || decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM @@ -585,6 +616,7 @@ protected void onStopped() { protected void onDisabled() { inputFormat = null; audioTrackNeedsConfigure = true; + setOutputStreamOffsetUs(C.TIME_UNSET); try { setSourceDrmSession(null); releaseDecoder(); @@ -599,6 +631,19 @@ protected void onStreamChanged(Format[] formats, long startPositionUs, long offs throws ExoPlaybackException { super.onStreamChanged(formats, startPositionUs, offsetUs); firstStreamSampleRead = false; + if (outputStreamOffsetUs == C.TIME_UNSET) { + setOutputStreamOffsetUs(offsetUs); + } else { + if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) { + Log.w( + TAG, + "Too many stream changes, so dropping offset: " + + pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]); + } else { + pendingOutputStreamOffsetCount++; + } + pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1] = offsetUs; + } } @Override diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java index 9ce4140d4e2..703d148be63 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java @@ -142,6 +142,11 @@ public void setPreferredDevice(@Nullable AudioDeviceInfo audioDeviceInfo) { sink.setPreferredDevice(audioDeviceInfo); } + @Override + public void setOutputStreamOffsetUs(long outputStreamOffsetUs) { + sink.setOutputStreamOffsetUs(outputStreamOffsetUs); + } + @Override public void enableTunnelingV21() { sink.enableTunnelingV21(); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index 62eab7dde7a..d76edd8840f 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -733,6 +733,11 @@ protected void renderToEndOfStream() throws ExoPlaybackException { } } + @Override + protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) { + audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs); + } + @Override public void handleMessage(@MessageType int messageType, @Nullable Object message) throws ExoPlaybackException { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index ead8d64163f..2bc8d2fbdf9 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -400,7 +400,7 @@ public MediaCodecRenderer( pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; pendingOutputStreamSwitchTimesUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; outputStreamStartPositionUs = C.TIME_UNSET; - outputStreamOffsetUs = C.TIME_UNSET; + setOutputStreamOffsetUs(C.TIME_UNSET); // MediaCodec outputs audio buffers in native endian: // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers // and code called from MediaCodecAudioRenderer.processOutputBuffer expects this endianness. @@ -649,7 +649,7 @@ protected void onStreamChanged(Format[] formats, long startPositionUs, long offs if (this.outputStreamOffsetUs == C.TIME_UNSET) { checkState(this.outputStreamStartPositionUs == C.TIME_UNSET); this.outputStreamStartPositionUs = startPositionUs; - this.outputStreamOffsetUs = offsetUs; + setOutputStreamOffsetUs(offsetUs); } else { if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) { Log.w( @@ -686,7 +686,7 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb } formatQueue.clear(); if (pendingOutputStreamOffsetCount != 0) { - outputStreamOffsetUs = pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]; + setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]); outputStreamStartPositionUs = pendingOutputStreamStartPositionsUs[pendingOutputStreamOffsetCount - 1]; pendingOutputStreamOffsetCount = 0; @@ -705,7 +705,7 @@ public void setPlaybackSpeed(float currentPlaybackSpeed, float targetPlaybackSpe protected void onDisabled() { inputFormat = null; outputStreamStartPositionUs = C.TIME_UNSET; - outputStreamOffsetUs = C.TIME_UNSET; + setOutputStreamOffsetUs(C.TIME_UNSET); pendingOutputStreamOffsetCount = 0; flushOrReleaseCodec(); } @@ -1586,7 +1586,7 @@ protected void onProcessedOutputBuffer(long presentationTimeUs) { while (pendingOutputStreamOffsetCount != 0 && presentationTimeUs >= pendingOutputStreamSwitchTimesUs[0]) { outputStreamStartPositionUs = pendingOutputStreamStartPositionsUs[0]; - outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0]; + setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]); pendingOutputStreamOffsetCount--; System.arraycopy( pendingOutputStreamStartPositionsUs, @@ -1636,6 +1636,17 @@ protected DecoderReuseEvaluation canReuseCodec( DISCARD_REASON_REUSE_NOT_IMPLEMENTED); } + /** + * Called after the output stream offset changes. + * + *

The default implementation is a no-op. + * + * @param outputStreamOffsetUs The output stream offset in microseconds. + */ + protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) { + // Do nothing + } + @Override public boolean isEnded() { return outputStreamEnded; @@ -2044,6 +2055,13 @@ protected final long getOutputStreamOffsetUs() { return outputStreamOffsetUs; } + private void setOutputStreamOffsetUs(long outputStreamOffsetUs) { + this.outputStreamOffsetUs = outputStreamOffsetUs; + if (outputStreamOffsetUs != C.TIME_UNSET) { + onOutputStreamOffsetUsChanged(outputStreamOffsetUs); + } + } + /** Returns whether this renderer supports the given {@link Format Format's} DRM scheme. */ protected static boolean supportsFormatDrm(Format format) { return format.cryptoType == C.CRYPTO_TYPE_NONE || format.cryptoType == C.CRYPTO_TYPE_FRAMEWORK; diff --git a/library/core/src/test/java/com/google/android/exoplayer2/audio/DecoderAudioRendererTest.java b/library/core/src/test/java/com/google/android/exoplayer2/audio/DecoderAudioRendererTest.java index 9b3e0a55843..bbed66f3948 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/audio/DecoderAudioRendererTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/audio/DecoderAudioRendererTest.java @@ -146,7 +146,8 @@ public void immediatelyReadEndOfStreamPlaysAudioSinkToEndOfStream() throws Excep } @Test - public void firstSampleOfStreamSignalsDiscontinuityToAudioSink() throws Exception { + public void firstSampleOfStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink() + throws Exception { when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); when(mockAudioSink.isEnded()).thenReturn(true); InOrder inOrderAudioSink = inOrder(mockAudioSink); @@ -177,12 +178,15 @@ public void firstSampleOfStreamSignalsDiscontinuityToAudioSink() throws Exceptio audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); } + inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0); inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); } @Test - public void firstSampleOfReplacementStreamSignalsDiscontinuityToAudioSink() throws Exception { + public void + firstSampleOfReplacementStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink() + throws Exception { when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); when(mockAudioSink.isEnded()).thenReturn(true); InOrder inOrderAudioSink = inOrder(mockAudioSink); @@ -233,9 +237,11 @@ public void firstSampleOfReplacementStreamSignalsDiscontinuityToAudioSink() thro audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); } + inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0); inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); + inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(1_000_000); inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java b/library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java index 63490438c56..5cd28566485 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java @@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.robolectric.Shadows.shadowOf; @@ -57,6 +58,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; @@ -324,6 +326,61 @@ protected void onOutputFormatChanged(Format format, @Nullable MediaFormat mediaF verify(audioRendererEventListener).onAudioSinkError(error); } + @Test + public void render_callsAudioSinkSetOutputStreamOffset_whenReplaceStream() throws Exception { + FakeSampleStream fakeSampleStream1 = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + AUDIO_AAC, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_000), + END_OF_STREAM_ITEM)); + fakeSampleStream1.writeData(/* startPositionUs= */ 0); + FakeSampleStream fakeSampleStream2 = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + AUDIO_AAC, + ImmutableList.of( + oneByteSample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_001_000), + END_OF_STREAM_ITEM)); + fakeSampleStream2.writeData(/* startPositionUs= */ 0); + mediaCodecAudioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {AUDIO_AAC}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + + mediaCodecAudioRenderer.start(); + while (!mediaCodecAudioRenderer.hasReadStreamToEnd()) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + mediaCodecAudioRenderer.replaceStream( + new Format[] {AUDIO_AAC}, + fakeSampleStream2, + /* startPositionUs= */ 1_000_000, + /* offsetUs= */ 1_000_000); + mediaCodecAudioRenderer.setCurrentStreamFinal(); + while (!mediaCodecAudioRenderer.isEnded()) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + + InOrder inOrderAudioSink = inOrder(audioSink); + inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(0); + inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(1_000_000); + } + @Test public void supportsFormat_withEac3JocMediaAndEac3Decoder_returnsTrue() throws Exception { Format mediaFormat =