Skip to content

Commit

Permalink
Provide access to original media timestamps in AudioSink.
Browse files Browse the repository at this point in the history 
* Add `setOutputStreamOffsetUs(long)` method in `AudioSink`.
* Add private methods `setOutputStreamOffsetUs(long)` method in `MediaCodecRenderer` and `DecoderAudioRenderer`.
* Add protected method `onOutputStreamOffsetUs(long)` method in `MediaCodecRenderer`, in which:
  * `MediaCodecRenderer` itself will be no-op for this method.
  * `MediaCodecAudioRenderer` will propagate this value to its `audioSink`.
* Add logics in `DecoderAudioRenderer` to calculate `outputStreamOffsetUs`.

PiperOrigin-RevId: 479265429
  • Loading branch information
@tianyif @marcbaechinger
tianyif authored and marcbaechinger committed Oct 20, 2022
1 parent 0889589 commit 4c73241
Show file tree
Hide file tree
Showing 7 changed files with 152 additions and 8 deletions.
8 changes: 8 additions & 0 deletions library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,14 @@ boolean handleBuffer(ByteBuffer buffer, long presentationTimeUs, int encodedAcce
@RequiresApi(23)
default void setPreferredDevice(@Nullable AudioDeviceInfo audioDeviceInfo) {}

/**
* Sets the offset that is added to the media timestamp before it is passed as {@code
* presentationTimeUs} in {@link #handleBuffer(ByteBuffer, long, int)}.
*
* @param outputStreamOffsetUs The output stream offset in microseconds.
*/
default void setOutputStreamOffsetUs(long outputStreamOffsetUs) {}

/**
* Enables tunneling, if possible. The sink is reset if tunneling was previously disabled.
* Enabling tunneling is only possible if the sink is based on a platform {@link AudioTrack}, and
Expand Down
47 changes: 46 additions & 1 deletion library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,11 @@ public abstract class DecoderAudioRenderer<
* end of stream signal to indicate that it has output any remaining buffers before we release it.
*/
private static final int REINITIALIZATION_STATE_WAIT_END_OF_STREAM = 2;
/**
* Generally there is zero or one pending output stream offset. We track more offsets to allow for
* pending output streams that have fewer frames than the codec latency.
*/
private static final int MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT = 10;

private final EventDispatcher eventDispatcher;
private final AudioSink audioSink;
Expand Down Expand Up @@ -147,6 +152,9 @@ public abstract class DecoderAudioRenderer<
private boolean allowPositionDiscontinuity;
private boolean inputStreamEnded;
private boolean outputStreamEnded;
private long outputStreamOffsetUs;
private final long[] pendingOutputStreamOffsetsUs;
private int pendingOutputStreamOffsetCount;

public DecoderAudioRenderer() {
this(/* eventHandler= */ null, /* eventListener= */ null);
Expand Down Expand Up @@ -206,6 +214,8 @@ public DecoderAudioRenderer(
flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance();
decoderReinitializationState = REINITIALIZATION_STATE_NONE;
audioTrackNeedsConfigure = true;
setOutputStreamOffsetUs(C.TIME_UNSET);
pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
}

/**
Expand Down Expand Up @@ -390,7 +400,7 @@ private boolean drainOutputBuffer()
audioSink.handleDiscontinuity();
}
if (outputBuffer.isFirstSample()) {
audioSink.handleDiscontinuity();
processFirstSampleOfStream();
}
}

Expand Down Expand Up @@ -436,6 +446,27 @@ private boolean drainOutputBuffer()
return false;
}

private void processFirstSampleOfStream() {
audioSink.handleDiscontinuity();
if (pendingOutputStreamOffsetCount != 0) {
setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]);
pendingOutputStreamOffsetCount--;
System.arraycopy(
pendingOutputStreamOffsetsUs,
/* srcPos= */ 1,
pendingOutputStreamOffsetsUs,
/* destPos= */ 0,
pendingOutputStreamOffsetCount);
}
}

private void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
this.outputStreamOffsetUs = outputStreamOffsetUs;
if (outputStreamOffsetUs != C.TIME_UNSET) {
audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs);
}
}

private boolean feedInputBuffer() throws DecoderException, ExoPlaybackException {
if (decoder == null
|| decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM
Expand Down Expand Up @@ -585,6 +616,7 @@ protected void onStopped() {
protected void onDisabled() {
inputFormat = null;
audioTrackNeedsConfigure = true;
setOutputStreamOffsetUs(C.TIME_UNSET);
try {
setSourceDrmSession(null);
releaseDecoder();
Expand All @@ -599,6 +631,19 @@ protected void onStreamChanged(Format[] formats, long startPositionUs, long offs
throws ExoPlaybackException {
super.onStreamChanged(formats, startPositionUs, offsetUs);
firstStreamSampleRead = false;
if (outputStreamOffsetUs == C.TIME_UNSET) {
setOutputStreamOffsetUs(offsetUs);
} else {
if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) {
Log.w(
TAG,
"Too many stream changes, so dropping offset: "
+ pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]);
} else {
pendingOutputStreamOffsetCount++;
}
pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1] = offsetUs;
}
}

@Override
Expand Down
5 changes: 5 additions & 0 deletions library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,11 @@ public void setPreferredDevice(@Nullable AudioDeviceInfo audioDeviceInfo) {
sink.setPreferredDevice(audioDeviceInfo);
}

@Override
public void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
sink.setOutputStreamOffsetUs(outputStreamOffsetUs);
}

@Override
public void enableTunnelingV21() {
sink.enableTunnelingV21();
Expand Down
5 changes: 5 additions & 0 deletions library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -733,6 +733,11 @@ protected void renderToEndOfStream() throws ExoPlaybackException {
}
}

@Override
protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) {
audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs);
}

@Override
public void handleMessage(@MessageType int messageType, @Nullable Object message)
throws ExoPlaybackException {
Expand Down
28 changes: 23 additions & 5 deletions library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ public MediaCodecRenderer(
pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
pendingOutputStreamSwitchTimesUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
outputStreamStartPositionUs = C.TIME_UNSET;
outputStreamOffsetUs = C.TIME_UNSET;
setOutputStreamOffsetUs(C.TIME_UNSET);
// MediaCodec outputs audio buffers in native endian:
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers
// and code called from MediaCodecAudioRenderer.processOutputBuffer expects this endianness.
Expand Down Expand Up @@ -649,7 +649,7 @@ protected void onStreamChanged(Format[] formats, long startPositionUs, long offs
if (this.outputStreamOffsetUs == C.TIME_UNSET) {
checkState(this.outputStreamStartPositionUs == C.TIME_UNSET);
this.outputStreamStartPositionUs = startPositionUs;
this.outputStreamOffsetUs = offsetUs;
setOutputStreamOffsetUs(offsetUs);
} else {
if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) {
Log.w(
Expand Down Expand Up @@ -686,7 +686,7 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb
}
formatQueue.clear();
if (pendingOutputStreamOffsetCount != 0) {
outputStreamOffsetUs = pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1];
setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]);
outputStreamStartPositionUs =
pendingOutputStreamStartPositionsUs[pendingOutputStreamOffsetCount - 1];
pendingOutputStreamOffsetCount = 0;
Expand All @@ -705,7 +705,7 @@ public void setPlaybackSpeed(float currentPlaybackSpeed, float targetPlaybackSpe
protected void onDisabled() {
inputFormat = null;
outputStreamStartPositionUs = C.TIME_UNSET;
outputStreamOffsetUs = C.TIME_UNSET;
setOutputStreamOffsetUs(C.TIME_UNSET);
pendingOutputStreamOffsetCount = 0;
flushOrReleaseCodec();
}
Expand Down Expand Up @@ -1586,7 +1586,7 @@ protected void onProcessedOutputBuffer(long presentationTimeUs) {
while (pendingOutputStreamOffsetCount != 0
&& presentationTimeUs >= pendingOutputStreamSwitchTimesUs[0]) {
outputStreamStartPositionUs = pendingOutputStreamStartPositionsUs[0];
outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0];
setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]);
pendingOutputStreamOffsetCount--;
System.arraycopy(
pendingOutputStreamStartPositionsUs,
Expand Down Expand Up @@ -1636,6 +1636,17 @@ protected DecoderReuseEvaluation canReuseCodec(
DISCARD_REASON_REUSE_NOT_IMPLEMENTED);
}

/**
* Called after the output stream offset changes.
*
* <p>The default implementation is a no-op.
*
* @param outputStreamOffsetUs The output stream offset in microseconds.
*/
protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) {
// Do nothing
}

@Override
public boolean isEnded() {
return outputStreamEnded;
Expand Down Expand Up @@ -2044,6 +2055,13 @@ protected final long getOutputStreamOffsetUs() {
return outputStreamOffsetUs;
}

private void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
this.outputStreamOffsetUs = outputStreamOffsetUs;
if (outputStreamOffsetUs != C.TIME_UNSET) {
onOutputStreamOffsetUsChanged(outputStreamOffsetUs);
}
}

/** Returns whether this renderer supports the given {@link Format Format's} DRM scheme. */
protected static boolean supportsFormatDrm(Format format) {
return format.cryptoType == C.CRYPTO_TYPE_NONE || format.cryptoType == C.CRYPTO_TYPE_FRAMEWORK;
Expand Down
10 changes: 8 additions & 2 deletions library/core/src/test/java/com/google/android/exoplayer2/audio/DecoderAudioRendererTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,8 @@ public void immediatelyReadEndOfStreamPlaysAudioSinkToEndOfStream() throws Excep
}

@Test
public void firstSampleOfStreamSignalsDiscontinuityToAudioSink() throws Exception {
public void firstSampleOfStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink()
throws Exception {
when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
when(mockAudioSink.isEnded()).thenReturn(true);
InOrder inOrderAudioSink = inOrder(mockAudioSink);
Expand Down Expand Up @@ -177,12 +178,15 @@ public void firstSampleOfStreamSignalsDiscontinuityToAudioSink() throws Exceptio
audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
}

inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0);
inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
}

@Test
public void firstSampleOfReplacementStreamSignalsDiscontinuityToAudioSink() throws Exception {
public void
firstSampleOfReplacementStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink()
throws Exception {
when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
when(mockAudioSink.isEnded()).thenReturn(true);
InOrder inOrderAudioSink = inOrder(mockAudioSink);
Expand Down Expand Up @@ -233,9 +237,11 @@ public void firstSampleOfReplacementStreamSignalsDiscontinuityToAudioSink() thro
audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
}

inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0);
inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(1_000_000);
inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
}

Expand Down
57 changes: 57 additions & 0 deletions ...y/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.robolectric.Shadows.shadowOf;
Expand Down Expand Up @@ -57,6 +58,7 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;
Expand Down Expand Up @@ -324,6 +326,61 @@ protected void onOutputFormatChanged(Format format, @Nullable MediaFormat mediaF
verify(audioRendererEventListener).onAudioSinkError(error);
}

@Test
public void render_callsAudioSinkSetOutputStreamOffset_whenReplaceStream() throws Exception {
FakeSampleStream fakeSampleStream1 =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
AUDIO_AAC,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 1_000),
END_OF_STREAM_ITEM));
fakeSampleStream1.writeData(/* startPositionUs= */ 0);
FakeSampleStream fakeSampleStream2 =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
AUDIO_AAC,
ImmutableList.of(
oneByteSample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 1_001_000),
END_OF_STREAM_ITEM));
fakeSampleStream2.writeData(/* startPositionUs= */ 0);
mediaCodecAudioRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {AUDIO_AAC},
fakeSampleStream1,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0);

mediaCodecAudioRenderer.start();
while (!mediaCodecAudioRenderer.hasReadStreamToEnd()) {
mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
}
mediaCodecAudioRenderer.replaceStream(
new Format[] {AUDIO_AAC},
fakeSampleStream2,
/* startPositionUs= */ 1_000_000,
/* offsetUs= */ 1_000_000);
mediaCodecAudioRenderer.setCurrentStreamFinal();
while (!mediaCodecAudioRenderer.isEnded()) {
mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
}

InOrder inOrderAudioSink = inOrder(audioSink);
inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(0);
inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(1_000_000);
}

@Test
public void supportsFormat_withEac3JocMediaAndEac3Decoder_returnsTrue() throws Exception {
Format mediaFormat =
Expand Down

0 comments on commit 4c73241

Please sign in to comment.