Skip to content

Commit

Permalink
Remove late frame dropping in FrameProcessor
Browse files Browse the repository at this point in the history
Currently, a frame is dropped if it's requested release time is in the past.
This mode was added to support previewing. However, in normal ExoPlayer
playback, slightly late frames (<30ms late) are also rendered. On MediaCodec
side, this means calling `releaseOutputBuffer` with a release time in the
past.

PiperOrigin-RevId: 479615291
  • Loading branch information
claincly authored and marcbaechinger committed Oct 20, 2022
1 parent a6b9772 commit 2188685
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ public void controlledFrameRelease_withOneFrameRequestImmediateRelease_releasesF
}

@Test
public void controlledFrameRelease_withLateFrame_dropsFrame() throws Exception {
public void controlledFrameRelease_withLateFrame_releasesFrame() throws Exception {
long originalPresentationTimeUs = 1234;
long releaseTimeBeforeCurrentTimeNs = System.nanoTime() - 345678;
AtomicLong actualPresentationTimeUs = new AtomicLong();
Expand All @@ -175,7 +175,9 @@ public void controlledFrameRelease_withLateFrame_dropsFrame() throws Exception {

assertThat(frameProcessingException.get()).isNull();
assertThat(actualPresentationTimeUs.get()).isEqualTo(originalPresentationTimeUs);
assertThat(outputReleaseTimesNs).isEmpty();
assertThat(outputReleaseTimesNs).hasSize(1);
// The actual release time is determined by the FrameProcessor when releasing the frame.
assertThat(outputReleaseTimesNs.remove()).isAtLeast(releaseTimeBeforeCurrentTimeNs);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,10 +150,7 @@ public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
frameProcessorListener.onOutputFrameAvailable(offsetPresentationTimeUs);
if (releaseFramesAutomatically) {
renderFrameToSurfaces(
inputTexture,
presentationTimeUs,
/* releaseTimeNs= */ offsetPresentationTimeUs * 1000,
/* dropLateFrame= */ false);
inputTexture, presentationTimeUs, /* releaseTimeNs= */ offsetPresentationTimeUs * 1000);
} else {
availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
}
Expand All @@ -169,21 +166,11 @@ public void releaseOutputFrame(TextureInfo outputTexture) {
@WorkerThread
public void releaseOutputFrame(long releaseTimeNs) {
checkState(!releaseFramesAutomatically);

boolean dropLateFrame = true;
if (releaseTimeNs == FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY) {
dropLateFrame = false;
releaseTimeNs = System.nanoTime();
} else if (releaseTimeNs == FrameProcessor.DROP_OUTPUT_FRAME) {
releaseTimeNs = C.TIME_UNSET;
}

Pair<TextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
renderFrameToSurfaces(
/* inputTexture= */ oldestAvailableFrame.first,
/* presentationTimeUs= */ oldestAvailableFrame.second,
releaseTimeNs,
dropLateFrame);
releaseTimeNs);
}

@Override
Expand Down Expand Up @@ -254,13 +241,9 @@ public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfac
}

private void renderFrameToSurfaces(
TextureInfo inputTexture,
long presentationTimeUs,
long releaseTimeNs,
boolean dropLateFrame) {
TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) {
try {
maybeRenderFrameToOutputSurface(
inputTexture, presentationTimeUs, releaseTimeNs, dropLateFrame);
maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
} catch (FrameProcessingException | GlUtil.GlException e) {
frameProcessorListener.onFrameProcessingError(
FrameProcessingException.from(e, presentationTimeUs));
Expand All @@ -270,10 +253,11 @@ private void renderFrameToSurfaces(
}

private synchronized void maybeRenderFrameToOutputSurface(
TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs, boolean dropLateFrame)
TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs)
throws FrameProcessingException, GlUtil.GlException {
if (!ensureConfigured(inputTexture.width, inputTexture.height)) {
return; // Drop frames when there is no output surface.
if (releaseTimeNs == FrameProcessor.DROP_OUTPUT_FRAME
|| !ensureConfigured(inputTexture.width, inputTexture.height)) {
return; // Drop frames when requested, or there is no output surface.
}

EGLSurface outputEglSurface = this.outputEglSurface;
Expand All @@ -289,10 +273,12 @@ private synchronized void maybeRenderFrameToOutputSurface(
GlUtil.clearOutputFrame();
matrixTextureProcessor.drawFrame(inputTexture.texId, presentationTimeUs);

if (dropLateFrame && System.nanoTime() > releaseTimeNs) {
return;
}
EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, releaseTimeNs);
EGLExt.eglPresentationTimeANDROID(
eglDisplay,
outputEglSurface,
releaseTimeNs == FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY
? System.nanoTime()
: releaseTimeNs);
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package androidx.media3.common;

import android.content.Context;
import android.opengl.EGLExt;
import android.view.Surface;
import androidx.annotation.Nullable;
import java.util.List;
Expand Down Expand Up @@ -172,10 +173,12 @@ interface Listener {
* false} using the {@link Factory} and should be called exactly once for each frame that becomes
* {@linkplain Listener#onOutputFrameAvailable(long) available}.
*
* @param releaseTimeNs The release time to use for the frame, in nanoseconds. Use {@link
* #DROP_OUTPUT_FRAME} to drop the frame, or {@link #RELEASE_OUTPUT_FRAME_IMMEDIATELY} to
* release the frame immediately. If {@code releaseTimeNs} is after {@link System#nanoTime()}
* at the time of the release, the frame is also dropped.
* <p>The {@code releaseTimeNs} may be passed to {@link EGLExt#eglPresentationTimeANDROID}
* depending on the implementation.
*
* @param releaseTimeNs The release time to use for the frame, in nanoseconds. The release time
* can be before of after the current system time. Use {@link #DROP_OUTPUT_FRAME} to drop the
* frame, or {@link #RELEASE_OUTPUT_FRAME_IMMEDIATELY} to release the frame immediately.
*/
void releaseOutputFrame(long releaseTimeNs);

Expand Down

0 comments on commit 2188685

Please sign in to comment.