Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement an FfmpegVideoRenderer #7132

Open
wants to merge 7 commits into
base: dev-v2
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ extensions/flac/src/main/jni/flac

# FFmpeg extension
extensions/ffmpeg/src/main/jni/ffmpeg
extensions/ffmpeg/.cxx
extensions/ffmpeg/src/main/jni/include

# Cronet extension
extensions/cronet/jniLibs/*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.ext.ffmpeg;

import android.util.Log;
import android.view.Surface;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.SimpleDecoder;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoDecoderInputBuffer;
import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer;
import java.nio.ByteBuffer;
import java.util.List;

/**
* Ffmpeg Video decoder.
*/
/* package */ final class FfmpegVideoDecoder
extends
SimpleDecoder<VideoDecoderInputBuffer, VideoDecoderOutputBuffer, FfmpegDecoderException> {

private static final String TAG = "FfmpegVideoDecoder";

// LINT.IfChange
private static final int VIDEO_DECODER_SUCCESS = 0;
private static final int VIDEO_DECODER_ERROR_INVALID_DATA = -1;
private static final int VIDEO_DECODER_ERROR_OTHER = -2;
private static final int VIDEO_DECODER_ERROR_READ_FRAME = -3;
// LINT.ThenChange(../../../../../../../jni/ffmpeg_jni.cc)

private final String codecName;
private long nativeContext;
@Nullable private final byte[] extraData;
private Format format;

@C.VideoOutputMode private volatile int outputMode;

/**
* Creates a Ffmpeg video Decoder.
*
* @param numInputBuffers Number of input buffers.
* @param numOutputBuffers Number of output buffers.
* @param initialInputBufferSize The initial size of each input buffer, in bytes.
* @param threads Number of threads libgav1 will use to decode.
* @throws FfmpegDecoderException Thrown if an exception occurs when initializing the
* decoder.
*/
public FfmpegVideoDecoder(
int numInputBuffers, int numOutputBuffers, int initialInputBufferSize, int threads, Format format)
throws FfmpegDecoderException {
super(
new VideoDecoderInputBuffer[numInputBuffers],
new VideoDecoderOutputBuffer[numOutputBuffers]);
if (!FfmpegLibrary.isAvailable()) {
throw new FfmpegDecoderException("Failed to load decoder native library.");
}
codecName = Assertions.checkNotNull(FfmpegLibrary.getCodecName(format.sampleMimeType));
extraData = getExtraData(format.sampleMimeType, format.initializationData);
this.format = format;
nativeContext = ffmpegInitialize(codecName, extraData, threads);
if (nativeContext == 0) {
throw new FfmpegDecoderException("Failed to initialize decoder.");
}
setInitialInputBufferSize(initialInputBufferSize);
}

/**
* Returns FFmpeg-compatible codec-specific initialization data ("extra data"), or {@code null} if
* not required.
*/
@Nullable
private static byte[] getExtraData(String mimeType, List<byte[]> initializationData) {
switch (mimeType) {
case MimeTypes.VIDEO_H264:
byte[] sps = initializationData.get(0);
byte[] pps = initializationData.get(1);
byte[] extraData = new byte[sps.length + pps.length];
System.arraycopy(sps, 0, extraData, 0, sps.length);
System.arraycopy(pps, 0, extraData, sps.length, pps.length);
return extraData;
case MimeTypes.VIDEO_H265:
return initializationData.get(0);
default:
// Other codecs do not require extra data.
return null;
}
}

@Override
public String getName() {
return "ffmpeg" + FfmpegLibrary.getVersion() + "-" + codecName;
}

/**
* Sets the output mode for frames rendered by the decoder.
*
* @param outputMode The output mode.
*/
public void setOutputMode(@C.VideoOutputMode int outputMode) {
this.outputMode = outputMode;
}

@Override
protected VideoDecoderInputBuffer createInputBuffer() {
return new VideoDecoderInputBuffer();
}

@Override
protected VideoDecoderOutputBuffer createOutputBuffer() {
return new VideoDecoderOutputBuffer(this::releaseOutputBuffer);
}

@Override
@Nullable
protected FfmpegDecoderException decode(
VideoDecoderInputBuffer inputBuffer, VideoDecoderOutputBuffer outputBuffer, boolean reset) {
if (reset) {
nativeContext = ffmpegReset(nativeContext);
if (nativeContext == 0) {
return new FfmpegDecoderException("Error resetting (see logcat).");
}
}

// send packet
ByteBuffer inputData = Util.castNonNull(inputBuffer.data);
int inputSize = inputData.limit();
// enqueue origin data
int sendPacketResult = ffmpegSendPacket(nativeContext, inputData, inputSize,
inputBuffer.timeUs);
if (sendPacketResult == VIDEO_DECODER_ERROR_INVALID_DATA) {
outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY);
return null;
} else if (sendPacketResult == VIDEO_DECODER_ERROR_READ_FRAME) {
// need read frame
Log.d(TAG, "VIDEO_DECODER_ERROR_READ_FRAME: " + "timeUs=" + inputBuffer.timeUs);
} else if (sendPacketResult == VIDEO_DECODER_ERROR_OTHER) {
return new FfmpegDecoderException("ffmpegDecode error: (see logcat)");
}

// receive frame
boolean decodeOnly = inputBuffer.isDecodeOnly();
// We need to dequeue the decoded frame from the decoder even when the input data is
// decode-only.
int getFrameResult = ffmpegReceiveFrame(nativeContext, outputMode, outputBuffer, decodeOnly);
if (getFrameResult == VIDEO_DECODER_ERROR_OTHER) {
return new FfmpegDecoderException("ffmpegDecode error: (see logcat)");
}

if (getFrameResult == VIDEO_DECODER_ERROR_INVALID_DATA) {
outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY);
}

if (!decodeOnly) {
outputBuffer.colorInfo = inputBuffer.colorInfo;
}

return null;
}

@Override
protected FfmpegDecoderException createUnexpectedDecodeException(Throwable error) {
return new FfmpegDecoderException("Unexpected decode error", error);
}

@Override
public void release() {
super.release();
ffmpegRelease(nativeContext);
nativeContext = 0;
}

/**
* Renders output buffer to the given surface. Must only be called when in {@link
* C#VIDEO_OUTPUT_MODE_SURFACE_YUV} mode.
*
* @param outputBuffer Output buffer.
* @param surface Output surface.
* @throws FfmpegDecoderException Thrown if called with invalid output mode or frame
* rendering fails.
*/
public void renderToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface)
throws FfmpegDecoderException {
if (outputBuffer.mode != C.VIDEO_OUTPUT_MODE_SURFACE_YUV) {
throw new FfmpegDecoderException("Invalid output mode.");
}
if (ffmpegRenderFrame(
nativeContext, surface,
outputBuffer, outputBuffer.width, outputBuffer.height) == VIDEO_DECODER_ERROR_OTHER) {
throw new FfmpegDecoderException(
"Buffer render error: ");
}
}

private native long ffmpegInitialize(String codecName, @Nullable byte[] extraData, int threads);

private native long ffmpegReset(long context);

private native void ffmpegRelease(long context);

private native int ffmpegRenderFrame(
long context, Surface surface, VideoDecoderOutputBuffer outputBuffer,
int displayedWidth,
int displayedHeight);

/**
* Decodes the encoded data passed.
*
* @param context Decoder context.
* @param encodedData Encoded data.
* @param length Length of the data buffer.
* @return {@link #VIDEO_DECODER_SUCCESS} if successful, {@link #VIDEO_DECODER_ERROR_OTHER} if an
* error occurred.
*/
private native int ffmpegSendPacket(long context, ByteBuffer encodedData, int length,
long inputTime);

/**
* Gets the decoded frame.
*
* @param context Decoder context.
* @param outputBuffer Output buffer for the decoded frame.
* @return {@link #VIDEO_DECODER_SUCCESS} if successful, {@link #VIDEO_DECODER_ERROR_INVALID_DATA}
* if successful but the frame is decode-only, {@link #VIDEO_DECODER_ERROR_OTHER} if an error
* occurred.
*/
private native int ffmpegReceiveFrame(
long context, int outputMode, VideoDecoderOutputBuffer outputBuffer, boolean decodeOnly);

}
Loading