Skip to content

Commit

Permalink
Move program initialization to texture processor constructor.
Browse files Browse the repository at this point in the history 
Once the more advanced GlTextureProcessor interface exists,
it will be possible to change the output size of a GlTextureProcessor
between frames. To keep the re-configuration based on the frame sizes
minimal, things indepedent of the frame size, such as the GlProgram,
can be initialized in the constructor.

PiperOrigin-RevId: 451997584
  • Loading branch information
@hmsch @marcbaechinger
hmsch authored and marcbaechinger committed May 31, 2022
1 parent 5cdac65 commit 87ab96d
Show file tree
Hide file tree
Showing 14 changed files with 266 additions and 250 deletions.
47 changes: 23 additions & 24 deletions demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import androidx.media3.transformer.SingleFrameGlTextureProcessor;
import java.io.IOException;
import java.util.Locale;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

/**
* A {@link SingleFrameGlTextureProcessor} that overlays a bitmap with a logo and timer on each
Expand All @@ -57,16 +56,20 @@

private final Paint paint;
private final Bitmap overlayBitmap;
private final Bitmap logoBitmap;
private final Canvas overlayCanvas;
private final GlProgram glProgram;

private float bitmapScaleX;
private float bitmapScaleY;
private int bitmapTexId;
private @MonotonicNonNull Size outputSize;
private @MonotonicNonNull Bitmap logoBitmap;
private @MonotonicNonNull GlProgram glProgram;

public BitmapOverlayProcessor() {
/**
* Creates a new instance.
*
* @throws IOException If a problem occurs while reading shader files.
*/
public BitmapOverlayProcessor(Context context) throws IOException {
paint = new Paint();
paint.setTextSize(64);
paint.setAntiAlias(true);
Expand All @@ -75,19 +78,6 @@ public BitmapOverlayProcessor() {
overlayBitmap =
Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888);
overlayCanvas = new Canvas(overlayBitmap);
}

@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException {
if (inputWidth > inputHeight) {
bitmapScaleX = inputWidth / (float) inputHeight;
bitmapScaleY = 1f;
} else {
bitmapScaleX = 1f;
bitmapScaleY = inputHeight / (float) inputWidth;
}
outputSize = new Size(inputWidth, inputHeight);

try {
logoBitmap =
Expand All @@ -106,19 +96,27 @@ public void initialize(Context context, int inputTexId, int inputWidth, int inpu
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1);
glProgram.setFloatUniform("uScaleX", bitmapScaleX);
glProgram.setFloatUniform("uScaleY", bitmapScaleY);
}

@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
public Size configure(int inputWidth, int inputHeight) {
if (inputWidth > inputHeight) {
bitmapScaleX = inputWidth / (float) inputHeight;
bitmapScaleY = 1f;
} else {
bitmapScaleX = 1f;
bitmapScaleY = inputHeight / (float) inputWidth;
}

glProgram.setFloatUniform("uScaleX", bitmapScaleX);
glProgram.setFloatUniform("uScaleY", bitmapScaleY);

return new Size(inputWidth, inputHeight);
}

@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use();

Expand All @@ -137,6 +135,7 @@ public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
flipBitmapVertically(overlayBitmap));
GlUtil.checkGlError();

glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
Expand Down
42 changes: 17 additions & 25 deletions ...transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
package androidx.media3.demo.transformer;

import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkStateNotNull;

import android.content.Context;
import android.opengl.GLES20;
Expand All @@ -26,7 +25,6 @@
import androidx.media3.transformer.FrameProcessingException;
import androidx.media3.transformer.SingleFrameGlTextureProcessor;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

/**
* A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are
Expand All @@ -41,14 +39,9 @@
private static final String FRAGMENT_SHADER_PATH = "fragment_shader_vignette_es2.glsl";
private static final float DIMMING_PERIOD_US = 5_600_000f;

private float centerX;
private float centerY;
private float minInnerRadius;
private float deltaInnerRadius;
private float outerRadius;

private @MonotonicNonNull Size outputSize;
private @MonotonicNonNull GlProgram glProgram;
private final GlProgram glProgram;
private final float minInnerRadius;
private final float deltaInnerRadius;

/**
* Creates a new instance.
Expand All @@ -61,29 +54,27 @@
*
* <p>The parameters are given in normalized texture coordinates from 0 to 1.
*
* @param context The {@link Context}.
* @param centerX The x-coordinate of the center of the effect.
* @param centerY The y-coordinate of the center of the effect.
* @param minInnerRadius The lower bound of the radius that is unaffected by the effect.
* @param maxInnerRadius The upper bound of the radius that is unaffected by the effect.
* @param outerRadius The radius after which all pixels are black.
* @throws IOException If a problem occurs while reading shader files.
*/
public PeriodicVignetteProcessor(
float centerX, float centerY, float minInnerRadius, float maxInnerRadius, float outerRadius) {
Context context,
float centerX,
float centerY,
float minInnerRadius,
float maxInnerRadius,
float outerRadius)
throws IOException {
checkArgument(minInnerRadius <= maxInnerRadius);
checkArgument(maxInnerRadius <= outerRadius);
this.centerX = centerX;
this.centerY = centerY;
this.minInnerRadius = minInnerRadius;
this.deltaInnerRadius = maxInnerRadius - minInnerRadius;
this.outerRadius = outerRadius;
}

@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException {
outputSize = new Size(inputWidth, inputHeight);
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
glProgram.setFloatsUniform("uCenter", new float[] {centerX, centerY});
glProgram.setFloatsUniform("uOuterRadius", new float[] {outerRadius});
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
Expand All @@ -94,14 +85,15 @@ public void initialize(Context context, int inputTexId, int inputWidth, int inpu
}

@Override
public Size getOutputSize() {
return checkStateNotNull(outputSize);
public Size configure(int inputWidth, int inputHeight) {
return new Size(inputWidth, inputHeight);
}

@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
try {
checkStateNotNull(glProgram).use();
glProgram.use();
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US;
float innerRadius =
minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta));
Expand Down
9 changes: 6 additions & 3 deletions demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static androidx.media3.common.util.Assertions.checkNotNull;

import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.net.Uri;
Expand Down Expand Up @@ -274,12 +275,13 @@ private Transformer createTransformer(@Nullable Bundle bundle, String filePath)
try {
Class<?> clazz = Class.forName("androidx.media3.demo.transformer.MediaPipeProcessor");
Constructor<?> constructor =
clazz.getConstructor(String.class, String.class, String.class);
clazz.getConstructor(Context.class, String.class, String.class, String.class);
effects.add(
() -> {
(Context context) -> {
try {
return (SingleFrameGlTextureProcessor)
constructor.newInstance(
context,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb",
/* inputStreamName= */ "input_video",
/* outputStreamName= */ "output_video");
Expand All @@ -294,8 +296,9 @@ private Transformer createTransformer(@Nullable Bundle bundle, String filePath)
}
if (selectedEffects[2]) {
effects.add(
() ->
(Context context) ->
new PeriodicVignetteProcessor(
context,
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat(
Expand Down
38 changes: 14 additions & 24 deletions ...ansformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,49 +63,36 @@ protected void loadLibrary(String name) {
private static final String COPY_VERTEX_SHADER_NAME = "vertex_shader_copy_es2.glsl";
private static final String COPY_FRAGMENT_SHADER_NAME = "shaders/fragment_shader_copy_es2.glsl";

private final String graphName;
private final String inputStreamName;
private final String outputStreamName;
private final ConditionVariable frameProcessorConditionVariable;
private final FrameProcessor frameProcessor;
private final GlProgram glProgram;

private @MonotonicNonNull FrameProcessor frameProcessor;
private int inputWidth;
private int inputHeight;
private int inputTexId;
private @MonotonicNonNull GlProgram glProgram;
private @MonotonicNonNull TextureFrame outputFrame;
private @MonotonicNonNull RuntimeException frameProcessorPendingError;

/**
* Creates a new texture processor that wraps a MediaPipe graph.
*
* @param context The {@link Context}.
* @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph.
* @throws IOException If a problem occurs while reading shader files or initializing MediaPipe
* resources.
*/
public MediaPipeProcessor(String graphName, String inputStreamName, String outputStreamName) {
checkState(LOADER.isAvailable());
this.graphName = graphName;
this.inputStreamName = inputStreamName;
this.outputStreamName = outputStreamName;
frameProcessorConditionVariable = new ConditionVariable();
}

@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
public MediaPipeProcessor(
Context context, String graphName, String inputStreamName, String outputStreamName)
throws IOException {
this.inputTexId = inputTexId;
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME);
checkState(LOADER.isAvailable());

frameProcessorConditionVariable = new ConditionVariable();
AndroidAssetUtil.initializeNativeAssetManager(context);

EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor =
new FrameProcessor(
context, eglManager.getNativeContext(), graphName, inputStreamName, outputStreamName);

// Unblock drawFrame when there is an output frame or an error.
frameProcessor.setConsumer(
frame -> {
Expand All @@ -117,15 +104,18 @@ public void initialize(Context context, int inputTexId, int inputWidth, int inpu
frameProcessorPendingError = error;
frameProcessorConditionVariable.open();
});
glProgram = new GlProgram(context, COPY_VERTEX_SHADER_NAME, COPY_FRAGMENT_SHADER_NAME);
}

@Override
public Size getOutputSize() {
public Size configure(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
return new Size(inputWidth, inputHeight);
}

@Override
public void drawFrame(long presentationTimeUs) throws FrameProcessingException {
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
frameProcessorConditionVariable.close();

// Pass the input frame to MediaPipe.
Expand Down
9 changes: 3 additions & 6 deletions ...transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ private FrameProcessorChain createFrameProcessorChainWithFakeTextureProcessors(
throws FrameProcessingException {
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
for (Size element : textureProcessorOutputSizes) {
effects.add(() -> new FakeTextureProcessor(element));
effects.add((Context context) -> new FakeTextureProcessor(element));
}
return FrameProcessorChain.create(
getApplicationContext(),
Expand All @@ -144,15 +144,12 @@ private FakeTextureProcessor(Size outputSize) {
}

@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) {}

@Override
public Size getOutputSize() {
public Size configure(int inputWidth, int inputHeight) {
return outputSize;
}

@Override
public void drawFrame(long presentationTimeNs) {}
public void drawFrame(int inputTexId, long presentationTimeNs) {}

@Override
public void release() {}
Expand Down
Loading

0 comments on commit 87ab96d

Please sign in to comment.