Apply priority/operating rate settings for video encoding.

- Added setter to disable this feature. - Added accompanying tests. - Plan to run tests on the same set of settings on H265. PiperOrigin-RevId: 460238673
androidx · Jul 12, 2022 · 18f4068 · 18f4068
1 parent 4eb34e4
commit 18f4068
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 12 deletions.
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java
@@ -27,6 +27,7 @@
 import android.media.MediaFormat;
 import android.view.Surface;
 import androidx.annotation.Nullable;
+import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.MimeTypes;
@@ -277,6 +278,11 @@ public String getName() {
     return mediaCodec.getName();
   }
 
+  @VisibleForTesting
+  /* package */ MediaFormat getConfigurationMediaFormat() {
+    return configurationMediaFormat;
+  }
+
   /**
    * Attempts to dequeue an output buffer if there is no output buffer pending. Does nothing
    * otherwise.

diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
@@ -20,7 +20,6 @@
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
-import static androidx.media3.common.util.Util.SDK_INT;
 import static java.lang.Math.abs;
 import static java.lang.Math.floor;
 import static java.lang.Math.round;
@@ -47,6 +46,9 @@
 @UnstableApi
 public final class DefaultEncoderFactory implements Codec.EncoderFactory {
   private static final int DEFAULT_FRAME_RATE = 30;
+  /** Best effort, or as-fast-as-possible priority setting for {@link MediaFormat#KEY_PRIORITY}. */
+  private static final int PRIORITY_BEST_EFFORT = 1;
+
   private static final String TAG = "DefaultEncoderFactory";
 
   /** A builder for {@link DefaultEncoderFactory} instances. */
@@ -254,7 +256,7 @@ public Codec createForVideoEncoding(Format format, List<String> allowedMimeTypes
 
     if (supportedVideoEncoderSettings.profile != VideoEncoderSettings.NO_VALUE
         && supportedVideoEncoderSettings.level != VideoEncoderSettings.NO_VALUE
-        && SDK_INT >= 23) {
+        && Util.SDK_INT >= 23) {
       // Set profile and level at the same time to maximize compatibility, or the encoder will pick
       // the values.
       mediaFormat.setInteger(MediaFormat.KEY_PROFILE, supportedVideoEncoderSettings.profile);
@@ -285,12 +287,17 @@ public Codec createForVideoEncoding(Format format, List<String> allowedMimeTypes
 
     if (Util.SDK_INT >= 23) {
       // Setting operating rate and priority is supported from API 23.
-      if (supportedVideoEncoderSettings.operatingRate != VideoEncoderSettings.NO_VALUE) {
-        mediaFormat.setInteger(
-            MediaFormat.KEY_OPERATING_RATE, supportedVideoEncoderSettings.operatingRate);
-      }
-      if (supportedVideoEncoderSettings.priority != VideoEncoderSettings.NO_VALUE) {
-        mediaFormat.setInteger(MediaFormat.KEY_PRIORITY, supportedVideoEncoderSettings.priority);
+      if (supportedVideoEncoderSettings.operatingRate == VideoEncoderSettings.NO_VALUE
+          && supportedVideoEncoderSettings.priority == VideoEncoderSettings.NO_VALUE) {
+        adjustMediaFormatForEncoderPerformanceSettings(mediaFormat);
+      } else {
+        if (supportedVideoEncoderSettings.operatingRate != VideoEncoderSettings.NO_VALUE) {
+          mediaFormat.setInteger(
+              MediaFormat.KEY_OPERATING_RATE, supportedVideoEncoderSettings.operatingRate);
+        }
+        if (supportedVideoEncoderSettings.priority != VideoEncoderSettings.NO_VALUE) {
+          mediaFormat.setInteger(MediaFormat.KEY_PRIORITY, supportedVideoEncoderSettings.priority);
+        }
       }
     }
 
@@ -462,6 +469,28 @@ public VideoEncoderQueryResult(
     }
   }
 
+  /**
+   * Applies empirical {@link MediaFormat#KEY_PRIORITY} and {@link MediaFormat#KEY_OPERATING_RATE}
+   * settings for better encoder performance.
+   *
+   * <p>The adjustment is applied in-place to {@code mediaFormat}.
+   */
+  private static void adjustMediaFormatForEncoderPerformanceSettings(MediaFormat mediaFormat) {
+    // TODO(b/213477153) Verify priority/operating rate settings work for non-AVC codecs.
+    if (Util.SDK_INT < 25) {
+      // Not setting priority and operating rate achieves better encoding performance.
+      return;
+    }
+
+    mediaFormat.setInteger(MediaFormat.KEY_PRIORITY, PRIORITY_BEST_EFFORT);
+
+    if (Util.SDK_INT == 26) {
+      mediaFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, DEFAULT_FRAME_RATE);
+    } else {
+      mediaFormat.setInteger(MediaFormat.KEY_OPERATING_RATE, Integer.MAX_VALUE);
+    }
+  }
+
   /**
    * Applying suggested profile/level settings from
    * https://developer.android.com/guide/topics/media/sharing-video#b-frames_and_encoding_profiles

diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoEncoderSettings.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoEncoderSettings.java
@@ -180,7 +180,8 @@ public Builder setiFrameIntervalSeconds(float iFrameIntervalSeconds) {
     }
 
     /**
-     * Sets encoding operating rate and priority. The default values are {@link #NO_VALUE}.
+     * Sets encoding operating rate and priority. The default values are {@link #NO_VALUE}, which is
+     * treated as configuring the encoder for maximum throughput.
      *
      * @param operatingRate The {@link MediaFormat#KEY_OPERATING_RATE operating rate}.
      * @param priority The {@link MediaFormat#KEY_PRIORITY priority}.

diff --git a/...ries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java b/...ries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java
@@ -30,6 +30,7 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.robolectric.annotation.Config;
 import org.robolectric.shadows.MediaCodecInfoBuilder;
 import org.robolectric.shadows.ShadowMediaCodecList;
 
@@ -40,6 +41,10 @@ public class DefaultEncoderFactoryTest {
 
   @Before
   public void setUp() {
+    createShadowH264Encoder();
+  }
+
+  private static void createShadowH264Encoder() {
     MediaFormat avcFormat = new MediaFormat();
     avcFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_AVC);
     MediaCodecInfo.CodecProfileLevel profileLevel = new MediaCodecInfo.CodecProfileLevel();
@@ -48,17 +53,26 @@ public void setUp() {
     // blocks will be left for encoding height 1088.
     profileLevel.level = MediaCodecInfo.CodecProfileLevel.AVCLevel4;
 
+    createShadowVideoEncoder(avcFormat, profileLevel, "test.transformer.avc.encoder");
+  }
+
+  private static void createShadowVideoEncoder(
+      MediaFormat supportedFormat,
+      MediaCodecInfo.CodecProfileLevel supportedProfileLevel,
+      String name) {
+    // ShadowMediaCodecList is static. The added encoders will be visible for every test.
     ShadowMediaCodecList.addCodec(
         MediaCodecInfoBuilder.newBuilder()
-            .setName("test.transformer.avc.encoder")
+            .setName(name)
             .setIsEncoder(true)
             .setCapabilities(
                 MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder()
-                    .setMediaFormat(avcFormat)
+                    .setMediaFormat(supportedFormat)
                     .setIsEncoder(true)
                     .setColorFormats(
                         new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible})
-                    .setProfileLevels(new MediaCodecInfo.CodecProfileLevel[] {profileLevel})
+                    .setProfileLevels(
+                        new MediaCodecInfo.CodecProfileLevel[] {supportedProfileLevel})
                     .build())
             .build());
   }
@@ -117,6 +131,29 @@ public void createForVideoEncoding_withFallbackOnAndUnsupportedResolution_config
     assertThat(actualVideoFormat.height).isEqualTo(1080);
   }
 
+  @Config(sdk = 29)
+  @Test
+  public void
+      createForVideoEncoding_withH264Encoding_configuresEncoderWithCorrectPerformanceSettings()
+          throws Exception {
+    Format requestedVideoFormat = createVideoFormat(MimeTypes.VIDEO_H264, 1920, 1080, 30);
+    Codec videoEncoder =
+        new DefaultEncoderFactory.Builder(context)
+            .build()
+            .createForVideoEncoding(
+                requestedVideoFormat,
+                /* allowedMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264));
+
+    assertThat(videoEncoder).isInstanceOf(DefaultCodec.class);
+    MediaFormat configurationMediaFormat =
+        ((DefaultCodec) videoEncoder).getConfigurationMediaFormat();
+    assertThat(configurationMediaFormat.containsKey(MediaFormat.KEY_PRIORITY)).isTrue();
+    assertThat(configurationMediaFormat.getInteger(MediaFormat.KEY_PRIORITY)).isEqualTo(1);
+    assertThat(configurationMediaFormat.containsKey(MediaFormat.KEY_OPERATING_RATE)).isTrue();
+    assertThat(configurationMediaFormat.getInteger(MediaFormat.KEY_OPERATING_RATE))
+        .isEqualTo(Integer.MAX_VALUE);
+  }
+
   @Test
   public void createForVideoEncoding_withNoSupportedEncoder_throws() {
     Format requestedVideoFormat = createVideoFormat(MimeTypes.VIDEO_H264, 1920, 1080, 30);