From 0db07c6791620d06db9f524bf0f47e90df635756 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Mon, 18 Jul 2022 10:09:35 +0000 Subject: [PATCH] Make minor fixes to HDR handling - Update profile selection logic to pick an HDR-compatible profile when doing HDR editing on H.264/AVC videos. - Handle doing the capabilities check for all MIME types that support HDR (not just H.265/HEVC). - Fix a bug where we would pass an HDR input color format to the encoder when using tone-mapping. - Tweak how `EncoderWrapper` works so decisions at made at construction time. Manually tested cases: - Transformation of an SDR video. - Transformation of an HDR video to AVC (which triggers fallback/tone-mapping on a device that doesn't support HDR editing for AVC). - Transformation of an HDR video with HDR editing. PiperOrigin-RevId: 461572973 --- .../transformer/DefaultEncoderFactory.java | 13 +- .../exoplayer2/transformer/EncoderUtil.java | 80 +++++++++++++ .../VideoTranscodingSamplePipeline.java | 113 +++++++----------- .../transformer/VideoEncoderWrapperTest.java | 3 +- 4 files changed, 136 insertions(+), 73 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java index 90bb6f99afc..2bf673eb3ee 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java @@ -273,7 +273,7 @@ public Codec createForVideoEncoding(Format format, List allowedMimeTypes } if (mimeType.equals(MimeTypes.VIDEO_H264)) { - adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo); + adjustMediaFormatForH264EncoderSettings(format.colorInfo, encoderInfo, mediaFormat); } MediaFormatUtil.maybeSetColorInfo(mediaFormat, encoderSupportedFormat.colorInfo); @@ -523,12 +523,21 @@ private static void adjustMediaFormatForEncoderPerformanceSettings(MediaFormat m *

The adjustment is applied in-place to {@code mediaFormat}. */ private static void adjustMediaFormatForH264EncoderSettings( - MediaFormat mediaFormat, MediaCodecInfo encoderInfo) { + @Nullable ColorInfo colorInfo, MediaCodecInfo encoderInfo, MediaFormat mediaFormat) { // TODO(b/210593256): Remove overriding profile/level (before API 29) after switching to in-app // muxing. String mimeType = MimeTypes.VIDEO_H264; if (Util.SDK_INT >= 29) { int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh; + if (colorInfo != null) { + int colorTransfer = colorInfo.colorTransfer; + ImmutableList codecProfiles = + EncoderUtil.getCodecProfilesForHdrFormat(mimeType, colorTransfer); + if (!codecProfiles.isEmpty()) { + // Default to the most compatible profile, which is first in the list. + expectedEncodingProfile = codecProfiles.get(0); + } + } int supportedEncodingLevel = EncoderUtil.findHighestSupportedEncodingLevel( encoderInfo, mimeType, expectedEncodingProfile); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java index 1e0373aea2a..a59902bb214 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java @@ -31,10 +31,13 @@ import androidx.annotation.DoNotInline; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.C.ColorTransfer; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.util.MediaFormatUtil; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.ColorInfo; import com.google.common.base.Ascii; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -65,6 +68,83 @@ public static ImmutableSet getSupportedVideoMimeTypes() { return checkNotNull(MIME_TYPE_TO_ENCODERS.get()).keySet(); } + /** + * Returns the names of encoders that support HDR editing for the given format, or an empty list + * if the format is unknown or not supported for HDR encoding. + */ + public static ImmutableList getSupportedEncoderNamesForHdrEditing( + String mimeType, @Nullable ColorInfo colorInfo) { + if (Util.SDK_INT < 31 || colorInfo == null) { + return ImmutableList.of(); + } + + @ColorTransfer int colorTransfer = colorInfo.colorTransfer; + ImmutableList profiles = getCodecProfilesForHdrFormat(mimeType, colorTransfer); + ImmutableList.Builder resultBuilder = ImmutableList.builder(); + ImmutableList mediaCodecInfos = + EncoderSelector.DEFAULT.selectEncoderInfos(mimeType); + for (int i = 0; i < mediaCodecInfos.size(); i++) { + MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i); + if (mediaCodecInfo.isAlias() + || !EncoderUtil.isFeatureSupported( + mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) { + continue; + } + for (MediaCodecInfo.CodecProfileLevel codecProfileLevel : + mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) { + if (profiles.contains(codecProfileLevel.profile)) { + resultBuilder.add(mediaCodecInfo.getName()); + } + } + } + return resultBuilder.build(); + } + + /** + * Returns the {@linkplain MediaCodecInfo.CodecProfileLevel#profile profile} constants that can be + * used to encode the given HDR format, if supported by the device (this method does not check + * device capabilities). If multiple profiles are returned, they are ordered by expected level of + * compatibility, with the most widely compatible profile first. + */ + @SuppressWarnings("InlinedApi") // Safe use of inlined constants from newer API versions. + public static ImmutableList getCodecProfilesForHdrFormat( + String mimeType, @ColorTransfer int colorTransfer) { + // TODO(b/239174610): Add a way to determine profiles for DV and HDR10+. + switch (mimeType) { + case MimeTypes.VIDEO_VP9: + if (colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084) { + // Profiles support both HLG and PQ. + return ImmutableList.of( + MediaCodecInfo.CodecProfileLevel.VP9Profile2HDR, + MediaCodecInfo.CodecProfileLevel.VP9Profile3HDR); + } + break; + case MimeTypes.VIDEO_H264: + if (colorTransfer == C.COLOR_TRANSFER_HLG) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AVCProfileHigh10); + } + break; + case MimeTypes.VIDEO_H265: + if (colorTransfer == C.COLOR_TRANSFER_HLG) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10); + } else if (colorTransfer == C.COLOR_TRANSFER_ST2084) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10HDR10); + } + break; + case MimeTypes.VIDEO_AV1: + if (colorTransfer == C.COLOR_TRANSFER_HLG) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10); + } else if (colorTransfer == C.COLOR_TRANSFER_ST2084) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10); + } + break; + default: + break; + } + // There are no profiles defined for the HDR format, or it's invalid. + return ImmutableList.of(); + } + /** Returns whether the {@linkplain MediaCodecInfo encoder} supports the given resolution. */ public static boolean isSizeSupported( MediaCodecInfo encoderInfo, String mimeType, int width, int height) { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index ecc7eaa076e..68eb5445372 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -17,25 +17,20 @@ package com.google.android.exoplayer2.transformer; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; -import static com.google.android.exoplayer2.util.Assertions.checkState; import android.content.Context; import android.media.MediaCodec; -import android.media.MediaCodecInfo; import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; -import com.google.android.exoplayer2.util.Log; -import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.ColorInfo; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; @@ -105,17 +100,6 @@ public VideoTranscodingSamplePipeline( transformationRequest, fallbackListener); - boolean enableRequestSdrToneMapping = transformationRequest.enableRequestSdrToneMapping; - // TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly - // processed as SDR, because the inputFormat.colorInfo reports the wrong value. - boolean useHdr = - transformationRequest.enableHdrEditing && ColorInfo.isHdr(inputFormat.colorInfo); - if (useHdr && !encoderWrapper.supportsHdr()) { - useHdr = false; - enableRequestSdrToneMapping = true; - encoderWrapper.signalFallbackToSdr(); - } - try { frameProcessor = GlEffectsFrameProcessor.create( @@ -153,7 +137,7 @@ public void onFrameProcessingEnded() { // HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This // implies that the OpenGL EXT_YUV_target extension is supported and hence the // GlEffectsFrameProcessor also supports HDR. - useHdr); + /* useHdr= */ encoderWrapper.isHdrEditingEnabled()); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessingException( e, TransformationException.ERROR_CODE_GL_INIT_FAILED); @@ -161,9 +145,11 @@ public void onFrameProcessingEnded() { frameProcessor.setInputFrameInfo( new FrameInfo(decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio)); + boolean isToneMappingRequired = + ColorInfo.isHdr(inputFormat.colorInfo) && !encoderWrapper.isHdrEditingEnabled(); decoder = decoderFactory.createForVideoDecoding( - inputFormat, frameProcessor.getInputSurface(), enableRequestSdrToneMapping); + inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired); // TODO(b/236316454): Check in the decoder output format whether tone-mapping was actually // applied and throw an exception if not. maxPendingFrameCount = decoder.getMaxPendingFrameCount(); @@ -331,14 +317,14 @@ private boolean isDecodeOnlyBuffer(long presentationTimeUs) { private final List allowedOutputMimeTypes; private final TransformationRequest transformationRequest; private final FallbackListener fallbackListener; - private final HashSet hdrMediaCodecNames; + private final String requestedOutputMimeType; + private final ImmutableList supportedEncoderNamesForHdrEditing; private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo; private volatile @MonotonicNonNull Codec encoder; private volatile int outputRotationDegrees; private volatile boolean releaseEncoder; - private boolean fallbackToSdr; public EncoderWrapper( Codec.EncoderFactory encoderFactory, @@ -346,14 +332,26 @@ public EncoderWrapper( List allowedOutputMimeTypes, TransformationRequest transformationRequest, FallbackListener fallbackListener) { - this.encoderFactory = encoderFactory; this.inputFormat = inputFormat; this.allowedOutputMimeTypes = allowedOutputMimeTypes; this.transformationRequest = transformationRequest; this.fallbackListener = fallbackListener; - hdrMediaCodecNames = new HashSet<>(); + requestedOutputMimeType = + transformationRequest.videoMimeType != null + ? transformationRequest.videoMimeType + : checkNotNull(inputFormat.sampleMimeType); + supportedEncoderNamesForHdrEditing = + EncoderUtil.getSupportedEncoderNamesForHdrEditing( + requestedOutputMimeType, inputFormat.colorInfo); + } + + /** Returns whether the wrapped encoder is expecting HDR input for the HDR editing use case. */ + public boolean isHdrEditingEnabled() { + return transformationRequest.enableHdrEditing + && !transformationRequest.enableRequestSdrToneMapping + && !supportedEncoderNamesForHdrEditing.isEmpty(); } @Nullable @@ -378,37 +376,39 @@ public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight) outputRotationDegrees = 90; } + boolean isInputToneMapped = ColorInfo.isHdr(inputFormat.colorInfo) && !isHdrEditingEnabled(); Format requestedEncoderFormat = new Format.Builder() .setWidth(requestedWidth) .setHeight(requestedHeight) .setRotationDegrees(0) .setFrameRate(inputFormat.frameRate) - .setSampleMimeType( - transformationRequest.videoMimeType != null - ? transformationRequest.videoMimeType - : inputFormat.sampleMimeType) - .setColorInfo(fallbackToSdr ? null : inputFormat.colorInfo) + .setSampleMimeType(requestedOutputMimeType) + .setColorInfo(isInputToneMapped ? null : inputFormat.colorInfo) .build(); encoder = encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); - if (!hdrMediaCodecNames.isEmpty() && !hdrMediaCodecNames.contains(encoder.getName())) { - Log.d( - TAG, - "Selected encoder " - + encoder.getName() - + " does not report sufficient HDR capabilities"); - } Format encoderSupportedFormat = encoder.getConfigurationFormat(); + if (isHdrEditingEnabled()) { + if (!requestedOutputMimeType.equals(encoderSupportedFormat.sampleMimeType)) { + throw createEncodingException( + new IllegalStateException("MIME type fallback unsupported with HDR editing"), + encoderSupportedFormat); + } else if (!supportedEncoderNamesForHdrEditing.contains(encoder.getName())) { + throw createEncodingException( + new IllegalStateException("Selected encoder doesn't support HDR editing"), + encoderSupportedFormat); + } + } fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( transformationRequest, /* hasOutputFormatRotation= */ flipOrientation, requestedEncoderFormat, encoderSupportedFormat, - fallbackToSdr)); + isInputToneMapped)); encoderSurfaceInfo = new SurfaceInfo( @@ -468,41 +468,14 @@ public void release() { releaseEncoder = true; } - /** - * Checks whether at least one MediaCodec encoder on the device has sufficient capabilities to - * encode HDR (only checks support for HLG at this time). - */ - public boolean supportsHdr() { - if (Util.SDK_INT < 31) { - return false; - } - - // The only output MIME type that Transformer currently supports that can be used with HDR - // is H265/HEVC. So we assume that the EncoderFactory will pick this if HDR is requested. - String mimeType = MimeTypes.VIDEO_H265; - - List mediaCodecInfos = EncoderSelector.DEFAULT.selectEncoderInfos(mimeType); - for (int i = 0; i < mediaCodecInfos.size(); i++) { - MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i); - if (EncoderUtil.isFeatureSupported( - mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) { - for (MediaCodecInfo.CodecProfileLevel capabilities : - mediaCodecInfo.getCapabilitiesForType(MimeTypes.VIDEO_H265).profileLevels) { - // TODO(b/227624622): What profile to check depends on the HDR format. Once other - // formats besides HLG are supported, check the corresponding profiles here. - if (capabilities.profile == MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10) { - return hdrMediaCodecNames.add(mediaCodecInfo.getCanonicalName()); - } - } - } - } - return !hdrMediaCodecNames.isEmpty(); - } - - public void signalFallbackToSdr() { - checkState(encoder == null, "Fallback to SDR is only allowed before encoder initialization"); - fallbackToSdr = true; - hdrMediaCodecNames.clear(); + private TransformationException createEncodingException(Exception cause, Format format) { + return TransformationException.createForCodec( + cause, + /* isVideo= */ true, + /* isDecoder= */ false, + format, + checkNotNull(encoder).getName(), + TransformationException.ERROR_CODE_ENCODING_FAILED); } } } diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java index ba1ff80e406..4d4bd5a78f9 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java @@ -27,6 +27,7 @@ import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.util.Clock; import com.google.android.exoplayer2.util.ListenerSet; +import com.google.android.exoplayer2.util.MimeTypes; import com.google.common.collect.ImmutableList; import java.util.List; import org.junit.Before; @@ -47,7 +48,7 @@ public final class VideoEncoderWrapperTest { private final VideoTranscodingSamplePipeline.EncoderWrapper encoderWrapper = new VideoTranscodingSamplePipeline.EncoderWrapper( fakeEncoderFactory, - /* inputFormat= */ new Format.Builder().build(), + /* inputFormat= */ new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H265).build(), /* allowedOutputMimeTypes= */ ImmutableList.of(), emptyTransformationRequest, fallbackListener);