diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 9d82264d79..2e29916261 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -232,6 +232,7 @@ if (is_android) { "api/org/webrtc/WrappedNativeVideoEncoder.java", "api/org/webrtc/YuvConverter.java", "api/org/webrtc/YuvHelper.java", + "api/org/webrtc/ResolutionAdjustment.java", "src/java/org/webrtc/EglBase10Impl.java", "src/java/org/webrtc/EglBase14Impl.java", "src/java/org/webrtc/GlGenericDrawer.java", @@ -363,6 +364,8 @@ if (is_android) { "api/org/webrtc/DefaultVideoDecoderFactory.java", "api/org/webrtc/DefaultVideoEncoderFactory.java", "api/org/webrtc/WrappedVideoDecoderFactory.java", + "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java", + "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java", ] deps = [ @@ -394,6 +397,8 @@ if (is_android) { sources = [ "api/org/webrtc/HardwareVideoDecoderFactory.java", "api/org/webrtc/HardwareVideoEncoderFactory.java", + "api/org/webrtc/HardwareVideoEncoderWrapper.java", + "api/org/webrtc/HardwareVideoEncoderWrapperFactory.java", "api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java", "src/java/org/webrtc/AndroidVideoDecoder.java", "src/java/org/webrtc/BaseBitrateAdjuster.java", diff --git a/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java new file mode 100644 index 0000000000..65720ef6a0 --- /dev/null +++ b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +import java.util.Arrays; +import java.util.LinkedHashSet; + +/** + * The main difference with the standard [DefaultAlignedVideoEncoderFactory] is that this fixes + * issues with resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can + * set the alignment by setting [resolutionAdjustment]. Internally the resolution during streaming + * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the + * standard [DefaultVideoEncoderFactory] and it will use the SW encoder if HW fails + * or is not available. + * + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072 + * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnco + * derFactoryWrapper.kt#L18 + */ +public class DefaultAlignedVideoEncoderFactory implements VideoEncoderFactory { + private final VideoEncoderFactory hardwareVideoEncoderFactory; + private final VideoEncoderFactory softwareVideoEncoderFactory; + + public DefaultAlignedVideoEncoderFactory( + EglBase.Context eglContext, + boolean enableIntelVp8Encoder, + boolean enableH264HighProfile, + ResolutionAdjustment resolutionAdjustment + ) { + HardwareVideoEncoderFactory defaultFactory = + new HardwareVideoEncoderFactory(eglContext, enableIntelVp8Encoder, enableH264HighProfile); + hardwareVideoEncoderFactory = (resolutionAdjustment == ResolutionAdjustment.NONE) ? + defaultFactory : + new HardwareVideoEncoderWrapperFactory(defaultFactory, resolutionAdjustment.getValue()); + softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory(); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + VideoEncoder softwareEncoder = softwareVideoEncoderFactory.createEncoder(info); + VideoEncoder hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info); + if (hardwareEncoder != null && softwareEncoder != null) { + return new VideoEncoderFallback(softwareEncoder, hardwareEncoder); + } + return hardwareEncoder != null ? hardwareEncoder : softwareEncoder; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + LinkedHashSet supportedCodecInfos = new LinkedHashSet<>(); + supportedCodecInfos.addAll(Arrays.asList(softwareVideoEncoderFactory.getSupportedCodecs())); + supportedCodecInfos.addAll(Arrays.asList(hardwareVideoEncoderFactory.getSupportedCodecs())); + return supportedCodecInfos.toArray(new VideoCodecInfo[0]); + } +} + diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java new file mode 100644 index 0000000000..ac3b0c4684 --- /dev/null +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207 + * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco + * derWrapperFactory.kt + */ +class HardwareVideoEncoderWrapper implements VideoEncoder { + + private static final String TAG = "HardwareVideoEncoderWrapper"; + + private final VideoEncoder internalEncoder; + private final int alignment; + + public HardwareVideoEncoderWrapper(VideoEncoder internalEncoder, int alignment) { + this.internalEncoder = internalEncoder; + this.alignment = alignment; + } + + private static class CropSizeCalculator { + + private static final String TAG = "CropSizeCalculator"; + + private final int alignment; + private final int originalWidth; + private final int originalHeight; + private final int cropX; + private final int cropY; + + public CropSizeCalculator(int alignment, int originalWidth, int originalHeight) { + this.alignment = alignment; + this.originalWidth = originalWidth; + this.originalHeight = originalHeight; + this.cropX = originalWidth % alignment; + this.cropY = originalHeight % alignment; + if (originalWidth != 0 && originalHeight != 0) { + Logging.v(TAG, "init(): alignment=" + alignment + + " size=" + originalWidth + "x" + originalHeight + " => " + getCroppedWidth() + "x" + getCroppedHeight()); + } + } + + public int getCroppedWidth() { + return originalWidth - cropX; + } + + public int getCroppedHeight() { + return originalHeight - cropY; + } + + public boolean isCropRequired() { + return cropX != 0 || cropY != 0; + } + + public boolean hasFrameSizeChanged(int nextWidth, int nextHeight) { + if (originalWidth == nextWidth && originalHeight == nextHeight) { + return false; + } else { + Logging.v(TAG, "frame size has changed: " + + originalWidth + "x" + originalHeight + " => " + nextWidth + "x" + nextHeight); + return true; + } + } + } + + private CropSizeCalculator calculator = new CropSizeCalculator(1, 0, 0); + + private VideoCodecStatus retryWithoutCropping(int width, int height, Runnable retryFunc) { + Logging.v(TAG, "retrying without resolution adjustment"); + calculator = new CropSizeCalculator(1, width, height); + retryFunc.run(); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus initEncode(VideoEncoder.Settings originalSettings, VideoEncoder.Callback callback) { + calculator = new CropSizeCalculator(alignment, originalSettings.width, originalSettings.height); + if (!calculator.isCropRequired()) { + return internalEncoder.initEncode(originalSettings, callback); + } else { + VideoEncoder.Settings croppedSettings = new VideoEncoder.Settings( + originalSettings.numberOfCores, + calculator.getCroppedWidth(), + calculator.getCroppedHeight(), + originalSettings.startBitrate, + originalSettings.maxFramerate, + originalSettings.numberOfSimulcastStreams, + originalSettings.automaticResizeOn, + originalSettings.capabilities + ); + try { + VideoCodecStatus result = internalEncoder.initEncode(croppedSettings, callback); + if (result == VideoCodecStatus.FALLBACK_SOFTWARE) { + Logging.e(TAG, "internalEncoder.initEncode() returned FALLBACK_SOFTWARE: " + + "croppedSettings " + croppedSettings); + return retryWithoutCropping( + originalSettings.width, + originalSettings.height, + () -> internalEncoder.initEncode(originalSettings, callback) + ); + } else { + return result; + } + } catch (Exception e) { + Logging.e(TAG, "internalEncoder.initEncode() failed", e); + return retryWithoutCropping( + originalSettings.width, + originalSettings.height, + () -> internalEncoder.initEncode(originalSettings, callback) + ); + } + } + } + + @Override + public VideoCodecStatus release() { + return internalEncoder.release(); + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, VideoEncoder.EncodeInfo encodeInfo) { + if (calculator.hasFrameSizeChanged(frame.getBuffer().getWidth(), frame.getBuffer().getHeight())) { + calculator = new CropSizeCalculator(alignment, frame.getBuffer().getWidth(), frame.getBuffer().getHeight()); + } + if (!calculator.isCropRequired()) { + return internalEncoder.encode(frame, encodeInfo); + } else { + int croppedWidth = calculator.getCroppedWidth(); + int croppedHeight = calculator.getCroppedHeight(); + VideoFrame.Buffer croppedBuffer = frame.getBuffer().cropAndScale( + calculator.cropX / 2, + calculator.cropY / 2, + croppedWidth, + croppedHeight, + croppedWidth, + croppedHeight + ); + VideoFrame croppedFrame = new VideoFrame(croppedBuffer, frame.getRotation(), frame.getTimestampNs()); + try { + VideoCodecStatus result = internalEncoder.encode(croppedFrame, encodeInfo); + if (result == VideoCodecStatus.FALLBACK_SOFTWARE) { + Logging.e(TAG, "internalEncoder.encode() returned FALLBACK_SOFTWARE"); + return retryWithoutCropping( + frame.getBuffer().getWidth(), + frame.getBuffer().getHeight(), + () -> internalEncoder.encode(frame, encodeInfo) + ); + } else { + return result; + } + } catch (Exception e) { + Logging.e(TAG, "internalEncoder.encode() failed", e); + return retryWithoutCropping( + frame.getBuffer().getWidth(), + frame.getBuffer().getHeight(), + () -> internalEncoder.encode(frame, encodeInfo) + ); + } finally { + croppedBuffer.release(); + } + } + } + + @Override + public VideoCodecStatus setRateAllocation(VideoEncoder.BitrateAllocation allocation, int frameRate) { + return internalEncoder.setRateAllocation(allocation, frameRate); + } + + @Override + public VideoEncoder.ScalingSettings getScalingSettings() { + return internalEncoder.getScalingSettings(); + } + + @Override + public String getImplementationName() { + return internalEncoder.getImplementationName(); + } + + @Override + public long createNativeVideoEncoder() { + return internalEncoder.createNativeVideoEncoder(); + } + + @Override + public boolean isHardwareEncoder() { + return internalEncoder.isHardwareEncoder(); + } + + @Override + public VideoCodecStatus setRates(VideoEncoder.RateControlParameters rcParameters) { + return internalEncoder.setRates(rcParameters); + } + + @Override + public VideoEncoder.ResolutionBitrateLimits[] getResolutionBitrateLimits() { + return internalEncoder.getResolutionBitrateLimits(); + } + + @Override + public VideoEncoder.EncoderInfo getEncoderInfo() { + return internalEncoder.getEncoderInfo(); + } +} + + diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java new file mode 100644 index 0000000000..2b5b1c8f7f --- /dev/null +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207 + * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco + * derWrapperFactory.kt + */ +class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory { + + private static final String TAG = "HardwareVideoEncoderWrapperFactory"; + + private final HardwareVideoEncoderFactory factory; + private final int resolutionPixelAlignment; + + public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) { + this.factory = factory; + this.resolutionPixelAlignment = resolutionPixelAlignment; + if (resolutionPixelAlignment == 0) { + throw new IllegalArgumentException("resolutionPixelAlignment should not be 0"); + } + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) { + try { + VideoEncoder encoder = factory.createEncoder(videoCodecInfo); + if (encoder == null) { + return null; + } + return new HardwareVideoEncoderWrapper(encoder, resolutionPixelAlignment); + } catch (Exception e) { + Logging.e(TAG, "createEncoder failed", e); + return null; + } + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return factory.getSupportedCodecs(); + } +} \ No newline at end of file diff --git a/sdk/android/api/org/webrtc/ResolutionAdjustment.java b/sdk/android/api/org/webrtc/ResolutionAdjustment.java new file mode 100644 index 0000000000..0ed741db74 --- /dev/null +++ b/sdk/android/api/org/webrtc/ResolutionAdjustment.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Resolution alignment values. Generally the MULTIPLE_OF_16 is recommended + * for both VP8 and H264 + */ +public enum ResolutionAdjustment { + NONE(1), + MULTIPLE_OF_2(2), + MULTIPLE_OF_4(4), + MULTIPLE_OF_8(8), + MULTIPLE_OF_16(16); + + private final int value; + + private ResolutionAdjustment(int value) { + this.value = value; + } + + public int getValue() { + return value; + } +} + diff --git a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java new file mode 100644 index 0000000000..ab9faaa0c4 --- /dev/null +++ b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * The main difference with the standard SimulcastVideoEncoderFactory is that this fixes issues + * with simulcasting resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can + * set the alignment by setting resolutionAdjustment. Internally the resolutions during simulcast + * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the + * standard SimulcastVideoEncoderFactory and it will use the SW encoder if HW fails + * or is not available. + * + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072 + * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnc + * oderFactoryWrapper.kt#L18 + */ +public class SimulcastAlignedVideoEncoderFactory implements VideoEncoderFactory { + private static class StreamEncoderWrapper implements VideoEncoder { + private static final String TAG = "StreamEncoderWrapper"; + private final VideoEncoder encoder; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private VideoEncoder.Settings streamSettings; + + public StreamEncoderWrapper(VideoEncoder encoder) { + this.encoder = encoder; + } + + @Override + public VideoCodecStatus initEncode(VideoEncoder.Settings settings, VideoEncoder.Callback callback) { + streamSettings = settings; + Callable callable = () -> { + Logging.v(TAG, "initEncode() thread=" + Thread.currentThread().getName() + " [" + Thread.currentThread().getId() + "]"); + Logging.v(TAG, " encoder=" + encoder.getImplementationName()); + Logging.v(TAG, " streamSettings:"); + Logging.v(TAG, " numberOfCores=" + settings.numberOfCores); + Logging.v(TAG, " width=" + settings.width); + Logging.v(TAG, " height=" + settings.height); + Logging.v(TAG, " startBitrate=" + settings.startBitrate); + Logging.v(TAG, " maxFramerate=" + settings.maxFramerate); + Logging.v(TAG, " automaticResizeOn=" + settings.automaticResizeOn); + Logging.v(TAG, " numberOfSimulcastStreams=" + settings.numberOfSimulcastStreams); + Logging.v(TAG, " lossNotification=" + settings.capabilities.lossNotification); + return encoder.initEncode(settings, callback); + }; + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus release() { + Callable callable = () -> encoder.release(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, VideoEncoder.EncodeInfo encodeInfo) { + Callable callable = () -> { + if (streamSettings != null) { + if (frame.getBuffer().getWidth() == streamSettings.width) { + return encoder.encode(frame, encodeInfo); + } else { + int originalWidth = frame.getBuffer().getWidth(); + int originalHeight = frame.getBuffer().getHeight(); + VideoFrame.Buffer scaledBuffer = frame.getBuffer().cropAndScale( + 0, 0, originalWidth, originalHeight, + streamSettings.width, streamSettings.height + ); + VideoFrame scaledFrame = new VideoFrame(scaledBuffer, frame.getRotation(), frame.getTimestampNs()); + VideoCodecStatus result = encoder.encode(scaledFrame, encodeInfo); + scaledBuffer.release(); + return result; + } + } else { + return VideoCodecStatus.ERROR; + } + }; + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus setRateAllocation(VideoEncoder.BitrateAllocation allocation, int frameRate) { + Callable callable = () -> encoder.setRateAllocation(allocation, frameRate); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoEncoder.ScalingSettings getScalingSettings() { + Callable callable = () -> encoder.getScalingSettings(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return null; + } + } + + @Override + public String getImplementationName() { + Callable callable = () -> encoder.getImplementationName(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return null; + } + } + } + + private static class StreamEncoderWrapperFactory implements VideoEncoderFactory { + private final VideoEncoderFactory factory; + + public StreamEncoderWrapperFactory(VideoEncoderFactory factory) { + this.factory = factory; + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) { + VideoEncoder encoder = factory.createEncoder(videoCodecInfo); + if (encoder == null) { + return null; + } + return new StreamEncoderWrapper(encoder); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return factory.getSupportedCodecs(); + } + } + + private final VideoEncoderFactory primary; + private final VideoEncoderFactory fallback; + private final SimulcastVideoEncoderFactory delegate; + + public SimulcastAlignedVideoEncoderFactory(EglBase.Context sharedContext, boolean enableIntelVp8Encoder, boolean enableH264HighProfile, ResolutionAdjustment resolutionAdjustment) { + HardwareVideoEncoderFactory hardwareVideoEncoderFactory = new HardwareVideoEncoderFactory(sharedContext, enableIntelVp8Encoder, enableH264HighProfile); + VideoEncoderFactory encoderFactory; + if (resolutionAdjustment == ResolutionAdjustment.NONE) { + encoderFactory = hardwareVideoEncoderFactory; + } else { + encoderFactory = new HardwareVideoEncoderWrapperFactory(hardwareVideoEncoderFactory, resolutionAdjustment.getValue()); + } + primary = new StreamEncoderWrapperFactory(encoderFactory); + fallback = new SoftwareVideoEncoderFactory(); + delegate = new SimulcastVideoEncoderFactory(primary, fallback); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + return delegate.createEncoder(info); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return delegate.getSupportedCodecs(); + } +} + +