Skip to content
Permalink
Browse files
Add support for WebCodecs encoder bitrate related parameters
https://bugs.webkit.org/show_bug.cgi?id=246399
rdar://problem/101075781

Reviewed by Eric Carlson.

Pass parameters received from JS to VPx and remote encoders.

* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp:
* Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp:
* Source/WebCore/Configurations/WebCore.xcconfig:
* Source/WebCore/Modules/webcodecs/WebCodecsVideoEncoder.cpp:
(WebCore::WebCodecsVideoEncoder::configure):
* Source/WebCore/platform/LibWebRTCVPXVideoEncoder.cpp:
(WebCore::LibWebRTCVPXInternalVideoEncoder::LibWebRTCVPXInternalVideoEncoder):
* Source/WebCore/platform/VideoEncoder.h:
* Source/WebKit/WebProcess/GPU/media/RemoteVideoCodecFactory.cpp:
(WebKit::RemoteVideoCodecFactory::createEncoder):
(WebKit::RemoteVideoEncoder::RemoteVideoEncoder):
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp:
(WebKit::LibWebRTCCodecs::createEncoder):
(WebKit::LibWebRTCCodecs::createEncoderAndWaitUntilReady):
(WebKit::LibWebRTCCodecs::createEncoderInternal):
(WebKit::LibWebRTCCodecs::gpuProcessConnectionDidClose):
* Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h:

Canonical link: https://commits.webkit.org/255476@main
  • Loading branch information
youennf committed Oct 13, 2022
1 parent 6692b0d commit 4c1cec83cdb9501fb0048965b2dee53b10730d49
Show file tree
Hide file tree
Showing 10 changed files with 34 additions and 11 deletions.
@@ -352,3 +352,7 @@ __ZTVN6webrtc20DecodedImageCallbackE
__ZN6webrtc10VP8Encoder6CreateEv
__ZN6webrtc10VP9Encoder6CreateEv
__ZN6webrtc10VideoCodecC1Ev
__ZN6webrtc12VideoEncoder21RateControlParametersC1ERKNS_22VideoBitrateAllocationEd
__ZN6webrtc12VideoEncoder21RateControlParametersD1Ev
__ZN6webrtc22VideoBitrateAllocation10SetBitrateEmmj
__ZN6webrtc22VideoBitrateAllocationC1Ev
@@ -352,3 +352,7 @@ __ZTVN6webrtc20DecodedImageCallbackE
__ZN6webrtc10VP8Encoder6CreateEv
__ZN6webrtc10VP9Encoder6CreateEv
__ZN6webrtc10VideoCodecC1Ev
__ZN6webrtc12VideoEncoder21RateControlParametersC1ERKNS_22VideoBitrateAllocationEd
__ZN6webrtc12VideoEncoder21RateControlParametersD1Ev
__ZN6webrtc22VideoBitrateAllocation10SetBitrateEmmj
__ZN6webrtc22VideoBitrateAllocationC1Ev
@@ -352,3 +352,7 @@ __ZTVN6webrtc20DecodedImageCallbackE
__ZN6webrtc10VP8Encoder6CreateEv
__ZN6webrtc10VP9Encoder6CreateEv
__ZN6webrtc10VideoCodecC1Ev
__ZN6webrtc12VideoEncoder21RateControlParametersC1ERKNS_22VideoBitrateAllocationEd
__ZN6webrtc12VideoEncoder21RateControlParametersD1Ev
__ZN6webrtc22VideoBitrateAllocation10SetBitrateEmmj
__ZN6webrtc22VideoBitrateAllocationC1Ev
@@ -159,7 +159,7 @@ WK_IOSURFACE_ACCELERATOR_LDFLAGS_watchos = $(WK_IOSURFACE_ACCELERATOR_LDFLAGS_ip
WK_IOSURFACE_ACCELERATOR_LDFLAGS_appletvos = $(WK_IOSURFACE_ACCELERATOR_LDFLAGS_iphoneos);

WK_LIBWEBRTC_LDFLAGS = $(WK_LIBWEBRTC_LDFLAGS_$(ENABLE_WEB_RTC));
WK_LIBWEBRTC_LDFLAGS_ENABLE_WEB_RTC = -framework VideoToolbox -framework CoreMedia -weak-lwebrtc -Xlinker --no-demangle;
WK_LIBWEBRTC_LDFLAGS_ENABLE_WEB_RTC = -framework VideoToolbox -framework CoreMedia -weak-lwebrtc;

WK_MOBILE_CORE_SERVICES_LDFLAGS = $(WK_MOBILE_CORE_SERVICES_LDFLAGS_$(WK_COCOA_TOUCH));
WK_MOBILE_CORE_SERVICES_LDFLAGS_cocoatouch = -framework MobileCoreServices;
@@ -102,7 +102,7 @@ ExceptionOr<void> WebCodecsVideoEncoder::configure(WebCodecsVideoEncoderConfig&&
};
}
bool useAnnexB = config.avc && config.avc->format == AvcBitstreamFormat::Annexb;
VideoEncoder::create(config.codec, { config.width, config.height, useAnnexB }, [this, weakThis = WeakPtr { *this }](auto&& result) {
VideoEncoder::create(config.codec, { config.width, config.height, useAnnexB, config.bitrate.value_or(0), config.framerate.value_or(0), config.latencyMode == LatencyMode::Realtime }, [this, weakThis = WeakPtr { *this }](auto&& result) {
if (!weakThis)
return;

@@ -147,6 +147,11 @@ LibWebRTCVPXInternalVideoEncoder::LibWebRTCVPXInternalVideoEncoder(LibWebRTCVPXV
, m_postTaskCallback(WTFMove(postTaskCallback))
, m_internalEncoder(createInternalEncoder(type))
{
if (config.bitRate) {
webrtc::VideoBitrateAllocation allocation;
allocation.SetBitrate(0, 0, config.bitRate);
m_internalEncoder->SetRates({ allocation, config.frameRate });
}
m_internalEncoder->RegisterEncodeCompleteCallback(this);

// FIXME: Check InitEncode result.
@@ -42,6 +42,9 @@ class VideoEncoder {
uint64_t width { 0 };
uint64_t height { 0 };
bool useAnnexB { false };
uint64_t bitRate { 0 };
double frameRate { 0 };
bool isRealtime { true };
};
struct ActiveConfiguration {
String codec;
@@ -151,7 +151,7 @@ void RemoteVideoCodecFactory::createEncoder(const String& codec, const WebCore::
VideoEncoder::createLocalEncoder(codec, config, WTFMove(createCallback), WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postTaskCallback));
return;
}
WebProcess::singleton().libWebRTCCodecs().createEncoderAndWaitUntilReady(*type, { }, config.useAnnexB, [config, createCallback = WTFMove(createCallback), descriptionCallback = WTFMove(descriptionCallback), outputCallback = WTFMove(outputCallback), postTaskCallback = WTFMove(postTaskCallback)](auto& internalEncoder) mutable {
WebProcess::singleton().libWebRTCCodecs().createEncoderAndWaitUntilReady(*type, { }, config.isRealtime, config.useAnnexB, [config, createCallback = WTFMove(createCallback), descriptionCallback = WTFMove(descriptionCallback), outputCallback = WTFMove(outputCallback), postTaskCallback = WTFMove(postTaskCallback)](auto& internalEncoder) mutable {
auto callbacks = RemoteVideoEncoderCallbacks::create(WTFMove(descriptionCallback), WTFMove(outputCallback), WTFMove(postTaskCallback));
UniqueRef<VideoEncoder> encoder = makeUniqueRef<RemoteVideoEncoder>(internalEncoder, config, callbacks.copyRef());
callbacks->postTask([createCallback = WTFMove(createCallback), encoder = WTFMove(encoder)]() mutable {
@@ -225,6 +225,7 @@ RemoteVideoEncoder::RemoteVideoEncoder(LibWebRTCCodecs::Encoder& encoder, const
, m_callbacks(WTFMove(callbacks))
{
WebProcess::singleton().libWebRTCCodecs().initializeEncoder(m_internalEncoder, config.width, config.height, 0, 0, 0, 0);
WebProcess::singleton().libWebRTCCodecs().setEncodeRates(m_internalEncoder, config.bitRate, config.frameRate);

WebProcess::singleton().libWebRTCCodecs().registerEncodedVideoFrameCallback(m_internalEncoder, [callbacks = m_callbacks](auto&& encodedChunk, bool isKeyFrame, auto timestamp) {
callbacks->notifyEncodedChunk(Vector<uint8_t> { encodedChunk }, isKeyFrame, timestamp);
@@ -456,21 +456,22 @@ static inline webrtc::VideoCodecType toWebRTCCodecType(VideoCodecType type)

LibWebRTCCodecs::Encoder* LibWebRTCCodecs::createEncoder(VideoCodecType type, const std::map<std::string, std::string>& parameters)
{
return createEncoderInternal(type, parameters, true, [](auto&) { });
return createEncoderInternal(type, parameters, true, true, [](auto&) { });
}

void LibWebRTCCodecs::createEncoderAndWaitUntilReady(VideoCodecType type, const std::map<std::string, std::string>& parameters, bool useAnnexB, Function<void(Encoder&)>&& callback)
void LibWebRTCCodecs::createEncoderAndWaitUntilReady(VideoCodecType type, const std::map<std::string, std::string>& parameters, bool useAnnexB, bool isRealtime, Function<void(Encoder&)>&& callback)
{
createEncoderInternal(type, parameters, useAnnexB, WTFMove(callback));
createEncoderInternal(type, parameters, useAnnexB, isRealtime, WTFMove(callback));
}

LibWebRTCCodecs::Encoder* LibWebRTCCodecs::createEncoderInternal(VideoCodecType type, const std::map<std::string, std::string>& formatParameters, bool useAnnexB, Function<void(Encoder&)>&& callback)
LibWebRTCCodecs::Encoder* LibWebRTCCodecs::createEncoderInternal(VideoCodecType type, const std::map<std::string, std::string>& formatParameters, bool useAnnexB, bool isRealtime, Function<void(Encoder&)>&& callback)
{
auto encoder = makeUnique<Encoder>();
auto* result = encoder.get();
encoder->identifier = VideoEncoderIdentifier::generateThreadSafe();
encoder->type = type;
encoder->useAnnexB = useAnnexB;
encoder->isRealtime = isRealtime;

auto parameters = WTF::map(formatParameters, [](auto& entry) {
return std::pair { String::fromUTF8(entry.first.data(), entry.first.length()), String::fromUTF8(entry.second.data(), entry.second.length()) };
@@ -488,7 +489,7 @@ LibWebRTCCodecs::Encoder* LibWebRTCCodecs::createEncoderInternal(VideoCodecType

{
Locker locker { m_encodersConnectionLock };
connection->send(Messages::LibWebRTCCodecsProxy::CreateEncoder { encoder->identifier, encoder->type, parameters, DeprecatedGlobalSettings::webRTCH264LowLatencyEncoderEnabled(), encoder->useAnnexB }, 0);
connection->send(Messages::LibWebRTCCodecsProxy::CreateEncoder { encoder->identifier, encoder->type, parameters, encoder->isRealtime, encoder->useAnnexB }, 0);
setEncoderConnection(*encoder, connection.ptr());
}

@@ -708,7 +709,7 @@ void LibWebRTCCodecs::gpuProcessConnectionDidClose(GPUProcessConnection&)

Locker locker { m_encodersConnectionLock };
for (auto& encoder : m_encoders.values()) {
connection->send(Messages::LibWebRTCCodecsProxy::CreateEncoder { encoder->identifier, encoder->type, encoder->parameters, DeprecatedGlobalSettings::webRTCH264LowLatencyEncoderEnabled(), encoder->useAnnexB }, 0);
connection->send(Messages::LibWebRTCCodecsProxy::CreateEncoder { encoder->identifier, encoder->type, encoder->parameters, encoder->isRealtime, encoder->useAnnexB }, 0);
if (encoder->initializationData)
connection->send(Messages::LibWebRTCCodecsProxy::InitializeEncoder { encoder->identifier, encoder->initializationData->width, encoder->initializationData->height, encoder->initializationData->startBitRate, encoder->initializationData->maxBitRate, encoder->initializationData->minBitRate, encoder->initializationData->maxFrameRate }, 0);
setEncoderConnection(*encoder, connection.get());
@@ -121,10 +121,11 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC
SharedVideoFrameWriter sharedVideoFrameWriter;
bool hasSentInitialEncodeRates { false };
bool useAnnexB { true };
bool isRealtime { true };
};

Encoder* createEncoder(VideoCodecType, const std::map<std::string, std::string>&);
void createEncoderAndWaitUntilReady(VideoCodecType, const std::map<std::string, std::string>&, bool useAnnexB, Function<void(Encoder&)>&&);
void createEncoderAndWaitUntilReady(VideoCodecType, const std::map<std::string, std::string>&, bool useAnnexB, bool isRealtime, Function<void(Encoder&)>&&);
int32_t releaseEncoder(Encoder&);
int32_t initializeEncoder(Encoder&, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate);
int32_t encodeFrame(Encoder&, const WebCore::VideoFrame&, uint32_t timestamp, bool shouldEncodeAsKeyFrame);
@@ -169,7 +170,7 @@ class LibWebRTCCodecs : public IPC::WorkQueueMessageReceiver, public GPUProcessC
WorkQueue& workQueue() const { return m_queue; }

Decoder* createDecoderInternal(VideoCodecType, Function<void(Decoder&)>&&);
Encoder* createEncoderInternal(VideoCodecType, const std::map<std::string, std::string>&, bool useAnnexB, Function<void(Encoder&)>&&);
Encoder* createEncoderInternal(VideoCodecType, const std::map<std::string, std::string>&, bool useAnnexB, bool isRealtime, Function<void(Encoder&)>&&);
template<typename Frame> int32_t encodeFrameInternal(Encoder&, const Frame&, bool shouldEncodeAsKeyFrame, WebCore::VideoFrame::Rotation, MediaTime, uint32_t);

private:

0 comments on commit 4c1cec8

Please sign in to comment.