Skip to content

Commit

Permalink
[GStreamer][WebRTC] Basic implementation of rtpSender.setParameters()
Browse files Browse the repository at this point in the history
https://bugs.webkit.org/show_bug.cgi?id=259445

Reviewed by Xabier Rodriguez-Calvar.

This allows JS to set the video maxFrameRate and maxBitrate on the outgoing video tracks. It is not
yet supported for Canvas capture tracks.

* LayoutTests/platform/glib/TestExpectations:
* Source/WebCore/Modules/mediastream/gstreamer/GStreamerPeerConnectionBackend.cpp:
(WebCore::GStreamerPeerConnectionBackend::addTrack):
* Source/WebCore/Modules/mediastream/gstreamer/GStreamerRtpSenderBackend.cpp:
(WebCore::GStreamerRtpSenderBackend::GStreamerRtpSenderBackend):
(WebCore::m_rtcSender):
(WebCore::GStreamerRtpSenderBackend::getParameters const):
(WebCore::validateModifiedParameters):
(WebCore::GStreamerRtpSenderBackend::setParameters):
* Source/WebCore/Modules/mediastream/gstreamer/GStreamerRtpSenderBackend.h:
* Source/WebCore/Modules/mediastream/gstreamer/GStreamerWebRTCUtils.cpp:
(WebCore::toRTCRtpSendParameters):
(WebCore::fromRTCSendParameters):
* Source/WebCore/Modules/mediastream/gstreamer/GStreamerWebRTCUtils.h:
* Source/WebCore/platform/mediastream/gstreamer/RealtimeIncomingVideoSourceGStreamer.cpp:
(WebCore::RealtimeIncomingVideoSourceGStreamer::settings):
(WebCore::RealtimeIncomingVideoSourceGStreamer::settingsDidChange):
(WebCore::RealtimeIncomingVideoSourceGStreamer::dispatchSample):
* Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceGStreamer.cpp:
(WebCore::RealtimeOutgoingAudioSourceGStreamer::setParameters):
* Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceGStreamer.h:
* Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingMediaSourceGStreamer.cpp:
(WebCore::RealtimeOutgoingMediaSourceGStreamer::parameters):
* Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingMediaSourceGStreamer.h:
(WebCore::RealtimeOutgoingMediaSourceGStreamer::fillEncodingParameters):
(WebCore::RealtimeOutgoingMediaSourceGStreamer::setParameters):
* Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingVideoSourceGStreamer.cpp:
(WebCore::RealtimeOutgoingVideoSourceGStreamer::RealtimeOutgoingVideoSourceGStreamer):
(WebCore::RealtimeOutgoingVideoSourceGStreamer::setPayloadType):
(WebCore::RealtimeOutgoingVideoSourceGStreamer::setParameters):
(WebCore::RealtimeOutgoingVideoSourceGStreamer::fillEncodingParameters):
* Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingVideoSourceGStreamer.h:

Canonical link: https://commits.webkit.org/266354@main
  • Loading branch information
philn committed Jul 27, 2023
1 parent 74de0d0 commit 72651b6
Show file tree
Hide file tree
Showing 15 changed files with 277 additions and 24 deletions.
1 change: 0 additions & 1 deletion LayoutTests/platform/glib/TestExpectations
Original file line number Diff line number Diff line change
Expand Up @@ -1788,7 +1788,6 @@ webkit.org/b/235885 webrtc/datachannel/getStats-no-prflx-remote-candidate.html [
webkit.org/b/235885 fast/mediastream/RTCPeerConnection-statsSelector.html [ Skip ]

webrtc/video-av1.html [ Skip ]
webrtc/video-maxFramerate.html [ Failure ]

# GStreamer's DTLS agent currently generates RSA certificates only. DTLS 1.2 is not supported yet (AFAIK).
webrtc/datachannel/dtls10.html [ Failure ]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1110,6 +1110,9 @@ ExceptionOr<GStreamerMediaEndpoint::Backends> GStreamerMediaEndpoint::createTran
}
gst_structure_take_value(initData.get(), "encodings", &encodingsValue);

auto transactionId = createVersion4UUIDString();
gst_structure_set(initData.get(), "transaction-id", G_TYPE_STRING, transactionId.ascii().data(), nullptr);

GRefPtr<GstWebRTCRTPTransceiver> rtcTransceiver;
g_signal_emit_by_name(m_webrtcBin.get(), "add-transceiver", direction, caps.get(), &rtcTransceiver.outPtr());
if (!rtcTransceiver)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ static inline RefPtr<RTCRtpSender> findExistingSender(const Vector<RefPtr<RTCRtp
ExceptionOr<Ref<RTCRtpSender>> GStreamerPeerConnectionBackend::addTrack(MediaStreamTrack& track, FixedVector<String>&& mediaStreamIds)
{
GST_DEBUG_OBJECT(m_endpoint->pipeline(), "Adding new track.");
auto senderBackend = WTF::makeUnique<GStreamerRtpSenderBackend>(*this, nullptr, nullptr);
auto senderBackend = WTF::makeUnique<GStreamerRtpSenderBackend>(*this, nullptr);
if (!m_endpoint->addTrack(*senderBackend, track, mediaStreamIds))
return Exception { TypeError, "Unable to add track"_s };

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,9 @@ static void ensureDebugCategoryIsRegistered()
});
}

GStreamerRtpSenderBackend::GStreamerRtpSenderBackend(GStreamerPeerConnectionBackend& backend, GRefPtr<GstWebRTCRTPSender>&& rtcSender, GUniquePtr<GstStructure>&& initData)
GStreamerRtpSenderBackend::GStreamerRtpSenderBackend(GStreamerPeerConnectionBackend& backend, GRefPtr<GstWebRTCRTPSender>&& rtcSender)
: m_peerConnectionBackend(WeakPtr { &backend })
, m_rtcSender(WTFMove(rtcSender))
, m_initData(WTFMove(initData))
{
ensureDebugCategoryIsRegistered();
GST_DEBUG_OBJECT(m_rtcSender.get(), "constructed without associated source");
Expand Down Expand Up @@ -162,17 +161,94 @@ bool GStreamerRtpSenderBackend::replaceTrack(RTCRtpSender& sender, MediaStreamTr

RTCRtpSendParameters GStreamerRtpSenderBackend::getParameters() const
{
return toRTCRtpSendParameters(m_initData.get());
switchOn(m_source, [&](const Ref<RealtimeOutgoingAudioSourceGStreamer>& source) {
m_currentParameters = source->parameters();
}, [&](const Ref<RealtimeOutgoingVideoSourceGStreamer>& source) {
m_currentParameters = source->parameters();
}, [](const std::nullptr_t&) {
});

GST_DEBUG_OBJECT(m_rtcSender.get(), "Current parameters: %" GST_PTR_FORMAT, m_currentParameters.get());
if (!m_currentParameters)
return toRTCRtpSendParameters(m_initData.get());

return toRTCRtpSendParameters(m_currentParameters.get());
}

static bool validateModifiedParameters(const RTCRtpSendParameters& newParameters, const RTCRtpSendParameters& oldParameters)
{
if (oldParameters.transactionId != newParameters.transactionId)
return false;

if (oldParameters.encodings.size() != newParameters.encodings.size())
return false;

for (size_t i = 0; i < oldParameters.encodings.size(); ++i) {
if (oldParameters.encodings[i].rid != newParameters.encodings[i].rid)
return false;
}

if (oldParameters.headerExtensions.size() != newParameters.headerExtensions.size())
return false;

for (size_t i = 0; i < oldParameters.headerExtensions.size(); ++i) {
const auto& oldExtension = oldParameters.headerExtensions[i];
const auto& newExtension = newParameters.headerExtensions[i];
if (oldExtension.uri != newExtension.uri || oldExtension.id != newExtension.id)
return false;
}

if (oldParameters.rtcp.cname != newParameters.rtcp.cname)
return false;

if (!!oldParameters.rtcp.reducedSize != !!newParameters.rtcp.reducedSize)
return false;

if (oldParameters.rtcp.reducedSize && *oldParameters.rtcp.reducedSize != *newParameters.rtcp.reducedSize)
return false;

if (oldParameters.codecs.size() != newParameters.codecs.size())
return false;

for (size_t i = 0; i < oldParameters.codecs.size(); ++i) {
const auto& oldCodec = oldParameters.codecs[i];
const auto& newCodec = newParameters.codecs[i];
if (oldCodec.payloadType != newCodec.payloadType
|| oldCodec.mimeType != newCodec.mimeType
|| oldCodec.clockRate != newCodec.clockRate
|| oldCodec.channels != newCodec.channels
|| oldCodec.sdpFmtpLine != newCodec.sdpFmtpLine)
return false;
}

return true;
}

void GStreamerRtpSenderBackend::setParameters(const RTCRtpSendParameters&, DOMPromiseDeferred<void>&& promise)
void GStreamerRtpSenderBackend::setParameters(const RTCRtpSendParameters& parameters, DOMPromiseDeferred<void>&& promise)
{
if (!m_rtcSender) {
if (!hasSource()) {
promise.reject(NotSupportedError);
return;
}

notImplemented();
if (!m_currentParameters) {
promise.reject(Exception { InvalidStateError, "getParameters must be called before setParameters"_s });
return;
}

if (!validateModifiedParameters(parameters, toRTCRtpSendParameters(m_currentParameters.get()))) {
promise.reject(InvalidModificationError, "parameters are not valid"_s);
return;
}

auto newParameters(fromRTCSendParameters(parameters));
switchOn(m_source, [&](Ref<RealtimeOutgoingAudioSourceGStreamer>& source) {
source->setParameters(WTFMove(newParameters));
}, [&](Ref<RealtimeOutgoingVideoSourceGStreamer>& source) {
source->setParameters(WTFMove(newParameters));
}, [](const std::nullptr_t&) {
});

promise.resolve();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class GStreamerPeerConnectionBackend;
class GStreamerRtpSenderBackend final : public RTCRtpSenderBackend {
WTF_MAKE_FAST_ALLOCATED;
public:
GStreamerRtpSenderBackend(GStreamerPeerConnectionBackend&, GRefPtr<GstWebRTCRTPSender>&&, GUniquePtr<GstStructure>&& initData);
GStreamerRtpSenderBackend(GStreamerPeerConnectionBackend&, GRefPtr<GstWebRTCRTPSender>&&);
using Source = std::variant<std::nullptr_t, Ref<RealtimeOutgoingAudioSourceGStreamer>, Ref<RealtimeOutgoingVideoSourceGStreamer>>;
GStreamerRtpSenderBackend(GStreamerPeerConnectionBackend&, GRefPtr<GstWebRTCRTPSender>&&, Source&&, GUniquePtr<GstStructure>&& initData);

Expand Down Expand Up @@ -86,6 +86,7 @@ class GStreamerRtpSenderBackend final : public RTCRtpSenderBackend {
GRefPtr<GstWebRTCRTPSender> m_rtcSender;
Source m_source;
GUniquePtr<GstStructure> m_initData;
mutable GUniquePtr<GstStructure> m_currentParameters;
};

} // namespace WebCore
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ GUniquePtr<GstStructure> fromRTCEncodingParameters(const RTCRtpEncodingParameter
"rid", G_TYPE_STRING, parameters.rid.utf8().data(), "bitrate-priority", G_TYPE_DOUBLE, toWebRTCBitRatePriority(parameters.priority), nullptr));

if (parameters.ssrc)
gst_structure_set(rtcParameters.get(), "ssrc", G_TYPE_ULONG, parameters.ssrc, nullptr);
gst_structure_set(rtcParameters.get(), "ssrc", G_TYPE_UINT, parameters.ssrc, nullptr);

if (parameters.maxBitrate)
gst_structure_set(rtcParameters.get(), "max-bitrate", G_TYPE_ULONG, parameters.maxBitrate, nullptr);
Expand Down Expand Up @@ -181,6 +181,8 @@ RTCRtpSendParameters toRTCRtpSendParameters(const GstStructure* rtcParameters)
return { };

RTCRtpSendParameters parameters;
parameters.transactionId = makeString(gst_structure_get_string(rtcParameters, "transaction-id"));

auto* encodings = gst_structure_get_value(rtcParameters, "encodings");
unsigned size = gst_value_list_get_size(encodings);
for (unsigned i = 0; i < size; i++) {
Expand All @@ -193,6 +195,22 @@ RTCRtpSendParameters toRTCRtpSendParameters(const GstStructure* rtcParameters)
return parameters;
}

GUniquePtr<GstStructure> fromRTCSendParameters(const RTCRtpSendParameters& parameters)
{
GUniquePtr<GstStructure> gstParameters(gst_structure_new("send-parameters", "transaction-id", G_TYPE_STRING, parameters.transactionId.ascii().data(), nullptr));
GValue encodingsValue = G_VALUE_INIT;
g_value_init(&encodingsValue, GST_TYPE_LIST);
for (auto& encoding : parameters.encodings) {
auto encodingData = fromRTCEncodingParameters(encoding);
GValue value = G_VALUE_INIT;
g_value_init(&value, GST_TYPE_STRUCTURE);
gst_value_set_structure(&value, encodingData.get());
gst_value_list_append_value(&encodingsValue, &value);
g_value_unset(&value);
}
gst_structure_take_value(gstParameters.get(), "encodings", &encodingsValue);
return gstParameters;
}

static void ensureDebugCategoryInitialized()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ RefPtr<RTCError> toRTCError(GError*);

GUniquePtr<GstStructure> fromRTCEncodingParameters(const RTCRtpEncodingParameters&);
RTCRtpSendParameters toRTCRtpSendParameters(const GstStructure*);
GUniquePtr<GstStructure> fromRTCSendParameters(const RTCRtpSendParameters&);

std::optional<Ref<RTCCertificate>> generateCertificate(Ref<SecurityOrigin>&&, const PeerConnectionBackend::CertificateInformation&);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -585,7 +585,7 @@ static void webkit_video_encoder_class_init(WebKitVideoEncoderClass* klass)
Encoders::registerEncoder(X264, "x264enc", "h264parse", "video/x-h264",
"video/x-h264,alignment=au,stream-format=byte-stream",
[](WebKitVideoEncoder* self) {
g_object_set(self->priv->encoder.get(), "key-int-max", 15, "threads", NUMBER_OF_THREADS, nullptr);
g_object_set(self->priv->encoder.get(), "key-int-max", 15, "threads", NUMBER_OF_THREADS, "b-adapt", FALSE, "vbv-buf-capacity", 120, nullptr);
g_object_set(self->priv->parser.get(), "config-interval", 1, nullptr);

const auto* structure = gst_caps_get_structure(self->priv->encodedCaps.get(), 0);
Expand Down Expand Up @@ -673,6 +673,8 @@ static void webkit_video_encoder_class_init(WebKitVideoEncoderClass* klass)
});

auto setVpxEncoderInputFormat = [](auto* self) {
g_object_set(self->priv->encoder.get(), "buffer-initial-size", 100, "buffer-optimal-size", 120, "buffer-size" , 150, "max-intra-bitrate", 250, nullptr);
gst_util_set_object_arg(G_OBJECT(self->priv->encoder.get()), "error-resilient", "default");
auto inputCaps = adoptGRef(gst_caps_new_any());
const auto* structure = gst_caps_get_structure(self->priv->encodedCaps.get(), 0);
if (const char* profileString = gst_structure_get_string(structure, "profile")) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,6 @@ RealtimeIncomingVideoSourceGStreamer::RealtimeIncomingVideoSourceGStreamer(AtomS
gst_element_set_name(bin(), makeString("incoming-video-source-", sourceCounter.exchangeAdd(1)).ascii().data());
GST_DEBUG_OBJECT(bin(), "New incoming video source created");

RealtimeMediaSourceSupportedConstraints constraints;
constraints.setSupportsWidth(true);
constraints.setSupportsHeight(true);
m_currentSettings = RealtimeMediaSourceSettings { };
m_currentSettings->setSupportedConstraints(WTFMove(constraints));

auto sinkPad = adoptGRef(gst_element_get_static_pad(bin(), "sink"));
gst_pad_add_probe(sinkPad.get(), static_cast<GstPadProbeType>(GST_PAD_PROBE_TYPE_BUFFER), [](GstPad*, GstPadProbeInfo* info, gpointer) -> GstPadProbeReturn {
auto videoFrameTimeMetadata = std::make_optional<VideoFrameTimeMetadata>({ });
Expand All @@ -74,14 +68,22 @@ const RealtimeMediaSourceSettings& RealtimeIncomingVideoSourceGStreamer::setting
if (m_currentSettings)
return m_currentSettings.value();

RealtimeMediaSourceSettings settings;
RealtimeMediaSourceSupportedConstraints constraints;
constraints.setSupportsWidth(true);
constraints.setSupportsHeight(true);

RealtimeMediaSourceSettings settings;
auto& size = this->size();
settings.setWidth(size.width());
settings.setHeight(size.height());
if (!size.isZero()) {
constraints.setSupportsWidth(true);
constraints.setSupportsHeight(true);
settings.setWidth(size.width());
settings.setHeight(size.height());
}

if (double frameRate = this->frameRate()) {
constraints.setSupportsFrameRate(true);
settings.setFrameRate(frameRate);
}

settings.setSupportedConstraints(constraints);

m_currentSettings = WTFMove(settings);
Expand All @@ -90,13 +92,25 @@ const RealtimeMediaSourceSettings& RealtimeIncomingVideoSourceGStreamer::setting

void RealtimeIncomingVideoSourceGStreamer::settingsDidChange(OptionSet<RealtimeMediaSourceSettings::Flag> settings)
{
if (settings.containsAny({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height }))
if (settings.containsAny({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height, RealtimeMediaSourceSettings::Flag::FrameRate }))
m_currentSettings = std::nullopt;
}

void RealtimeIncomingVideoSourceGStreamer::dispatchSample(GRefPtr<GstSample>&& sample)
{
auto* buffer = gst_sample_get_buffer(sample.get());
auto* caps = gst_sample_get_caps(sample.get());
if (auto size = getVideoResolutionFromCaps(caps))
setSize({ static_cast<int>(size->width()), static_cast<int>(size->height()) });

int frameRateNumerator, frameRateDenominator;
auto* structure = gst_caps_get_structure(caps, 0);
if (gst_structure_get_fraction(structure, "framerate", &frameRateNumerator, &frameRateDenominator)) {
double framerate;
gst_util_fraction_to_double(frameRateNumerator, frameRateDenominator, &framerate);
setFrameRate(framerate);
}

videoFrameAvailable(VideoFrameGStreamer::create(WTFMove(sample), size(), fromGstClockTime(GST_BUFFER_PTS(buffer))), { });
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,11 @@ void RealtimeOutgoingAudioSourceGStreamer::linkOutgoingSource()
g_object_set(m_inputSelector.get(), "active-pad", sinkPad.get(), nullptr);
}

void RealtimeOutgoingAudioSourceGStreamer::setParameters(GUniquePtr<GstStructure>&& parameters)
{
m_parameters = WTFMove(parameters);
}

#undef GST_CAT_DEFAULT

} // namespace WebCore
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class RealtimeOutgoingAudioSourceGStreamer final : public RealtimeOutgoingMediaS
static Ref<RealtimeOutgoingAudioSourceGStreamer> create(const RefPtr<UniqueSSRCGenerator>& ssrcGenerator, const String& mediaStreamId, MediaStreamTrack& track) { return adoptRef(*new RealtimeOutgoingAudioSourceGStreamer(ssrcGenerator, mediaStreamId, track)); }

bool setPayloadType(const GRefPtr<GstCaps>&) final;
void setParameters(GUniquePtr<GstStructure>&&) final;

protected:
explicit RealtimeOutgoingAudioSourceGStreamer(const RefPtr<UniqueSSRCGenerator>&, const String& mediaStreamId, MediaStreamTrack&);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
#include <gst/webrtc/webrtc.h>
#undef GST_USE_UNSTABLE_API

#include <wtf/UUID.h>

GST_DEBUG_CATEGORY(webkit_webrtc_outgoing_media_debug);
#define GST_CAT_DEFAULT webkit_webrtc_outgoing_media_debug

Expand Down Expand Up @@ -240,6 +242,33 @@ void RealtimeOutgoingMediaSourceGStreamer::setSinkPad(GRefPtr<GstPad>&& pad)
g_object_get(m_transceiver.get(), "sender", &m_sender.outPtr(), nullptr);
}

GUniquePtr<GstStructure> RealtimeOutgoingMediaSourceGStreamer::parameters()
{
if (!m_parameters) {
auto transactionId = createVersion4UUIDString();
m_parameters.reset(gst_structure_new("send-parameters", "transaction-id", G_TYPE_STRING, transactionId.ascii().data(), nullptr));

GUniquePtr<GstStructure> encodingParameters(gst_structure_new("encoding-parameters", "active", G_TYPE_BOOLEAN, TRUE, nullptr));

if (m_payloader) {
uint32_t ssrc;
g_object_get(m_payloader.get(), "ssrc", &ssrc, nullptr);
gst_structure_set(encodingParameters.get(), "ssrc", G_TYPE_UINT, ssrc, nullptr);
}
fillEncodingParameters(encodingParameters);

GValue encodingsValue = G_VALUE_INIT;
g_value_init(&encodingsValue, GST_TYPE_LIST);
GValue value = G_VALUE_INIT;
g_value_init(&value, GST_TYPE_STRUCTURE);
gst_value_set_structure(&value, encodingParameters.get());
gst_value_list_append_value(&encodingsValue, &value);
g_value_unset(&value);
gst_structure_take_value(m_parameters.get(), "encodings", &encodingsValue);
}
return GUniquePtr<GstStructure>(gst_structure_copy(m_parameters.get()));
}

#undef GST_CAT_DEFAULT

} // namespace WebCore
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ class RealtimeOutgoingMediaSourceGStreamer : public ThreadSafeRefCounted<Realtim
virtual bool setPayloadType(const GRefPtr<GstCaps>&) { return false; }
virtual void teardown() { }

GUniquePtr<GstStructure> parameters();
virtual void fillEncodingParameters(const GUniquePtr<GstStructure>&) { }
virtual void setParameters(GUniquePtr<GstStructure>&&) { }

protected:
explicit RealtimeOutgoingMediaSourceGStreamer(const RefPtr<UniqueSSRCGenerator>&, const String& mediaStreamId, MediaStreamTrack&);

Expand Down Expand Up @@ -86,6 +90,7 @@ class RealtimeOutgoingMediaSourceGStreamer : public ThreadSafeRefCounted<Realtim
GRefPtr<GstWebRTCRTPSender> m_sender;
GRefPtr<GstPad> m_webrtcSinkPad;
RefPtr<UniqueSSRCGenerator> m_ssrcGenerator;
GUniquePtr<GstStructure> m_parameters;

private:
void sourceMutedChanged();
Expand Down
Loading

0 comments on commit 72651b6

Please sign in to comment.