Skip to content

Commit

Permalink
[GStreamer][LibWebRTC] Timestamp handling improvements in video decod…
Browse files Browse the repository at this point in the history 
…er factory

https://bugs.webkit.org/show_bug.cgi?id=273757

Reviewed by Xabier Rodriguez-Calvar.

The buffers injected into the parser pipeline are now timestamped by appsrc. The RTP timestamps are
required by the LibWebRTC generic decoder in order to match parsed frames with input buffers, so
they are attached on each buffers using a reference timestamp meta.

* Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoDecoderFactory.cpp:
(WebCore::GStreamerWebRTCVideoDecoder::GStreamerWebRTCVideoDecoder):
(WebCore::GStreamerWebRTCVideoDecoder::pullSample):
* Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoFrameLibWebRTC.cpp:
(WebCore::ensureDebugCategoryIsRegistered):
(WebCore::convertGStreamerSampleToLibWebRTCVideoFrame):
(WebCore::GStreamerVideoFrameLibWebRTC::ToI420):
* Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoFrameLibWebRTC.h:

Canonical link: https://commits.webkit.org/278447@main
  • Loading branch information
@philn
philn committed May 7, 2024
1 parent b18dd9e commit c47f18c
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 62 deletions.
74 changes: 17 additions & 57 deletions Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoDecoderFactory.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

#include "config.h"

#if ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER)
#if USE(LIBWEBRTC) && USE(GSTREAMER)
#include "GStreamerVideoDecoderFactory.h"

#include "GStreamerQuirks.h"
Expand All @@ -44,22 +44,15 @@ GST_DEBUG_CATEGORY(webkit_webrtcdec_debug);

namespace WebCore {

typedef struct {
uint64_t timestamp;
int64_t renderTimeMs;
} InputTimestamps;

class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
public:
GStreamerWebRTCVideoDecoder()
: m_pictureId(0)
, m_width(0)
: m_width(0)
, m_height(0)
, m_requireParse(false)
, m_needsKeyframe(true)
, m_firstBufferPts(GST_CLOCK_TIME_NONE)
, m_firstBufferDts(GST_CLOCK_TIME_NONE)
{
m_rtpTimestampCaps = adoptGRef(gst_caps_new_empty_simple("timestamp/x-rtp"));
}

static void decodebinPadAddedCb(GstElement*, GstPad* srcpad, GstPad* sinkpad)
Expand Down Expand Up @@ -102,6 +95,7 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
bool Configure(const webrtc::VideoDecoder::Settings& codecSettings) override
{
m_src = makeElement("appsrc");
g_object_set(m_src, "is-live", TRUE, "do-timestamp", TRUE, "max-buffers", 2, "max-bytes", 0, nullptr);

GRefPtr<GstCaps> caps = nullptr;
auto capsfilter = CreateFilter();
Expand Down Expand Up @@ -153,8 +147,7 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {

m_sink = makeElement("appsink");
gst_app_sink_set_emit_signals(GST_APP_SINK(m_sink), true);
// This is an decoder, everything should happen as fast as possible and not
// be synced on the clock.
// This is a decoder, everything should happen as fast as possible and not be synced on the clock.
g_object_set(m_sink, "sync", false, nullptr);

gst_bin_add_many(GST_BIN(pipeline()), m_src, decoder, capsfilter, m_sink, nullptr);
Expand Down Expand Up @@ -203,9 +196,7 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
return WEBRTC_VIDEO_CODEC_OK;
}

int32_t Decode(const webrtc::EncodedImage& inputImage,
bool,
int64_t renderTimeMs) override
int32_t Decode(const webrtc::EncodedImage& inputImage, int64_t) override
{
if (m_needsKeyframe) {
if (inputImage._frameType != webrtc::VideoFrameType::kVideoFrameKey) {
Expand All @@ -217,33 +208,15 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {

if (!m_src) {
GST_ERROR("No source set, can't decode.");

return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}

// No renderTime provided, probably some issue with the WebRTC clock. Use a monotonically
// incrementing counter instead.
static int64_t s_forgedRenderTime { 0 };
if (!renderTimeMs) {
renderTimeMs = s_forgedRenderTime;
s_forgedRenderTime += 30 * GST_MSECOND;
}

if (!GST_CLOCK_TIME_IS_VALID(m_firstBufferPts)) {
GRefPtr<GstPad> srcpad = adoptGRef(gst_element_get_static_pad(m_src, "src"));
m_firstBufferPts = (static_cast<guint64>(renderTimeMs)) * GST_MSECOND;
m_firstBufferDts = (static_cast<guint64>(inputImage.RtpTimestamp())) * GST_MSECOND;
}
// FIXME: Use a GstBufferPool.
GST_TRACE_OBJECT(pipeline(), "Pushing encoded image with RTP timestamp %u", inputImage.RtpTimestamp());
auto buffer = adoptGRef(gstBufferNewWrappedFast(fastMemDup(inputImage.data(), inputImage.size()), inputImage.size()));

// FIXME- Use a GstBufferPool.
auto buffer = adoptGRef(gstBufferNewWrappedFast(fastMemDup(inputImage.data(), inputImage.size()),
inputImage.size()));
GST_BUFFER_DTS(buffer.get()) = (static_cast<guint64>(inputImage.RtpTimestamp()) * GST_MSECOND) - m_firstBufferDts;
GST_BUFFER_PTS(buffer.get()) = (static_cast<guint64>(renderTimeMs) * GST_MSECOND) - m_firstBufferPts;
InputTimestamps timestamps = { inputImage.RtpTimestamp(), renderTimeMs };
m_dtsPtsMap[GST_BUFFER_PTS(buffer.get())] = timestamps;
gst_buffer_add_reference_timestamp_meta(buffer.get(), m_rtpTimestampCaps.get(), inputImage.RtpTimestamp(), GST_CLOCK_TIME_NONE);

GST_LOG_OBJECT(pipeline(), "%" G_GINT64_FORMAT " Decoding: %" GST_PTR_FORMAT, renderTimeMs, buffer.get());
auto sample = adoptGRef(gst_sample_new(buffer.get(), GetCapsForFrame(inputImage), nullptr, nullptr));
switch (gst_app_src_push_sample(GST_APP_SRC(m_src), sample.get())) {
case GST_FLOW_OK:
Expand All @@ -265,21 +238,11 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {
return WEBRTC_VIDEO_CODEC_OK;
}
auto buffer = gst_sample_get_buffer(sample.get());

// Make sure that the frame.timestamp == previsouly input_frame._timeStamp
// as it is required by the VideoDecoder baseclass.
auto timestamps = m_dtsPtsMap[GST_BUFFER_PTS(buffer)];
m_dtsPtsMap.erase(GST_BUFFER_PTS(buffer));

auto frame(convertGStreamerSampleToLibWebRTCVideoFrame(WTFMove(sample), webrtc::kVideoRotation_0,
timestamps.timestamp, timestamps.renderTimeMs));

GST_BUFFER_DTS(buffer) = GST_CLOCK_TIME_NONE;
GST_LOG_OBJECT(pipeline(), "Output decoded frame! %d -> %" GST_PTR_FORMAT,
frame->timestamp(), buffer);

m_imageReadyCb->Decoded(*frame.get(), absl::optional<int32_t>(), absl::optional<uint8_t>());

auto meta = gst_buffer_get_reference_timestamp_meta(buffer, m_rtpTimestampCaps.get());
RELEASE_ASSERT(meta);
auto frame = convertGStreamerSampleToLibWebRTCVideoFrame(WTFMove(sample), meta->timestamp);
GST_TRACE_OBJECT(pipeline(), "Pulled video frame with RTP timestamp %u from %" GST_PTR_FORMAT, static_cast<uint32_t>(meta->timestamp), buffer);
m_imageReadyCb->Decoded(frame);
return WEBRTC_VIDEO_CODEC_OK;
}

Expand Down Expand Up @@ -337,7 +300,6 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {

protected:
GRefPtr<GstCaps> m_caps;
gint m_pictureId;
gint m_width;
gint m_height;
bool m_requireParse = false;
Expand All @@ -350,9 +312,7 @@ class GStreamerWebRTCVideoDecoder : public webrtc::VideoDecoder {

webrtc::DecodedImageCallback* m_imageReadyCb;

StdMap<GstClockTime, InputTimestamps> m_dtsPtsMap;
GstClockTime m_firstBufferPts;
GstClockTime m_firstBufferDts;
GRefPtr<GstCaps> m_rtpTimestampCaps;
};

class H264Decoder : public GStreamerWebRTCVideoDecoder {
Expand Down Expand Up @@ -490,4 +450,4 @@ std::vector<webrtc::SdpVideoFormat> GStreamerVideoDecoderFactory::GetSupportedFo

} // namespace WebCore

#endif // ENABLE(VIDEO) && ENABLE(MEDIA_STREAM) && USE(LIBWEBRTC) && USE(GSTREAMER)
#endif // USE(LIBWEBRTC) && USE(GSTREAMER)
27 changes: 23 additions & 4 deletions Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoFrameLibWebRTC.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,17 @@

namespace WebCore {

GST_DEBUG_CATEGORY(webkit_libwebrtc_video_frame_debug);
#define GST_CAT_DEFAULT webkit_libwebrtc_video_frame_debug

static void ensureDebugCategoryIsRegistered()
{
static std::once_flag debugRegisteredFlag;
std::call_once(debugRegisteredFlag, [] {
GST_DEBUG_CATEGORY_INIT(webkit_libwebrtc_video_frame_debug, "webkitlibwebrtcvideoframe", 0, "WebKit LibWebRTC Video Frame");
});
}

GRefPtr<GstSample> convertLibWebRTCVideoFrameToGStreamerSample(const webrtc::VideoFrame& frame)
{
RELEASE_ASSERT(frame.video_frame_buffer()->type() != webrtc::VideoFrameBuffer::Type::kNative);
Expand Down Expand Up @@ -56,10 +67,15 @@ GRefPtr<GstSample> convertLibWebRTCVideoFrameToGStreamerSample(const webrtc::Vid
return sample;
}

std::unique_ptr<webrtc::VideoFrame> convertGStreamerSampleToLibWebRTCVideoFrame(GRefPtr<GstSample>&& sample, webrtc::VideoRotation rotation, int64_t timestamp, int64_t renderTimeMs)
webrtc::VideoFrame convertGStreamerSampleToLibWebRTCVideoFrame(GRefPtr<GstSample>&& sample, uint32_t rtpTimestamp)
{
auto frameBuffer(GStreamerVideoFrameLibWebRTC::create(WTFMove(sample)));
return std::unique_ptr<webrtc::VideoFrame>(new webrtc::VideoFrame(WTFMove(frameBuffer), timestamp, renderTimeMs, rotation));
webrtc::VideoFrame::Builder builder;
auto buffer = gst_sample_get_buffer(sample.get());
auto pts = GST_BUFFER_PTS(buffer);
return builder.set_video_frame_buffer(GStreamerVideoFrameLibWebRTC::create(WTFMove(sample)))
.set_timestamp_rtp(rtpTimestamp)
.set_timestamp_us(pts)
.build();
}

rtc::scoped_refptr<webrtc::VideoFrameBuffer> GStreamerVideoFrameLibWebRTC::create(GRefPtr<GstSample>&& sample)
Expand All @@ -74,6 +90,7 @@ rtc::scoped_refptr<webrtc::VideoFrameBuffer> GStreamerVideoFrameLibWebRTC::creat

rtc::scoped_refptr<webrtc::I420BufferInterface> GStreamerVideoFrameLibWebRTC::ToI420()
{
ensureDebugCategoryIsRegistered();
GstMappedFrame inFrame(m_sample, GST_MAP_READ);
if (!inFrame) {
GST_WARNING("Could not map input frame");
Expand Down Expand Up @@ -111,4 +128,6 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> GStreamerVideoFrameLibWebRTC::To

}

#endif // USE(LIBWEBRTC)
#undef GST_CAT_DEFAULT

#endif // USE(LIBWEBRTC) && USE(GSTREAMER)
2 changes: 1 addition & 1 deletion Source/WebCore/platform/mediastream/libwebrtc/gstreamer/GStreamerVideoFrameLibWebRTC.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ namespace WebCore {

WARN_UNUSED_RETURN GRefPtr<GstSample> convertLibWebRTCVideoFrameToGStreamerSample(const webrtc::VideoFrame&);

std::unique_ptr<webrtc::VideoFrame> convertGStreamerSampleToLibWebRTCVideoFrame(GRefPtr<GstSample>&&, webrtc::VideoRotation, int64_t timestamp, int64_t renderTimeMs);
webrtc::VideoFrame convertGStreamerSampleToLibWebRTCVideoFrame(GRefPtr<GstSample>&&, uint32_t rtpTimestamp);

class GStreamerVideoFrameLibWebRTC : public rtc::RefCountedObject<webrtc::VideoFrameBuffer> {
public:
Expand Down

0 comments on commit c47f18c

Please sign in to comment.