Skip to content

Commit 469467e

Browse files
youennfyouennf
committed
Safari WebRTC screen sharing via getDisplayMedia starts at extremely low quality and takes ~30 seconds to become legible for remote participants
rdar://175425085 https://bugs.webkit.org/show_bug.cgi?id=312472 Reviewed by Jean-Yves Avenard. By default, peer connection should favour resolution for screenshare and frame rate for camera content. We make sure to let know the libwebrtc backend that sent content is screenshare by implementing RealtimeOutgoingVideoSource::is_screencast based on the capture device type. We also make sure to pipe content hint information to the libwebrtc backend, which can allow canvas tracks to select easily between the two behaviors. Covered by added test. Test: webrtc/getDisplayMedia-pc-resolution.html * LayoutTests/webrtc/getDisplayMedia-pc-resolution-expected.txt: Added. * LayoutTests/webrtc/getDisplayMedia-pc-resolution.html: Added. * Source/ThirdParty/libwebrtc/Configurations/libwebrtc.exp: * Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.cpp: (WebCore::LibWebRTCMediaEndpoint::addTrack): (WebCore::LibWebRTCMediaEndpoint::createSourceAndRTCTrack): * Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpSenderBackend.cpp: (WebCore::LibWebRTCRtpSenderBackend::replaceTrack): * Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCUtils.cpp: (WebCore::toWebRTCContentHint): * Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCUtils.h: * Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.cpp: (WebCore::RealtimeOutgoingVideoSource::RealtimeOutgoingVideoSource): (WebCore::RealtimeOutgoingVideoSource::setSource): (WebCore::RealtimeOutgoingVideoSource::is_screencast const): * Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.h: Canonical link: https://commits.webkit.org/313072@main
1 parent 9bc8066 commit 469467e

12 files changed

Lines changed: 194 additions & 33 deletions

File tree

‎LayoutTests/platform/glib/TestExpectations‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2834,6 +2834,8 @@ fast/mediastream/video-rotation2.html [ Failure ]
28342834

28352835
http/wpt/mediastream/webrtc-vp9-colorspace.html [ Failure ]
28362836

2837+
webrtc/getDisplayMedia-pc-resolution.html [ Failure ]
2838+
28372839
# Regressions introduced by https://commits.webkit.org/263750@main
28382840
fast/mediastream/apply-constraints-advanced.html [ Failure ]
28392841
fast/mediastream/apply-constraints-video.html [ Failure ]

‎LayoutTests/platform/ios/TestExpectations‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3026,6 +3026,7 @@ fast/mediastream/getDisplayMedia-max-constraints5.html [ Skip ]
30263026
webkit.org/b/263466 fast/mediastream/getDisplayMedia-max-constraints4.html [ Failure ]
30273027
webrtc/getDisplayMedia-pc.html [ Skip ]
30283028
webrtc/getDisplayMedia-odd-size.html [ Skip ]
3029+
webrtc/getDisplayMedia-pc-resolution.html [ Skip ]
30293030

30303031
fast/mediastream/getDisplayMedia-size.html [ Skip ]
30313032

‎LayoutTests/webrtc/getDisplayMedia-pc-resolution-expected.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
2+
3+
PASS Setup
4+
PASS getUserMedia() streaming to PeerConnection should favor frame rate
5+
PASS getDisplayMedia() streaming to PeerConnection should favor frame resolution
6+
PASS getUserMedia() streaming to PeerConnection can favor frame resolution with content hint
7+
PASS getDisplayMedia() streaming to PeerConnection can favor frame rate with content hint
8+
PASS getDisplayMedia() streaming to PeerConnection (while sender was created for video) should favor frame resolution
9+
PASS Cleanup
10+

‎LayoutTests/webrtc/getDisplayMedia-pc-resolution.html‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
<!doctype html><!-- webkit-test-runner [ PeerConnectionVideoScalingAdaptationDisabled=false ] -->
2+
<html>
3+
<head>
4+
<meta charset="utf-8">
5+
<script src="../resources/testharness.js"></script>
6+
<script src="../resources/testharnessreport.js"></script>
7+
<script src="../fast/mediastream/resources/getDisplayMedia-utils.js"></script>
8+
</head>
9+
<body>
10+
<video id="video" autoplay=""></video>
11+
<script src ="routines.js"></script>
12+
<script>
13+
async function waitFor(duration)
14+
{
15+
return new Promise((resolve) => setTimeout(resolve, duration));
16+
}
17+
18+
function getInboundRTPStats(pc)
19+
{
20+
return pc.getStats().then((report) => {
21+
var stats;
22+
report.forEach((statItem) => {
23+
if (statItem.type === "inbound-rtp") {
24+
stats = statItem;
25+
}
26+
});
27+
return stats;
28+
});
29+
}
30+
31+
async function getFirstInboundRtpReport(pc)
32+
{
33+
const stats = await getInboundRTPStats(pc);
34+
if (!stats || !stats.framesDecoded)
35+
return getFirstInboundRtpReport(pc);
36+
return stats;
37+
}
38+
39+
async function testVideoAdaptation(localStreamCallback)
40+
{
41+
var pc1, pc2;
42+
await new Promise((resolve, reject) => {
43+
createConnections(async (firstConnection) => {
44+
pc1 = firstConnection;
45+
localStreamCallback(firstConnection);
46+
const sender = firstConnection.getSenders()[0];
47+
const parameters = sender.getParameters();
48+
// We severly limit bandwidth to trigger bandwidth reduction via frame rate or resolution decrease.
49+
parameters.encodings[0].maxBitrate = 83000;
50+
await sender.setParameters(parameters);
51+
}, (secondConnection) => {
52+
pc2 = secondConnection;
53+
secondConnection.ontrack = (trackEvent) => {
54+
resolve(trackEvent.streams[0]);
55+
};
56+
});
57+
setTimeout(() => reject("Test timed out"), 5000);
58+
});
59+
60+
return getFirstInboundRtpReport(pc2);
61+
}
62+
63+
let gumStream, gdmStream;
64+
promise_test(async () => {
65+
gumStream = await navigator.mediaDevices.getUserMedia({ video: { width: 1920, height:1080 } });
66+
gdmStream = await callGetDisplayMedia({ video: { width: 1920, height:1080 } });
67+
}, "Setup");
68+
69+
promise_test(async (test) => {
70+
const results = await testVideoAdaptation(pc => {
71+
pc.addTrack(gumStream.getVideoTracks()[0], gumStream);
72+
});
73+
assert_less_than(results.frameWidth, 1920);
74+
assert_less_than(results.frameHeight, 1080);
75+
}, "getUserMedia() streaming to PeerConnection should favor frame rate");
76+
77+
promise_test(async (test) => {
78+
const results = await testVideoAdaptation(pc => {
79+
pc.addTrack(gdmStream.getVideoTracks()[0], gdmStream);
80+
});
81+
assert_equals(results.frameWidth, 1920);
82+
assert_equals(results.frameHeight, 1080);
83+
}, "getDisplayMedia() streaming to PeerConnection should favor frame resolution");
84+
85+
promise_test(async (test) => {
86+
gumStream.getTracks()[0].contentHint = "text";
87+
test.add_cleanup(async () => gumStream.getTracks()[0].contentHint = "");
88+
const results = await testVideoAdaptation(pc => {
89+
pc.addTrack(gumStream.getVideoTracks()[0], gumStream);
90+
});
91+
assert_equals(results.frameWidth, 1920);
92+
assert_equals(results.frameHeight, 1080);
93+
}, "getUserMedia() streaming to PeerConnection can favor frame resolution with content hint");
94+
95+
promise_test(async (test) => {
96+
gdmStream.getTracks()[0].contentHint = "motion";
97+
test.add_cleanup(async () => gdmStream.getTracks()[0].contentHint = "");
98+
const results = await testVideoAdaptation(pc => {
99+
pc.addTrack(gdmStream.getVideoTracks()[0], gdmStream);
100+
});
101+
assert_less_than(results.frameWidth, 1920);
102+
assert_less_than(results.frameHeight, 1080);
103+
}, "getDisplayMedia() streaming to PeerConnection can favor frame rate with content hint");
104+
105+
promise_test(async (test) => {
106+
const results = await testVideoAdaptation(async pc => {
107+
pc.addTrack(gumStream.getVideoTracks()[0], gumStream);
108+
pc.getSenders()[0].replaceTrack(gdmStream.getTracks()[0]);
109+
});
110+
assert_equals(results.frameWidth, 1920);
111+
assert_equals(results.frameHeight, 1080);
112+
}, "getDisplayMedia() streaming to PeerConnection (while sender was created for video) should favor frame resolution");
113+
114+
promise_test(async test => {
115+
test.add_cleanup(async () => gumStream.getTracks().forEach(track => track.stop()));
116+
test.add_cleanup(async () => gdmStream.getTracks().forEach(track => track.stop()));
117+
}, "Cleanup");
118+
119+
</script>
120+
</body>
121+
</html>

‎Source/ThirdParty/libwebrtc/Configurations/libwebrtc.exp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,3 +422,4 @@ __ZNK6webrtc7Network5CloneEv
422422
__ZN6webrtc10ColorSpaceC1ENS0_9PrimaryIDENS0_10TransferIDENS0_8MatrixIDENS0_7RangeIDE
423423
__ZN6webrtc10ColorSpaceC1EOS0_
424424
__ZN6webrtc10ColorSpaceaSERKS0_
425+
__ZN6webrtc25MediaStreamTrackInterface10kVideoKindE

‎Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.cpp‎

Lines changed: 13 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -242,27 +242,7 @@ bool LibWebRTCMediaEndpoint::addTrack(LibWebRTCRtpSenderBackend& sender, MediaSt
242242

243243
ALWAYS_LOG(LOGIDENTIFIER, "Adding "_s, track.privateTrack().type() == RealtimeMediaSource::Type::Audio ? "audio"_s : "video"_s, " track with id "_s, track.id());
244244

245-
LibWebRTCRtpSenderBackend::Source source;
246-
webrtc::scoped_refptr<webrtc::MediaStreamTrackInterface> rtcTrack;
247-
switch (track.privateTrack().type()) {
248-
case RealtimeMediaSource::Type::Audio: {
249-
auto audioSource = RealtimeOutgoingAudioSource::create(track.privateTrack());
250-
rtcTrack = m_peerConnectionFactory->CreateAudioTrack(track.id().utf8().data(), audioSource.ptr());
251-
source = WTF::move(audioSource);
252-
break;
253-
}
254-
case RealtimeMediaSource::Type::Video: {
255-
auto videoSource = RealtimeOutgoingVideoSource::create(track.privateTrack());
256-
257-
RefPtr context = m_peerConnectionBackend->connection().scriptExecutionContext();
258-
if (context && context->settingsValues().peerConnectionVideoScalingAdaptationDisabled)
259-
videoSource->disableVideoScaling();
260-
261-
rtcTrack = m_peerConnectionFactory->CreateVideoTrack(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> (videoSource.ptr()), track.id().utf8().data());
262-
source = WTF::move(videoSource);
263-
break;
264-
}
265-
}
245+
auto [source, rtcTrack] = createSourceAndRTCTrack(track);
266246

267247
sender.setSource(WTF::move(source));
268248
if (RefPtr rtpSender = sender.rtcSender()) {
@@ -425,14 +405,14 @@ ExceptionOr<LibWebRTCMediaEndpoint::Backends> LibWebRTCMediaEndpoint::addTransce
425405
return createTransceiverBackends(type, fromRtpTransceiverInit(init, type), nullptr, ignoreNegotiationNeededFlag);
426406
}
427407

428-
std::pair<LibWebRTCRtpSenderBackend::Source, Ref<webrtc::MediaStreamTrackInterface>> LibWebRTCMediaEndpoint::createSourceAndRTCTrack(MediaStreamTrack& track)
408+
std::pair<LibWebRTCRtpSenderBackend::Source, webrtc::scoped_refptr<webrtc::MediaStreamTrackInterface>> LibWebRTCMediaEndpoint::createSourceAndRTCTrack(MediaStreamTrack& track)
429409
{
430410
LibWebRTCRtpSenderBackend::Source source;
431-
RefPtr<webrtc::MediaStreamTrackInterface> rtcTrack;
411+
webrtc::scoped_refptr<webrtc::MediaStreamTrackInterface> rtcTrack;
432412
switch (track.privateTrack().type()) {
433413
case RealtimeMediaSource::Type::Audio: {
434414
Ref audioSource = RealtimeOutgoingAudioSource::create(track.privateTrack());
435-
rtcTrack = toRef(m_peerConnectionFactory->CreateAudioTrack(track.id().utf8().data(), audioSource.ptr()));
415+
rtcTrack = m_peerConnectionFactory->CreateAudioTrack(track.id().utf8().data(), audioSource.ptr());
436416
source = WTF::move(audioSource);
437417
break;
438418
}
@@ -443,12 +423,14 @@ std::pair<LibWebRTCRtpSenderBackend::Source, Ref<webrtc::MediaStreamTrackInterfa
443423
if (context && context->settingsValues().peerConnectionVideoScalingAdaptationDisabled)
444424
videoSource->disableVideoScaling();
445425

446-
rtcTrack = toRef(m_peerConnectionFactory->CreateVideoTrack(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> (videoSource.ptr()), track.id().utf8().data()));
426+
auto rtcVideoTrack = m_peerConnectionFactory->CreateVideoTrack(webrtc::scoped_refptr<webrtc::VideoTrackSourceInterface> (videoSource.ptr()), track.id().utf8().data());
427+
rtcVideoTrack->set_content_hint(toWebRTCContentHint(track.privateTrack().contentHint()));
428+
rtcTrack = WTF::move(rtcVideoTrack);
447429
source = WTF::move(videoSource);
448430
break;
449431
}
450432
}
451-
return std::make_pair(WTF::move(source), rtcTrack.releaseNonNull());
433+
return std::make_pair(WTF::move(source), WTF::move(rtcTrack));
452434
}
453435

454436
ExceptionOr<LibWebRTCMediaEndpoint::Backends> LibWebRTCMediaEndpoint::addTransceiver(MediaStreamTrack& track, const RTCRtpTransceiverInit& init, PeerConnectionBackend::IgnoreNegotiationNeededFlag ignoreNegotiationNeededFlag)
@@ -457,15 +439,15 @@ ExceptionOr<LibWebRTCMediaEndpoint::Backends> LibWebRTCMediaEndpoint::addTransce
457439
ALWAYS_LOG(LOGIDENTIFIER, "Adding "_s, track.kind().string(), " ", direction, " transceiver for track "_s, track.id());
458440

459441
auto type = track.source().type() == RealtimeMediaSource::Type::Audio ? webrtc::MediaType::AUDIO : webrtc::MediaType::VIDEO;
460-
auto sourceAndTrack = createSourceAndRTCTrack(track);
461-
return createTransceiverBackends(webrtc::scoped_refptr { sourceAndTrack.second.ptr() }, fromRtpTransceiverInit(init, type), WTF::move(sourceAndTrack.first), ignoreNegotiationNeededFlag);
442+
auto [source, rtcTrack] = createSourceAndRTCTrack(track);
443+
return createTransceiverBackends(WTF::move(rtcTrack), fromRtpTransceiverInit(init, type), WTF::move(source), ignoreNegotiationNeededFlag);
462444
}
463445

464446
void LibWebRTCMediaEndpoint::setSenderSourceFromTrack(LibWebRTCRtpSenderBackend& sender, MediaStreamTrack& track)
465447
{
466-
auto sourceAndTrack = createSourceAndRTCTrack(track);
467-
sender.setSource(WTF::move(sourceAndTrack.first));
468-
protect(sender.rtcSender())->SetTrack(sourceAndTrack.second.ptr());
448+
auto [source, rtcTrack] = createSourceAndRTCTrack(track);
449+
sender.setSource(WTF::move(source));
450+
protect(sender.rtcSender())->SetTrack(rtcTrack.get());
469451
}
470452

471453
std::unique_ptr<LibWebRTCRtpTransceiverBackend> LibWebRTCMediaEndpoint::transceiverBackendFromSender(LibWebRTCRtpSenderBackend& backend)

‎Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ class LibWebRTCMediaEndpoint final
174174
return result ? webrtc::RefCountReleaseStatus::kOtherRefsRemained : webrtc::RefCountReleaseStatus::kDroppedLastRef;
175175
}
176176

177-
std::pair<LibWebRTCRtpSenderBackend::Source, Ref<webrtc::MediaStreamTrackInterface>> createSourceAndRTCTrack(MediaStreamTrack&);
177+
std::pair<LibWebRTCRtpSenderBackend::Source, webrtc::scoped_refptr<webrtc::MediaStreamTrackInterface>> createSourceAndRTCTrack(MediaStreamTrack&);
178178
RefPtr<RealtimeMediaSource> sourceFromNewReceiver(webrtc::RtpReceiverInterface&);
179179

180180
#if !RELEASE_LOG_DISABLED

‎Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpSenderBackend.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,14 @@ bool LibWebRTCRtpSenderBackend::replaceTrack(RTCRtpSender& sender, MediaStreamTr
111111
}
112112

113113
if (sender.track()) {
114+
if (auto rtcTrack = protect(m_rtcSender)->track()) {
115+
if (rtcTrack->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
116+
// This is a cast from a webrtc type, not much we can do to make it safe.
117+
SUPPRESS_MEMORY_UNSAFE_CAST webrtc::scoped_refptr<webrtc::VideoTrackInterface> videoTrack { static_cast<webrtc::VideoTrackInterface*>(rtcTrack.get()) };
118+
videoTrack->set_content_hint(toWebRTCContentHint(track->privateTrack().contentHint()));
119+
}
120+
}
121+
114122
switchOn(m_source, [&](Ref<RealtimeOutgoingAudioSource>& source) {
115123
ASSERT(track->source().type() == RealtimeMediaSource::Type::Audio);
116124
source->stop();

‎Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCUtils.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#if ENABLE(WEB_RTC) && USE(LIBWEBRTC)
2929

3030
#include "LibWebRTCMacros.h"
31+
#include "MediaStreamTrackHintValue.h"
3132
#include "RTCDtlsTransportState.h"
3233
#include "RTCError.h"
3334
#include "RTCIceCandidate.h"
@@ -531,6 +532,27 @@ RefPtr<RTCError> toRTCError(const webrtc::RTCError& rtcError)
531532
return RTCError::create(*detail, String::fromLatin1(rtcError.message()));
532533
}
533534

535+
webrtc::VideoTrackInterface::ContentHint toWebRTCContentHint(MediaStreamTrackHintValue value)
536+
{
537+
ASSERT(value != MediaStreamTrackHintValue::Speech);
538+
ASSERT(value != MediaStreamTrackHintValue::Music);
539+
switch (value) {
540+
case MediaStreamTrackHintValue::Speech:
541+
return webrtc::VideoTrackInterface::ContentHint::kNone;
542+
case MediaStreamTrackHintValue::Music:
543+
return webrtc::VideoTrackInterface::ContentHint::kNone;
544+
case MediaStreamTrackHintValue::Empty:
545+
return webrtc::VideoTrackInterface::ContentHint::kNone;
546+
case MediaStreamTrackHintValue::Motion:
547+
return webrtc::VideoTrackInterface::ContentHint::kFluid;
548+
case MediaStreamTrackHintValue::Detail:
549+
return webrtc::VideoTrackInterface::ContentHint::kDetailed;
550+
case MediaStreamTrackHintValue::Text:
551+
return webrtc::VideoTrackInterface::ContentHint::kText;
552+
}
553+
return webrtc::VideoTrackInterface::ContentHint::kNone;
554+
}
555+
534556
} // namespace WebCore
535557

536558
#endif // ENABLE(WEB_RTC) && USE(LIBWEBRTC)

‎Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCUtils.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
IGNORE_CLANG_WARNINGS_BEGIN("non-modular-include-in-module")
3333
#include <webrtc/api/media_types.h>
3434
WTF_IGNORE_WARNINGS_IN_THIRD_PARTY_CODE_BEGIN
35+
#include <webrtc/api/media_stream_interface.h>
3536
#include <webrtc/api/stats/rtcstats_objects.h>
3637
WTF_IGNORE_WARNINGS_IN_THIRD_PARTY_CODE_END
3738
IGNORE_CLANG_WARNINGS_END
@@ -83,6 +84,9 @@ RTCPriorityType toRTCPriorityType(webrtc::PriorityValue);
8384
RTCPriorityType NODELETE toRTCPriorityType(webrtc::Priority);
8485
webrtc::Priority NODELETE fromRTCPriorityType(RTCPriorityType);
8586

87+
enum class MediaStreamTrackHintValue : uint8_t;
88+
webrtc::VideoTrackInterface::ContentHint toWebRTCContentHint(MediaStreamTrackHintValue);
89+
8690
inline String fromStdString(const std::string& value)
8791
{
8892
return String::fromUTF8(value);

0 commit comments

Comments
 (0)