Skip to content

Commit

Permalink
Update applyConstraints() to restart video content source
Browse files Browse the repository at this point in the history 
Restart the video content source (desktop, window, and tab capture)
with the new constraints when applyConstraints is called. This fixes
an issue where the source frame rate would remain at a high fps even
though applyConstraints() was called with a low max fps. This resulted
in a high percentage of dropped frames and unnecessarily high CPU usage.

The feature is enabled by default but can be forced off by disabling
the feature ApplyConstraintsRestartsVideoContentSources.


Bug: chromium:1417195
Change-Id: Ia4b43fe1852120ea928aec3c65a0cfd0049f22ab
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4314019
Reviewed-by: Elad Alon <eladalon@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Commit-Queue: Johannes Kron <kron@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1117569}
  • Loading branch information
Johannes Kron authored and Chromium LUCI CQ committed Mar 15, 2023
1 parent 529e1c6 commit a9e8bd8
Show file tree
Hide file tree
Showing 10 changed files with 597 additions and 59 deletions.
136 changes: 119 additions & 17 deletions third_party/blink/renderer/modules/mediastream/apply_constraints_processor.cc
View file
Expand Up @@ -16,6 +16,7 @@
#include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_audio.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_video_content.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_video_device.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_utils.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_video_track.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_audio_source.h"
#include "third_party/blink/renderer/platform/mediastream/media_stream_source.h"
Expand All @@ -39,11 +40,19 @@ void RequestSucceeded(blink::ApplyConstraintsRequest* request) {

} // namespace

BASE_FEATURE(kApplyConstraintsRestartsVideoContentSources,
"ApplyConstraintsRestartsVideoContentSources",
base::FEATURE_ENABLED_BY_DEFAULT);

ApplyConstraintsProcessor::ApplyConstraintsProcessor(
LocalFrame* frame,
MediaDevicesDispatcherCallback media_devices_dispatcher_cb,
scoped_refptr<base::SingleThreadTaskRunner> task_runner)
: media_devices_dispatcher_cb_(std::move(media_devices_dispatcher_cb)),
task_runner_(std::move(task_runner)) {}
: frame_(frame),
media_devices_dispatcher_cb_(std::move(media_devices_dispatcher_cb)),
task_runner_(std::move(task_runner)) {
DCHECK(frame_);
}

ApplyConstraintsProcessor::~ApplyConstraintsProcessor() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
Expand Down Expand Up @@ -106,9 +115,21 @@ void ApplyConstraintsProcessor::ProcessVideoRequest() {
return;
}

// The crop version is lost if the capture is restarted, because of this we
// don't try to restart the source if cropTo() has ever been called.
const blink::MediaStreamDevice& device_info = video_source_->device();
if (device_info.type == blink::mojom::MediaStreamType::DEVICE_VIDEO_CAPTURE) {
ProcessVideoDeviceRequest();
} else if (base::FeatureList::IsEnabled(
kApplyConstraintsRestartsVideoContentSources) &&
video_source_->GetCropVersion() == 0 &&
(device_info.type ==
mojom::blink::MediaStreamType::GUM_DESKTOP_VIDEO_CAPTURE ||
device_info.type ==
mojom::blink::MediaStreamType::DISPLAY_VIDEO_CAPTURE ||
device_info.type == mojom::blink::MediaStreamType::
DISPLAY_VIDEO_CAPTURE_THIS_TAB)) {
ProcessVideoContentRequest();
} else {
FinalizeVideoRequest();
}
Expand All @@ -122,8 +143,8 @@ void ApplyConstraintsProcessor::ProcessVideoDeviceRequest() {
if (AbortIfVideoRequestStateInvalid())
return;

// TODO(guidou): Support restarting the source even if there is more than
// one track in the source. https://crbug.com/768205
// TODO(crbug.com/768205): Support restarting the source even if there is more
// than one track in the source.
if (video_source_->NumTracks() > 1U) {
FinalizeVideoRequest();
return;
Expand All @@ -139,17 +160,35 @@ void ApplyConstraintsProcessor::ProcessVideoDeviceRequest() {
// to know all the formats potentially supported by the source.
GetMediaDevicesDispatcher()->GetAllVideoInputDeviceFormats(
String(video_source_->device().id.data()),
WTF::BindOnce(&ApplyConstraintsProcessor::MaybeStopSourceForRestart,
WrapWeakPersistent(this)));
WTF::BindOnce(
&ApplyConstraintsProcessor::MaybeStopVideoDeviceSourceForRestart,
WrapWeakPersistent(this)));
}

void ApplyConstraintsProcessor::ProcessVideoContentRequest() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);

if (AbortIfVideoRequestStateInvalid()) {
return;
}

// TODO(crbug.com/768205): Support restarting the source even if there is more
// than one track in the source.
if (video_source_->NumTracks() > 1U) {
FinalizeVideoRequest();
return;
}

MaybeStopVideoContentSourceForRestart();
}

void ApplyConstraintsProcessor::MaybeStopSourceForRestart(
void ApplyConstraintsProcessor::MaybeStopVideoDeviceSourceForRestart(
const Vector<media::VideoCaptureFormat>& formats) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
if (AbortIfVideoRequestStateInvalid())
return;

blink::VideoCaptureSettings settings = SelectVideoSettings(formats);
blink::VideoCaptureSettings settings = SelectVideoDeviceSettings(formats);
if (!settings.HasValue()) {
ApplyConstraintsFailed(settings.failed_constraint_name());
return;
Expand All @@ -164,13 +203,38 @@ void ApplyConstraintsProcessor::MaybeStopSourceForRestart(
if (video_device_request_trace_)
video_device_request_trace_->AddStep("StopForRestart");

video_source_->StopForRestart(
WTF::BindOnce(&ApplyConstraintsProcessor::MaybeSourceStoppedForRestart,
WrapWeakPersistent(this)));
video_source_->StopForRestart(WTF::BindOnce(
&ApplyConstraintsProcessor::MaybeDeviceSourceStoppedForRestart,
WrapWeakPersistent(this)));
}
}

void ApplyConstraintsProcessor::MaybeSourceStoppedForRestart(
void ApplyConstraintsProcessor::MaybeStopVideoContentSourceForRestart() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
if (AbortIfVideoRequestStateInvalid()) {
return;
}

blink::VideoCaptureSettings settings = SelectVideoContentSettings();

if (!settings.HasValue()) {
ApplyConstraintsFailed(settings.failed_constraint_name());
return;
}

if (video_source_->GetCurrentFormat() == settings.Format()) {
video_source_->ReconfigureTrack(GetCurrentVideoTrack(),
settings.track_adapter_settings());
ApplyConstraintsSucceeded();
GetCurrentVideoTrack()->NotifyConstraintsConfigurationComplete();
} else {
video_source_->StopForRestart(WTF::BindOnce(
&ApplyConstraintsProcessor::MaybeRestartStoppedVideoContentSource,
WrapWeakPersistent(this)));
}
}

void ApplyConstraintsProcessor::MaybeDeviceSourceStoppedForRestart(
blink::MediaStreamVideoSource::RestartResult result) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);

Expand All @@ -188,11 +252,39 @@ void ApplyConstraintsProcessor::MaybeSourceStoppedForRestart(
DCHECK_EQ(result, blink::MediaStreamVideoSource::RestartResult::IS_STOPPED);
GetMediaDevicesDispatcher()->GetAvailableVideoInputDeviceFormats(
String(video_source_->device().id.data()),
WTF::BindOnce(&ApplyConstraintsProcessor::FindNewFormatAndRestart,
WTF::BindOnce(
&ApplyConstraintsProcessor::FindNewFormatAndRestartDeviceSource,
WrapWeakPersistent(this)));
}

void ApplyConstraintsProcessor::MaybeRestartStoppedVideoContentSource(
blink::MediaStreamVideoSource::RestartResult result) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);

if (AbortIfVideoRequestStateInvalid()) {
return;
}

if (result == blink::MediaStreamVideoSource::RestartResult::IS_RUNNING) {
FinalizeVideoRequest();
return;
}

DCHECK_EQ(result, blink::MediaStreamVideoSource::RestartResult::IS_STOPPED);

blink::VideoCaptureSettings settings = SelectVideoContentSettings();
// |settings| should have a value. If it does not due to some unexpected
// reason (perhaps a race with another renderer process), restart the source
// with the old format.
DCHECK(video_source_->GetCurrentFormat());
video_source_->Restart(
settings.HasValue() ? settings.Format()
: *video_source_->GetCurrentFormat(),
WTF::BindOnce(&ApplyConstraintsProcessor::MaybeSourceRestarted,
WrapWeakPersistent(this)));
}

void ApplyConstraintsProcessor::FindNewFormatAndRestart(
void ApplyConstraintsProcessor::FindNewFormatAndRestartDeviceSource(
const Vector<media::VideoCaptureFormat>& formats) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);

Expand All @@ -202,7 +294,7 @@ void ApplyConstraintsProcessor::FindNewFormatAndRestart(
if (video_device_request_trace_)
video_device_request_trace_->AddStep("Restart");

blink::VideoCaptureSettings settings = SelectVideoSettings(formats);
blink::VideoCaptureSettings settings = SelectVideoDeviceSettings(formats);
DCHECK(video_source_->GetCurrentFormat());
// |settings| should have a value. If it does not due to some unexpected
// reason (perhaps a race with another renderer process), restart the source
Expand Down Expand Up @@ -246,7 +338,7 @@ void ApplyConstraintsProcessor::FinalizeVideoRequest() {
} else {
format = GetCurrentVideoTrack()->GetComputedSourceFormat();
}
blink::VideoCaptureSettings settings = SelectVideoSettings({format});
blink::VideoCaptureSettings settings = SelectVideoDeviceSettings({format});

if (settings.HasValue()) {
if (settings.min_frame_rate().has_value()) {
Expand All @@ -262,7 +354,8 @@ void ApplyConstraintsProcessor::FinalizeVideoRequest() {
}
}

blink::VideoCaptureSettings ApplyConstraintsProcessor::SelectVideoSettings(
blink::VideoCaptureSettings
ApplyConstraintsProcessor::SelectVideoDeviceSettings(
Vector<media::VideoCaptureFormat> formats) {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
DCHECK(current_request_);
Expand Down Expand Up @@ -302,6 +395,15 @@ blink::VideoCaptureSettings ApplyConstraintsProcessor::SelectVideoSettings(
settings.height, settings.frame_rate);
}

blink::VideoCaptureSettings
ApplyConstraintsProcessor::SelectVideoContentSettings() {
DCHECK(video_source_);
gfx::Size screen_size = MediaStreamUtils::GetScreenSize(frame_);
return blink::SelectSettingsVideoContentCapture(
current_request_->Constraints(), video_source_->device().type,
screen_size.width(), screen_size.height());
}

blink::MediaStreamAudioSource*
ApplyConstraintsProcessor::GetCurrentAudioSource() {
DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
Expand Down
33 changes: 26 additions & 7 deletions third_party/blink/renderer/modules/mediastream/apply_constraints_processor.h
View file
Expand Up @@ -5,13 +5,15 @@
#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_APPLY_CONSTRAINTS_PROCESSOR_H_
#define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_APPLY_CONSTRAINTS_PROCESSOR_H_

#include "base/feature_list.h"
#include "base/functional/callback.h"
#include "base/task/single_thread_task_runner.h"
#include "base/threading/thread_checker.h"
#include "media/base/scoped_async_trace.h"
#include "media/capture/video_capture_types.h"
#include "third_party/blink/public/mojom/mediastream/media_devices.mojom-blink-forward.h"
#include "third_party/blink/public/web/modules/mediastream/media_stream_video_source.h"
#include "third_party/blink/renderer/core/frame/local_frame.h"
#include "third_party/blink/renderer/modules/mediastream/apply_constraints_request.h"
#include "third_party/blink/renderer/modules/mediastream/media_stream_constraints_util.h"
#include "third_party/blink/renderer/modules/modules_export.h"
Expand All @@ -22,6 +24,11 @@ namespace blink {
class MediaStreamAudioSource;
class MediaStreamVideoTrack;

// If this feature is enabled, a call to applyConstraints() on a video content
// source will make the source restart with the new format.
MODULES_EXPORT BASE_DECLARE_FEATURE(
kApplyConstraintsRestartsVideoContentSources);

// ApplyConstraintsProcessor is responsible for processing applyConstraints()
// requests. Only one applyConstraints() request can be processed at a time.
// ApplyConstraintsProcessor must be created, called and destroyed on the main
Expand All @@ -32,6 +39,7 @@ class MODULES_EXPORT ApplyConstraintsProcessor final
using MediaDevicesDispatcherCallback = base::RepeatingCallback<
blink::mojom::blink::MediaDevicesDispatcherHost*()>;
ApplyConstraintsProcessor(
LocalFrame* frame,
MediaDevicesDispatcherCallback media_devices_dispatcher_cb,
scoped_refptr<base::SingleThreadTaskRunner> task_runner);

Expand All @@ -48,28 +56,38 @@ class MODULES_EXPORT ApplyConstraintsProcessor final
void ProcessRequest(blink::ApplyConstraintsRequest* request,
base::OnceClosure callback);

void Trace(Visitor* visitor) const { visitor->Trace(current_request_); }
void Trace(Visitor* visitor) const {
visitor->Trace(current_request_);
visitor->Trace(frame_);
}

private:
// Helpers for video device-capture requests.
void ProcessVideoDeviceRequest();
void MaybeStopSourceForRestart(
void MaybeStopVideoDeviceSourceForRestart(
const Vector<media::VideoCaptureFormat>& formats);
void MaybeSourceStoppedForRestart(
void MaybeDeviceSourceStoppedForRestart(
blink::MediaStreamVideoSource::RestartResult result);
void FindNewFormatAndRestart(
void FindNewFormatAndRestartDeviceSource(
const Vector<media::VideoCaptureFormat>& formats);
void MaybeSourceRestarted(
blink::VideoCaptureSettings SelectVideoDeviceSettings(
Vector<media::VideoCaptureFormat> formats);

// Helpers for video content-capture requests.
void ProcessVideoContentRequest();
void MaybeStopVideoContentSourceForRestart();
void MaybeRestartStoppedVideoContentSource(
blink::MediaStreamVideoSource::RestartResult result);
blink::VideoCaptureSettings SelectVideoContentSettings();

// Helpers for all video requests.
void ProcessVideoRequest();
blink::MediaStreamVideoTrack* GetCurrentVideoTrack();
blink::MediaStreamVideoSource* GetCurrentVideoSource();
bool AbortIfVideoRequestStateInvalid(); // Returns true if aborted.
blink::VideoCaptureSettings SelectVideoSettings(
Vector<media::VideoCaptureFormat> formats);
void FinalizeVideoRequest();
void MaybeSourceRestarted(
blink::MediaStreamVideoSource::RestartResult result);

// Helpers for audio requests.
void ProcessAudioRequest();
Expand All @@ -96,6 +114,7 @@ class MODULES_EXPORT ApplyConstraintsProcessor final
blink::MediaStreamVideoSource* video_source_ = nullptr;
base::OnceClosure request_completed_cb_;

const Member<LocalFrame> frame_;
MediaDevicesDispatcherCallback media_devices_dispatcher_cb_;
THREAD_CHECKER(thread_checker_);

Expand Down

0 comments on commit a9e8bd8

Please sign in to comment.