Skip to content

Commit

Permalink
MediaRecorder: Wire is_screencast to media VideoEncoder implementation
Browse files Browse the repository at this point in the history 
media::VideoEncoder supports content_hint. This CL sets content_hint
to Screen if the media recorders records a screen content.

Bug: b:280361196
Test: None
Change-Id: I4595c5b06962ef51aee8983fc7516c5f471a2157
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4976381
Reviewed-by: Eugene Zemtsov <eugene@chromium.org>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: Markus Handell <handellm@google.com>
Cr-Commit-Position: refs/heads/main@{#1216801}
  • Loading branch information
Hirokazu Honda authored and Chromium LUCI CQ committed Oct 30, 2023
1 parent 37ba04e commit d3d49c2
Show file tree
Hide file tree
Showing 6 changed files with 176 additions and 46 deletions.
9 changes: 9 additions & 0 deletions third_party/blink/renderer/modules/mediarecorder/media_recorder_encoder_wrapper.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ MediaRecorderEncoderWrapper::MediaRecorderEncoderWrapper(
scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
media::VideoCodecProfile profile,
uint32_t bits_per_second,
bool is_screencast,
media::GpuVideoAcceleratorFactories* gpu_factories,
CreateEncoderCB create_encoder_cb,
VideoTrackRecorder::OnEncodedVideoCB on_encoded_video_cb,
Expand All @@ -62,6 +63,9 @@ MediaRecorderEncoderWrapper::MediaRecorderEncoderWrapper(
CHECK(base::Contains(kSupportedCodecs, codec_));
options_.bitrate = media::Bitrate::VariableBitrate(
bits_per_second, base::ClampMul(bits_per_second, 2u).RawValue());
options_.content_hint = is_screencast
? media::VideoEncoder::ContentHint::Screen
: media::VideoEncoder::ContentHint::Camera;
}

MediaRecorderEncoderWrapper::~MediaRecorderEncoderWrapper() {
Expand All @@ -74,6 +78,11 @@ bool MediaRecorderEncoderWrapper::CanEncodeAlphaChannel() const {
codec_ == media::VideoCodec::kVP9);
}

bool MediaRecorderEncoderWrapper::IsScreenContentEncodingForTesting() const {
return options_.content_hint.has_value() &&
*options_.content_hint == media::VideoEncoder::ContentHint::Screen;
}

void MediaRecorderEncoderWrapper::EnterErrorState(
const media::EncoderStatus& status) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
Expand Down
2 changes: 2 additions & 0 deletions third_party/blink/renderer/modules/mediarecorder/media_recorder_encoder_wrapper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ class MODULES_EXPORT MediaRecorderEncoderWrapper final
scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
media::VideoCodecProfile profile,
uint32_t bits_per_second,
bool is_screencast,
media::GpuVideoAcceleratorFactories* gpu_factories,
CreateEncoderCB create_encoder_cb,
VideoTrackRecorder::OnEncodedVideoCB on_encoded_video_cb,
Expand All @@ -45,6 +46,7 @@ class MODULES_EXPORT MediaRecorderEncoderWrapper final
base::WeakPtr<Encoder> GetWeakPtr() override {
return weak_factory_.GetWeakPtr();
}
bool IsScreenContentEncodingForTesting() const override;

private:
friend class MediaRecorderEncoderWrapperTest;
Expand Down
132 changes: 96 additions & 36 deletions third_party/blink/renderer/modules/mediarecorder/media_recorder_encoder_wrapper_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,15 @@ media::VideoEncoderOutput DefaultEncoderOutput() {
return output;
}

MATCHER_P2(MatchEncoderOptions, bitrate, frame_size, "encoder option matcher") {
MATCHER_P3(MatchEncoderOptions,
bitrate,
frame_size,
content_hint,
"encoder option matcher") {
return arg.bitrate.has_value() &&
arg.bitrate->mode() == media::Bitrate::Mode::kVariable &&
arg.bitrate->target_bps() == base::checked_cast<uint32_t>(bitrate) &&
arg.frame_size == frame_size;
*arg.content_hint == content_hint && arg.frame_size == frame_size;
}

MATCHER_P(MatchEncodeOption, key_frame, "encode option matcher") {
Expand Down Expand Up @@ -110,26 +114,7 @@ class MediaRecorderEncoderWrapperTest
public:
MediaRecorderEncoderWrapperTest()
: profile_(GetParam()),
codec_(media::VideoCodecProfileToVideoCodec(profile_)),
encoder_wrapper_(
scheduler::GetSingleThreadTaskRunnerForTesting(),
profile_,
kDefaultBitrate,
/*gpu_factories=*/nullptr,
WTF::BindRepeating(
&MediaRecorderEncoderWrapperTest::CreateMockVideoEncoder,
base::Unretained(this)),
WTF::BindRepeating(&MediaRecorderEncoderWrapperTest::OnEncodedVideo,
base::Unretained(this)),
WTF::BindRepeating(&MediaRecorderEncoderWrapperTest::OnError,
base::Unretained(this))) {
auto metrics_provider =
std::make_unique<media::MockVideoEncoderMetricsProvider>();
mock_metrics_provider_ = metrics_provider.get();
encoder_wrapper_.metrics_provider_ = std::move(metrics_provider);

SetupSuccessful720pEncoderInitialization();
}
codec_(media::VideoCodecProfileToVideoCodec(profile_)) {}

~MediaRecorderEncoderWrapperTest() override {
EXPECT_CALL(mock_encoder_, Dtor);
Expand All @@ -149,11 +134,34 @@ class MediaRecorderEncoderWrapperTest
&MediaRecorderEncoderWrapperTest::MockVideoEncoderWrapperDtor,
base::Unretained(this)));
}

void CreateEncoderWrapper(bool is_screencast) {
encoder_wrapper_ = std::make_unique<MediaRecorderEncoderWrapper>(
scheduler::GetSingleThreadTaskRunnerForTesting(), profile_,
kDefaultBitrate, is_screencast,
/*gpu_factories=*/nullptr,
WTF::BindRepeating(
&MediaRecorderEncoderWrapperTest::CreateMockVideoEncoder,
base::Unretained(this)),
WTF::BindRepeating(&MediaRecorderEncoderWrapperTest::OnEncodedVideo,
base::Unretained(this)),
WTF::BindRepeating(&MediaRecorderEncoderWrapperTest::OnError,
base::Unretained(this)));
EXPECT_EQ(is_screencast,
encoder_wrapper_->IsScreenContentEncodingForTesting());
auto metrics_provider =
std::make_unique<media::MockVideoEncoderMetricsProvider>();
mock_metrics_provider_ = metrics_provider.get();
encoder_wrapper_->metrics_provider_ = std::move(metrics_provider);

SetupSuccessful720pEncoderInitialization();
}

// EncodeFrame is a private function of MediaRecorderEncoderWrapper.
// It can be called only in MediaRecorderEncoderWrapperTest.
void EncodeFrame(scoped_refptr<media::VideoFrame> frame,
base::TimeTicks capture_timestamp) {
encoder_wrapper_.EncodeFrame(std::move(frame), capture_timestamp, false);
encoder_wrapper_->EncodeFrame(std::move(frame), capture_timestamp, false);
}

MOCK_METHOD(
Expand All @@ -169,8 +177,11 @@ class MediaRecorderEncoderWrapperTest

void SetupSuccessful720pEncoderInitialization() {
ON_CALL(mock_encoder_,
Initialize(profile_, MatchEncoderOptions(kDefaultBitrate, k720p), _,
_, _))
Initialize(
profile_,
MatchEncoderOptions(kDefaultBitrate, k720p,
media::VideoEncoder::ContentHint::Camera),
_, _, _))
.WillByDefault(WithArgs<3, 4>(
[this](media::VideoEncoder::OutputCB output_callback,
media::VideoEncoder::EncoderStatusCB initialize_done_cb) {
Expand Down Expand Up @@ -199,10 +210,11 @@ class MediaRecorderEncoderWrapperTest
media::MockVideoEncoder mock_encoder_;
raw_ptr<media::MockVideoEncoderMetricsProvider, DanglingUntriaged>
mock_metrics_provider_;
MediaRecorderEncoderWrapper encoder_wrapper_;
std::unique_ptr<MediaRecorderEncoderWrapper> encoder_wrapper_;
};

TEST_P(MediaRecorderEncoderWrapperTest, InitializesAndEncodesOneFrame) {
CreateEncoderWrapper(false);
InSequence s;
EXPECT_CALL(*this, CreateEncoder);
EXPECT_CALL(*mock_metrics_provider_, MockInitialize);
Expand All @@ -219,8 +231,39 @@ TEST_P(MediaRecorderEncoderWrapperTest, InitializesAndEncodesOneFrame) {
EXPECT_CALL(*this, MockVideoEncoderWrapperDtor);
}

TEST_P(MediaRecorderEncoderWrapperTest,
InitializesWithScreenCastAndEncodesOneFrame) {
CreateEncoderWrapper(true);
InSequence s;
EXPECT_CALL(*this, CreateEncoder);
EXPECT_CALL(*mock_metrics_provider_, MockInitialize);
ON_CALL(
mock_encoder_,
Initialize(profile_,
MatchEncoderOptions(kDefaultBitrate, k720p,
media::VideoEncoder::ContentHint::Screen),
_, _, _))
.WillByDefault(WithArgs<3, 4>(
[this](media::VideoEncoder::OutputCB output_callback,
media::VideoEncoder::EncoderStatusCB initialize_done_cb) {
this->output_cb = output_callback;
std::move(initialize_done_cb).Run(media::EncoderStatus::Codes::kOk);
}));
EXPECT_CALL(mock_encoder_, Encode);

EXPECT_CALL(*mock_metrics_provider_, MockIncrementEncodedFrameCount);
EXPECT_CALL(*this, OnEncodedVideo(MatchVideoParams(k720p, codec_),
MatchStringSize(kChunkSize),
MatchStringSize(0), _, _,
/*key_frame=*/true));
EncodeFrame(media::VideoFrame::CreateBlackFrame(k720p),
base::TimeTicks::Now());
EXPECT_CALL(*this, MockVideoEncoderWrapperDtor);
}

TEST_P(MediaRecorderEncoderWrapperTest,
EncodesTwoFramesWithoutRecreatingEncoder) {
CreateEncoderWrapper(false);
InSequence s;
const auto capture_timestamp1 = base::TimeTicks::Now();
EXPECT_CALL(*mock_metrics_provider_, MockIncrementEncodedFrameCount);
Expand Down Expand Up @@ -255,6 +298,7 @@ TEST_P(MediaRecorderEncoderWrapperTest,

TEST_P(MediaRecorderEncoderWrapperTest,
EncodeTwoFramesAndDelayEncodeDoneAndOutputCB) {
CreateEncoderWrapper(false);
InSequence s;
media::VideoEncoder::EncoderStatusCB encode_done_cb1;
const auto capture_timestamp1 = base::TimeTicks::Now();
Expand Down Expand Up @@ -295,6 +339,7 @@ TEST_P(MediaRecorderEncoderWrapperTest,
}

TEST_P(MediaRecorderEncoderWrapperTest, RecreatesEncoderOnNewResolution) {
CreateEncoderWrapper(false);
InSequence s;
EXPECT_CALL(*mock_metrics_provider_, MockIncrementEncodedFrameCount);
EncodeFrame(media::VideoFrame::CreateBlackFrame(k720p),
Expand All @@ -310,9 +355,12 @@ TEST_P(MediaRecorderEncoderWrapperTest, RecreatesEncoderOnNewResolution) {
EXPECT_CALL(
*mock_metrics_provider_,
MockInitialize(profile_, k360p, false, media::SVCScalabilityMode::kL1T1));
EXPECT_CALL(mock_encoder_,
Initialize(profile_, MatchEncoderOptions(kDefaultBitrate, k360p),
_, _, _))
EXPECT_CALL(
mock_encoder_,
Initialize(profile_,
MatchEncoderOptions(kDefaultBitrate, k360p,
media::VideoEncoder::ContentHint::Camera),
_, _, _))
.WillOnce(WithArgs<3, 4>(
[this](media::VideoEncoder::OutputCB output_cb,
media::VideoEncoder::EncoderStatusCB initialize_done_cb) {
Expand All @@ -337,10 +385,14 @@ TEST_P(MediaRecorderEncoderWrapperTest, RecreatesEncoderOnNewResolution) {
}

TEST_P(MediaRecorderEncoderWrapperTest, HandlesInitializeFailure) {
CreateEncoderWrapper(false);
InSequence s;
EXPECT_CALL(mock_encoder_,
Initialize(profile_, MatchEncoderOptions(kDefaultBitrate, k720p),
_, _, _))
EXPECT_CALL(
mock_encoder_,
Initialize(profile_,
MatchEncoderOptions(kDefaultBitrate, k720p,
media::VideoEncoder::ContentHint::Camera),
_, _, _))
.WillOnce(WithArgs<4>(
[](media::VideoEncoder::EncoderStatusCB initialize_done_cb) {
std::move(initialize_done_cb)
Expand All @@ -356,6 +408,7 @@ TEST_P(MediaRecorderEncoderWrapperTest, HandlesInitializeFailure) {
}

TEST_P(MediaRecorderEncoderWrapperTest, HandlesEncodeFailure) {
CreateEncoderWrapper(false);
InSequence s;
EXPECT_CALL(mock_encoder_, Encode(_, MatchEncodeOption(false), _))
.WillOnce(
Expand All @@ -373,6 +426,7 @@ TEST_P(MediaRecorderEncoderWrapperTest, HandlesEncodeFailure) {
}

TEST_P(MediaRecorderEncoderWrapperTest, HandlesFlushFailure) {
CreateEncoderWrapper(false);
InSequence s;
EXPECT_CALL(*mock_metrics_provider_, MockIncrementEncodedFrameCount);
EXPECT_CALL(mock_encoder_, Flush)
Expand All @@ -393,6 +447,7 @@ TEST_P(MediaRecorderEncoderWrapperTest, HandlesFlushFailure) {
}

TEST_P(MediaRecorderEncoderWrapperTest, NotCallOnEncodedVideoCBIfEncodeFail) {
CreateEncoderWrapper(false);
InSequence s;
EXPECT_CALL(mock_encoder_, Encode(_, MatchEncodeOption(false), _))
.WillOnce(WithArgs<2>(
Expand All @@ -414,6 +469,7 @@ TEST_P(MediaRecorderEncoderWrapperTest, NotCallOnEncodedVideoCBIfEncodeFail) {

TEST_P(MediaRecorderEncoderWrapperTest,
NotErrorCallbackTwiceByTwiceEncodeDoneFailure) {
CreateEncoderWrapper(false);
InSequence s;
media::VideoEncoder::EncoderStatusCB encode_done_cb1;
EXPECT_CALL(mock_encoder_, Encode)
Expand Down Expand Up @@ -443,10 +499,14 @@ TEST_P(MediaRecorderEncoderWrapperTest,
}

TEST_P(MediaRecorderEncoderWrapperTest, IgnoresEncodeAfterFailure) {
CreateEncoderWrapper(false);
InSequence s;
EXPECT_CALL(mock_encoder_,
Initialize(profile_, MatchEncoderOptions(kDefaultBitrate, k720p),
_, _, _))
EXPECT_CALL(
mock_encoder_,
Initialize(profile_,
MatchEncoderOptions(kDefaultBitrate, k720p,
media::VideoEncoder::ContentHint::Camera),
_, _, _))
.WillOnce(WithArgs<4>(
[](media::VideoEncoder::EncoderStatusCB initialize_done_cb) {
std::move(initialize_done_cb)
Expand All @@ -473,7 +533,7 @@ TEST_P(MediaRecorderEncoderWrapperTest, InitializesAndEncodesOneAlphaFrame) {
GTEST_SKIP() << "no alpha encoding is supported in"
<< media::GetCodecName(codec_);
}

CreateEncoderWrapper(false);
constexpr size_t kAlphaChunkSize = 2345;
EXPECT_CALL(*this, CreateEncoder).Times(2);
EXPECT_CALL(*mock_metrics_provider_, MockInitialize);
Expand Down
17 changes: 9 additions & 8 deletions third_party/blink/renderer/modules/mediarecorder/video_track_recorder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -820,6 +820,7 @@ VideoTrackRecorderImpl::CreateMediaVideoEncoder(
scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
CodecProfile codec_profile,
uint32_t bits_per_second,
bool is_screencast,
const OnEncodedVideoCB& on_encoded_video_cb,
bool create_vea_encoder) {
DCHECK_CALLED_ON_VALID_SEQUENCE(main_sequence_checker_);
Expand Down Expand Up @@ -858,10 +859,9 @@ VideoTrackRecorderImpl::CreateMediaVideoEncoder(

media::GpuVideoAcceleratorFactories* gpu_factories =
Platform::Current()->GetGpuFactories();
// TODO(crbug.com/1441395): Support |is_screencast|.
return std::make_unique<MediaRecorderEncoderWrapper>(
std::move(encoding_task_runner), video_codec_profile, bits_per_second,
create_vea_encoder ? gpu_factories : nullptr,
is_screencast, create_vea_encoder ? gpu_factories : nullptr,
create_vea_encoder
? GetCreateHardwareVideoEncoderCallback()
: GetCreateSoftwareVideoEncoderCallback(codec_profile.codec_id),
Expand All @@ -873,6 +873,7 @@ VideoTrackRecorderImpl::CreateSoftwareVideoEncoder(
scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
CodecProfile codec_profile,
uint32_t bits_per_second,
bool is_screencast,
const OnEncodedVideoCB& on_encoded_video_cb) {
DCHECK_CALLED_ON_VALID_SEQUENCE(main_sequence_checker_);
switch (codec_profile.codec_id) {
Expand All @@ -897,7 +898,7 @@ VideoTrackRecorderImpl::CreateSoftwareVideoEncoder(
return std::make_unique<MediaRecorderEncoderWrapper>(
std::move(encoding_task_runner),
codec_profile.profile.value_or(media::AV1PROFILE_PROFILE_MAIN),
bits_per_second,
bits_per_second, is_screencast,
/*gpu_factories=*/nullptr,
GetCreateSoftwareVideoEncoderCallback(CodecId::kAv1),
on_encoded_video_cb, std::move(on_error_cb));
Expand Down Expand Up @@ -1012,8 +1013,8 @@ void VideoTrackRecorderImpl::InitializeEncoderOnEncoderSupportKnown(
encoding_task_runner =
base::ThreadPool::CreateSequencedTaskRunner({base::MayBlock()});
encoder = CreateMediaVideoEncoder(encoding_task_runner, codec_profile,
bits_per_second, on_encoded_video_cb,
create_vea_encoder);
bits_per_second, is_screencast,
on_encoded_video_cb, create_vea_encoder);
} else {
if (create_vea_encoder) {
encoding_task_runner =
Expand All @@ -1024,9 +1025,9 @@ void VideoTrackRecorderImpl::InitializeEncoderOnEncoderSupportKnown(
} else {
encoding_task_runner =
base::ThreadPool::CreateSequencedTaskRunner({base::MayBlock()});
encoder =
CreateSoftwareVideoEncoder(encoding_task_runner, codec_profile,
bits_per_second, on_encoded_video_cb);
encoder = CreateSoftwareVideoEncoder(encoding_task_runner, codec_profile,
bits_per_second, is_screencast,
on_encoded_video_cb);
}
}

Expand Down
6 changes: 4 additions & 2 deletions third_party/blink/renderer/modules/mediarecorder/video_track_recorder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,11 +196,11 @@ class VideoTrackRecorder : public TrackRecorder<MediaStreamVideoSink> {

void SetPaused(bool paused);
virtual bool CanEncodeAlphaChannel() const;
virtual bool IsScreenContentEncodingForTesting() const { return false; }
virtual base::WeakPtr<Encoder> GetWeakPtr() = 0;
void ForceKeyFrameForNextFrameForTesting() {
request_key_frame_for_testing_ = true;
}

protected:
friend class VideoTrackRecorderTest;

Expand Down Expand Up @@ -378,6 +378,7 @@ class MODULES_EXPORT VideoTrackRecorderImpl : public VideoTrackRecorder {
scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
CodecProfile codec_profile,
uint32_t bits_per_second,
bool is_screencast,
const OnEncodedVideoCB& on_encoded_video_cb);
std::unique_ptr<Encoder> CreateHardwareVideoEncoder(
scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
Expand All @@ -386,11 +387,12 @@ class MODULES_EXPORT VideoTrackRecorderImpl : public VideoTrackRecorder {
uint32_t bits_per_second,
const OnEncodedVideoCB& on_encoded_video_cb,
bool use_import_mode,
bool create_vea_encoder);
bool is_screencast);
std::unique_ptr<Encoder> CreateMediaVideoEncoder(
scoped_refptr<base::SequencedTaskRunner> encoding_task_runner,
CodecProfile codec_profile,
uint32_t bits_per_second,
bool is_screencast,
const OnEncodedVideoCB& on_encoded_video_cb,
bool create_vea_encoder);
void OnHardwareEncoderError();
Expand Down

0 comments on commit d3d49c2

Please sign in to comment.