Skip to content

Commit

Permalink
WebCodecs/VideoEncoder: Inject VideoEncoderMetricsProvider
Browse files Browse the repository at this point in the history 
This CL collects the video encoder usage in webcodecs by using
VideoEncoderMetricsProvider.

Bug: b:275663480
Test: Media.VideoEncoderMetrics is shown in chrome://ukm in
https://w3c.github.io/webcodecs/samples/capture-to-file/capture-to-file.html

Change-Id: I82ff8a82216ba004c3bae4aa1c0c8ff9933b03ad
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4613331
Reviewed-by: Eugene Zemtsov <eugene@chromium.org>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1191920}
  • Loading branch information
Hirokazu Honda authored and Chromium LUCI CQ committed Sep 4, 2023
1 parent dd1d7c6 commit 9e683ce
Show file tree
Hide file tree
Showing 3 changed files with 342 additions and 52 deletions.
123 changes: 95 additions & 28 deletions third_party/blink/renderer/modules/webcodecs/video_encoder.cc
View file
Expand Up @@ -31,12 +31,14 @@
#include "media/base/video_encoder.h"
#include "media/base/video_util.h"
#include "media/media_buildflags.h"
#include "media/mojo/clients/mojo_video_encoder_metrics_provider.h"
#include "media/video/gpu_video_accelerator_factories.h"
#include "media/video/h264_level_limits.h"
#include "media/video/offloading_video_encoder.h"
#include "media/video/video_encode_accelerator_adapter.h"
#include "media/video/video_encoder_fallback.h"
#include "third_party/blink/public/common/features.h"
#include "third_party/blink/public/common/thread_safe_browser_interface_broker_proxy.h"
#include "third_party/blink/public/mojom/use_counter/metrics/web_feature.mojom-blink.h"
#include "third_party/blink/public/platform/platform.h"
#include "third_party/blink/public/platform/task_type.h"
Expand All @@ -61,6 +63,8 @@
#include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_support.h"
#include "third_party/blink/renderer/bindings/modules/v8/v8_video_pixel_format.h"
#include "third_party/blink/renderer/core/dom/dom_exception.h"
#include "third_party/blink/renderer/core/frame/local_dom_window.h"
#include "third_party/blink/renderer/core/frame/local_frame.h"
#include "third_party/blink/renderer/core/streams/readable_stream.h"
#include "third_party/blink/renderer/core/streams/writable_stream.h"
#include "third_party/blink/renderer/modules/event_modules.h"
Expand Down Expand Up @@ -670,7 +674,9 @@ std::unique_ptr<media::VideoEncoder> CreateOpenH264VideoEncoder() {
// with a weak |this|. It's needed in to avoid a persistent reference cycle.
std::unique_ptr<media::VideoEncoder> VideoEncoder::CreateSoftwareVideoEncoder(
VideoEncoder* self,
bool fallback,
media::VideoCodec codec) {
DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_);
if (!self)
return nullptr;
std::unique_ptr<media::VideoEncoder> result;
Expand All @@ -690,31 +696,41 @@ std::unique_ptr<media::VideoEncoder> VideoEncoder::CreateSoftwareVideoEncoder(
}
if (!result)
return nullptr;
if (fallback) {
CHECK(self->encoder_metrics_provider_);
self->encoder_metrics_provider_->Initialize(
self->active_config_->profile, self->active_config_->options.frame_size,
/*is_hardware_encoder=*/false,
self->active_config_->options.scalability_mode.value_or(
media::SVCScalabilityMode::kL1T1));
}
return std::make_unique<media::OffloadingVideoEncoder>(std::move(result));
}

std::unique_ptr<media::VideoEncoder> VideoEncoder::CreateMediaVideoEncoder(
const ParsedConfig& config,
media::GpuVideoAcceleratorFactories* gpu_factories) {
media::GpuVideoAcceleratorFactories* gpu_factories,
bool& is_platform_encoder) {
is_platform_encoder = true;
if (config.hw_pref == HardwarePreference::kPreferHardware ||
config.hw_pref == HardwarePreference::kNoPreference ||
MayHaveOSSoftwareEncoder(config.profile)) {
auto result = CreateAcceleratedVideoEncoder(config.profile, config.options,
gpu_factories, config.hw_pref);

if (config.hw_pref == HardwarePreference::kPreferHardware) {
return result;
} else if (result) {
// 'no-preference' or 'prefer-software' and we have OS software encoders.
return std::make_unique<media::VideoEncoderFallback>(
std::move(result),
ConvertToBaseOnceCallback(CrossThreadBindOnce(
&VideoEncoder::CreateSoftwareVideoEncoder,
MakeUnwrappingCrossThreadWeakHandle(this), config.codec)));
std::move(result), ConvertToBaseOnceCallback(CrossThreadBindOnce(
&VideoEncoder::CreateSoftwareVideoEncoder,
MakeUnwrappingCrossThreadWeakHandle(this),
/*fallback=*/true, config.codec)));
}
}

return CreateSoftwareVideoEncoder(this, config.codec);
is_platform_encoder = false;
return CreateSoftwareVideoEncoder(this, /*fallback=*/false, config.codec);
}

void VideoEncoder::ContinueConfigureWithGpuFactories(
Expand All @@ -723,15 +739,15 @@ void VideoEncoder::ContinueConfigureWithGpuFactories(
DCHECK(active_config_);
DCHECK_EQ(request->type, Request::Type::kConfigure);
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);

media_encoder_ = CreateMediaVideoEncoder(*active_config_, gpu_factories);
bool is_platform_encoder = false;
media_encoder_ = CreateMediaVideoEncoder(*active_config_, gpu_factories,
is_platform_encoder);
if (!media_encoder_) {
HandleError(logger_->MakeOperationError(
"Encoder creation error.",
media::EncoderStatus(
media::EncoderStatus::Codes::kEncoderInitializationError,
"Unable to create encoder (most likely unsupported "
"codec/acceleration requirement combination)")));
ReportError("Encoder creation error.",
media::EncoderStatus(
media::EncoderStatus::Codes::kEncoderInitializationError,
"Unable to create encoder (most likely unsupported "
"codec/acceleration requirement combination)"));
request->EndTracing();
return;
}
Expand Down Expand Up @@ -771,8 +787,7 @@ void VideoEncoder::ContinueConfigureWithGpuFactories(
break;
}

self->HandleError(
self->logger_->MakeOperationError(error_message, std::move(status)));
self->ReportError(error_message.c_str(), std::move(status));
} else {
UMA_HISTOGRAM_ENUMERATION("Blink.WebCodecs.VideoEncoder.Codec", codec,
media::VideoCodec::kMaxValue);
Expand All @@ -782,7 +797,14 @@ void VideoEncoder::ContinueConfigureWithGpuFactories(
self->blocking_request_in_progress_ = false;
self->ProcessRequests();
};

if (!encoder_metrics_provider_) {
encoder_metrics_provider_ = CreateVideoEncoderMetricsProvider();
}
encoder_metrics_provider_->Initialize(
active_config_->profile, active_config_->options.frame_size,
is_platform_encoder,
active_config_->options.scalability_mode.value_or(
media::SVCScalabilityMode::kL1T1));
media_encoder_->Initialize(
active_config_->profile, active_config_->options, std::move(info_cb),
std::move(output_cb),
Expand All @@ -791,6 +813,26 @@ void VideoEncoder::ContinueConfigureWithGpuFactories(
MakeUnwrappingCrossThreadHandle(request), active_config_->codec)));
}

std::unique_ptr<media::VideoEncoderMetricsProvider>
VideoEncoder::CreateVideoEncoderMetricsProvider() const {
mojo::PendingRemote<media::mojom::VideoEncoderMetricsProvider>
video_encoder_metrics_provider;
LocalDOMWindow* window = DomWindow();
LocalFrame* local_frame = window ? window->GetFrame() : nullptr;
// There is no DOM frame if WebCodecs runs in a service worker.
if (local_frame) {
local_frame->GetBrowserInterfaceBroker().GetInterface(
video_encoder_metrics_provider.InitWithNewPipeAndPassReceiver());
} else {
Platform::Current()->GetBrowserInterfaceBroker()->GetInterface(
video_encoder_metrics_provider.InitWithNewPipeAndPassReceiver());
}
return base::MakeRefCounted<media::MojoVideoEncoderMetricsProviderFactory>(
media::mojom::VideoEncoderUseCase::kWebCodecs,
std::move(video_encoder_metrics_provider))
->CreateVideoEncoderMetricsProvider();
}

bool VideoEncoder::CanReconfigure(ParsedConfig& original_config,
ParsedConfig& new_config) {
// Reconfigure is intended for things that don't require changing underlying
Expand All @@ -814,6 +856,20 @@ void VideoEncoder::Trace(Visitor* visitor) const {
Base::Trace(visitor);
}

void VideoEncoder::ReportError(const char* error_message,
const media::EncoderStatus& status) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
CHECK(!status.is_ok());
// ReportError() can be called before |encoder_metrics_provider_| is created
// in media::VideoEncoder::Initialize() (e.g. there is no available
// media::VideoEncoder). Since the case is about webrtc::VideoEncoder failure,
// we don't record it.
if (encoder_metrics_provider_) {
encoder_metrics_provider_->SetError(status);
}
HandleError(logger_->MakeOperationError(error_message, status));
}

bool VideoEncoder::ReadyToProcessNextRequest() {
if (active_encodes_ >= max_active_encodes_)
return false;
Expand Down Expand Up @@ -1079,8 +1135,7 @@ void VideoEncoder::OnEncodeDone(Request* request, media::EncoderStatus status) {

active_encodes_--;
if (!status.is_ok()) {
HandleError(
logger_->MakeEncodingError("Encoding error.", std::move(status)));
ReportError("Encoding error.", std::move(status));
}
request->EndTracing();
ProcessRequests();
Expand All @@ -1091,7 +1146,6 @@ void VideoEncoder::ProcessConfigure(Request* request) {
DCHECK_EQ(request->type, Request::Type::kConfigure);
DCHECK(active_config_);
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);

request->StartTracing();

blocking_request_in_progress_ = true;
Expand All @@ -1114,7 +1168,6 @@ void VideoEncoder::ProcessReconfigure(Request* request) {
DCHECK(active_config_);
DCHECK(media_encoder_);
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);

request->StartTracing();

auto reconf_done_callback = [](VideoEncoder* self, Request* req,
Expand Down Expand Up @@ -1142,15 +1195,15 @@ void VideoEncoder::ProcessReconfigure(Request* request) {

auto flush_done_callback = [](VideoEncoder* self, Request* req,
decltype(reconf_done_callback) reconf_callback,
bool is_platform_encoder,
media::EncoderStatus status) {
if (!self || self->reset_count_ != req->reset_count) {
req->EndTracing(/*aborted=*/true);
return;
}
DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_);
if (!status.is_ok()) {
self->HandleError(self->logger_->MakeOperationError(
"Encoder initialization error.", std::move(status)));
self->ReportError("Encoder initialization error.", std::move(status));
self->blocking_request_in_progress_ = false;
req->EndTracing();
return;
Expand All @@ -1165,6 +1218,15 @@ void VideoEncoder::ProcessReconfigure(Request* request) {
MakeUnwrappingCrossThreadHandle(self->active_config_.Get()),
self->reset_count_));

if (!self->encoder_metrics_provider_) {
self->encoder_metrics_provider_ =
self->CreateVideoEncoderMetricsProvider();
}
self->encoder_metrics_provider_->Initialize(
self->active_config_->profile, self->active_config_->options.frame_size,
is_platform_encoder,
self->active_config_->options.scalability_mode.value_or(
media::SVCScalabilityMode::kL1T1));
self->first_output_after_configure_ = true;
self->media_encoder_->ChangeOptions(
self->active_config_->options, std::move(output_cb),
Expand All @@ -1174,14 +1236,15 @@ void VideoEncoder::ProcessReconfigure(Request* request) {
};

blocking_request_in_progress_ = true;
media_encoder_->Flush(WTF::BindOnce(flush_done_callback,
MakeUnwrappingCrossThreadWeakHandle(this),
MakeUnwrappingCrossThreadHandle(request),
std::move(reconf_done_callback)));
media_encoder_->Flush(WTF::BindOnce(
flush_done_callback, MakeUnwrappingCrossThreadWeakHandle(this),
MakeUnwrappingCrossThreadHandle(request), std::move(reconf_done_callback),
is_platform_encoder_));
}

void VideoEncoder::OnMediaEncoderInfoChanged(
const media::VideoEncoderInfo& encoder_info) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (encoder_info.is_hardware_accelerated)
ApplyCodecPressure();
else
Expand All @@ -1193,6 +1256,7 @@ void VideoEncoder::OnMediaEncoderInfoChanged(
log->SetProperty<media::MediaLogProperty::kIsPlatformVideoEncoder>(
encoder_info.is_hardware_accelerated);

is_platform_encoder_ = encoder_info.is_hardware_accelerated;
max_active_encodes_ = ComputeMaxActiveEncodes(encoder_info.frame_delay,
encoder_info.input_capacity);
// We may have increased our capacity for active encodes.
Expand All @@ -1204,6 +1268,7 @@ void VideoEncoder::CallOutputCallback(
uint32_t reset_count,
media::VideoEncoderOutput output,
absl::optional<media::VideoEncoder::CodecDescription> codec_desc) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(active_config);
if (!script_state_->ContextIsValid() || !output_callback_ ||
state_.AsEnum() != V8CodecState::Enum::kConfigured ||
Expand Down Expand Up @@ -1299,6 +1364,8 @@ void VideoEncoder::CallOutputCallback(
metadata->setDecoderConfig(decoder_config);
}

encoder_metrics_provider_->IncrementEncodedFrameCount();

TRACE_EVENT_BEGIN1(kCategory, GetTraceNames()->output.c_str(), "timestamp",
chunk->timestamp());

Expand Down
17 changes: 16 additions & 1 deletion third_party/blink/renderer/modules/webcodecs/video_encoder.h
View file
Expand Up @@ -22,6 +22,7 @@

namespace media {
class GpuVideoAcceleratorFactories;
class VideoEncoderMetricsProvider;
class VideoEncoder;
struct VideoEncoderOutput;
} // namespace media
Expand Down Expand Up @@ -88,6 +89,12 @@ class MODULES_EXPORT VideoEncoder : public EncoderBase<VideoEncoderTraits> {
// GarbageCollected override.
void Trace(Visitor*) const override;

void ReportError(const char* error_message,
const media::EncoderStatus& status);

std::unique_ptr<media::VideoEncoderMetricsProvider> encoder_metrics_provider_
GUARDED_BY_CONTEXT(sequence_checker_);

protected:
using Base = EncoderBase<VideoEncoderTraits>;
using ParsedConfig = VideoEncoderTraits::ParsedConfig;
Expand All @@ -113,16 +120,21 @@ class MODULES_EXPORT VideoEncoder : public EncoderBase<VideoEncoderTraits> {
scoped_refptr<media::VideoFrame> result_frame);
static std::unique_ptr<media::VideoEncoder> CreateSoftwareVideoEncoder(
VideoEncoder* self,
bool fallback,
media::VideoCodec codec);

ParsedConfig* ParseConfig(const VideoEncoderConfig*,
ExceptionState&) override;
bool VerifyCodecSupport(ParsedConfig*, ExceptionState&) override;

// Virtual for UTs.
// Returns the VideoEncoder.
virtual std::unique_ptr<media::VideoEncoder> CreateMediaVideoEncoder(
const ParsedConfig& config,
media::GpuVideoAcceleratorFactories* gpu_factories);
media::GpuVideoAcceleratorFactories* gpu_factories,
bool& is_platform_encoder);
virtual std::unique_ptr<media::VideoEncoderMetricsProvider>
CreateVideoEncoderMetricsProvider() const;

void ContinueConfigureWithGpuFactories(
Request* request,
Expand Down Expand Up @@ -153,6 +165,9 @@ class MODULES_EXPORT VideoEncoder : public EncoderBase<VideoEncoderTraits> {
// The current upper limit on |active_encodes_|.
int max_active_encodes_;

// True if a running video encoder is hardware accelerated.
bool is_platform_encoder_ = false;

// Per-frame metadata to be applied to outputs, linked by timestamp.
struct FrameMetadata {
base::TimeDelta duration;
Expand Down

0 comments on commit 9e683ce

Please sign in to comment.