Skip to content

Commit

Permalink
media/gpu/vea_perf_tests: Collect bitrate and its accuracy metrics
Browse files Browse the repository at this point in the history 
Bitstream quality metrics (e.g PSNR and SSIM) should be compared
taking into account bitrate accuracy. This CL collects bitrate
and the difference between specified bitrate and practical bitrate
in the bitstream quality test case.

Bug: b:220225711
Test: video_encode_accelerator_tests on volteer
Change-Id: Ic28635a4bd3862f4f7c2404b19844940f99362b8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3488653
Reviewed-by: Miguel Casas-Sanchez <mcasas@chromium.org>
Commit-Queue: Hirokazu Honda <hiroh@chromium.org>
Cr-Commit-Position: refs/heads/main@{#988380}
  • Loading branch information
Hirokazu Honda authored and Chromium LUCI CQ committed Apr 4, 2022
1 parent db897c4 commit 4972d31
Showing 1 changed file with 78 additions and 28 deletions.
106 changes: 78 additions & 28 deletions media/gpu/video_encode_accelerator_perf_tests.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
// found in the LICENSE file.

#include <algorithm>
#include <iomanip>
#include <iostream>
#include <map>
#include <numeric>
#include <vector>
Expand Down Expand Up @@ -318,7 +320,11 @@ struct BitstreamQualityMetrics {
const SSIMVideoFrameValidator* const ssim_validator,
const absl::optional<size_t>& spatial_idx,
const absl::optional<size_t>& temporal_idx);
void Output();

void Output(uint32_t target_bitrate, uint32_t actual_bitrate);

absl::optional<size_t> spatial_idx;
absl::optional<size_t> temporal_idx;

private:
struct QualityStats {
Expand All @@ -336,28 +342,30 @@ struct BitstreamQualityMetrics {
static QualityStats ComputeQualityStats(
const std::map<size_t, double>& values);

void WriteToConsole() const;
void WriteToFile() const;
void WriteToConsole(const std::string& svc_text,
const BitstreamQualityMetrics::QualityStats& psnr_stats,
const BitstreamQualityMetrics::QualityStats& ssim_stats,
uint32_t target_bitrate,
uint32_t actual_bitrate) const;
void WriteToFile(const std::string& svc_text,
const BitstreamQualityMetrics::QualityStats& psnr_stats,
const BitstreamQualityMetrics::QualityStats& ssim_stats,
uint32_t target_bitrate,
uint32_t actual_bitrate) const;

std::string svc_text;
const PSNRVideoFrameValidator* const psnr_validator;
const SSIMVideoFrameValidator* const ssim_validator;

QualityStats psnr_stats;
QualityStats ssim_stats;
};

BitstreamQualityMetrics::BitstreamQualityMetrics(
const PSNRVideoFrameValidator* const psnr_validator,
const SSIMVideoFrameValidator* const ssim_validator,
const absl::optional<size_t>& spatial_idx,
const absl::optional<size_t>& temporal_idx)
: psnr_validator(psnr_validator), ssim_validator(ssim_validator) {
if (spatial_idx)
svc_text += "L" + base::NumberToString(*spatial_idx + 1);
if (temporal_idx)
svc_text += "T" + base::NumberToString(*temporal_idx + 1);
}
: spatial_idx(spatial_idx),
temporal_idx(temporal_idx),
psnr_validator(psnr_validator),
ssim_validator(ssim_validator) {}

// static
BitstreamQualityMetrics::QualityStats
Expand Down Expand Up @@ -387,15 +395,36 @@ BitstreamQualityMetrics::ComputeQualityStats(
return stats;
}

void BitstreamQualityMetrics::Output() {
psnr_stats = ComputeQualityStats(psnr_validator->GetPSNRValues());
ssim_stats = ComputeQualityStats(ssim_validator->GetSSIMValues());
WriteToConsole();
WriteToFile();
void BitstreamQualityMetrics::Output(uint32_t target_bitrate,
uint32_t actual_bitrate) {
std::string svc_text;
if (spatial_idx)
svc_text += "L" + base::NumberToString(*spatial_idx + 1);
if (temporal_idx)
svc_text += "T" + base::NumberToString(*temporal_idx + 1);

auto psnr_stats = ComputeQualityStats(psnr_validator->GetPSNRValues());
auto ssim_stats = ComputeQualityStats(ssim_validator->GetSSIMValues());

WriteToConsole(svc_text, psnr_stats, ssim_stats, target_bitrate,
actual_bitrate);
WriteToFile(svc_text, psnr_stats, ssim_stats, target_bitrate, actual_bitrate);
}

void BitstreamQualityMetrics::WriteToConsole() const {
void BitstreamQualityMetrics::WriteToConsole(
const std::string& svc_text,
const BitstreamQualityMetrics::QualityStats& psnr_stats,
const BitstreamQualityMetrics::QualityStats& ssim_stats,
uint32_t target_bitrate,
uint32_t actual_bitrate) const {
const auto default_ssize = std::cout.precision();
std::cout << "[ Result " << svc_text << "]" << std::endl;
std::cout << "Bitrate: " << actual_bitrate << std::endl;
std::cout << "Bitrate deviation: " << std::fixed << std::setprecision(2)
<< (actual_bitrate * 100.0 / target_bitrate) - 100.0 << " %"
<< std::endl;

std::cout << std::fixed << std::setprecision(4);
std::cout << "SSIM - average: " << ssim_stats.avg << std::endl;
std::cout << "SSIM - percentile 25: " << ssim_stats.percentile_25
<< std::endl;
Expand All @@ -410,9 +439,15 @@ void BitstreamQualityMetrics::WriteToConsole() const {
<< std::endl;
std::cout << "PSNR - percentile 75: " << psnr_stats.percentile_75
<< std::endl;
std::cout.precision(default_ssize);
}

void BitstreamQualityMetrics::WriteToFile() const {
void BitstreamQualityMetrics::WriteToFile(
const std::string& svc_text,
const BitstreamQualityMetrics::QualityStats& psnr_stats,
const BitstreamQualityMetrics::QualityStats& ssim_stats,
uint32_t target_bitrate,
uint32_t actual_bitrate) const {
base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder());
if (!DirectoryExists(output_folder_path))
base::CreateDirectory(output_folder_path);
Expand All @@ -421,14 +456,13 @@ void BitstreamQualityMetrics::WriteToFile() const {
base::Value metrics(base::Value::Type::DICTIONARY);
if (!svc_text.empty())
metrics.SetKey("SVC", base::Value(svc_text));
metrics.SetKey("Bitrate",
base::Value(base::checked_cast<int>(actual_bitrate)));
metrics.SetKey(
"Bitrate deviation",
base::Value((actual_bitrate * 100.0 / target_bitrate) - 100.0));
metrics.SetKey("SSIMAverage", base::Value(ssim_stats.avg));
metrics.SetKey("SSIMPercentile25", base::Value(ssim_stats.percentile_25));
metrics.SetKey("SSIMPercentile50", base::Value(ssim_stats.percentile_50));
metrics.SetKey("SSIMPercentile75", base::Value(psnr_stats.percentile_75));
metrics.SetKey("PSNRAverage", base::Value(psnr_stats.avg));
metrics.SetKey("PSNRPercentile25", base::Value(psnr_stats.percentile_25));
metrics.SetKey("PSNRPercentile50", base::Value(psnr_stats.percentile_50));
metrics.SetKey("PSNRPercentile75", base::Value(psnr_stats.percentile_75));
// Write ssim values bitstream delivery times to json.
base::Value ssim_values(base::Value::Type::LIST);
for (double value : ssim_stats.values_in_order)
Expand Down Expand Up @@ -664,8 +698,24 @@ TEST_F(VideoEncoderTest, MeasureProducedBitstreamQuality) {
EXPECT_EQ(encoder->GetFrameReleasedCount(), kNumFramesToEncodeForPerformance);
EXPECT_TRUE(encoder->WaitForBitstreamProcessors());

for (auto& metrics : quality_metrics_)
metrics.Output();
const VideoEncoderStats stats = encoder->GetStats();
for (auto& metrics : quality_metrics_) {
absl::optional<size_t> spatial_idx = metrics.spatial_idx;
absl::optional<size_t> temporal_idx = metrics.temporal_idx;
uint32_t target_bitrate = 0;
uint32_t actual_bitrate = 0;
if (!spatial_idx && !temporal_idx) {
target_bitrate = g_env->Bitrate().GetSumBps();
actual_bitrate = stats.Bitrate();
} else {
CHECK(spatial_idx && temporal_idx);
target_bitrate =
g_env->Bitrate().GetBitrateBps(*spatial_idx, *temporal_idx);
actual_bitrate = stats.LayerBitrate(*spatial_idx, *temporal_idx);
}

metrics.Output(target_bitrate, actual_bitrate);
}
}

// TODO(b/211783279) The |performance_evaluator_| only keeps track of the last
Expand Down

0 comments on commit 4972d31

Please sign in to comment.