From 601e2e3612bbf80c7187f4aaf313c4d6019fde37 Mon Sep 17 00:00:00 2001 From: Justin Green Date: Fri, 16 Dec 2022 22:34:57 +0000 Subject: [PATCH] media/gpu: Add Log likelihood ratio frame validator Add a log likelihood ratio frame validation mechanism, but do not turn it on. We want to collect data on what values indicate a true positive before we pick a threshold value. Bug: 262772938 Test: Ran video.EncodeAccelPerf.h264_1080p on an Asurada. Change-Id: I9b48d9def4f34416ca4377cc4da892ef943a9c0e Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4112789 Reviewed-by: Frank Liberato Commit-Queue: Justin Green Cr-Commit-Position: refs/heads/main@{#1084573} --- media/gpu/test/image_quality_metrics.cc | 106 ++++++++++++++++++ media/gpu/test/image_quality_metrics.h | 10 ++ media/gpu/test/video_frame_validator.cc | 69 ++++++++++++ media/gpu/test/video_frame_validator.h | 44 ++++++++ .../video_encode_accelerator_perf_tests.cc | 47 +++++++- 5 files changed, 271 insertions(+), 5 deletions(-) diff --git a/media/gpu/test/image_quality_metrics.cc b/media/gpu/test/image_quality_metrics.cc index 52dab7b8b61c6..1e3c41d13c237 100644 --- a/media/gpu/test/image_quality_metrics.cc +++ b/media/gpu/test/image_quality_metrics.cc @@ -194,6 +194,87 @@ double ComputeSimilarity(const VideoFrame* frame1, frame2->stride(1), frame2->visible_data(2), frame2->stride(2), frame1->visible_rect().width(), frame1->visible_rect().height()); } + +constexpr int kJointDistributionBitDepth = 4; +constexpr int kJointDistributionDim = 1 << kJointDistributionBitDepth; + +using DistributionTable = + double[kJointDistributionDim][kJointDistributionDim][kJointDistributionDim]; + +bool ComputeLogJointDistribution(const VideoFrame& frame, + DistributionTable& log_joint_distribution) { + ASSERT_TRUE_OR_RETURN(frame.IsMappable(), false); + ASSERT_TRUE_OR_RETURN(frame.format() == PIXEL_FORMAT_ARGB, false); + ASSERT_TRUE_OR_RETURN(frame.BitDepth() == 8, false); + + // Arbitrarily small number to fill the probability distribution table with so + // we don't have a problem with taking the log of 0. + static const double kMinProbabilityValue = 0.000000001; + + double normalization_factor = kJointDistributionDim * kJointDistributionDim * + kJointDistributionDim * + kMinProbabilityValue + + (double)frame.visible_rect().size().GetArea(); + + // Initialize distribution table to arbitrarily small probability value. + for (int i = 0; i < kJointDistributionDim; i++) { + for (int j = 0; j < kJointDistributionDim; j++) { + for (int k = 0; k < kJointDistributionDim; k++) { + log_joint_distribution[i][j][k] = kMinProbabilityValue; + } + } + } + + // Downsample the RGB values of the plane into 4-bits per channel, and use the + // downsampled color information to increment the corresponding element of the + // distribution table. + const uint8_t* row_ptr = frame.visible_data(0); + for (int y = 0; y < frame.visible_rect().height(); y++) { + for (int x = 0; x < frame.visible_rect().width(); x++) { + log_joint_distribution[row_ptr[4 * x + 1] >> kJointDistributionBitDepth] + [row_ptr[4 * x + 2] >> kJointDistributionBitDepth] + [row_ptr[4 * x + 3] >> + kJointDistributionBitDepth] += 1.0; + } + row_ptr += frame.stride(0); + } + + // Normalize the joint distribution so that it sums to 1.0 and then take the + // log. + for (int i = 0; i < kJointDistributionDim; i++) { + for (int j = 0; j < kJointDistributionDim; j++) { + for (int k = 0; k < kJointDistributionDim; k++) { + log_joint_distribution[i][j][k] /= normalization_factor; + log_joint_distribution[i][j][k] = log(log_joint_distribution[i][j][k]); + } + } + } + + return true; +} + +double ComputeLogProbability(const VideoFrame& frame, + DistributionTable& log_joint_distribution) { + ASSERT_TRUE_OR_RETURN(frame.IsMappable(), 0.0); + ASSERT_TRUE_OR_RETURN(frame.format() == PIXEL_FORMAT_ARGB, 0.0); + ASSERT_TRUE_OR_RETURN(frame.BitDepth() == 8, 0.0); + + double ret = 0.0; + + const uint8_t* row_ptr = frame.visible_data(0); + for (int y = 0; y < frame.visible_rect().height(); y++) { + for (int x = 0; x < frame.visible_rect().width(); x++) { + ret += log_joint_distribution + [row_ptr[4 * x + 1] >> kJointDistributionBitDepth] + [row_ptr[4 * x + 2] >> kJointDistributionBitDepth] + [row_ptr[4 * x + 3] >> kJointDistributionBitDepth]; + } + row_ptr += frame.stride(0); + } + + return ret; +} + } // namespace size_t CompareFramesWithErrorDiff(const VideoFrame& frame1, @@ -237,5 +318,30 @@ double ComputePSNR(const VideoFrame& frame1, const VideoFrame& frame2) { double ComputeSSIM(const VideoFrame& frame1, const VideoFrame& frame2) { return ComputeSimilarity(&frame1, &frame2, SimilarityMetrics::SSIM); } + +double ComputeLogLikelihoodRatio(scoped_refptr golden_frame, + scoped_refptr test_frame) { + if (golden_frame->format() != PIXEL_FORMAT_ARGB) { + golden_frame = ConvertVideoFrame(golden_frame.get(), PIXEL_FORMAT_ARGB); + } + + if (test_frame->format() != PIXEL_FORMAT_ARGB) { + test_frame = ConvertVideoFrame(test_frame.get(), PIXEL_FORMAT_ARGB); + } + + DistributionTable log_joint_distribution; + double golden_log_prob = 0.0; + ASSERT_TRUE_OR_RETURN( + ComputeLogJointDistribution(*golden_frame, log_joint_distribution), 0.0); + golden_log_prob = + ComputeLogProbability(*golden_frame, log_joint_distribution); + ASSERT_TRUE_OR_RETURN(golden_log_prob != 0.0, 0.0); + + double test_log_prob = 0.0; + test_log_prob = ComputeLogProbability(*test_frame, log_joint_distribution); + ASSERT_TRUE_OR_RETURN(test_log_prob != 0.0, 0.0); + + return test_log_prob / golden_log_prob; +} } // namespace test } // namespace media diff --git a/media/gpu/test/image_quality_metrics.h b/media/gpu/test/image_quality_metrics.h index 358c39f947c22..7cd88d2531108 100644 --- a/media/gpu/test/image_quality_metrics.h +++ b/media/gpu/test/image_quality_metrics.h @@ -7,6 +7,8 @@ #include +#include "base/memory/scoped_refptr.h" + namespace media { class VideoFrame; @@ -30,6 +32,14 @@ size_t CompareFramesWithErrorDiff(const VideoFrame& frame1, double ComputePSNR(const VideoFrame& frame1, const VideoFrame& frame2); double ComputeSSIM(const VideoFrame& frame1, const VideoFrame& frame2); +// Compute the log likelihood ratio between a golden frame and a test frame. +// This metric performs a statistical analysis on the distribution of colors in +// each frame, and looks for anomalies consistent with encoding or decoding +// bugs. More details on this algorithm can be found here: +// go/log-likelihood-artifact-detection +double ComputeLogLikelihoodRatio(scoped_refptr golden_frame, + scoped_refptr test_frame); + } // namespace test } // namespace media diff --git a/media/gpu/test/video_frame_validator.cc b/media/gpu/test/video_frame_validator.cc index 521ea3a0e7bf0..a70b05ed9a050 100644 --- a/media/gpu/test/video_frame_validator.cc +++ b/media/gpu/test/video_frame_validator.cc @@ -565,5 +565,74 @@ bool SSIMVideoFrameValidator::Passed() const { } return true; } + +struct LogLikelihoodRatioVideoFrameValidator:: + LogLikelihoodRatioMismatchedFrameInfo + : public VideoFrameValidator::MismatchedFrameInfo { + LogLikelihoodRatioMismatchedFrameInfo(size_t frame_index, double ratio) + : MismatchedFrameInfo(frame_index), ratio(ratio) {} + ~LogLikelihoodRatioMismatchedFrameInfo() override = default; + void Print() const override { + LOG(ERROR) << "frame_index: " << frame_index + << ", log likelihood ratio: " << ratio; + } + + double ratio; +}; + +// static +std::unique_ptr +LogLikelihoodRatioVideoFrameValidator::Create( + const GetModelFrameCB& get_model_frame_cb, + std::unique_ptr corrupt_frame_processor, + ValidationMode validation_mode, + double tolerance, + CropHelper crop_helper) { + auto video_frame_validator = + base::WrapUnique(new LogLikelihoodRatioVideoFrameValidator( + get_model_frame_cb, std::move(corrupt_frame_processor), + validation_mode, tolerance, std::move(crop_helper))); + if (!video_frame_validator->Initialize()) { + LOG(ERROR) << "Failed to initialize LogLikelihoodRatioVideoFrameValidator."; + return nullptr; + } + + return video_frame_validator; +} + +LogLikelihoodRatioVideoFrameValidator::LogLikelihoodRatioVideoFrameValidator( + const GetModelFrameCB& get_model_frame_cb, + std::unique_ptr corrupt_frame_processor, + ValidationMode validation_mode, + double tolerance, + CropHelper crop_helper) + : VideoFrameValidator(std::move(corrupt_frame_processor), + std::move(crop_helper)), + get_model_frame_cb_(get_model_frame_cb), + tolerance_(tolerance) {} + +LogLikelihoodRatioVideoFrameValidator:: + ~LogLikelihoodRatioVideoFrameValidator() = default; + +std::unique_ptr +LogLikelihoodRatioVideoFrameValidator::Validate( + scoped_refptr frame, + size_t frame_index) { + DCHECK_CALLED_ON_VALID_SEQUENCE(validator_thread_sequence_checker_); + auto model_frame = get_model_frame_cb_.Run(frame_index); + + CHECK(model_frame); + double ratio = ComputeLogLikelihoodRatio(model_frame, frame); + DVLOGF(4) << "frame_index: " << frame_index + << ", log likelihood ratio: " << ratio; + log_likelihood_ratios_[frame_index] = ratio; + if (ratio < tolerance_) { + return std::make_unique(frame_index, + ratio); + } + + return nullptr; +} + } // namespace test } // namespace media diff --git a/media/gpu/test/video_frame_validator.h b/media/gpu/test/video_frame_validator.h index 2c504b5a2b16a..fef2212db87a3 100644 --- a/media/gpu/test/video_frame_validator.h +++ b/media/gpu/test/video_frame_validator.h @@ -285,6 +285,50 @@ class SSIMVideoFrameValidator : public VideoFrameValidator { const ValidationMode validation_mode_; std::map ssim_; }; + +// Validate by computing the log likelihood ratio of the frame to be validate +// and the model frame acquired by |get_model_frame_cb_|. If the log likelihood +// ratio is less than or equal to |tolerance_|, the validation on the frame +// passes. +class LogLikelihoodRatioVideoFrameValidator : public VideoFrameValidator { + public: + // TODO (b/262772938): Find an actual tolerance value for this validator. This + // is just a placeholder until we get some results on the range of this + // metric. + constexpr static double kDefaultTolerance = 50.0; + + static std::unique_ptr Create( + const GetModelFrameCB& get_model_frame_cb, + std::unique_ptr corrupt_frame_processor = nullptr, + ValidationMode validation_mode = ValidationMode::kThreshold, + double tolerance = kDefaultTolerance, + CropHelper crop_helper = CropHelper()); + + const std::map& get_log_likelihood_ratio_values() const { + return log_likelihood_ratios_; + } + + ~LogLikelihoodRatioVideoFrameValidator() override; + + private: + struct LogLikelihoodRatioMismatchedFrameInfo; + + LogLikelihoodRatioVideoFrameValidator( + const GetModelFrameCB& get_model_frame_cb, + std::unique_ptr corrupt_frame_processor, + ValidationMode validation_mode, + double tolerance, + CropHelper crop_helper); + + std::unique_ptr Validate( + scoped_refptr frame, + size_t frame_index) override; + + const GetModelFrameCB get_model_frame_cb_; + const double tolerance_; + std::map log_likelihood_ratios_; +}; + } // namespace test } // namespace media diff --git a/media/gpu/video_encode_accelerator_perf_tests.cc b/media/gpu/video_encode_accelerator_perf_tests.cc index ce290e88a3016..223bcb99b2bc2 100644 --- a/media/gpu/video_encode_accelerator_perf_tests.cc +++ b/media/gpu/video_encode_accelerator_perf_tests.cc @@ -323,6 +323,8 @@ struct BitstreamQualityMetrics { const PSNRVideoFrameValidator* const psnr_validator, const SSIMVideoFrameValidator* const ssim_validator, const PSNRVideoFrameValidator* const bottom_row_psnr_validator, + const LogLikelihoodRatioVideoFrameValidator* const + log_likelihood_validator, const absl::optional& spatial_idx, const absl::optional& temporal_idx); @@ -352,6 +354,7 @@ struct BitstreamQualityMetrics { const BitstreamQualityMetrics::QualityStats& psnr_stats, const BitstreamQualityMetrics::QualityStats& ssim_stats, const BitstreamQualityMetrics::QualityStats& bottom_row_psnr_stats, + const BitstreamQualityMetrics::QualityStats& log_likelihood_stats, uint32_t target_bitrate, uint32_t actual_bitrate) const; void WriteToFile( @@ -359,25 +362,30 @@ struct BitstreamQualityMetrics { const BitstreamQualityMetrics::QualityStats& psnr_stats, const BitstreamQualityMetrics::QualityStats& ssim_stats, const BitstreamQualityMetrics::QualityStats& bottom_row_psnr_stats, + const BitstreamQualityMetrics::QualityStats& log_likelihood_stats, uint32_t target_bitrate, uint32_t actual_bitrate) const; const raw_ptr psnr_validator; const raw_ptr ssim_validator; const raw_ptr bottom_row_psnr_validator; + const raw_ptr + log_likelihood_validator; }; BitstreamQualityMetrics::BitstreamQualityMetrics( const PSNRVideoFrameValidator* const psnr_validator, const SSIMVideoFrameValidator* const ssim_validator, const PSNRVideoFrameValidator* const bottom_row_psnr_validator, + const LogLikelihoodRatioVideoFrameValidator* const log_likelihood_validator, const absl::optional& spatial_idx, const absl::optional& temporal_idx) : spatial_idx(spatial_idx), temporal_idx(temporal_idx), psnr_validator(psnr_validator), ssim_validator(ssim_validator), - bottom_row_psnr_validator(bottom_row_psnr_validator) {} + bottom_row_psnr_validator(bottom_row_psnr_validator), + log_likelihood_validator(log_likelihood_validator) {} // static BitstreamQualityMetrics::QualityStats @@ -419,11 +427,13 @@ void BitstreamQualityMetrics::Output(uint32_t target_bitrate, auto ssim_stats = ComputeQualityStats(ssim_validator->GetSSIMValues()); auto bottom_row_psnr_stats = ComputeQualityStats(bottom_row_psnr_validator->GetPSNRValues()); + auto log_likelihood_stats = ComputeQualityStats( + log_likelihood_validator->get_log_likelihood_ratio_values()); WriteToConsole(svc_text, psnr_stats, ssim_stats, bottom_row_psnr_stats, - target_bitrate, actual_bitrate); + log_likelihood_stats, target_bitrate, actual_bitrate); WriteToFile(svc_text, psnr_stats, ssim_stats, bottom_row_psnr_stats, - target_bitrate, actual_bitrate); + log_likelihood_stats, target_bitrate, actual_bitrate); } void BitstreamQualityMetrics::WriteToConsole( @@ -431,6 +441,7 @@ void BitstreamQualityMetrics::WriteToConsole( const BitstreamQualityMetrics::QualityStats& psnr_stats, const BitstreamQualityMetrics::QualityStats& ssim_stats, const BitstreamQualityMetrics::QualityStats& bottom_row_psnr_stats, + const BitstreamQualityMetrics::QualityStats& log_likelihood_stats, uint32_t target_bitrate, uint32_t actual_bitrate) const { const auto default_ssize = std::cout.precision(); @@ -464,6 +475,14 @@ void BitstreamQualityMetrics::WriteToConsole( << bottom_row_psnr_stats.percentile_50 << std::endl; std::cout << "Bottom row PSNR - percentile 75: " << bottom_row_psnr_stats.percentile_75 << std::endl; + std::cout << "Log likelihood ratio - average: " + << log_likelihood_stats.avg << std::endl; + std::cout << "Log likelihood ratio - percentile 25: " + << log_likelihood_stats.percentile_25 << std::endl; + std::cout << "Log likelihood ratio - percentile 50: " + << log_likelihood_stats.percentile_50 << std::endl; + std::cout << "Log likelihood ratio - percentile 75: " + << log_likelihood_stats.percentile_75 << std::endl; std::cout.precision(default_ssize); } @@ -472,6 +491,7 @@ void BitstreamQualityMetrics::WriteToFile( const BitstreamQualityMetrics::QualityStats& psnr_stats, const BitstreamQualityMetrics::QualityStats& ssim_stats, const BitstreamQualityMetrics::QualityStats& bottom_row_psnr_stats, + const BitstreamQualityMetrics::QualityStats& log_likelihood_stats, uint32_t target_bitrate, uint32_t actual_bitrate) const { base::FilePath output_folder_path = base::FilePath(g_env->OutputFolder()); @@ -491,6 +511,8 @@ void BitstreamQualityMetrics::WriteToFile( metrics.SetKey("PSNRAverage", base::Value(psnr_stats.avg)); metrics.SetKey("BottomRowPSNRAverage", base::Value(bottom_row_psnr_stats.avg)); + metrics.SetKey("LogLikelihoodRatioAverage", + base::Value(log_likelihood_stats.avg)); // Write ssim values bitstream delivery times to json. base::Value ssim_values(base::Value::Type::LIST); for (double value : ssim_stats.values_in_order) @@ -509,6 +531,13 @@ void BitstreamQualityMetrics::WriteToFile( bottom_row_psnr_values.Append(value); metrics.SetKey("BottomRowPSNRValues", std::move(bottom_row_psnr_values)); + // Write log likelihood ratio values to json. + base::Value log_likelihood_values(base::Value::Type::LIST); + for (double value : log_likelihood_stats.values_in_order) { + log_likelihood_values.Append(value); + } + metrics.SetKey("LogLikelihoodRatioValues", std::move(log_likelihood_values)); + // Write json to file. std::string metrics_str; ASSERT_TRUE(base::JSONWriter::WriteWithOptions( @@ -607,13 +636,21 @@ class VideoEncoderTest : public ::testing::Test { /*tolerance=*/0.0, base::BindRepeating(&BottomRowCrop, kDefaultBottomRowCropHeight)); LOG_ASSERT(bottom_row_psnr_validator); + auto log_likelihood_validator = + LogLikelihoodRatioVideoFrameValidator::Create( + get_model_frame_cb, + /*corrupt_frame_processor=*/nullptr, + VideoFrameValidator::ValidationMode::kAverage, + /*tolerance=*/0.0); + LOG_ASSERT(log_likelihood_validator); quality_metrics_.push_back(BitstreamQualityMetrics( psnr_validator.get(), ssim_validator.get(), - bottom_row_psnr_validator.get(), spatial_layer_index_to_decode, - temporal_layer_index_to_decode)); + bottom_row_psnr_validator.get(), log_likelihood_validator.get(), + spatial_layer_index_to_decode, temporal_layer_index_to_decode)); video_frame_processors.push_back(std::move(ssim_validator)); video_frame_processors.push_back(std::move(psnr_validator)); video_frame_processors.push_back(std::move(bottom_row_psnr_validator)); + video_frame_processors.push_back(std::move(log_likelihood_validator)); VideoDecoderConfig decoder_config( VideoCodecProfileToVideoCodec(profile), profile,