media/gpu: Add Log likelihood ratio frame validator

Add a log likelihood ratio frame validation mechanism, but do not turn
it on. We want to collect data on what values indicate a true positive
before we pick a threshold value.

Bug: 262772938
Test: Ran video.EncodeAccelPerf.h264_1080p on an Asurada.
Change-Id: I9b48d9def4f34416ca4377cc4da892ef943a9c0e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4112789
Reviewed-by: Frank Liberato <liberato@chromium.org>
Commit-Queue: Justin Green <greenjustin@google.com>
Cr-Commit-Position: refs/heads/main@{#1084573}

media/gpu/test/image_quality_metrics.cc

@@ -194,6 +194,87 @@ double ComputeSimilarity(const VideoFrame* frame1,
       frame2->stride(1), frame2->visible_data(2), frame2->stride(2),
       frame1->visible_rect().width(), frame1->visible_rect().height());
 }
+
+constexpr int kJointDistributionBitDepth = 4;
+constexpr int kJointDistributionDim = 1 << kJointDistributionBitDepth;
+
+using DistributionTable =
+    double[kJointDistributionDim][kJointDistributionDim][kJointDistributionDim];
+
+bool ComputeLogJointDistribution(const VideoFrame& frame,
+                                 DistributionTable& log_joint_distribution) {
+  ASSERT_TRUE_OR_RETURN(frame.IsMappable(), false);
+  ASSERT_TRUE_OR_RETURN(frame.format() == PIXEL_FORMAT_ARGB, false);
+  ASSERT_TRUE_OR_RETURN(frame.BitDepth() == 8, false);
+
+  // Arbitrarily small number to fill the probability distribution table with so
+  // we don't have a problem with taking the log of 0.
+  static const double kMinProbabilityValue = 0.000000001;
+
+  double normalization_factor = kJointDistributionDim * kJointDistributionDim *
+                                    kJointDistributionDim *
+                                    kMinProbabilityValue +
+                                (double)frame.visible_rect().size().GetArea();
+
+  // Initialize distribution table to arbitrarily small probability value.
+  for (int i = 0; i < kJointDistributionDim; i++) {
+    for (int j = 0; j < kJointDistributionDim; j++) {
+      for (int k = 0; k < kJointDistributionDim; k++) {
+        log_joint_distribution[i][j][k] = kMinProbabilityValue;
+      }
+    }
+  }
+
+  // Downsample the RGB values of the plane into 4-bits per channel, and use the
+  // downsampled color information to increment the corresponding element of the
+  // distribution table.
+  const uint8_t* row_ptr = frame.visible_data(0);
+  for (int y = 0; y < frame.visible_rect().height(); y++) {
+    for (int x = 0; x < frame.visible_rect().width(); x++) {
+      log_joint_distribution[row_ptr[4 * x + 1] >> kJointDistributionBitDepth]
+                            [row_ptr[4 * x + 2] >> kJointDistributionBitDepth]
+                            [row_ptr[4 * x + 3] >>
+                             kJointDistributionBitDepth] += 1.0;
+    }
+    row_ptr += frame.stride(0);
+  }
+
+  // Normalize the joint distribution so that it sums to 1.0 and then take the
+  // log.
+  for (int i = 0; i < kJointDistributionDim; i++) {
+    for (int j = 0; j < kJointDistributionDim; j++) {
+      for (int k = 0; k < kJointDistributionDim; k++) {
+        log_joint_distribution[i][j][k] /= normalization_factor;
+        log_joint_distribution[i][j][k] = log(log_joint_distribution[i][j][k]);
+      }
+    }
+  }
+
+  return true;
+}
+
+double ComputeLogProbability(const VideoFrame& frame,
+                             DistributionTable& log_joint_distribution) {
+  ASSERT_TRUE_OR_RETURN(frame.IsMappable(), 0.0);
+  ASSERT_TRUE_OR_RETURN(frame.format() == PIXEL_FORMAT_ARGB, 0.0);
+  ASSERT_TRUE_OR_RETURN(frame.BitDepth() == 8, 0.0);
+
+  double ret = 0.0;
+
+  const uint8_t* row_ptr = frame.visible_data(0);
+  for (int y = 0; y < frame.visible_rect().height(); y++) {
+    for (int x = 0; x < frame.visible_rect().width(); x++) {
+      ret += log_joint_distribution
+          [row_ptr[4 * x + 1] >> kJointDistributionBitDepth]
+          [row_ptr[4 * x + 2] >> kJointDistributionBitDepth]
+          [row_ptr[4 * x + 3] >> kJointDistributionBitDepth];
+    }
+    row_ptr += frame.stride(0);
+  }
+
+  return ret;
+}
+
 }  // namespace
 
 size_t CompareFramesWithErrorDiff(const VideoFrame& frame1,
@@ -237,5 +318,30 @@ double ComputePSNR(const VideoFrame& frame1, const VideoFrame& frame2) {
 double ComputeSSIM(const VideoFrame& frame1, const VideoFrame& frame2) {
   return ComputeSimilarity(&frame1, &frame2, SimilarityMetrics::SSIM);
 }
+
+double ComputeLogLikelihoodRatio(scoped_refptr<const VideoFrame> golden_frame,
+                                 scoped_refptr<const VideoFrame> test_frame) {
+  if (golden_frame->format() != PIXEL_FORMAT_ARGB) {
+    golden_frame = ConvertVideoFrame(golden_frame.get(), PIXEL_FORMAT_ARGB);
+  }
+
+  if (test_frame->format() != PIXEL_FORMAT_ARGB) {
+    test_frame = ConvertVideoFrame(test_frame.get(), PIXEL_FORMAT_ARGB);
+  }
+
+  DistributionTable log_joint_distribution;
+  double golden_log_prob = 0.0;
+  ASSERT_TRUE_OR_RETURN(
+      ComputeLogJointDistribution(*golden_frame, log_joint_distribution), 0.0);
+  golden_log_prob =
+      ComputeLogProbability(*golden_frame, log_joint_distribution);
+  ASSERT_TRUE_OR_RETURN(golden_log_prob != 0.0, 0.0);
+
+  double test_log_prob = 0.0;
+  test_log_prob = ComputeLogProbability(*test_frame, log_joint_distribution);
+  ASSERT_TRUE_OR_RETURN(test_log_prob != 0.0, 0.0);
+
+  return test_log_prob / golden_log_prob;
+}
 }  // namespace test
 }  // namespace media

media/gpu/test/image_quality_metrics.h

@@ -7,6 +7,8 @@
 
 #include <stdint.h>
 
+#include "base/memory/scoped_refptr.h"
+
 namespace media {
 
 class VideoFrame;
@@ -30,6 +32,14 @@ size_t CompareFramesWithErrorDiff(const VideoFrame& frame1,
 double ComputePSNR(const VideoFrame& frame1, const VideoFrame& frame2);
 double ComputeSSIM(const VideoFrame& frame1, const VideoFrame& frame2);
 
+// Compute the log likelihood ratio between a golden frame and a test frame.
+// This metric performs a statistical analysis on the distribution of colors in
+// each frame, and looks for anomalies consistent with encoding or decoding
+// bugs. More details on this algorithm can be found here:
+// go/log-likelihood-artifact-detection
+double ComputeLogLikelihoodRatio(scoped_refptr<const VideoFrame> golden_frame,
+                                 scoped_refptr<const VideoFrame> test_frame);
+
 }  // namespace test
 }  // namespace media
 

media/gpu/test/video_frame_validator.cc

@@ -565,5 +565,74 @@ bool SSIMVideoFrameValidator::Passed() const {
   }
   return true;
 }
+
+struct LogLikelihoodRatioVideoFrameValidator::
+    LogLikelihoodRatioMismatchedFrameInfo
+    : public VideoFrameValidator::MismatchedFrameInfo {
+  LogLikelihoodRatioMismatchedFrameInfo(size_t frame_index, double ratio)
+      : MismatchedFrameInfo(frame_index), ratio(ratio) {}
+  ~LogLikelihoodRatioMismatchedFrameInfo() override = default;
+  void Print() const override {
+    LOG(ERROR) << "frame_index: " << frame_index
+               << ", log likelihood ratio: " << ratio;
+  }
+
+  double ratio;
+};
+
+// static
+std::unique_ptr<LogLikelihoodRatioVideoFrameValidator>
+LogLikelihoodRatioVideoFrameValidator::Create(
+    const GetModelFrameCB& get_model_frame_cb,
+    std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+    ValidationMode validation_mode,
+    double tolerance,
+    CropHelper crop_helper) {
+  auto video_frame_validator =
+      base::WrapUnique(new LogLikelihoodRatioVideoFrameValidator(
+          get_model_frame_cb, std::move(corrupt_frame_processor),
+          validation_mode, tolerance, std::move(crop_helper)));
+  if (!video_frame_validator->Initialize()) {
+    LOG(ERROR) << "Failed to initialize LogLikelihoodRatioVideoFrameValidator.";
+    return nullptr;
+  }
+
+  return video_frame_validator;
+}
+
+LogLikelihoodRatioVideoFrameValidator::LogLikelihoodRatioVideoFrameValidator(
+    const GetModelFrameCB& get_model_frame_cb,
+    std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+    ValidationMode validation_mode,
+    double tolerance,
+    CropHelper crop_helper)
+    : VideoFrameValidator(std::move(corrupt_frame_processor),
+                          std::move(crop_helper)),
+      get_model_frame_cb_(get_model_frame_cb),
+      tolerance_(tolerance) {}
+
+LogLikelihoodRatioVideoFrameValidator::
+    ~LogLikelihoodRatioVideoFrameValidator() = default;
+
+std::unique_ptr<VideoFrameValidator::MismatchedFrameInfo>
+LogLikelihoodRatioVideoFrameValidator::Validate(
+    scoped_refptr<const VideoFrame> frame,
+    size_t frame_index) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(validator_thread_sequence_checker_);
+  auto model_frame = get_model_frame_cb_.Run(frame_index);
+
+  CHECK(model_frame);
+  double ratio = ComputeLogLikelihoodRatio(model_frame, frame);
+  DVLOGF(4) << "frame_index: " << frame_index
+            << ", log likelihood ratio: " << ratio;
+  log_likelihood_ratios_[frame_index] = ratio;
+  if (ratio < tolerance_) {
+    return std::make_unique<LogLikelihoodRatioMismatchedFrameInfo>(frame_index,
+                                                                   ratio);
+  }
+
+  return nullptr;
+}
+
 }  // namespace test
 }  // namespace media

media/gpu/test/video_frame_validator.h

@@ -285,6 +285,50 @@ class SSIMVideoFrameValidator : public VideoFrameValidator {
   const ValidationMode validation_mode_;
   std::map<size_t, double> ssim_;
 };
+
+// Validate by computing the log likelihood ratio of the frame to be validate
+// and the model frame acquired by |get_model_frame_cb_|. If the log likelihood
+// ratio is less than or equal to |tolerance_|, the validation on the frame
+// passes.
+class LogLikelihoodRatioVideoFrameValidator : public VideoFrameValidator {
+ public:
+  // TODO (b/262772938): Find an actual tolerance value for this validator. This
+  // is just a placeholder until we get some results on the range of this
+  // metric.
+  constexpr static double kDefaultTolerance = 50.0;
+
+  static std::unique_ptr<LogLikelihoodRatioVideoFrameValidator> Create(
+      const GetModelFrameCB& get_model_frame_cb,
+      std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor = nullptr,
+      ValidationMode validation_mode = ValidationMode::kThreshold,
+      double tolerance = kDefaultTolerance,
+      CropHelper crop_helper = CropHelper());
+
+  const std::map<size_t, double>& get_log_likelihood_ratio_values() const {
+    return log_likelihood_ratios_;
+  }
+
+  ~LogLikelihoodRatioVideoFrameValidator() override;
+
+ private:
+  struct LogLikelihoodRatioMismatchedFrameInfo;
+
+  LogLikelihoodRatioVideoFrameValidator(
+      const GetModelFrameCB& get_model_frame_cb,
+      std::unique_ptr<VideoFrameProcessor> corrupt_frame_processor,
+      ValidationMode validation_mode,
+      double tolerance,
+      CropHelper crop_helper);
+
+  std::unique_ptr<MismatchedFrameInfo> Validate(
+      scoped_refptr<const VideoFrame> frame,
+      size_t frame_index) override;
+
+  const GetModelFrameCB get_model_frame_cb_;
+  const double tolerance_;
+  std::map<size_t, double> log_likelihood_ratios_;
+};
+
 }  // namespace test
 }  // namespace media