e8+ and butteraugli: better HDR behavior (libjxl#3885)

Previously, the Butteraugli loop would use an intensity of 80 even for
HDR content, thereby lowering quality at efforts 8 and above.

butteraugli_main also did not handle the reference and actual images
having different intensity_targets, meaning that it was not possible to
compare a PQ PNG (which would automatically have an intensity_target of
10000) with a compressed JXL version using a lower intensity_target
(reflecting the fact that the image doesn’t actually get that bright).

ClassE_LasVegasStore, before:

    e1:  1.709 bpp, Butteraugli 1.9095990658, 3-norm 0.763096
    e2:  1.709 bpp, Butteraugli 1.9095990658, 3-norm 0.763096
    e3:  1.616 bpp, Butteraugli 1.9095990658, 3-norm 0.763096
    e4:  1.631 bpp, Butteraugli 1.9226754904, 3-norm 0.761637
    e5:  1.519 bpp, Butteraugli 2.5887553692, 3-norm 0.801795
    e6:  1.507 bpp, Butteraugli 2.4450392723, 3-norm 0.777965
    e7:  1.681 bpp, Butteraugli 1.4998193979, 3-norm 0.617420
    e8:  1.415 bpp, Butteraugli 1.9081192017, 3-norm 0.770197
    e9:  1.413 bpp, Butteraugli 1.9081192017, 3-norm 0.770197
    e10: 1.409 bpp, Butteraugli 1.9175151587, 3-norm 0.770864

After:

    e1:  1.709 bpp, Butteraugli 1.9095990658, 3-norm 0.763096
    e2:  1.709 bpp, Butteraugli 1.9095990658, 3-norm 0.763096
    e3:  1.616 bpp, Butteraugli 1.9095990658, 3-norm 0.763096
    e4:  1.631 bpp, Butteraugli 1.9226754904, 3-norm 0.761637
    e5:  1.519 bpp, Butteraugli 2.5887553692, 3-norm 0.801795
    e6:  1.507 bpp, Butteraugli 2.4450392723, 3-norm 0.777965
    e7:  1.681 bpp, Butteraugli 1.4998193979, 3-norm 0.617420
    e8:  1.721 bpp, Butteraugli 1.4964444637, 3-norm 0.590536
    e9:  1.727 bpp, Butteraugli 1.2693173885, 3-norm 0.585146
    e10: 1.719 bpp, Butteraugli 1.3858509064, 3-norm 0.589227

(cherry picked from commit 095f259)

lib/jxl/enc_adaptive_quantization.cc

@@ -951,7 +951,11 @@ Status FindBestQuantization(const FrameHeader& frame_header,
   const float butteraugli_target = cparams.butteraugli_distance;
   const float original_butteraugli = cparams.original_butteraugli_distance;
   ButteraugliParams params;
-  params.intensity_target = 80.f;
+  const auto& tf = frame_header.nonserialized_metadata->m.color_encoding.Tf();
+  params.intensity_target =
+      tf.IsPQ() || tf.IsHLG()
+          ? frame_header.nonserialized_metadata->m.IntensityTarget()
+          : 80.f;
   JxlButteraugliComparator comparator(params, cms);
   JXL_RETURN_IF_ERROR(comparator.SetLinearReferenceImage(linear));
   bool lower_is_better =

lib/jxl/enc_butteraugli_comparator.cc

@@ -27,6 +27,7 @@ Status JxlButteraugliComparator::SetReferenceImage(const ImageBundle& ref) {
                                         ref_linear_srgb->color(), params_));
   xsize_ = ref.xsize();
   ysize_ = ref.ysize();
+  intensity_target_ = ref.metadata()->IntensityTarget();
   return true;
 }
 
@@ -60,8 +61,29 @@ Status JxlButteraugliComparator::CompareWith(const ImageBundle& actual,
 
   JXL_ASSIGN_OR_RETURN(ImageF temp_diffmap,
                        ImageF::Create(memory_manager, xsize_, ysize_));
+  const Image3F* scaled_actual_linear_srgb = &actual_linear_srgb->color();
+  Image3F scaled_actual_linear_srgb_store;
+  if (intensity_target_ != 0 &&
+      actual.metadata()->IntensityTarget() != intensity_target_) {
+    scaled_actual_linear_srgb = &scaled_actual_linear_srgb_store;
+    JXL_ASSIGN_OR_RETURN(scaled_actual_linear_srgb_store,
+                         Image3F::Create(memory_manager, xsize_, ysize_));
+    const float scale =
+        actual.metadata()->IntensityTarget() / intensity_target_;
+    for (size_t c = 0; c < 3; ++c) {
+      for (size_t y = 0; y < ysize_; ++y) {
+        const float* JXL_RESTRICT source_row =
+            actual_linear_srgb->color().ConstPlaneRow(c, y);
+        float* JXL_RESTRICT scaled_row =
+            scaled_actual_linear_srgb_store.PlaneRow(c, y);
+        for (size_t x = 0; x < xsize_; ++x) {
+          scaled_row[x] = scale * source_row[x];
+        }
+      }
+    }
+  }
   JXL_RETURN_IF_ERROR(
-      comparator_->Diffmap(actual_linear_srgb->color(), temp_diffmap));
+      comparator_->Diffmap(*scaled_actual_linear_srgb, temp_diffmap));
 
   if (score != nullptr) {
     *score = ButteraugliScoreFromDiffmap(temp_diffmap, &params_);

lib/jxl/enc_butteraugli_comparator.h

@@ -39,6 +39,7 @@ class JxlButteraugliComparator : public Comparator {
   std::unique_ptr<ButteraugliComparator> comparator_;
   size_t xsize_ = 0;
   size_t ysize_ = 0;
+  float intensity_target_ = 0.f;
 };
 
 }  // namespace jxl

tools/butteraugli_main.cc

@@ -103,10 +103,10 @@ Status RunButteraugli(const char* pathname1, const char* pathname2,
     butteraugli_params.intensity_target = intensity_target;
   } else {
     const auto& transfer_function = io1.Main().c_current().Tf();
-    butteraugli_params.intensity_target = transfer_function.IsPQ() ? 10000.f
-                                          : transfer_function.IsHLG()
-                                              ? 1000.f
-                                              : 80.f;  // sRGB intensity target.
+    butteraugli_params.intensity_target =
+        transfer_function.IsPQ() || transfer_function.IsHLG()
+            ? io1.metadata.m.IntensityTarget()
+            : 80.f;  // sRGB intensity target.
   }
   const JxlCmsInterface& cms = *JxlGetDefaultCms();
   JxlButteraugliComparator comparator(butteraugli_params, cms);