[M102] Add visual features to CSPP pings

We want to achieve feature parity with PhishGuard pings, which
currently populate visual features for extended reporting users. So
this CL extracts the code used in PhishGuard to a common location
and uses in both PhishGuard and CSPP.

(cherry picked from commit df87724)

Bug: 1316024
Change-Id: Icc1f332963ea988531698a6e9c8935f6b3bb12ae
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3584779
Reviewed-by: Xinghui Lu <xinghuilu@chromium.org>
Reviewed-by: Brian Salomon <bsalomon@google.com>
Reviewed-by: James Maclean <wjmaclean@chromium.org>
Reviewed-by: Nico Weber <thakis@chromium.org>
Commit-Queue: Daniel Rubery <drubery@chromium.org>
Cr-Original-Commit-Position: refs/heads/main@{#994726}
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3603727
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
Cr-Commit-Position: refs/branch-heads/5005@{#178}
Cr-Branched-From: 5b4d945-refs/heads/main@{#992738}

components/BUILD.gn

@@ -567,7 +567,12 @@ test("components_unittests") {
     deps += [ "//components/pwg_encoder:unit_tests" ]
   }
 
-  # TODO(crbug / 1056278): Enable this on Fuchsia
+  if (safe_browsing_mode != 0) {
+    deps += [ "//components/safe_browsing/content/common:unit_tests" ]
+  }
+
+  # TODO(crbug / 1056278): Enable this on Fuchsia and merge with the above
+  # conditional.
   if (safe_browsing_mode != 0 && !is_fuchsia) {
     deps += [
       "//components/safe_browsing/content/browser/download:unittests",

components/safe_browsing/content/browser/BUILD.gn

@@ -219,11 +219,20 @@ source_set("client_side_detection") {
     "//components/safe_browsing/core/common/proto:csd_proto",
     "//components/security_interstitials/content:security_interstitial_page",
     "//components/variations",
+    "//components/zoom:zoom",
     "//content/public/browser",
     "//google_apis:google_apis",
     "//net:net",
     "//net/traffic_annotation:traffic_annotation",
     "//services/network/public/cpp",
     "//url:url",
   ]
+
+  if (safe_browsing_mode != 0) {
+    deps += [ "//components/safe_browsing/content/common:visual_utils" ]
+  }
+
+  if (is_android) {
+    deps += [ "//ui/android" ]
+  }
 }

components/safe_browsing/content/browser/DEPS

@@ -10,6 +10,7 @@ include_rules = [
   "+components/sessions/content",
   "+components/sessions/core/session_id.h",
   "+components/version_info",
+  "+components/zoom",
   "+content/public/browser",
   "+content/public/common",
   "+content/public/test",

components/safe_browsing/content/browser/client_side_detection_host.cc

@@ -25,18 +25,21 @@
 #include "components/safe_browsing/content/browser/client_side_phishing_model.h"
 #include "components/safe_browsing/content/common/safe_browsing.mojom-shared.h"
 #include "components/safe_browsing/content/common/safe_browsing.mojom.h"
+#include "components/safe_browsing/content/common/visual_utils.h"
 #include "components/safe_browsing/core/browser/db/allowlist_checker_client.h"
 #include "components/safe_browsing/core/browser/db/database_manager.h"
 #include "components/safe_browsing/core/browser/sync/sync_utils.h"
 #include "components/safe_browsing/core/common/features.h"
 #include "components/security_interstitials/content/unsafe_resource_util.h"
+#include "components/zoom/zoom_controller.h"
 #include "content/public/browser/browser_context.h"
 #include "content/public/browser/browser_task_traits.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/global_routing_id.h"
 #include "content/public/browser/navigation_handle.h"
 #include "content/public/browser/render_frame_host.h"
 #include "content/public/browser/render_process_host.h"
+#include "content/public/browser/render_widget_host_view.h"
 #include "content/public/browser/web_contents.h"
 #include "content/public/common/url_constants.h"
 #include "net/base/ip_endpoint.h"
@@ -45,6 +48,10 @@
 #include "third_party/blink/public/mojom/loader/referrer.mojom.h"
 #include "url/gurl.h"
 
+#if BUILDFLAG(IS_ANDROID)
+#include "ui/android/view_android.h"
+#endif
+
 using content::BrowserThread;
 using content::WebContents;
 
@@ -557,6 +564,34 @@ void ClientSideDetectionHost::PhishingDetectionDone(
                                                 GetDebugFeatureDirectory()));
     }
 
+#if BUILDFLAG(IS_ANDROID)
+    gfx::Size size;
+    content::RenderWidgetHostView* view =
+        web_contents()->GetRenderWidgetHostView();
+    if (view) {
+      gfx::SizeF viewport = view->GetNativeView()->viewport_size();
+      size = gfx::Size(static_cast<int>(viewport.width()),
+                       static_cast<int>(viewport.height()));
+    }
+    bool can_extract_visual_features = visual_utils::CanExtractVisualFeatures(
+        IsExtendedReportingEnabled(*delegate_->GetPrefs()),
+        web_contents()->GetBrowserContext()->IsOffTheRecord(), size);
+#else
+    gfx::Size size;
+    content::RenderWidgetHostView* view =
+        web_contents()->GetRenderWidgetHostView();
+    if (view) {
+      size = view->GetVisibleViewportSize();
+    }
+    bool can_extract_visual_features = visual_utils::CanExtractVisualFeatures(
+        IsExtendedReportingEnabled(*delegate_->GetPrefs()),
+        web_contents()->GetBrowserContext()->IsOffTheRecord(), size,
+        zoom::ZoomController::GetZoomLevelForWebContents(web_contents()));
+#endif
+    if (!can_extract_visual_features) {
+      verdict->clear_visual_features();
+    }
+
     if (IsEnhancedProtectionEnabled(*delegate_->GetPrefs()) &&
         base::FeatureList::IsEnabled(kClientSideDetectionReferrerChain)) {
       delegate_->AddReferrerChain(verdict.get(), current_url_,

components/safe_browsing/content/browser/password_protection/BUILD.gn

@@ -27,6 +27,7 @@ source_set("password_protection") {
       "//components/password_manager/core/browser:browser",
       "//components/safe_browsing/content/browser/web_ui:web_ui",
       "//components/safe_browsing/content/common:interfaces",
+      "//components/safe_browsing/content/common:visual_utils",
       "//components/safe_browsing/core/browser:referrer_chain_provider",
       "//components/safe_browsing/core/browser:safe_browsing_metrics_collector",
       "//components/safe_browsing/core/browser/db:allowlist_checker_client",

components/safe_browsing/content/browser/password_protection/password_protection_request_content.cc

@@ -18,7 +18,7 @@
 #include "url/gurl.h"
 
 #if BUILDFLAG(SAFE_BROWSING_AVAILABLE)
-#include "components/safe_browsing/core/common/visual_utils.h"
+#include "components/safe_browsing/content/common/visual_utils.h"
 #include "components/zoom/zoom_controller.h"
 #include "content/public/browser/render_widget_host_view.h"
 #include "content/public/browser/web_contents.h"
@@ -35,52 +35,8 @@ namespace {
 #if BUILDFLAG(SAFE_BROWSING_AVAILABLE)
 // The maximum time to wait for DOM features to be collected, in milliseconds.
 const int kDomFeatureTimeoutMs = 3000;
-
-// Parameters chosen to ensure privacy is preserved by visual features.
-#if BUILDFLAG(IS_ANDROID)
-const int kMinWidthForVisualFeatures = 258;
-const int kMinHeightForVisualFeatures = 258;
-#else
-const int kMinWidthForVisualFeatures = 576;
-const int kMinHeightForVisualFeatures = 576;
-#endif
-
-#endif  // BUILDFLAG(SAFE_BROWSING_AVAILABLE)
-
-#if BUILDFLAG(FULL_SAFE_BROWSING)
-// Parameters chosen to ensure privacy is preserved by visual features.
-const float kMaxZoomForVisualFeatures = 2.0;
-#endif  // BUILDFLAG(FULL_SAFE_BROWSING)
-
-#if BUILDFLAG(SAFE_BROWSING_AVAILABLE)
-std::unique_ptr<VisualFeatures> ExtractVisualFeatures(
-    const SkBitmap& screenshot) {
-  auto features = std::make_unique<VisualFeatures>();
-  visual_utils::GetHistogramForImage(screenshot,
-                                     features->mutable_color_histogram());
-  visual_utils::GetBlurredImage(screenshot, features->mutable_image());
-  return features;
-}
 #endif  // BUILDFLAG(SAFE_BROWSING_AVAILABLE)
 
-int GetMinWidthForVisualFeatures() {
-  if (base::FeatureList::IsEnabled(kVisualFeaturesSizes)) {
-    return base::GetFieldTrialParamByFeatureAsInt(
-        kVisualFeaturesSizes, "min_width", kMinWidthForVisualFeatures);
-  }
-
-  return kMinWidthForVisualFeatures;
-}
-
-int GetMinHeightForVisualFeatures() {
-  if (base::FeatureList::IsEnabled(kVisualFeaturesSizes)) {
-    return base::GetFieldTrialParamByFeatureAsInt(
-        kVisualFeaturesSizes, "min_height", kMinHeightForVisualFeatures);
-  }
-
-  return kMinHeightForVisualFeatures;
-}
-
 }  // namespace
 
 PasswordProtectionRequestContent::PasswordProtectionRequestContent(
@@ -264,21 +220,26 @@ void PasswordProtectionRequestContent::MaybeCollectVisualFeatures() {
   }
 #endif
 
-  bool can_collect_visual_features =
-      trigger_type() == LoginReputationClientRequest::UNFAMILIAR_LOGIN_PAGE &&
-      password_protection_service()->IsExtendedReporting() &&
-      !password_protection_service()->IsIncognito() &&
-      request_proto_->content_area_width() >= GetMinWidthForVisualFeatures() &&
-      request_proto_->content_area_height() >= GetMinHeightForVisualFeatures();
-#if !BUILDFLAG(IS_ANDROID)
-  can_collect_visual_features &=
-      zoom::ZoomController::GetZoomLevelForWebContents(web_contents_) <=
-      kMaxZoomForVisualFeatures;
+  bool can_extract_visual_features =
+#if BUILDFLAG(IS_ANDROID)
+      visual_utils::CanExtractVisualFeatures(
+          password_protection_service()->IsExtendedReporting(),
+          password_protection_service()->IsIncognito(),
+          gfx::Size(request_proto_->content_area_width(),
+                    request_proto_->content_area_height()));
+#else
+      visual_utils::CanExtractVisualFeatures(
+          password_protection_service()->IsExtendedReporting(),
+          password_protection_service()->IsIncognito(),
+          gfx::Size(request_proto_->content_area_width(),
+                    request_proto_->content_area_height()),
+          zoom::ZoomController::GetZoomLevelForWebContents(web_contents_));
 #endif
 
   // Once the DOM features are collected, either collect visual features, or go
   // straight to sending the ping.
-  if (can_collect_visual_features) {
+  if (trigger_type() == LoginReputationClientRequest::UNFAMILIAR_LOGIN_PAGE &&
+      can_extract_visual_features) {
     CollectVisualFeatures();
   } else {
     SendRequest();
@@ -310,7 +271,7 @@ void PasswordProtectionRequestContent::OnScreenshotTaken(
       FROM_HERE,
       {base::MayBlock(), base::TaskPriority::BEST_EFFORT,
        base::TaskShutdownBehavior::CONTINUE_ON_SHUTDOWN},
-      base::BindOnce(&ExtractVisualFeatures, screenshot),
+      base::BindOnce(&visual_utils::ExtractVisualFeatures, screenshot),
       base::BindOnce(
           &PasswordProtectionRequestContent::OnVisualFeatureCollectionDone,
           AsWeakPtr()));

components/safe_browsing/content/browser/web_ui/safe_browsing_ui.cc

@@ -974,6 +974,20 @@ std::string SerializeClientDownloadResponse(const ClientDownloadResponse& cdr) {
   return request_serialized;
 }
 
+base::Value SerializeVisualFeatures(const VisualFeatures& visual_features) {
+  base::Value image_dict(base::Value::Type::DICTIONARY);
+  const VisualFeatures::BlurredImage& image = visual_features.image();
+  image_dict.SetIntKey("width", image.width());
+  image_dict.SetIntKey("height", image.height());
+  image_dict.SetStringKey(
+      "data",
+      base::Base64Encode(base::as_bytes(base::make_span(image.data()))));
+
+  base::Value visual_dict(base::Value::Type::DICTIONARY);
+  visual_dict.SetKey("blurred_image", base::Value(std::move(image_dict)));
+  return visual_dict;
+}
+
 std::string SerializeClientPhishingRequest(
     const ClientPhishingRequestAndToken& cprat) {
   const ClientPhishingRequest& cpr = cprat.request;
@@ -1029,6 +1043,11 @@ std::string SerializeClientPhishingRequest(
   }
   dict.SetKey("tflite_model_scores", base::Value(std::move(tflite_scores)));
 
+  if (cpr.has_visual_features()) {
+    dict.SetKey("visual_features",
+                SerializeVisualFeatures(cpr.visual_features()));
+  }
+
   base::Value* request_tree = &dict;
   std::string request_serialized;
   JSONStringValueSerializer serializer(&request_serialized);
@@ -1629,6 +1648,11 @@ std::string SerializePGPing(
         SerializeReferringAppInfo(request.referring_app_info()));
   }
 
+  if (request.has_visual_features()) {
+    request_dict.SetKey("visual_features",
+                        SerializeVisualFeatures(request.visual_features()));
+  }
+
   request_dict.SetKey("scoped_oauth_token",
                       base::Value(request_and_token.token));
 

components/safe_browsing/content/common/BUILD.gn

@@ -83,4 +83,33 @@ if (safe_browsing_mode != 0) {
       "//testing/gtest:gtest",
     ]
   }
+
+  source_set("visual_utils") {
+    sources = [
+      "visual_utils.cc",
+      "visual_utils.h",
+    ]
+
+    deps = [
+      "//components/safe_browsing:buildflags",
+      "//components/safe_browsing/core/common:common",
+      "//components/safe_browsing/core/common/proto:client_model_proto",
+      "//components/safe_browsing/core/common/proto:csd_proto",
+      "//ui/gfx:color_utils",
+    ]
+  }
+
+  source_set("unit_tests") {
+    testonly = true
+    sources = [ "visual_utils_unittest.cc" ]
+
+    deps = [
+      ":visual_utils",
+      "//base/test:test_support",
+      "//components/safe_browsing/core/common",
+      "//testing/gmock:gmock",
+      "//testing/gtest:gtest",
+      "//ui/gfx:color_utils",
+    ]
+  }
 }

components/safe_browsing/content/common/DEPS

@@ -1,4 +1,7 @@
 include_rules = [
   "+components/safe_browsing/core/common",
   "+third_party/blink/public/mojom/loader/resource_load_info.mojom-shared.h",
+  "+third_party/skia/include",
+  "+skia/ext",
+  "+ui/gfx",
 ]