AI Chat: sniff subresource content via throttle to detect new content…

… metadata for same-page navigations (#22334)

* AI Chat: sniff subresource content via throttle to detect new content metadata for same-page navigations

* optimization: don't parse yt metadata (or fetch transcript) until an ai chat message is sent by the user

browser/brave_content_browser_client.cc

@@ -152,10 +152,12 @@ using extensions::ChromeContentBrowserClientExtensionsPart;
 
 #if BUILDFLAG(ENABLE_AI_CHAT)
 #include "brave/browser/ui/webui/ai_chat/ai_chat_ui.h"
+#include "brave/components/ai_chat/content/browser/ai_chat_tab_helper.h"
 #include "brave/components/ai_chat/content/browser/ai_chat_throttle.h"
 #include "brave/components/ai_chat/core/browser/utils.h"
 #include "brave/components/ai_chat/core/common/features.h"
 #include "brave/components/ai_chat/core/common/mojom/ai_chat.mojom.h"
+#include "brave/components/ai_chat/core/common/mojom/page_content_extractor.mojom.h"
 #if BUILDFLAG(IS_ANDROID)
 #include "brave/components/ai_chat/core/browser/android/ai_chat_iap_subscription_android.h"
 #endif
@@ -594,6 +596,19 @@ void BraveContentBrowserClient::
           &render_frame_host));
 #endif
 
+#if BUILDFLAG(ENABLE_AI_CHAT)
+  // AI Chat page content extraction renderer -> browser interface
+  associated_registry.AddInterface<ai_chat::mojom::PageContentExtractorHost>(
+      base::BindRepeating(
+          [](content::RenderFrameHost* render_frame_host,
+             mojo::PendingAssociatedReceiver<
+                 ai_chat::mojom::PageContentExtractorHost> receiver) {
+            ai_chat::AIChatTabHelper::BindPageContentExtractorHost(
+                render_frame_host, std::move(receiver));
+          },
+          &render_frame_host));
+#endif
+
   ChromeContentBrowserClient::
       RegisterAssociatedInterfaceBindersForRenderFrameHost(render_frame_host,
                                                            associated_registry);
@@ -814,6 +829,7 @@ void BraveContentBrowserClient::RegisterBrowserInterfaceBindersForFrame(
       user_prefs::UserPrefs::Get(render_frame_host->GetBrowserContext());
   if (ai_chat::IsAIChatEnabled(prefs) &&
       brave::IsRegularProfile(render_frame_host->GetBrowserContext())) {
+    // WebUI -> Browser interface
     content::RegisterWebUIControllerInterfaceBinder<ai_chat::mojom::PageHandler,
                                                     AIChatUI>(map);
   }

chromium_src/chrome/renderer/chrome_content_renderer_client.cc

@@ -6,6 +6,7 @@
 #include "brave/components/ai_chat/core/common/buildflags/buildflags.h"
 #include "brave/components/content_settings/renderer/brave_content_settings_agent_impl.h"
 #include "chrome/common/chrome_isolated_world_ids.h"
+#include "chrome/renderer/chrome_render_thread_observer.h"
 #include "components/dom_distiller/content/renderer/distillability_agent.h"
 #include "components/feed/content/renderer/rss_link_reader.h"
 #include "content/public/common/isolated_world_ids.h"
@@ -22,7 +23,8 @@ void RenderFrameWithBinderRegistryCreated(
     service_manager::BinderRegistry* registry) {
   new feed::RssLinkReader(render_frame, registry);
 #if BUILDFLAG(ENABLE_AI_CHAT)
-  if (ai_chat::features::IsAIChatEnabled()) {
+  if (ai_chat::features::IsAIChatEnabled() &&
+      !ChromeRenderThreadObserver::is_incognito_process()) {
     new ai_chat::PageContentExtractor(render_frame, registry,
                                       content::ISOLATED_WORLD_ID_GLOBAL,
                                       ISOLATED_WORLD_ID_BRAVE_INTERNAL);

components/ai_chat/content/browser/ai_chat_tab_helper.cc

@@ -5,6 +5,7 @@
 
 #include "brave/components/ai_chat/content/browser/ai_chat_tab_helper.h"
 
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <utility>
@@ -19,6 +20,7 @@
 #include "brave/components/ai_chat/content/browser/page_content_fetcher.h"
 #include "brave/components/ai_chat/core/browser/ai_chat_metrics.h"
 #include "brave/components/ai_chat/core/common/features.h"
+#include "brave/components/ai_chat/core/common/mojom/page_content_extractor.mojom.h"
 #include "brave/components/ai_chat/core/common/pref_names.h"
 #include "components/favicon/content/content_favicon_driver.h"
 #include "components/grit/brave_components_strings.h"
@@ -31,6 +33,8 @@
 #include "content/public/browser/navigation_entry.h"
 #include "content/public/browser/scoped_accessibility_mode.h"
 #include "content/public/browser/storage_partition.h"
+#include "content/public/browser/web_contents.h"
+#include "mojo/public/cpp/bindings/self_owned_receiver.h"
 #include "pdf/buildflags.h"
 #include "ui/accessibility/ax_mode.h"
 #include "ui/base/l10n/l10n_util.h"
@@ -59,6 +63,30 @@ void AIChatTabHelper::PDFA11yInfoLoadObserver::AccessibilityEventReceived(
 
 AIChatTabHelper::PDFA11yInfoLoadObserver::~PDFA11yInfoLoadObserver() = default;
 
+// static
+void AIChatTabHelper::BindPageContentExtractorHost(
+    content::RenderFrameHost* rfh,
+    mojo::PendingAssociatedReceiver<mojom::PageContentExtractorHost> receiver) {
+  CHECK(rfh);
+  if (!rfh->IsInPrimaryMainFrame()) {
+    DVLOG(4) << "Not binding extractor host to non-main frame";
+    return;
+  }
+  auto* sender = content::WebContents::FromRenderFrameHost(rfh);
+  if (!sender) {
+    DVLOG(1) << "Cannot bind extractor host, no valid WebContents";
+    return;
+  }
+  auto* tab_helper = AIChatTabHelper::FromWebContents(sender);
+  if (!tab_helper) {
+    DVLOG(1) << "Cannot bind extractor host, no AIChatTabHelper - "
+             << sender->GetVisibleURL();
+    return;
+  }
+  DVLOG(4) << "Binding extractor host to AIChatTabHelper";
+  tab_helper->BindPageContentExtractorReceiver(std::move(receiver));
+}
+
 AIChatTabHelper::AIChatTabHelper(
     content::WebContents* web_contents,
     AIChatMetrics* ai_chat_metrics,
@@ -127,6 +155,7 @@ void AIChatTabHelper::DidFinishNavigation(
   // and treating it as a "fresh page".
   is_same_document_navigation_ = navigation_handle->IsSameDocument();
   pending_navigation_id_ = navigation_handle->GetNavigationId();
+
   // Experimentally only call |OnNewPage| for same-page navigations _if_
   // it results in a page title change (see |TtileWasSet|).
   if (!is_same_document_navigation_) {
@@ -137,7 +166,7 @@ void AIChatTabHelper::DidFinishNavigation(
 void AIChatTabHelper::TitleWasSet(content::NavigationEntry* entry) {
   DVLOG(3) << __func__ << entry->GetTitle();
   if (is_same_document_navigation_) {
-    DVLOG(3) << "Same document navigation detected new \"page\" - calling "
+    DVLOG(2) << "Same document navigation detected new \"page\" - calling "
                 "OnNewPage()";
     // Page title modification after same-document navigation seems as good a
     // time as any to assume meaningful changes occured to the content.
@@ -190,6 +219,22 @@ void AIChatTabHelper::OnFaviconUpdated(
   OnFaviconImageDataChanged();
 }
 
+// mojom::PageContentExtractorHost
+void AIChatTabHelper::OnInterceptedPageContentChanged() {
+  // Maybe mark that the page changed, if we didn't detect it already via title
+  // change after a same-page navigation. This is the main benefit of this
+  // function.
+  if (is_same_document_navigation_) {
+    DVLOG(2) << "Same document navigation detected new \"page\" - calling "
+                "OnNewPage()";
+    // Page title modification after same-document navigation seems as good a
+    // time as any to assume meaningful changes occured to the content.
+    OnNewPage(pending_navigation_id_);
+    // Don't respond to further TitleWasSet
+    is_same_document_navigation_ = false;
+  }
+}
+
 // ai_chat::ConversationDriver
 
 GURL AIChatTabHelper::GetPageURL() const {
@@ -214,6 +259,12 @@ std::u16string AIChatTabHelper::GetPageTitle() const {
   return web_contents()->GetTitle();
 }
 
+void AIChatTabHelper::BindPageContentExtractorReceiver(
+    mojo::PendingAssociatedReceiver<mojom::PageContentExtractorHost> receiver) {
+  page_content_extractor_receiver_.reset();
+  page_content_extractor_receiver_.Bind(std::move(receiver));
+}
+
 WEB_CONTENTS_USER_DATA_KEY_IMPL(AIChatTabHelper);
 
 }  // namespace ai_chat

components/ai_chat/content/browser/ai_chat_tab_helper.h

@@ -15,11 +15,16 @@
 #include "brave/components/ai_chat/core/browser/conversation_driver.h"
 #include "brave/components/ai_chat/core/browser/engine/engine_consumer.h"
 #include "brave/components/ai_chat/core/common/mojom/ai_chat.mojom.h"
+#include "brave/components/ai_chat/core/common/mojom/page_content_extractor.mojom.h"
 #include "components/favicon/core/favicon_driver_observer.h"
 #include "components/prefs/pref_change_registrar.h"
 #include "content/public/browser/navigation_handle.h"
+#include "content/public/browser/render_frame_host.h"
 #include "content/public/browser/web_contents_observer.h"
 #include "content/public/browser/web_contents_user_data.h"
+#include "mojo/public/cpp/bindings/associated_receiver.h"
+#include "mojo/public/cpp/bindings/pending_associated_receiver.h"
+#include "mojo/public/cpp/bindings/receiver_set.h"
 #include "services/data_decoder/public/cpp/data_decoder.h"
 
 class PrefService;
@@ -34,15 +39,24 @@ class AIChatMetrics;
 // Provides context to an AI Chat conversation in the form of the Tab's content
 class AIChatTabHelper : public content::WebContentsObserver,
                         public content::WebContentsUserData<AIChatTabHelper>,
+                        public mojom::PageContentExtractorHost,
                         public favicon::FaviconDriverObserver,
                         public ConversationDriver {
  public:
+  static void BindPageContentExtractorHost(
+      content::RenderFrameHost* rfh,
+      mojo::PendingAssociatedReceiver<mojom::PageContentExtractorHost>
+          receiver);
+
   AIChatTabHelper(const AIChatTabHelper&) = delete;
   AIChatTabHelper& operator=(const AIChatTabHelper&) = delete;
   ~AIChatTabHelper() override;
 
   void SetOnPDFA11yInfoLoadedCallbackForTesting(base::OnceClosure cb);
 
+  // mojom::PageContentExtractorHost
+  void OnInterceptedPageContentChanged() override;
+
  private:
   friend class content::WebContentsUserData<AIChatTabHelper>;
 
@@ -92,6 +106,10 @@ class AIChatTabHelper : public content::WebContentsObserver,
                       std::string_view invalidation_token) override;
   std::u16string GetPageTitle() const override;
 
+  void BindPageContentExtractorReceiver(
+      mojo::PendingAssociatedReceiver<mojom::PageContentExtractorHost>
+          receiver);
+
   raw_ptr<AIChatMetrics> ai_chat_metrics_;
 
   bool is_same_document_navigation_ = false;
@@ -105,6 +123,9 @@ class AIChatTabHelper : public content::WebContentsObserver,
   // A scoper only used for PDF viewing.
   std::unique_ptr<content::ScopedAccessibilityMode> scoped_accessibility_mode_;
 
+  mojo::AssociatedReceiver<mojom::PageContentExtractorHost>
+      page_content_extractor_receiver_{this};
+
   base::WeakPtrFactory<AIChatTabHelper> weak_ptr_factory_{this};
   WEB_CONTENTS_USER_DATA_KEY_DECL();
 };

components/ai_chat/content/browser/page_content_fetcher.cc

@@ -85,11 +85,13 @@ net::NetworkTrafficAnnotationTag GetNetworkTrafficAnnotationTag() {
 
 class PageContentFetcher {
  public:
+  explicit PageContentFetcher(
+      scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory)
+      : url_loader_factory_(url_loader_factory) {}
+
   void Start(mojo::Remote<mojom::PageContentExtractor> content_extractor,
              std::string_view invalidation_token,
-             scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
              FetchPageContentCallback callback) {
-    url_loader_factory_ = url_loader_factory;
     content_extractor_ = std::move(content_extractor);
     if (!content_extractor_) {
       DeleteSelf();
@@ -104,17 +106,6 @@ class PageContentFetcher {
         std::move(callback), invalidation_token));
   }
 
- private:
-  void DeleteSelf() { delete this; }
-
-  void SendResultAndDeleteSelf(FetchPageContentCallback callback,
-                               std::string content = "",
-                               std::string invalidation_token = "",
-                               bool is_video = false) {
-    std::move(callback).Run(content, is_video, invalidation_token);
-    delete this;
-  }
-
   void OnTabContentResult(FetchPageContentCallback callback,
                           std::string_view invalidation_token,
                           mojom::PageContentPtr data) {
@@ -177,6 +168,17 @@ class PageContentFetcher {
                                  std::move(on_response), 2 * 1024 * 1024);
   }
 
+ private:
+  void DeleteSelf() { delete this; }
+
+  void SendResultAndDeleteSelf(FetchPageContentCallback callback,
+                               std::string content = "",
+                               std::string invalidation_token = "",
+                               bool is_video = false) {
+    std::move(callback).Run(content, is_video, invalidation_token);
+    delete this;
+  }
+
   void OnYoutubeTranscriptXMLParsed(
       FetchPageContentCallback callback,
       std::string invalidation_token,
@@ -411,13 +413,12 @@ void FetchPageContent(content::WebContents* web_contents,
   primary_rfh->GetRemoteInterfaces()->GetInterface(
       extractor.BindNewPipeAndPassReceiver());
 
-  auto* fetcher = new PageContentFetcher();
   auto* loader = web_contents->GetBrowserContext()
                      ->GetDefaultStoragePartition()
                      ->GetURLLoaderFactoryForBrowserProcess()
                      .get();
-  fetcher->Start(std::move(extractor), invalidation_token, loader,
-                 std::move(callback));
+  auto* fetcher = new PageContentFetcher(loader);
+  fetcher->Start(std::move(extractor), invalidation_token, std::move(callback));
 }
 
 }  // namespace ai_chat

components/ai_chat/core/browser/conversation_driver.cc

@@ -472,17 +472,42 @@ void ConversationDriver::OnGeneratePageContentComplete(
     std::string contents_text,
     bool is_video,
     std::string invalidation_token) {
-  VLOG(1) << "OnGeneratePageContentComplete";
-  VLOG(4) << "Contents(is_video=" << is_video
-          << ", invalidation_token=" << invalidation_token
-          << "): " << contents_text;
+  DVLOG(1) << "OnGeneratePageContentComplete";
+  DVLOG(4) << "Contents(is_video=" << is_video
+           << ", invalidation_token=" << invalidation_token
+           << "): " << contents_text;
   if (navigation_id != current_navigation_id_) {
     VLOG(1) << __func__ << " for a different navigation. Ignoring.";
     return;
   }
 
-  is_page_text_fetch_in_progress_ = false;
+  // Ignore if we received content from observer in the meantime
+  if (!is_page_text_fetch_in_progress_) {
+    DVLOG(1) << __func__
+             << " but already received contents from observer. Ignoring.";
+    return;
+  }
 
+  OnPageContentUpdated(contents_text, is_video, invalidation_token);
+
+  std::move(callback).Run(article_text_, is_video_,
+                          content_invalidation_token_);
+}
+
+void ConversationDriver::OnExistingGeneratePageContentComplete(
+    GetPageContentCallback callback) {
+  // Don't need to check navigation ID since existing event will be
+  // deleted when there's a new conversation.
+  DVLOG(1) << "Existing page content fetch completed, proceeding with "
+              "the results of that operation.";
+  std::move(callback).Run(article_text_, is_video_,
+                          content_invalidation_token_);
+}
+
+void ConversationDriver::OnPageContentUpdated(std::string contents_text,
+                                              bool is_video,
+                                              std::string invalidation_token) {
+  is_page_text_fetch_in_progress_ = false;
   // If invalidation token matches existing token, then
   // content was not re-fetched and we can use our existing cache.
   if (!invalidation_token.empty() &&
@@ -500,27 +525,12 @@ void ConversationDriver::OnGeneratePageContentComplete(
     OnPageHasContentChanged(BuildSiteInfo());
   }
 
-  on_page_text_fetch_complete_->Signal();
-  on_page_text_fetch_complete_ = std::make_unique<base::OneShotEvent>();
-
   if (contents_text.empty()) {
     VLOG(1) << __func__ << ": No data";
   }
 
-  VLOG(4) << "calling callback with text: " << article_text_;
-
-  std::move(callback).Run(article_text_, is_video_,
-                          content_invalidation_token_);
-}
-
-void ConversationDriver::OnExistingGeneratePageContentComplete(
-    GetPageContentCallback callback) {
-  // Don't need to check navigation ID since existing event will be
-  // deleted when there's a new conversation.
-  VLOG(1) << "Existing page content fetch completed, proceeding with "
-             "the results of that operation.";
-  std::move(callback).Run(article_text_, is_video_,
-                          content_invalidation_token_);
+  on_page_text_fetch_complete_->Signal();
+  on_page_text_fetch_complete_ = std::make_unique<base::OneShotEvent>();
 }
 
 void ConversationDriver::OnNewPage(int64_t navigation_id) {

components/ai_chat/core/browser/conversation_driver.h

@@ -146,6 +146,15 @@ class ConversationDriver {
 
   virtual void OnFaviconImageDataChanged();
 
+  // Implementer should call this when the content is updated in a way that
+  // will not be detected by the on-demand techniques used by GetPageContent.
+  // For example for sites where GetPageContent does not read the live DOM but
+  // reads static JS from HTML that doesn't change for same-page navigation and
+  // we need to intercept new JS data from subresource loads.
+  void OnPageContentUpdated(std::string content,
+                            bool is_video,
+                            std::string invalidation_token);
+
   // To be called when a page navigation is detected and a new conversation
   // is expected.
   void OnNewPage(int64_t navigation_id);