Skip to content

Commit

Permalink
[Lens] Add functionality to pass a full page screenshot to chrome://l…
Browse files Browse the repository at this point in the history
…ens page for display.

- This also starts the Lens region search controller when the page is done loading.
- This allows the region search controller to observe more than one web contents in its lifetime.
- Closes the web contents on capture complete if the static page feature is enabled.

Cq-Include-Trybots: luci.chrome.try:linux-chromeos-chrome,mac-chrome
Bug: b:242724302, b:242724636
Change-Id: I18a65da2564a07caad09e236b2957967e0f0ff5c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4018087
Reviewed-by: Ali Stanfield <stanfield@google.com>
Reviewed-by: Scott Violet <sky@chromium.org>
Commit-Queue: Juan Mojica <juanmojica@google.com>
Cr-Commit-Position: refs/heads/main@{#1070575}
  • Loading branch information
Juan Mojica authored and Chromium LUCI CQ committed Nov 11, 2022
1 parent 66273c7 commit 9b85c86
Show file tree
Hide file tree
Showing 12 changed files with 276 additions and 38 deletions.
3 changes: 3 additions & 0 deletions chrome/browser/lens/region_search/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ static_library("region_search") {
"lens_region_search_controller.cc",
"lens_region_search_controller.h",
]

# TODO(crbug/1383280): Remove allow_circular_includes_from when dependencies for controller are fixed.
allow_circular_includes_from = [ "//chrome/browser/ui" ]
deps = [
"//base",
"//chrome/browser/image_editor",
Expand Down
34 changes: 24 additions & 10 deletions chrome/browser/lens/region_search/lens_region_search_controller.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,28 +26,29 @@

namespace lens {

LensRegionSearchController::LensRegionSearchController(
content::WebContents* web_contents,
Browser* browser)
: content::WebContentsObserver(web_contents), browser_(browser) {
screenshot_flow_ =
std::make_unique<image_editor::ScreenshotFlow>(web_contents);
RegionSearchCapturedData::RegionSearchCapturedData() = default;
RegionSearchCapturedData::~RegionSearchCapturedData() = default;

LensRegionSearchController::LensRegionSearchController(Browser* browser)
: browser_(browser) {
weak_this_ = weak_factory_.GetWeakPtr();
}

LensRegionSearchController::~LensRegionSearchController() {
CloseWithReason(views::Widget::ClosedReason::kLostFocus);
}

void LensRegionSearchController::Start(bool use_fullscreen_capture,
void LensRegionSearchController::Start(content::WebContents* web_contents,
bool use_fullscreen_capture,
bool is_google_default_search_provider) {
is_google_default_search_provider_ = is_google_default_search_provider;
if (!web_contents() || !browser_)
if (!web_contents || !browser_)
return;

Observe(web_contents);
if (!screenshot_flow_)
screenshot_flow_ =
std::make_unique<image_editor::ScreenshotFlow>(web_contents());
std::make_unique<image_editor::ScreenshotFlow>(web_contents);

base::OnceCallback<void(const image_editor::ScreenshotCaptureResult&)>
callback = base::BindOnce(&LensRegionSearchController::OnCaptureCompleted,
Expand Down Expand Up @@ -235,6 +236,9 @@ void LensRegionSearchController::OnCaptureCompleted(
}

RecordCaptureResult(lens::LensRegionSearchCaptureResult::SUCCESS);
if (web_contents() && lens::features::IsLensRegionSearchStaticPageEnabled()) {
web_contents()->ClosePage();
}
}

void LensRegionSearchController::WebContentsDestroyed() {
Expand Down Expand Up @@ -272,8 +276,13 @@ void LensRegionSearchController::CloseWithReason(
if (bubble_widget_) {
std::exchange(bubble_widget_, nullptr)->CloseWithReason(reason);
}
if (screenshot_flow_)
if (screenshot_flow_) {
screenshot_flow_->CancelCapture();
screenshot_flow_.reset();
}
if (web_contents() && lens::features::IsLensRegionSearchStaticPageEnabled()) {
web_contents()->ClosePage();
}
}

bool LensRegionSearchController::IsOverlayUIVisibleForTesting() {
Expand All @@ -282,4 +291,9 @@ bool LensRegionSearchController::IsOverlayUIVisibleForTesting() {
return bubble_widget_->IsVisible() && screenshot_flow_->IsCaptureModeActive();
}

void LensRegionSearchController::SetWebContentsForTesting(
content::WebContents* web_contents) {
Observe(web_contents);
}

} // namespace lens
25 changes: 22 additions & 3 deletions chrome/browser/lens/region_search/lens_region_search_controller.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
#define CHROME_BROWSER_LENS_REGION_SEARCH_LENS_REGION_SEARCH_CONTROLLER_H_

#include "base/memory/raw_ptr.h"
#include "base/supports_user_data.h"
#include "chrome/browser/image_editor/screenshot_flow.h"
#include "chrome/browser/lens/metrics/lens_metrics.h"
#include "content/public/browser/web_contents_observer.h"
#include "ui/gfx/image/image.h"
#include "ui/views/widget/widget.h"

class Browser;
Expand All @@ -26,16 +28,16 @@ namespace lens {

class LensRegionSearchController : public content::WebContentsObserver {
public:
explicit LensRegionSearchController(content::WebContents* web_contents,
Browser* browser);
explicit LensRegionSearchController(Browser* browser);
~LensRegionSearchController() override;

// Creates and runs the drag and capture flow. When run, the user enters into
// a screenshot capture mode with the ability to draw a rectagular region
// around the web contents. When finished with selection, the region is
// converted into a PNG and sent to Lens. If `use_fullscreen_capture` is set
// to true, the whole screen will automatically be captured.
void Start(bool use_fullscreen_capture,
void Start(content::WebContents* web_contents,
bool use_fullscreen_capture,
bool is_google_default_search_provider);

// Closes the UI overlay and user education bubble if currently being shown.
Expand Down Expand Up @@ -75,6 +77,10 @@ class LensRegionSearchController : public content::WebContentsObserver {
// either of the UI elements is not visible, returns false.
bool IsOverlayUIVisibleForTesting();

// Sets the web contents for unit tests that do not launch the region search
// UI.
void SetWebContentsForTesting(content::WebContents* web_contents);

private:
void RecordCaptureResult(lens::LensRegionSearchCaptureResult result);

Expand All @@ -100,5 +106,18 @@ class LensRegionSearchController : public content::WebContentsObserver {

base::WeakPtrFactory<LensRegionSearchController> weak_factory_{this};
};

// Class to associate region search data with Profile across navigation. Used to
// support region search on a static WebUI page.
class RegionSearchCapturedData : public base::SupportsUserData::Data {
public:
RegionSearchCapturedData();
~RegionSearchCapturedData() override;
RegionSearchCapturedData(const RegionSearchCapturedData&) = delete;
RegionSearchCapturedData& operator=(const RegionSearchCapturedData&) = delete;

static constexpr char kDataKey[] = "region_search_data";
gfx::Image image;
};
} // namespace lens
#endif // CHROME_BROWSER_LENS_REGION_SEARCH_LENS_REGION_SEARCH_CONTROLLER_H_
19 changes: 9 additions & 10 deletions chrome/browser/renderer_context_menu/render_view_context_menu.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3640,24 +3640,23 @@ void RenderViewContextMenu::ExecRegionSearch(
return;
}

WebContents* web_contents = source_web_contents_;
if (base::FeatureList::IsEnabled(
lens::features::kEnableRegionSearchOnPdfViewer)) {
// We don't use `source_web_contents_` here because it doesn't work with
// the PDF reader.
web_contents = browser->tab_strip_model()->GetActiveWebContents();
}
if (!lens_region_search_controller_) {
WebContents* web_contents = source_web_contents_;
if (base::FeatureList::IsEnabled(
lens::features::kEnableRegionSearchOnPdfViewer)) {
// We don't use `source_web_contents_` here because it doesn't work with
// the PDF reader.
web_contents = browser->tab_strip_model()->GetActiveWebContents();
}
lens_region_search_controller_ =
std::make_unique<lens::LensRegionSearchController>(web_contents,
browser);
std::make_unique<lens::LensRegionSearchController>(browser);
}
// If Lens fullscreen search is enabled, we want to send every region search
// as a fullscreen capture.
bool use_fullscreen_capture =
GetMenuSourceType(event_flags) == ui::MENU_SOURCE_KEYBOARD ||
lens::features::IsLensFullscreenSearchEnabled();
lens_region_search_controller_->Start(use_fullscreen_capture,
lens_region_search_controller_->Start(web_contents, use_fullscreen_capture,
is_google_default_search_provider);
#endif // BUILDFLAG(GOOGLE_CHROME_BRANDING)
}
Expand Down
4 changes: 1 addition & 3 deletions chrome/browser/resources/lens/region_search_untrusted.html
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
<meta charset="utf-8">
</head>
<body>
<pre>
Placeholder text
</pre>
<img src="screenshot.png"></img>
</body>
</html>
2 changes: 2 additions & 0 deletions chrome/browser/ui/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -5480,6 +5480,8 @@ static_library("ui") {
"views/lens/lens_side_panel_navigation_helper.cc",
"views/lens/lens_side_panel_view.cc",
"views/lens/lens_side_panel_view.h",
"views/lens/lens_static_page_controller.cc",
"views/lens/lens_static_page_controller.h",
"views/side_panel/lens/lens_side_panel_coordinator.cc",
"views/side_panel/lens/lens_side_panel_coordinator.h",
"views/side_panel/lens/lens_unified_side_panel_view.cc",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,15 @@ class LensRegionSearchControllerTest : public TestWithBrowserView {

// Create an active web contents.
AddTab(browser_view()->browser(), GURL("about:blank"));
controller_ = std::make_unique<LensRegionSearchController>(
browser_view()->GetActiveWebContents(), browser_view()->browser());
controller_ =
std::make_unique<LensRegionSearchController>(browser_view()->browser());
controller_->SetWebContentsForTesting(
browser_view()->GetActiveWebContents());
}

void TearDown() override {
TestWithBrowserView::TearDown();
controller_.reset();
}

protected:
Expand Down
16 changes: 7 additions & 9 deletions chrome/browser/ui/views/lens/lens_side_panel_helper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
#include "chrome/browser/ui/views/frame/top_container_view.h"
#include "chrome/browser/ui/views/lens/lens_region_search_instructions_view.h"
#include "chrome/browser/ui/views/lens/lens_side_panel_controller.h"
#include "chrome/browser/ui/views/lens/lens_static_page_controller.h"
#include "chrome/browser/ui/views/side_panel/lens/lens_side_panel_coordinator.h"
#include "chrome/browser/ui/views/side_panel/side_panel_coordinator.h"
#include "chrome/common/webui_url_constants.h"
#include "components/lens/lens_entrypoints.h"
#include "components/lens/lens_features.h"
#include "components/lens/lens_rendering_environment.h"
Expand Down Expand Up @@ -121,15 +121,13 @@ content::WebContents* GetLensUnifiedSidePanelWebContentsForTesting(
}

void OpenLensStaticPage(Browser* browser) {
// TODO(juanmojica): Expand this function to simulate the current region
// search experience in the new tab.
DCHECK(browser);
GURL url(chrome::kChromeUILensURL);
content::OpenURLParams params(
url, content::Referrer(), WindowOpenDisposition::NEW_FOREGROUND_TAB,
ui::PAGE_TRANSITION_LINK, /*is_renderer_initiated=*/false);
params.initiator_origin = url::Origin::Create(url);
browser->OpenURL(params);
auto lens_static_page_data = std::make_unique<lens::LensStaticPageData>();
lens_static_page_data->lens_static_page_controller =
std::make_unique<lens::LensStaticPageController>(browser);
lens_static_page_data->lens_static_page_controller->OpenStaticPage();
browser->SetUserData(LensStaticPageData::kDataKey,
std::move(lens_static_page_data));
}

} // namespace lens
90 changes: 90 additions & 0 deletions chrome/browser/ui/views/lens/lens_static_page_controller.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright 2022 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/browser/ui/views/lens/lens_static_page_controller.h"

#include "chrome/browser/lens/region_search/lens_region_search_controller.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/common/webui_url_constants.h"
#include "ui/base/webui/web_ui_util.h"
#include "ui/gfx/image/image.h"
#include "ui/snapshot/snapshot.h"

namespace lens {

LensStaticPageData::LensStaticPageData() = default;
LensStaticPageData::~LensStaticPageData() = default;

LensStaticPageController::LensStaticPageController(Browser* browser)
: browser_(browser) {}
LensStaticPageController::~LensStaticPageController() = default;

void LensStaticPageController::OpenStaticPage() {
DCHECK(browser_);
// Take a screenshot of the active web contents and save it to profile user
// data.
content::WebContents* active_web_contents =
browser_->tab_strip_model()->GetActiveWebContents();
gfx::Rect fullscreen_size = gfx::Rect(active_web_contents->GetSize());
// TODO(crbug/1383279): Refactor screenshot code shared here with code in
// image_editor::ScreenshotFlow.
#if BUILDFLAG(IS_MAC)
const gfx::NativeView& native_view =
active_web_contents->GetContentNativeView();
gfx::Image img;
bool rval = ui::GrabViewSnapshot(native_view, fullscreen_size, &img);
// If |img| is empty, clients should treat it as a canceled action, but
// we have a DCHECK for development as we expected this call to succeed.
DCHECK(rval);
LoadChromeLens(img);
#else
ui::GrabWindowSnapshotAsyncCallback load_url_callback = base::BindOnce(
&LensStaticPageController::LoadChromeLens, base::Unretained(this));
const gfx::NativeWindow& native_window = active_web_contents->GetNativeView();
ui::GrabWindowSnapshotAsync(native_window, fullscreen_size,
std::move(load_url_callback));
#endif
}

void LensStaticPageController::LoadChromeLens(gfx::Image image) {
DCHECK(browser_);
auto region_search_data = std::make_unique<RegionSearchCapturedData>();
region_search_data->image = image;
browser_->profile()->SetUserData(RegionSearchCapturedData::kDataKey,
std::move(region_search_data));

// After we have set the image data, we can open the WebUI tab and finish
// displaying the region search experience.
GURL url(chrome::kChromeUILensURL);
content::OpenURLParams params(
url, content::Referrer(), WindowOpenDisposition::NEW_FOREGROUND_TAB,
ui::PAGE_TRANSITION_LINK, /*is_renderer_initiated=*/false);
params.initiator_origin = url::Origin::Create(url);
content::WebContents* new_tab = browser_->OpenURL(params);
// Observe the new web contents in order to start the region search controller
// once it is properly loaded (this prevents the region search controller from
// closing prematurely).
Observe(new_tab);
}

void LensStaticPageController::DocumentOnLoadCompletedInPrimaryMainFrame() {
StartRegionSearch(web_contents());
}

void LensStaticPageController::StartRegionSearch(
content::WebContents* contents) {
DCHECK(contents);
DCHECK(browser_);
if (!lens_region_search_controller_) {
lens_region_search_controller_ =
std::make_unique<lens::LensRegionSearchController>(browser_);
}
lens_region_search_controller_->Start(
contents,
/*use_fullscreen_capture=*/false,
/*is_google_default_search_provider=*/true);
}

} // namespace lens

0 comments on commit 9b85c86

Please sign in to comment.