Skip to content

/ brave-core

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for ABP filter redirects as mock responses #3419

Merged
merged 4 commits into from Nov 19, 2019
+318 −353
Merged

Add support for ABP filter redirects as mock responses #3419

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
898f5d7
Use ABP filter redirects as mock responses
bbondy Sep 12, 2019
48ea270
Add unittest for mock redirect based responses
bbondy Nov 11, 2019
58f8190
Add browsertest for redirect content support
bbondy Nov 12, 2019
7e2ccbc
Various format fixes for ABP redirects
bbondy Nov 18, 2019
File filter...
Filter file types
Jump to…
Jump to file
Failed to load files.

Always

Just for now

2 DEPS
View file
Open in desktop
@@ -1,7 +1,7 @@
use_relative_paths = True

deps = {
"vendor/adblock_rust_ffi": "https://github.com/brave/adblock-rust-ffi.git@2360813922ceed63e3798737a268858c11e24d80",
"vendor/adblock_rust_ffi": "https://github.com/brave/adblock-rust-ffi.git@89127a30655eaf54cf73794309846084ea8b91b9",
"vendor/autoplay-whitelist": "https://github.com/brave/autoplay-whitelist.git@ea527a4d36051daedb34421e129c98eda06cb5d3",
"vendor/extension-whitelist": "https://github.com/brave/extension-whitelist.git@7843f62e26a23c51336330e220e9d7992680aae9",
"vendor/hashset-cpp": "https://github.com/brave/hashset-cpp.git@6eab0271d014ff09bd9f38abe1e0c117e13e9aa9",
5 app/brave_generated_resources.grd
View file
Open in desktop
@@ -649,10 +649,5 @@ By installing this extension, you are agreeing to the Google Widevine Terms of U
You can read <ph name="BEGIN_LINK_BUILD_INSTRUCTIONS">&lt;a target="_blank" href="$3"&gt;</ph>instructions on how to download and build for yourself<ph name="END_LINK_BUILD_INSTRUCTIONS">&lt;/a&gt;</ph> the specific <ph name="BEGIN_LINK_SOURCE_CODE">&lt;a target="_blank" href="$4"&gt;</ph>source code used to create this copy<ph name="END_LINK_SOURCE_CODE">&lt;/a&gt;</ph>.
</message>
</messages>
<includes>
<include name="IDR_BRAVE_GOOGLE_ANALYTICS_POLYFILL" file="resources/js/google_analytics_polyfill.js" type="BINDATA" />
<include name="IDR_BRAVE_TAG_SERVICES_POLYFILL" file="resources/js/tag_services_polyfill.js" type="BINDATA" />
<include name="IDR_BRAVE_TAG_MANAGER_POLYFILL" file="resources/js/tag_manager_polyfill.js" type="BINDATA" />
</includes>
</release>
</grit>
50 app/resources/js/google_analytics_polyfill.js
View file
Open in desktop

This file was deleted.

1 app/resources/js/tag_manager_polyfill.js
View file
Open in desktop

This file was deleted.

1 app/resources/js/tag_services_polyfill.js
View file
Open in desktop

This file was deleted.

88 browser/net/brave_ad_block_tp_network_delegate_helper.cc
View file
Open in desktop
@@ -29,98 +29,27 @@ using content::ResourceType;

namespace brave {

std::string GetGoogleAnalyticsPolyfillJS() {
static std::string base64_output;
if (base64_output.length() != 0) {
return base64_output;
}
std::string str = ui::ResourceBundle::GetSharedInstance().GetRawDataResource(
IDR_BRAVE_GOOGLE_ANALYTICS_POLYFILL).as_string();
Base64UrlEncode(str, base::Base64UrlEncodePolicy::OMIT_PADDING,
&base64_output);
base64_output = std::string(kJSDataURLPrefix) + base64_output;
return base64_output;
}

std::string GetGoogleTagManagerPolyfillJS() {
static std::string base64_output;
if (base64_output.length() != 0) {
return base64_output;
}
std::string str = ui::ResourceBundle::GetSharedInstance().GetRawDataResource(
IDR_BRAVE_TAG_MANAGER_POLYFILL).as_string();
Base64UrlEncode(str, base::Base64UrlEncodePolicy::OMIT_PADDING,
&base64_output);
base64_output = std::string(kJSDataURLPrefix) + base64_output;
return base64_output;
}

std::string GetGoogleTagServicesPolyfillJS() {
static std::string base64_output;
if (base64_output.length() != 0) {
return base64_output;
}
std::string str = ui::ResourceBundle::GetSharedInstance().GetRawDataResource(
IDR_BRAVE_TAG_SERVICES_POLYFILL).as_string();
Base64UrlEncode(str, base::Base64UrlEncodePolicy::OMIT_PADDING,
&base64_output);
base64_output = std::string(kJSDataURLPrefix) + base64_output;
return base64_output;
}

bool GetPolyfillForAdBlock(bool allow_brave_shields, bool allow_ads,
const GURL& tab_origin, const GURL& gurl, std::string* new_url_spec) {
// Polyfills which are related to adblock should only apply when shields
// are up.
if (!allow_brave_shields || allow_ads) {
return false;
}

static URLPattern analytics(URLPattern::SCHEME_ALL,
kGoogleAnalyticsPattern);
static URLPattern tag_manager(URLPattern::SCHEME_ALL,
kGoogleTagManagerPattern);
static URLPattern tag_services(URLPattern::SCHEME_ALL,
kGoogleTagServicesPattern);
if (analytics.MatchesURL(gurl)) {
std::string&& data_url = GetGoogleAnalyticsPolyfillJS();
*new_url_spec = data_url;
return true;
}

if (tag_manager.MatchesURL(gurl)) {
std::string&& data_url = GetGoogleTagManagerPolyfillJS();
*new_url_spec = data_url;
return true;
}

if (tag_services.MatchesURL(gurl)) {
std::string&& data_url = GetGoogleTagServicesPolyfillJS();
*new_url_spec = data_url;
return true;
}

return false;
}

void ShouldBlockAdOnTaskRunner(std::shared_ptr<BraveRequestInfo> ctx) {
bool did_match_exception = false;
std::string tab_host = ctx->tab_origin.host();
if (!g_brave_browser_process->ad_block_service()->ShouldStartRequest(
ctx->request_url, ctx->resource_type, tab_host,
&did_match_exception, &ctx->cancel_request_explicitly)) {
&did_match_exception, &ctx->cancel_request_explicitly,
&ctx->mock_data_url)) {
ctx->blocked_by = kAdBlocked;
} else if (!did_match_exception &&
!g_brave_browser_process->ad_block_regional_service_manager()
->ShouldStartRequest(ctx->request_url, ctx->resource_type,
tab_host, &did_match_exception,
&ctx->cancel_request_explicitly)) {
&ctx->cancel_request_explicitly,
&ctx->mock_data_url)) {
ctx->blocked_by = kAdBlocked;
} else if (!did_match_exception &&
!g_brave_browser_process->ad_block_custom_filters_service()
->ShouldStartRequest(ctx->request_url, ctx->resource_type,
tab_host, &did_match_exception,
&ctx->cancel_request_explicitly)) {
&ctx->cancel_request_explicitly,
&ctx->mock_data_url)) {
ctx->blocked_by = kAdBlocked;
}
}
@@ -164,11 +93,6 @@ int OnBeforeURLRequest_AdBlockTPPreWork(
return net::OK;
}

if (GetPolyfillForAdBlock(ctx->allow_brave_shields, ctx->allow_ads,
ctx->tab_origin, ctx->request_url, &ctx->new_url_spec)) {
return net::OK;
}

// Most blocked resources have been moved to our ad block lists.
// This is only for special cases like the PDFjs ping which can
// occur before the ad block lists are fully loaded.
14 browser/net/brave_ad_block_tp_network_delegate_helper.h
View file
Open in desktop
@@ -1,9 +1,12 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
/* Copyright (c) 2019 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_H_
#define BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_H_
#ifndef BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_HELPER_H_
#define BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_HELPER_H_

#include <memory>

#include "brave/browser/net/url_context.h"

@@ -13,9 +16,6 @@ int OnBeforeURLRequest_AdBlockTPPreWork(
const ResponseCallback& next_callback,
std::shared_ptr<BraveRequestInfo> ctx);

bool GetPolyfillForAdBlock(bool allow_brave_shields, bool allow_ads,
const GURL& tab_origin, const GURL& gurl, std::string* new_url_spec);

} // namespace brave

#endif // BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_H_
#endif // BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_HELPER_H_
75 browser/net/brave_ad_block_tp_network_delegate_helper_unittest.cc
View file
Open in desktop
@@ -32,18 +32,6 @@ TEST(BraveAdBlockTPNetworkDelegateHelperTest, EmptyRequestURL) {
EXPECT_EQ(rc, net::OK);
}

TEST(BraveAdBlockTPNetworkDelegateHelperTest, RedirectsToStubs) {
const std::vector<const GURL> urls(
{GURL(kGoogleTagManagerPattern), GURL(kGoogleTagServicesPattern)});
for (const auto& url : urls) {
auto request_info = std::make_shared<brave::BraveRequestInfo>(url);
int rc =
OnBeforeURLRequest_AdBlockTPPreWork(ResponseCallback(), request_info);
EXPECT_EQ(rc, net::OK);
EXPECT_TRUE(GURL(request_info->new_url_spec).SchemeIs("data"));
}
}

TEST(BraveAdBlockTPNetworkDelegateHelperTest, Blocking) {
const std::vector<const GURL> urls({
GURL("https://pdfjs.robwu.nl/ping"),
@@ -56,66 +44,3 @@ TEST(BraveAdBlockTPNetworkDelegateHelperTest, Blocking) {
EXPECT_EQ(rc, net::OK);
}
}

TEST(BraveAdBlockTPNetworkDelegateHelperTest, GetPolyfill) {
using brave::GetPolyfillForAdBlock;

const GURL tab_origin("https://test.com");
const GURL google_analytics_url(kGoogleAnalyticsPattern);
const GURL tag_manager_url(kGoogleTagManagerPattern);
const GURL tag_services_url(kGoogleTagServicesPattern);
const GURL normal_url("https://a.com");

std::string out_url_spec;
// Shields up, block ads, google analytics should get polyfill
ASSERT_TRUE(GetPolyfillForAdBlock(true, false, tab_origin,
google_analytics_url, &out_url_spec));
// Shields up, block ads, tag manager should get polyfill
ASSERT_TRUE(GetPolyfillForAdBlock(true, false, tab_origin, tag_manager_url,
&out_url_spec));
// Shields up, block ads, tag services should get polyfill
ASSERT_TRUE(GetPolyfillForAdBlock(true, false, tab_origin, tag_services_url,
&out_url_spec));
// Shields up, block ads, normal URL should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(true, false, tab_origin, normal_url,
&out_url_spec));

// Shields up, allow ads, google analytics should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(true, true, tab_origin,
google_analytics_url, &out_url_spec));
// Shields up, allow ads, tag manager should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(true, true, tab_origin, tag_manager_url,
&out_url_spec));
// Shields up, allow ads, tag services should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(true, true, tab_origin, tag_services_url,
&out_url_spec));
// Shields up, allow ads, normal URL should NOT get polyfill
ASSERT_FALSE(
GetPolyfillForAdBlock(true, true, tab_origin, normal_url, &out_url_spec));

// Shields down, allow ads, google analytics should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(false, true, tab_origin,
google_analytics_url, &out_url_spec));
// Shields down, allow ads, tag manager should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(false, true, tab_origin, tag_manager_url,
&out_url_spec));
// Shields down, allow ads, tag services should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(false, true, tab_origin, tag_services_url,
&out_url_spec));
// Shields down, allow ads, normal URL should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(false, true, tab_origin, normal_url,
&out_url_spec));

// Shields down, block ads, google analytics should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(false, false, tab_origin,
google_analytics_url, &out_url_spec));
// Shields down, block ads, tag manager should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(false, false, tab_origin, tag_manager_url,
&out_url_spec));
// Shields down, block ads, tag services should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(false, false, tab_origin, tag_services_url,
&out_url_spec));
// Shields down, block ads, normal URL should NOT get polyfill
ASSERT_FALSE(GetPolyfillForAdBlock(false, false, tab_origin, normal_url,
&out_url_spec));
}
22 browser/net/brave_proxying_url_loader_factory.cc
View file
Open in desktop
@@ -110,7 +110,7 @@ void BraveProxyingURLLoaderFactory::InProgressRequest::RestartInternal() {

base::RepeatingCallback<void(int)> continuation =
base::BindRepeating(&InProgressRequest::ContinueToBeforeSendHeaders,
weak_factory_.GetWeakPtr());
weak_factory_.GetWeakPtr());
redirect_url_ = GURL();
ctx_ = std::make_shared<brave::BraveRequestInfo>();
brave::BraveRequestInfo::FillCTX(request_, render_process_id_,
@@ -211,8 +211,8 @@ void BraveProxyingURLLoaderFactory::InProgressRequest::OnUploadProgress(
std::move(callback));
}

void BraveProxyingURLLoaderFactory::InProgressRequest::
OnReceiveCachedMetadata(mojo_base::BigBuffer data) {
void BraveProxyingURLLoaderFactory::InProgressRequest::OnReceiveCachedMetadata(
mojo_base::BigBuffer data) {
target_client_->OnReceiveCachedMetadata(std::move(data));
}

@@ -337,7 +337,8 @@ void BraveProxyingURLLoaderFactory::InProgressRequest::
}
network::ResourceResponseHead response;
std::string response_data;
brave_shields::MakeStubResponse(request_, &response, &response_data);
brave_shields::MakeStubResponse(ctx_->mock_data_url, request_, &response,
&response_data);

target_client_->OnReceiveResponse(response);

@@ -626,10 +627,8 @@ bool BraveProxyingURLLoaderFactory::MaybeProxyRequest(
network::mojom::URLLoaderFactoryPtrInfo target_factory_info;
*factory_receiver = mojo::MakeRequest(&target_factory_info);


ResourceContextData::StartProxying(
browser_context,
render_process_id,
browser_context, render_process_id,
render_frame_host ? render_frame_host->GetFrameTreeNodeId() : 0,
std::move(proxied_receiver), std::move(target_factory_info));
return true;
@@ -651,11 +650,10 @@ void BraveProxyingURLLoaderFactory::CreateLoaderAndStart(
// unique, so we don't use it for identity here.
const uint64_t brave_request_id = request_id_generator_->Generate();

auto result = requests_.emplace(
std::make_unique<InProgressRequest>(
this, brave_request_id, request_id, routing_id, render_process_id_,
frame_tree_node_id_, options, request, browser_context_,
traffic_annotation, std::move(loader_request), std::move(client)));
auto result = requests_.emplace(std::make_unique<InProgressRequest>(
this, brave_request_id, request_id, routing_id, render_process_id_,
frame_tree_node_id_, options, request, browser_context_,
traffic_annotation, std::move(loader_request), std::move(client)));
(*result.first)->Restart();
}

2 browser/net/url_context.h
View file
Open in desktop
@@ -90,6 +90,8 @@ struct BraveRequestInfo {
const base::ListValue* referral_headers_list = nullptr;
BlockedBy blocked_by = kNotBlocked;
bool cancel_request_explicitly = false;
std::string mock_data_url;

// Default to invalid type for resource_type, so delegate helpers
// can properly detect that the info couldn't be obtained.
// TODO(iefremov): Replace with something like |WebRequestResourceType| to
6 common/network_constants.cc
View file
Open in desktop
@@ -44,14 +44,8 @@ const char kCRLSetPrefix3[] =
const char kCRLSetPrefix4[] =
"*://storage.googleapis.com/update-delta/hfnkpimlhhgieaddgfemjhofmfblmnib"
"/*crxd";
const char kGoogleAnalyticsPattern[] =
"https://www.google-analytics.com/analytics.js";
const char kChromeCastPrefix[] =
"*://*.gvt1.com/edgedl/chromewebstore/*pkedcjkdefgpdelpbcmbmeomcjbeemfm*";
const char kGoogleTagManagerPattern[] =
"https://www.googletagmanager.com/gtm.js";
const char kGoogleTagServicesPattern[] =
"https://www.googletagservices.com/tag/js/gpt.js";
const char kForbesPattern[] = "https://www.forbes.com/*";
const char kForbesExtraCookies[] =
"forbes_ab=true; welcomeAd=true; adblock_session=Off; "
3 common/network_constants.h
View file
Open in desktop
@@ -25,9 +25,6 @@ extern const char kEmptyDataURI[];
extern const char kEmptyImageDataURI[];
extern const char kJSDataURLPrefix[];
extern const char kGeoLocationsPattern[];
extern const char kGoogleAnalyticsPattern[];
extern const char kGoogleTagManagerPattern[];
extern const char kGoogleTagServicesPattern[];
extern const char kForbesPattern[];
extern const char kForbesExtraCookies[];
extern const char kSafeBrowsingPrefix[];
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.