Add BraveNetworkDelegate and tie in HTTPS Everywhere

browser/BUILD.gn

@@ -24,5 +24,6 @@ source_set("browser") {
 
   deps = [
     "//brave/components/brave_shields/browser:brave_shields",
+    "net",
   ]
 }

browser/net/BUILD.gn

@@ -0,0 +1,11 @@
+import("//build/config/features.gni")
+
+source_set("net") {
+  sources = [
+    "brave_network_delegate.cc",
+    "brave_network_delegate.h",
+  ]
+  public_deps = [
+    "//third_party/boringssl",
+  ]
+}

browser/net/brave_network_delegate.cc

@@ -0,0 +1,144 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "brave/browser/net/brave_network_delegate.h"
+
+#include "brave/browser/brave_browser_process_impl.h"
+#include "brave/components/brave_shields/browser/https_everywhere_service.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/url_request/url_request.h"
+
+struct OnBeforeURLRequestContext {
+  OnBeforeURLRequestContext() {}
+  ~OnBeforeURLRequestContext() {}
+  GURL request_url;
+  std::string new_url_spec;
+  uint64_t request_identifier = 0;
+
+  DISALLOW_COPY_AND_ASSIGN(OnBeforeURLRequestContext);
+};
+
+class PendingRequests {
+public:
+  void Insert(const uint64_t &request_identifier) {
+    pending_requests_.insert(request_identifier);
+  }
+  void Destroy(const uint64_t &request_identifier) {
+    pending_requests_.erase(request_identifier);
+  }
+  bool IsPendingAndAlive(const uint64_t &request_identifier) {
+    bool isPending = pending_requests_.find(request_identifier) != pending_requests_.end();
+    return isPending;
+  }
+private:
+  std::set<uint64_t> pending_requests_;
+  //no need synchronization, should be executed in the same thread content::BrowserThread::IO
+};
+
+BraveNetworkDelegate::BraveNetworkDelegate(
+    extensions::EventRouterForwarder* event_router,
+    BooleanPrefMember* enable_referrers) :
+    ChromeNetworkDelegate(event_router, enable_referrers) {
+  pending_requests_.reset(new PendingRequests());
+}
+
+BraveNetworkDelegate::~BraveNetworkDelegate() {
+}
+
+int BraveNetworkDelegate::OnBeforeURLRequest(net::URLRequest* request,
+    const net::CompletionCallback& callback,
+    GURL* new_url) {
+  std::shared_ptr<OnBeforeURLRequestContext> ctx(new OnBeforeURLRequestContext());
+  if (request) {
+    ctx->request_identifier = request->identifier();
+  }
+  return OnBeforeURLRequest_HttpsePreFileWork(request, callback, new_url, ctx);
+}
+
+int BraveNetworkDelegate::OnBeforeURLRequest_HttpsePreFileWork(
+  net::URLRequest* request,
+  const net::CompletionCallback& callback,
+  GURL* new_url,
+  std::shared_ptr<OnBeforeURLRequestContext> ctx) {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+
+  bool isValidURL = true;
+  if (request) {
+    isValidURL = request->url().is_valid();
+    std::string scheme = request->url().scheme();
+    if (scheme.length()) {
+      std::transform(scheme.begin(), scheme.end(), scheme.begin(), ::tolower);
+      if ("http" != scheme && "https" != scheme) {
+        isValidURL = false;
+      }
+    }
+  }
+
+  if (isValidURL) {
+    ctx->new_url_spec = g_browser_process->https_everywhere_service()
+      ->GetHTTPSURLFromCacheOnly(&request->url(), request->identifier());
+    if (ctx->new_url_spec == request->url().spec()) {
+      ctx->request_url = request->url();
+      content::BrowserThread::PostTaskAndReply(
+        content::BrowserThread::FILE, FROM_HERE,
+        base::Bind(&BraveNetworkDelegate::OnBeforeURLRequest_HttpseFileWork,
+            base::Unretained(this), base::Unretained(request), ctx),
+        base::Bind(base::IgnoreResult(
+            &BraveNetworkDelegate::OnBeforeURLRequest_HttpsePostFileWork),
+            base::Unretained(this), base::Unretained(request),
+            callback, new_url, ctx)
+          );
+      pending_requests_->Insert(request->identifier());
+      return net::ERR_IO_PENDING;
+    }
+  }
+
+  return ChromeNetworkDelegate::OnBeforeURLRequest(request, callback, new_url);
+}
+
+void BraveNetworkDelegate::OnBeforeURLRequest_HttpseFileWork(
+    net::URLRequest* request, std::shared_ptr<OnBeforeURLRequestContext> ctx) {
+  base::ThreadRestrictions::AssertIOAllowed();
+  DCHECK_CURRENTLY_ON(content::BrowserThread::FILE);
+  DCHECK(ctx->request_identifier != 0);
+  ctx->new_url_spec = g_browser_process->https_everywhere_service()->
+    GetHTTPSURL(&ctx->request_url, ctx->request_identifier);
+}
+
+int BraveNetworkDelegate::OnBeforeURLRequest_HttpsePostFileWork(
+    net::URLRequest* request,
+    const net::CompletionCallback& callback,
+    GURL* new_url,
+    std::shared_ptr<OnBeforeURLRequestContext> ctx) {
+
+  DCHECK_CURRENTLY_ON(content::BrowserThread::IO);
+
+  if (PendedRequestIsDestroyedOrCancelled(ctx.get(), request)) {
+    return net::OK;
+  }
+
+  if (!ctx->new_url_spec.empty() &&
+    ctx->new_url_spec != request->url().spec()) {
+    *new_url = GURL(ctx->new_url_spec);
+    // TODO: dispatch about HTTPS upgrade enabled
+  }
+
+  int rv =
+    ChromeNetworkDelegate::OnBeforeURLRequest(request, callback, new_url);
+  if (rv != net::ERR_IO_PENDING) {
+    callback.Run(rv);
+  }
+  return rv;
+}
+
+bool BraveNetworkDelegate::PendedRequestIsDestroyedOrCancelled(
+    OnBeforeURLRequestContext* ctx, net::URLRequest* request) {
+
+  if (!pending_requests_->IsPendingAndAlive(ctx->request_identifier)
+    || request->status().status() == net::URLRequestStatus::CANCELED) {
+    return true;
+  }
+
+  return false;
+}

browser/net/brave_network_delegate.h

@@ -0,0 +1,63 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef BRAVE_BROWSER_NET_BRAVE_NETWORK_DELEGATE_H_
+#define BRAVE_BROWSER_NET_BRAVE_NETWORK_DELEGATE_H_
+
+#include "chrome/browser/net/chrome_network_delegate.h"
+
+template<class T> class PrefMember;
+
+typedef PrefMember<bool> BooleanPrefMember;
+struct OnBeforeURLRequestContext;
+class PendingRequests;
+
+namespace extensions {
+class EventRouterForwarder;
+}
+
+namespace net {
+class URLRequest;
+}
+
+// BraveNetworkDelegate is the central point from within the Brave code to
+// add hooks into the network stack.
+class BraveNetworkDelegate : public ChromeNetworkDelegate {
+ public:
+  // |enable_referrers| (and all of the other optional PrefMembers) should be
+  // initialized on the UI thread (see below) beforehand. This object's owner is
+  // responsible for cleaning them up at shutdown.
+  BraveNetworkDelegate(extensions::EventRouterForwarder* event_router,
+                        BooleanPrefMember* enable_referrers);
+  ~BraveNetworkDelegate() override;
+  // NetworkDelegate implementation.
+  int OnBeforeURLRequest(net::URLRequest* request,
+                         const net::CompletionCallback& callback,
+                         GURL* new_url) override;
+
+ protected:
+  int OnBeforeURLRequest_HttpsePreFileWork(
+      net::URLRequest* request,
+      const net::CompletionCallback& callback,
+      GURL* new_url,
+      std::shared_ptr<OnBeforeURLRequestContext> ctx);
+  void OnBeforeURLRequest_HttpseFileWork(
+      net::URLRequest* request,
+      std::shared_ptr<OnBeforeURLRequestContext> ctx);
+  int OnBeforeURLRequest_HttpsePostFileWork(
+      net::URLRequest* request,
+      const net::CompletionCallback& callback,
+      GURL* new_url,
+      std::shared_ptr<OnBeforeURLRequestContext> ctx);
+  bool PendedRequestIsDestroyedOrCancelled(
+            OnBeforeURLRequestContext* ctx,
+            net::URLRequest* request);
+  // (TODO)find a better way to handle last first party
+  // This is a hack from Android
+  GURL last_first_party_url_;
+  std::auto_ptr<PendingRequests> pending_requests_;
+  DISALLOW_COPY_AND_ASSIGN(BraveNetworkDelegate);
+};
+
+#endif  // BRAVE_BROWSER_NET_BRAVE_NETWORK_DELEGATE_H_

components/brave_shields/browser/base_brave_shields_service.cc

@@ -33,6 +33,10 @@ BaseBraveShieldsService::BaseBraveShieldsService(
 BaseBraveShieldsService::~BaseBraveShieldsService() {
 }
 
+bool BaseBraveShieldsService::IsInitialized() const {
+  return initialized_;
+}
+
 void BaseBraveShieldsService::DownloadDATFile() {
   web_request_.reset(new DATFileWebRequest(
     file_name_,

components/brave_shields/browser/base_brave_shields_service.h

@@ -27,6 +27,7 @@ class BaseBraveShieldsService {
   virtual ~BaseBraveShieldsService();
   bool Start();
   void Stop();
+  bool IsInitialized() const;
   virtual bool ShouldStartRequest(const GURL& url,
       content::ResourceType resource_type,
       const std::string& tab_host);

components/brave_shields/browser/brave_resource_dispatcher_host_delegate.cc

@@ -6,6 +6,7 @@
 
 #include "brave/components/brave_shields/browser/ad_block_service.h"
 #include "brave/components/brave_shields/browser/brave_shields_resource_throttle.h"
+#include "brave/components/brave_shields/browser/https_everywhere_service.h"
 #include "brave/components/brave_shields/browser/tracking_protection_service.h"
 #include "chrome/browser/browser_process.h"
 
@@ -14,6 +15,7 @@ using content::ResourceType;
 BraveResourceDispatcherHostDelegate::BraveResourceDispatcherHostDelegate() {
   g_browser_process->tracking_protection_service()->Start();
   g_browser_process->ad_block_service()->Start();
+  g_browser_process->https_everywhere_service()->Start();
 }
 
 BraveResourceDispatcherHostDelegate::~BraveResourceDispatcherHostDelegate() {

components/brave_shields/browser/https_everywhere_service.cc

@@ -23,6 +23,7 @@
 #include "brave/components/brave_shields/browser/dat_file_util.h"
 #include "third_party/leveldatabase/src/include/leveldb/db.h"
 #include "third_party/re2/src/re2/re2.h"
+#include "third_party/zlib/google/zip.h"
 
 #define DAT_FILE "httpse.leveldb.zip"
 // TODO: Repalce this with the real version at runtime
@@ -90,31 +91,37 @@ void HTTPSEverywhereService::Cleanup() {
 }
 
 bool HTTPSEverywhereService::Init() {
-   if (level_db_) {
-     return true;
-   }
-  base::FilePath db_file_path = GetDATFilePath(DAT_FILE);
+  base::FilePath zip_db_file_path = GetDATFilePath(DAT_FILE);
+
+  base::FilePath unzipped_level_db_path = zip_db_file_path.RemoveExtension();
+  base::FilePath destination = zip_db_file_path.DirName();
+  if (!zip::Unzip(zip_db_file_path, destination)) {
+    LOG(ERROR) << "Unzip error for HTTPSE";
+    return false;
+  }
+
   leveldb::Options options;
   leveldb::Status status = leveldb::DB::Open(options,
-      db_file_path.value().c_str(),
+      unzipped_level_db_path.value().c_str(),
       &level_db_);
   if (!status.ok() || !level_db_) {
     if (level_db_) {
       delete level_db_;
       level_db_ = nullptr;
     }
 
-    LOG(ERROR) << "level db open error " << db_file_path.value().c_str();
+    LOG(ERROR) << "Level db open error "
+      << unzipped_level_db_path.value().c_str()
+      << ", error: " << status.ToString();
     return false;
   }
-
   return true;
 }
 
 std::string HTTPSEverywhereService::GetHTTPSURL(
     const GURL* url, const uint64_t &request_identifier) {
   base::ThreadRestrictions::AssertIOAllowed();
-  if (nullptr == url || url->scheme() == "https") {
+  if (!IsInitialized() || url->scheme() == "https") {
     return url->spec();
   }
   if (!ShouldHTTPSERedirect(request_identifier)) {
@@ -145,7 +152,7 @@ std::string HTTPSEverywhereService::GetHTTPSURL(
 std::string HTTPSEverywhereService::GetHTTPSURLFromCacheOnly(
     const GURL* url,
     const uint64_t &request_identifier) {
-  if (nullptr == url || url->scheme() == "https") {
+  if (!IsInitialized() || url->scheme() == "https") {
     return url->spec();
   }
   if (!ShouldHTTPSERedirect(request_identifier)) {
@@ -207,7 +214,6 @@ std::string HTTPSEverywhereService::ApplyHTTPSRule(
     const std::string& rule) {
   std::unique_ptr<base::Value> json_object = base::JSONReader::Read(rule);
   if (nullptr == json_object.get()) {
-    LOG(ERROR) << "ApplyHTTPSRule: incorrect json rule";
     return "";
   }
 
@@ -326,7 +332,7 @@ std::string HTTPSEverywhereService::CorrecttoRuleToRE2Engine(
 // The brave shields factory. Using the Brave Shields as a singleton
 // is the job of the browser process.
 // TODO(bbondy): consider making this a singleton.
-std::unique_ptr<BaseBraveShieldsService> HTTPSEverywhereServiceFactory() {
+std::unique_ptr<HTTPSEverywhereService> HTTPSEverywhereServiceFactory() {
   return base::MakeUnique<HTTPSEverywhereService>();
 }
 

components/brave_shields/browser/https_everywhere_service.h

@@ -38,16 +38,16 @@ class HTTPSEverywhereService : public BaseBraveShieldsService {
  public:
    HTTPSEverywhereService();
    ~HTTPSEverywhereService() override;
+  std::string GetHTTPSURL(const GURL* url, const uint64_t &request_id);
+  std::string GetHTTPSURLFromCacheOnly(const GURL* url,
+      const uint64_t &request_id);
 
  protected:
   bool Init() override;
   void Cleanup() override;
 
   void AddHTTPSEUrlToRedirectList(const uint64_t &request_id);
   bool ShouldHTTPSERedirect(const uint64_t &request_id);
-  std::string GetHTTPSURL(const GURL* url, const uint64_t &request_id);
-  std::string GetHTTPSURLFromCacheOnly(const GURL* url,
-      const uint64_t &request_id);
   std::string ApplyHTTPSRule(const std::string& originalUrl,
       const std::string& rule);
   std::string CorrecttoRuleToRE2Engine(const std::string& to);
@@ -60,7 +60,7 @@ class HTTPSEverywhereService : public BaseBraveShieldsService {
 };
 
 // Creates the HTTPSEverywhereService
-std::unique_ptr<BaseBraveShieldsService> HTTPSEverywhereServiceFactory();
+std::unique_ptr<HTTPSEverywhereService> HTTPSEverywhereServiceFactory();
 
 }  // namespace brave_shields