Skip to content

Commit

Permalink
Move HSTS/CookieStorage to Arc<RwLock> from Ipc
Browse files Browse the repository at this point in the history 
#7421
  • Loading branch information
@samfoo
samfoo committed Sep 19, 2015
1 parent c91c018 commit 6f573d5
Show file tree
Hide file tree
Showing 6 changed files with 247 additions and 188 deletions.
107 changes: 62 additions & 45 deletions components/net/http_loader.rs
View file
Expand Up @@ -3,35 +3,36 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */




use cookie;
use cookie_storage::CookieStorage;
use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, NetworkEvent}; use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, NetworkEvent};
use file_loader; use file_loader;
use flate2::read::{DeflateDecoder, GzDecoder}; use flate2::read::{DeflateDecoder, GzDecoder};
use hsts::secure_url; use hsts::{secure_url, HSTSList, HSTSEntry};
use hyper::Error as HttpError; use hyper::Error as HttpError;
use hyper::client::{Request, Response, Pool}; use hyper::client::{Request, Response, Pool};
use hyper::header::{AcceptEncoding, Accept, ContentLength, ContentType, Host}; use hyper::header::{AcceptEncoding, Accept, ContentLength, ContentType, Host};
use hyper::header::{Location, qitem, StrictTransportSecurity, UserAgent}; use hyper::header::{Location, qitem, StrictTransportSecurity, UserAgent, SetCookie};
use hyper::header::{Quality, QualityItem, Headers, ContentEncoding, Encoding}; use hyper::header::{Quality, QualityItem, Headers, ContentEncoding, Encoding, Header};
use hyper::http::RawStatus; use hyper::http::RawStatus;
use hyper::method::Method; use hyper::method::Method;
use hyper::mime::{Mime, TopLevel, SubLevel}; use hyper::mime::{Mime, TopLevel, SubLevel};
use hyper::net::{Fresh, HttpsConnector, Openssl}; use hyper::net::{Fresh, HttpsConnector, Openssl};
use hyper::status::{StatusCode, StatusClass}; use hyper::status::{StatusCode, StatusClass};
use ipc_channel::ipc::{self, IpcSender};
use log; use log;
use mime_classifier::MIMEClassifier; use mime_classifier::MIMEClassifier;
use net_traits::ProgressMsg::{Payload, Done}; use net_traits::ProgressMsg::{Payload, Done};
use net_traits::hosts::replace_hosts; use net_traits::hosts::replace_hosts;
use net_traits::{ControlMsg, CookieSource, LoadData, Metadata, LoadConsumer, IncludeSubdomains}; use net_traits::{CookieSource, LoadData, Metadata, LoadConsumer, IncludeSubdomains};
use openssl::ssl::{SslContext, SslMethod, SSL_VERIFY_PEER}; use openssl::ssl::{SslContext, SslMethod, SSL_VERIFY_PEER};
use resource_task::{start_sending_opt, start_sending_sniffed_opt}; use resource_task::{start_sending_opt, start_sending_sniffed_opt};
use std::borrow::ToOwned; use std::borrow::ToOwned;
use std::boxed::FnBox; use std::boxed::FnBox;
use std::collections::HashSet; use std::collections::HashSet;
use std::error::Error; use std::error::Error;
use std::io::{self, Read, Write}; use std::io::{self, Read, Write};
use std::sync::Arc; use std::sync::mpsc::Sender;
use std::sync::mpsc::{Sender, channel}; use std::sync::{Arc, RwLock};
use url::{Url, UrlParser}; use url::{Url, UrlParser};
use util::resource_files::resources_dir_path; use util::resource_files::resources_dir_path;
use util::task::spawn_named; use util::task::spawn_named;
Expand All @@ -50,13 +51,21 @@ pub fn create_http_connector() -> Arc<Pool<Connector>> {
Arc::new(Pool::with_connector(Default::default(), connector)) Arc::new(Pool::with_connector(Default::default(), connector))
} }


pub fn factory(resource_mgr_chan: IpcSender<ControlMsg>, pub fn factory(hsts_list: Arc<RwLock<HSTSList>>,
cookie_jar: Arc<RwLock<CookieStorage>>,
devtools_chan: Option<Sender<DevtoolsControlMsg>>, devtools_chan: Option<Sender<DevtoolsControlMsg>>,
connector: Arc<Pool<Connector>>) connector: Arc<Pool<Connector>>)
-> Box<FnBox(LoadData, LoadConsumer, Arc<MIMEClassifier>, String) + Send> { -> Box<FnBox(LoadData, LoadConsumer, Arc<MIMEClassifier>, String) + Send> {
box move |load_data: LoadData, senders, classifier, user_agent| { box move |load_data: LoadData, senders, classifier, user_agent| {
spawn_named(format!("http_loader for {}", load_data.url.serialize()), move || { spawn_named(format!("http_loader for {}", load_data.url.serialize()), move || {
load_for_consumer(load_data, senders, classifier, connector, resource_mgr_chan, devtools_chan, user_agent) load_for_consumer(load_data,
senders,
classifier,
connector,
hsts_list,
cookie_jar,
devtools_chan,
user_agent)
}) })
} }
} }
Expand Down Expand Up @@ -98,14 +107,15 @@ fn load_for_consumer(load_data: LoadData,
start_chan: LoadConsumer, start_chan: LoadConsumer,
classifier: Arc<MIMEClassifier>, classifier: Arc<MIMEClassifier>,
connector: Arc<Pool<Connector>>, connector: Arc<Pool<Connector>>,
resource_mgr_chan: IpcSender<ControlMsg>, hsts_list: Arc<RwLock<HSTSList>>,
cookie_jar: Arc<RwLock<CookieStorage>>,
devtools_chan: Option<Sender<DevtoolsControlMsg>>, devtools_chan: Option<Sender<DevtoolsControlMsg>>,
user_agent: String) { user_agent: String) {


let factory = NetworkHttpRequestFactory { let factory = NetworkHttpRequestFactory {
connector: connector, connector: connector,
}; };
match load::<WrappedHttpRequest>(load_data, resource_mgr_chan, devtools_chan, &factory, user_agent) { match load::<WrappedHttpRequest>(load_data, hsts_list, cookie_jar, devtools_chan, &factory, user_agent) {
Err(LoadError::UnsupportedScheme(url)) => { Err(LoadError::UnsupportedScheme(url)) => {
let s = format!("{} request, but we don't support that scheme", &*url.scheme); let s = format!("{} request, but we don't support that scheme", &*url.scheme);
send_error(url, s, start_chan) send_error(url, s, start_chan)
Expand Down Expand Up @@ -313,61 +323,66 @@ fn set_default_accept(headers: &mut Headers) {
} }
} }


fn set_request_cookies(url: Url, headers: &mut Headers, resource_mgr_chan: &IpcSender<ControlMsg>) { fn set_request_cookies(url: Url, headers: &mut Headers, cookie_jar: Arc<RwLock<CookieStorage>>) {
let (tx, rx) = ipc::channel().unwrap(); let mut cookie_jar = cookie_jar.write().unwrap();
resource_mgr_chan.send(ControlMsg::GetCookiesForUrl(url, tx, CookieSource::HTTP)).unwrap(); if let Some(cookie_list) = cookie_jar.cookies_for_url(&url, CookieSource::HTTP) {
if let Some(cookie_list) = rx.recv().unwrap() {
let mut v = Vec::new(); let mut v = Vec::new();
v.push(cookie_list.into_bytes()); v.push(cookie_list.into_bytes());
headers.set_raw("Cookie".to_owned(), v); headers.set_raw("Cookie".to_owned(), v);
} }
} }


fn set_cookies_from_response(url: Url, response: &HttpResponse, resource_mgr_chan: &IpcSender<ControlMsg>) { fn set_cookie_for_url(cookie_jar: Arc<RwLock<CookieStorage>>,
if let Some(cookies) = response.headers().get_raw("set-cookie") { request: Url,
for cookie in cookies.iter() { cookie_val: String) {
if let Ok(cookies) = String::from_utf8(cookie.clone()) { let mut cookie_jar = cookie_jar.write().unwrap();
resource_mgr_chan.send(ControlMsg::SetCookiesForUrl(url.clone(), let source = CookieSource::HTTP;
cookies, let header = Header::parse_header(&[cookie_val.into_bytes()]);
CookieSource::HTTP)).unwrap();
if let Ok(SetCookie(cookies)) = header {
for bare_cookie in cookies {
if let Some(cookie) = cookie::Cookie::new_wrapped(bare_cookie, &request, source) {
cookie_jar.push(cookie, source);
} }
} }
} }
} }


fn request_must_be_secured(url: &Url, resource_mgr_chan: &IpcSender<ControlMsg>) -> bool { fn set_cookies_from_response(url: Url, response: &HttpResponse, cookie_jar: Arc<RwLock<CookieStorage>>) {
let (tx, rx) = ipc::channel().unwrap(); if let Some(cookies) = response.headers().get_raw("set-cookie") {
resource_mgr_chan.send( for cookie in cookies.iter() {
ControlMsg::GetHostMustBeSecured(url.domain().unwrap().to_string(), tx) if let Ok(cookie_value) = String::from_utf8(cookie.clone()) {
).unwrap(); set_cookie_for_url(cookie_jar.clone(),

url.clone(),
rx.recv().unwrap() cookie_value);
}
}
}
} }


fn update_sts_list_from_response(url: &Url, response: &HttpResponse, resource_mgr_chan: &IpcSender<ControlMsg>) { fn update_sts_list_from_response(url: &Url, response: &HttpResponse, hsts_list: Arc<RwLock<HSTSList>>) {
if url.scheme != "https" { if url.scheme != "https" {
return; return;
} }


if let Some(header) = response.headers().get::<StrictTransportSecurity>() { if let Some(header) = response.headers().get::<StrictTransportSecurity>() {
if let Some(host) = url.domain() { if let Some(host) = url.domain() {
info!("adding host {} to the strict transport security list", host); let mut hsts_list = hsts_list.write().unwrap();
info!("- max-age {}", header.max_age);

let include_subdomains = if header.include_subdomains { let include_subdomains = if header.include_subdomains {
info!("- includeSubdomains");
IncludeSubdomains::Included IncludeSubdomains::Included
} else { } else {
IncludeSubdomains::NotIncluded IncludeSubdomains::NotIncluded
}; };


let msg = ControlMsg::SetHSTSEntryForHost( if let Some(entry) = HSTSEntry::new(host.to_string(), include_subdomains, Some(header.max_age)) {
host.to_string(), info!("adding host {} to the strict transport security list", host);
include_subdomains, info!("- max-age {}", header.max_age);
header.max_age if header.include_subdomains {
); info!("- includeSubdomains");
}


resource_mgr_chan.send(msg).unwrap(); hsts_list.push(entry);
}
} }
} }
} }
Expand Down Expand Up @@ -452,7 +467,8 @@ fn send_response_to_devtools(devtools_chan: Option<Sender<DevtoolsControlMsg>>,
} }


pub fn load<A>(load_data: LoadData, pub fn load<A>(load_data: LoadData,
resource_mgr_chan: IpcSender<ControlMsg>, hsts_list: Arc<RwLock<HSTSList>>,
cookie_jar: Arc<RwLock<CookieStorage>>,
devtools_chan: Option<Sender<DevtoolsControlMsg>>, devtools_chan: Option<Sender<DevtoolsControlMsg>>,
request_factory: &HttpRequestFactory<R=A>, request_factory: &HttpRequestFactory<R=A>,
user_agent: String) user_agent: String)
Expand Down Expand Up @@ -484,7 +500,8 @@ pub fn load<A>(load_data: LoadData,
loop { loop {
iters = iters + 1; iters = iters + 1;


if &*url.scheme == "http" && request_must_be_secured(&url, &resource_mgr_chan) { // if &*url.scheme == "http" && request_must_be_secured(&url, &resource_mgr_chan) {
if &*url.scheme == "http" && hsts_list.read().unwrap().is_host_secure(url.domain().unwrap()) {
info!("{} is in the strict transport security list, requesting secure host", url); info!("{} is in the strict transport security list, requesting secure host", url);
url = secure_url(&url); url = secure_url(&url);
} }
Expand Down Expand Up @@ -523,7 +540,7 @@ pub fn load<A>(load_data: LoadData,


set_default_accept(&mut request_headers); set_default_accept(&mut request_headers);
set_default_accept_encoding(&mut request_headers); set_default_accept_encoding(&mut request_headers);
set_request_cookies(doc_url.clone(), &mut request_headers, &resource_mgr_chan); set_request_cookies(doc_url.clone(), &mut request_headers, cookie_jar.clone());


let request_id = uuid::Uuid::new_v4().to_simple_string(); let request_id = uuid::Uuid::new_v4().to_simple_string();


Expand Down Expand Up @@ -601,8 +618,8 @@ pub fn load<A>(load_data: LoadData,
} }
} }


set_cookies_from_response(doc_url.clone(), &response, &resource_mgr_chan); set_cookies_from_response(doc_url.clone(), &response, cookie_jar.clone());
update_sts_list_from_response(&url, &response, &resource_mgr_chan); update_sts_list_from_response(&url, &response, hsts_list.clone());


// --- Loop if there's a redirect // --- Loop if there's a redirect
if response.status().class() == StatusClass::Redirection { if response.status().class() == StatusClass::Redirection {
Expand Down
66 changes: 23 additions & 43 deletions components/net/resource_task.rs
View file
Expand Up @@ -8,30 +8,25 @@ use about_loader;
use cookie; use cookie;
use cookie_storage::CookieStorage; use cookie_storage::CookieStorage;
use data_loader; use data_loader;
use devtools_traits::{DevtoolsControlMsg};
use file_loader; use file_loader;
use hsts::{HSTSList, preload_hsts_domains};
use http_loader::{self, create_http_connector, Connector}; use http_loader::{self, create_http_connector, Connector};
use mime_classifier::{ApacheBugFlag, MIMEClassifier, NoSniffFlag};
use net_traits::ProgressMsg::Done;
use net_traits::{ControlMsg, LoadData, LoadResponse, LoadConsumer, CookieSource};
use net_traits::{Metadata, ProgressMsg, ResourceTask, AsyncResponseTarget, ResponseAction};
use url::Url;
use util::opts;
use util::task::spawn_named;

use hsts::{HSTSList, HSTSEntry, preload_hsts_domains};

use devtools_traits::{DevtoolsControlMsg};
use hyper::client::pool::Pool; use hyper::client::pool::Pool;
use hyper::header::{ContentType, Header, SetCookie}; use hyper::header::{ContentType, Header, SetCookie};
use hyper::mime::{Mime, TopLevel, SubLevel}; use hyper::mime::{Mime, TopLevel, SubLevel};
use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use ipc_channel::ipc::{self, IpcReceiver, IpcSender};

use mime_classifier::{ApacheBugFlag, MIMEClassifier, NoSniffFlag};
use net_traits::ProgressMsg::Done;
use net_traits::{ControlMsg, LoadData, LoadResponse, LoadConsumer, CookieSource};
use net_traits::{Metadata, ProgressMsg, ResourceTask, AsyncResponseTarget, ResponseAction};
use std::borrow::ToOwned; use std::borrow::ToOwned;
use std::boxed::FnBox; use std::boxed::FnBox;

use std::sync::{Arc};

use std::sync::mpsc::{channel, Sender}; use std::sync::mpsc::{channel, Sender};
use std::sync::{Arc, RwLock};
use url::Url;
use util::opts;
use util::task::spawn_named;


pub enum ProgressSender { pub enum ProgressSender {
Channel(IpcSender<ProgressMsg>), Channel(IpcSender<ProgressMsg>),
Expand Down Expand Up @@ -147,10 +142,9 @@ pub fn new_resource_task(user_agent: String,
}; };


let (setup_chan, setup_port) = ipc::channel().unwrap(); let (setup_chan, setup_port) = ipc::channel().unwrap();
let setup_chan_clone = setup_chan.clone();
spawn_named("ResourceManager".to_owned(), move || { spawn_named("ResourceManager".to_owned(), move || {
let resource_manager = ResourceManager::new( let resource_manager = ResourceManager::new(
user_agent, setup_chan_clone, hsts_preload, devtools_chan user_agent, hsts_preload, devtools_chan
); );


let mut channel_manager = ResourceChannelManager { let mut channel_manager = ResourceChannelManager {
Expand Down Expand Up @@ -179,15 +173,9 @@ impl ResourceChannelManager {
self.resource_manager.set_cookies_for_url(request, cookie_list, source) self.resource_manager.set_cookies_for_url(request, cookie_list, source)
} }
ControlMsg::GetCookiesForUrl(url, consumer, source) => { ControlMsg::GetCookiesForUrl(url, consumer, source) => {
consumer.send(self.resource_manager.cookie_storage.cookies_for_url(&url, source)).unwrap(); let cookie_jar = self.resource_manager.cookie_storage.clone();
} let mut cookie_jar = cookie_jar.write().unwrap();
ControlMsg::SetHSTSEntryForHost(host, include_subdomains, max_age) => { consumer.send(cookie_jar.cookies_for_url(&url, source)).unwrap();
if let Some(entry) = HSTSEntry::new(host, include_subdomains, Some(max_age)) {
self.resource_manager.add_hsts_entry(entry)
}
}
ControlMsg::GetHostMustBeSecured(host, consumer) => {
consumer.send(self.resource_manager.is_host_sts(&*host)).unwrap();
} }
ControlMsg::Exit => { ControlMsg::Exit => {
break break
Expand All @@ -199,26 +187,23 @@ impl ResourceChannelManager {


pub struct ResourceManager { pub struct ResourceManager {
user_agent: String, user_agent: String,
cookie_storage: CookieStorage, cookie_storage: Arc<RwLock<CookieStorage>>,
resource_task: IpcSender<ControlMsg>,
mime_classifier: Arc<MIMEClassifier>, mime_classifier: Arc<MIMEClassifier>,
devtools_chan: Option<Sender<DevtoolsControlMsg>>, devtools_chan: Option<Sender<DevtoolsControlMsg>>,
hsts_list: HSTSList, hsts_list: Arc<RwLock<HSTSList>>,
connector: Arc<Pool<Connector>>, connector: Arc<Pool<Connector>>,
} }


impl ResourceManager { impl ResourceManager {
pub fn new(user_agent: String, pub fn new(user_agent: String,
resource_task: IpcSender<ControlMsg>,
hsts_list: HSTSList, hsts_list: HSTSList,
devtools_channel: Option<Sender<DevtoolsControlMsg>>) -> ResourceManager { devtools_channel: Option<Sender<DevtoolsControlMsg>>) -> ResourceManager {
ResourceManager { ResourceManager {
user_agent: user_agent, user_agent: user_agent,
cookie_storage: CookieStorage::new(), cookie_storage: Arc::new(RwLock::new(CookieStorage::new())),
resource_task: resource_task,
mime_classifier: Arc::new(MIMEClassifier::new()), mime_classifier: Arc::new(MIMEClassifier::new()),
devtools_chan: devtools_channel, devtools_chan: devtools_channel,
hsts_list: hsts_list, hsts_list: Arc::new(RwLock::new(hsts_list)),
connector: create_http_connector(), connector: create_http_connector(),
} }
} }
Expand All @@ -230,20 +215,14 @@ impl ResourceManager {
if let Ok(SetCookie(cookies)) = header { if let Ok(SetCookie(cookies)) = header {
for bare_cookie in cookies { for bare_cookie in cookies {
if let Some(cookie) = cookie::Cookie::new_wrapped(bare_cookie, &request, source) { if let Some(cookie) = cookie::Cookie::new_wrapped(bare_cookie, &request, source) {
self.cookie_storage.push(cookie, source); let cookie_jar = self.cookie_storage.clone();
let mut cookie_jar = cookie_jar.write().unwrap();
cookie_jar.push(cookie, source);
} }
} }
} }
} }


pub fn add_hsts_entry(&mut self, entry: HSTSEntry) {
self.hsts_list.push(entry);
}

pub fn is_host_sts(&self, host: &str) -> bool {
self.hsts_list.is_host_secure(host)
}

fn load(&mut self, load_data: LoadData, consumer: LoadConsumer) { fn load(&mut self, load_data: LoadData, consumer: LoadConsumer) {


fn from_factory(factory: fn(LoadData, LoadConsumer, Arc<MIMEClassifier>)) fn from_factory(factory: fn(LoadData, LoadConsumer, Arc<MIMEClassifier>))
Expand All @@ -256,7 +235,8 @@ impl ResourceManager {
let loader = match &*load_data.url.scheme { let loader = match &*load_data.url.scheme {
"file" => from_factory(file_loader::factory), "file" => from_factory(file_loader::factory),
"http" | "https" | "view-source" => "http" | "https" | "view-source" =>
http_loader::factory(self.resource_task.clone(), http_loader::factory(self.hsts_list.clone(),
self.cookie_storage.clone(),
self.devtools_chan.clone(), self.devtools_chan.clone(),
self.connector.clone()), self.connector.clone()),
"data" => from_factory(data_loader::factory), "data" => from_factory(data_loader::factory),
Expand Down
3 changes: 0 additions & 3 deletions components/net_traits/lib.rs
View file
Expand Up @@ -159,9 +159,6 @@ pub enum ControlMsg {
SetCookiesForUrl(Url, String, CookieSource), SetCookiesForUrl(Url, String, CookieSource),
/// Retrieve the stored cookies for a given URL /// Retrieve the stored cookies for a given URL
GetCookiesForUrl(Url, IpcSender<Option<String>>, CookieSource), GetCookiesForUrl(Url, IpcSender<Option<String>>, CookieSource),
/// Store a domain's STS information
SetHSTSEntryForHost(String, IncludeSubdomains, u64),
GetHostMustBeSecured(String, IpcSender<bool>),
Exit Exit
} }


Expand Down

0 comments on commit 6f573d5

Please sign in to comment.