Added devtools support to fetch for XHR + Manish's XHR ident fix

added unit test for request fetch with devtools

added devtools/fetch test

components/devtools/actors/network_event.rs

@@ -43,6 +43,7 @@ pub struct NetworkEventActor {
     pub name: String,
     request: HttpRequest,
     response: HttpResponse,
+    is_xhr: bool,
 }
 
 #[derive(Serialize)]
@@ -340,7 +341,8 @@ impl NetworkEventActor {
                 headers: None,
                 status: None,
                 body: None,
-            }
+            },
+            is_xhr: false,
         }
     }
 
@@ -353,6 +355,7 @@ impl NetworkEventActor {
         self.request.timeStamp = request.timeStamp;
         self.request.connect_time = request.connect_time;
         self.request.send_time = request.send_time;
+        self.is_xhr = request.is_xhr;
     }
 
     pub fn add_response(&mut self, response: DevtoolsHttpResponse) {
@@ -369,7 +372,7 @@ impl NetworkEventActor {
             method: format!("{}", self.request.method),
             startedDateTime: format!("{}", self.request.startedDateTime.rfc3339()),
             timeStamp: self.request.timeStamp,
-            isXHR: false,
+            isXHR: self.is_xhr,
             private: false,
         }
     }

components/devtools_traits/lib.rs

@@ -298,6 +298,7 @@ pub struct HttpRequest {
     pub timeStamp: i64,
     pub connect_time: u64,
     pub send_time: u64,
+    pub is_xhr: bool,
 }
 
 #[derive(Debug, PartialEq)]

components/net/fetch/methods.rs

@@ -4,10 +4,12 @@
 
 use connector::create_http_connector;
 use data_loader::decode;
+use devtools_traits::DevtoolsControlMsg;
 use fetch::cors_cache::CORSCache;
 use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies};
 use http_loader::{NetworkHttpRequestFactory, ReadResult, StreamedResponse, obtain_response, read_block};
 use http_loader::{auth_from_cache, determine_request_referrer};
+use http_loader::{send_response_to_devtools, send_request_to_devtools};
 use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials};
 use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods};
 use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod, Basic};
@@ -20,7 +22,7 @@ use hyper::status::StatusCode;
 use mime_guess::guess_mime_type;
 use msg::constellation_msg::ReferrerPolicy;
 use net_traits::FetchTaskTarget;
-use net_traits::request::{CacheMode, CredentialsMode};
+use net_traits::request::{CacheMode, CredentialsMode, Destination};
 use net_traits::request::{RedirectMode, Referer, Request, RequestMode, ResponseTainting};
 use net_traits::request::{Type, Origin, Window};
 use net_traits::response::{HttpsState, TerminationReason};
@@ -36,6 +38,7 @@ use std::sync::mpsc::{channel, Sender, Receiver};
 use unicase::UniCase;
 use url::{Origin as UrlOrigin, Url};
 use util::thread::spawn_named;
+use uuid;
 
 pub type Target = Option<Box<FetchTaskTarget + Send>>;
 
@@ -47,6 +50,7 @@ enum Data {
 pub struct FetchContext {
     pub state: HttpState,
     pub user_agent: String,
+    pub devtools_chan: Option<Sender<DevtoolsControlMsg>>,
 }
 
 type DoneChannel = Option<(Sender<Data>, Receiver<Data>)>;
@@ -210,7 +214,8 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
                   request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) {
                 request.response_tainting.set(ResponseTainting::CORSTainting);
                 request.redirect_mode.set(RedirectMode::Error);
-                let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan, context);
+                let response = http_fetch(request.clone(), cache, true, true, false,
+                                          target, done_chan, context);
                 if response.is_network_error() {
                     // TODO clear cache entries using request
                 }
@@ -888,7 +893,8 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
     // Step 18
     if response.is_none() {
-        response = Some(http_network_fetch(http_request.clone(), credentials_flag, done_chan));
+        response = Some(http_network_fetch(http_request.clone(), credentials_flag,
+                                           done_chan, context.devtools_chan.clone()));
     }
     let response = response.unwrap();
 
@@ -924,7 +930,8 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 /// [HTTP network fetch](https://fetch.spec.whatwg.org/#http-network-fetch)
 fn http_network_fetch(request: Rc<Request>,
                       _credentials_flag: bool,
-                      done_chan: &mut DoneChannel) -> Response {
+                      done_chan: &mut DoneChannel,
+                      devtools_chan: Option<Sender<DevtoolsControlMsg>>) -> Response {
     // TODO: Implement HTTP network fetch spec
 
     // Step 1
@@ -944,14 +951,22 @@ fn http_network_fetch(request: Rc<Request>,
     let url = request.current_url();
     let cancellation_listener = CancellationListener::new(None);
 
+    let request_id = uuid::Uuid::new_v4().simple().to_string();
+
+    // XHR uses the default destination; other kinds of fetches (which haven't been implemented yet)
+    // do not. Once we support other kinds of fetches we'll need to be more fine grained here
+    // since things like image fetches are classified differently by devtools
+    let is_xhr = request.destination == Destination::None;
     let wrapped_response = obtain_response(&factory, &url, &request.method.borrow(),
                                            &request.headers.borrow(),
                                            &cancellation_listener, &request.body.borrow(), &request.method.borrow(),
-                                           &None, request.redirect_count.get() + 1, &None, "");
+                                           &request.pipeline_id.get(), request.redirect_count.get() + 1,
+                                           &devtools_chan, &request_id, is_xhr);
 
+    let pipeline_id = request.pipeline_id.get();
     let mut response = Response::new();
     match wrapped_response {
-        Ok((res, _)) => {
+        Ok((res, msg)) => {
             response.url = Some(url.clone());
             response.status = Some(res.response.status);
             response.raw_status = Some(res.response.status_raw().clone());
@@ -963,10 +978,29 @@ fn http_network_fetch(request: Rc<Request>,
             *done_chan = Some(channel());
             let meta = response.metadata().expect("Response metadata should exist at this stage");
             let done_sender = done_chan.as_ref().map(|ch| ch.0.clone());
+            let devtools_sender = devtools_chan.clone();
+            let meta_status = meta.status.clone();
+            let meta_headers = meta.headers.clone();
             spawn_named(format!("fetch worker thread"), move || {
                 match StreamedResponse::from_http_response(box res, meta) {
                     Ok(mut res) => {
                         *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]);
+
+                        if let Some(ref sender) = devtools_sender {
+                            if let Some(m) = msg {
+                                send_request_to_devtools(m, &sender);
+                            }
+
+                            // --- Tell devtools that we got a response
+                            // Send an HttpResponse message to devtools with the corresponding request_id
+                            if let Some(pipeline_id) = pipeline_id {
+                                send_response_to_devtools(
+                                    &sender, request_id.into(),
+                                    meta_headers, meta_status,
+                                    pipeline_id);
+                            }
+                        }
+
                         loop {
                             match read_block(&mut res) {
                                 Ok(ReadResult::Payload(chunk)) => {
@@ -994,7 +1028,6 @@ fn http_network_fetch(request: Rc<Request>,
                                     break;
                                 }
                             }
-
                         }
                     }
                     Err(_) => {
@@ -1064,9 +1097,11 @@ fn http_network_fetch(request: Rc<Request>,
 }
 
 /// [CORS preflight fetch](https://fetch.spec.whatwg.org#cors-preflight-fetch)
-fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache, context: &FetchContext) -> Response {
+fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache,
+                        context: &FetchContext) -> Response {
     // Step 1
-    let mut preflight = Request::new(request.current_url(), Some(request.origin.borrow().clone()), false);
+    let mut preflight = Request::new(request.current_url(), Some(request.origin.borrow().clone()),
+                                     false, request.pipeline_id.get());
     *preflight.method.borrow_mut() = Method::Options;
     preflight.initiator = request.initiator.clone();
     preflight.type_ = request.type_.clone();

components/net/http_loader.rs

@@ -600,7 +600,8 @@ fn prepare_devtools_request(request_id: String,
                             pipeline_id: PipelineId,
                             now: Tm,
                             connect_time: u64,
-                            send_time: u64) -> ChromeToDevtoolsControlMsg {
+                            send_time: u64,
+                            is_xhr: bool) -> ChromeToDevtoolsControlMsg {
     let request = DevtoolsHttpRequest {
         url: url,
         method: method,
@@ -611,18 +612,19 @@ fn prepare_devtools_request(request_id: String,
         timeStamp: now.to_timespec().sec,
         connect_time: connect_time,
         send_time: send_time,
+        is_xhr: is_xhr,
     };
     let net_event = NetworkEvent::HttpRequest(request);
 
     ChromeToDevtoolsControlMsg::NetworkEvent(request_id, net_event)
 }
 
-fn send_request_to_devtools(msg: ChromeToDevtoolsControlMsg,
+pub fn send_request_to_devtools(msg: ChromeToDevtoolsControlMsg,
                             devtools_chan: &Sender<DevtoolsControlMsg>) {
     devtools_chan.send(DevtoolsControlMsg::FromChrome(msg)).unwrap();
 }
 
-fn send_response_to_devtools(devtools_chan: &Sender<DevtoolsControlMsg>,
+pub fn send_response_to_devtools(devtools_chan: &Sender<DevtoolsControlMsg>,
                              request_id: String,
                              headers: Option<Headers>,
                              status: Option<RawStatus>,
@@ -742,14 +744,16 @@ pub fn obtain_response<A>(request_factory: &HttpRequestFactory<R=A>,
                           pipeline_id: &Option<PipelineId>,
                           iters: u32,
                           devtools_chan: &Option<Sender<DevtoolsControlMsg>>,
-                          request_id: &str)
+                          request_id: &str,
+                          is_xhr: bool)
                           -> Result<(A::R, Option<ChromeToDevtoolsControlMsg>), LoadError>
                           where A: HttpRequest + 'static  {
     let null_data = None;
     let response;
     let connection_url = replace_hosts(&url);
     let mut msg;
 
+
     // loop trying connections in connection pool
     // they may have grown stale (disconnected), in which case we'll get
     // a ConnectionAborted error. this loop tries again with a new
@@ -809,7 +813,7 @@ pub fn obtain_response<A>(request_factory: &HttpRequestFactory<R=A>,
                     request_id.clone().into(),
                     url.clone(), method.clone(), headers,
                     request_body.clone(), pipeline_id, time::now(),
-                    connect_end - connect_start, send_end - send_start))
+                    connect_end - connect_start, send_end - send_start, is_xhr))
             } else {
                 None
             }
@@ -987,7 +991,7 @@ pub fn load<A, B>(load_data: &LoadData,
         let (response, msg) =
             try!(obtain_response(request_factory, &doc_url, &method, &request_headers,
                                  &cancel_listener, &load_data.data, &load_data.method,
-                                 &load_data.pipeline_id, iters, &devtools_chan, &request_id));
+                                 &load_data.pipeline_id, iters, &devtools_chan, &request_id, false));
 
         process_response_headers(&response, &doc_url, &http_state.cookie_jar, &http_state.hsts_list, &load_data);
 
@@ -1087,7 +1091,7 @@ pub fn load<A, B>(load_data: &LoadData,
         if let Some(pipeline_id) = load_data.pipeline_id {
             if let Some(ref chan) = devtools_chan {
                 send_response_to_devtools(
-                    chan, request_id,
+                    &chan, request_id,
                     metadata.headers.clone(), metadata.status.clone(),
                     pipeline_id);
             }

components/net/resource_thread.rs

@@ -574,7 +574,7 @@ impl CoreResourceManager {
                 return
             }
         };
-        debug!("resource_thread: loading url: {}", load_data.url);
+        debug!("loading url: {}", load_data.url);
 
         loader.call_box((load_data,
                          consumer,
@@ -593,14 +593,15 @@ impl CoreResourceManager {
             blocked_content: BLOCKED_CONTENT_RULES.clone(),
         };
         let ua = self.user_agent.clone();
+        let dc = self.devtools_chan.clone();
         spawn_named(format!("fetch thread for {}", init.url), move || {
             let request = Request::from_init(init);
             // XXXManishearth: Check origin against pipeline id (also ensure that the mode is allowed)
             // todo load context / mimesniff in fetch
             // todo referrer policy?
             // todo service worker stuff
             let mut target = Some(Box::new(sender) as Box<FetchTaskTarget + Send + 'static>);
-            let context = FetchContext { state: http_state, user_agent: ua };
+            let context = FetchContext { state: http_state, user_agent: ua, devtools_chan: dc };
             fetch(Rc::new(request), &mut target, context);
         })
     }

components/net_traits/request.rs

@@ -4,7 +4,7 @@
 
 use hyper::header::Headers;
 use hyper::method::Method;
-use msg::constellation_msg::ReferrerPolicy;
+use msg::constellation_msg::{PipelineId, ReferrerPolicy};
 use std::cell::{Cell, RefCell};
 use std::mem::swap;
 use url::{Origin as UrlOrigin, Url};
@@ -130,6 +130,7 @@ pub struct RequestInit {
     // XXXManishearth these should be part of the client object
     pub referer_url: Option<Url>,
     pub referrer_policy: Option<ReferrerPolicy>,
+    pub pipeline_id: Option<PipelineId>,
 }
 
 /// A [Request](https://fetch.spec.whatwg.org/#requests) as defined by the Fetch spec
@@ -159,6 +160,7 @@ pub struct Request {
     /// https://fetch.spec.whatwg.org/#concept-request-referrer
     pub referer: RefCell<Referer>,
     pub referrer_policy: Cell<Option<ReferrerPolicy>>,
+    pub pipeline_id: Cell<Option<PipelineId>>,
     pub synchronous: bool,
     pub mode: RequestMode,
     pub use_cors_preflight: bool,
@@ -178,7 +180,8 @@ pub struct Request {
 impl Request {
     pub fn new(url: Url,
                origin: Option<Origin>,
-               is_service_worker_global_scope: bool) -> Request {
+               is_service_worker_global_scope: bool,
+               pipeline_id: Option<PipelineId>) -> Request {
         Request {
             method: RefCell::new(Method::Get),
             local_urls_only: false,
@@ -198,6 +201,7 @@ impl Request {
             same_origin_data: Cell::new(false),
             referer: RefCell::new(Referer::Client),
             referrer_policy: Cell::new(None),
+            pipeline_id: Cell::new(pipeline_id),
             synchronous: false,
             mode: RequestMode::NoCORS,
             use_cors_preflight: false,
@@ -216,7 +220,7 @@ impl Request {
     pub fn from_init(init: RequestInit) -> Request {
         let mut req = Request::new(init.url,
                                    Some(Origin::Origin(init.origin.origin())),
-                                   false);
+                                   false, init.pipeline_id);
         *req.method.borrow_mut() = init.method;
         *req.headers.borrow_mut() = init.headers;
         req.unsafe_request = init.unsafe_request;
@@ -234,14 +238,16 @@ impl Request {
             Referer::NoReferer
         };
         req.referrer_policy.set(init.referrer_policy);
+        req.pipeline_id.set(init.pipeline_id);
         req
     }
 
     /// https://html.spec.whatwg.org/multipage/#create-a-potential-cors-request
     pub fn potential_cors_request(url: Url,
                                   cors_attribute_state: Option<CORSSettings>,
                                   is_service_worker_global_scope: bool,
-                                  same_origin_fallback: bool) -> Request {
+                                  same_origin_fallback: bool,
+                                  pipeline_id: Option<PipelineId>) -> Request {
         Request {
             method: RefCell::new(Method::Get),
             local_urls_only: false,
@@ -281,6 +287,7 @@ impl Request {
             url_list: RefCell::new(vec![url]),
             redirect_count: Cell::new(0),
             response_tainting: Cell::new(ResponseTainting::Basic),
+            pipeline_id: Cell::new(pipeline_id),
             done: Cell::new(false)
         }
     }

components/script/dom/xmlhttprequest.rs

@@ -594,6 +594,7 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
             origin: self.global().r().get_url(),
             referer_url: self.referrer_url.clone(),
             referrer_policy: self.referrer_policy.clone(),
+            pipeline_id: self.pipeline_id(),
         };
 
         if bypass_cross_origin_check {