Auto merge of #14042 - servo:fetch-unit-data, r=nox

Rewrite the data_loader test with fetch.

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/14042)
<!-- Reviewable:end -->

components/net/lib.rs

@@ -57,7 +57,7 @@ pub mod connector;
 pub mod content_blocker;
 pub mod cookie;
 pub mod cookie_storage;
-pub mod data_loader;
+mod data_loader;
 pub mod file_loader;
 pub mod filemanager_thread;
 pub mod hsts;

tests/unit/net/cookie.rs

@@ -2,8 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-extern crate cookie as cookie_rs;
-
+use cookie_rs;
 use net::cookie::Cookie;
 use net::cookie_storage::CookieStorage;
 use net_traits::CookieSource;

tests/unit/net/data_loader.rs

@@ -2,64 +2,54 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-extern crate hyper;
-extern crate hyper_serde;
-
+use fetch_sync;
+use hyper::header::ContentType;
+use hyper::mime::{Attr, Mime, SubLevel, TopLevel, Value};
 use hyper_serde::Serde;
-use ipc_channel::ipc;
-use msg::constellation_msg::{PipelineId, ReferrerPolicy};
-use net_traits::{LoadContext, LoadData, LoadOrigin, NetworkError};
-use net_traits::LoadConsumer::Channel;
-use net_traits::ProgressMsg::{Done, Payload};
-use self::hyper::header::ContentType;
-use self::hyper::mime::{Attr, Mime, SubLevel, TopLevel, Value};
+use net_traits::{FetchMetadata, FilteredMetadata, NetworkError};
+use net_traits::request::{Origin, Request};
+use net_traits::response::ResponseBody;
+use std::ops::Deref;
 use url::Url;
 
-struct DataLoadTest;
-
-impl LoadOrigin for DataLoadTest {
-    fn referrer_url(&self) -> Option<Url> {
-        None
-    }
-    fn referrer_policy(&self) -> Option<ReferrerPolicy> {
-        None
-    }
-    fn pipeline_id(&self) -> Option<PipelineId> {
-        None
-    }
-}
-
 #[cfg(test)]
 fn assert_parse(url:          &'static str,
                 content_type: Option<ContentType>,
-                charset:      Option<String>,
-                data:         Option<Vec<u8>>) {
-    use net::data_loader::load;
-    use net::mime_classifier::MimeClassifier;
-    use net::resource_thread::CancellationListener;
-    use std::sync::Arc;
-
-    let (start_chan, start_port) = ipc::channel().unwrap();
-    let classifier = Arc::new(MimeClassifier::new());
-    load(LoadData::new(LoadContext::Browsing, Url::parse(url).unwrap(), &DataLoadTest),
-         Channel(start_chan),
-         classifier, CancellationListener::new(None));
-
-    let response = start_port.recv().unwrap();
-    assert_eq!(&response.metadata.content_type.map(Serde::into_inner),
-               &content_type);
-    assert_eq!(&response.metadata.charset,      &charset);
-
-    let progress = response.progress_port.recv().unwrap();
+                charset:      Option<&str>,
+                data:         Option<&[u8]>) {
+    let url = Url::parse(url).unwrap();
+    let origin = Origin::Origin(url.origin());
+    let request = Request::new(url, Some(origin), false, None);
+
+    let response = fetch_sync(request, None);
 
     match data {
+        Some(data) => {
+            assert!(!response.is_network_error());
+            assert_eq!(response.headers.len(), 1);
+
+            let header_content_type = response.headers.get::<ContentType>();
+            assert_eq!(header_content_type, content_type.as_ref());
+
+            let metadata = match response.metadata() {
+                Ok(FetchMetadata::Filtered { filtered: FilteredMetadata::Transparent(m), .. }) => m,
+                result => panic!(result),
+            };
+            assert_eq!(metadata.content_type.map(Serde::into_inner), content_type);
+            assert_eq!(metadata.charset.as_ref().map(String::deref), charset);
+
+            let resp_body = response.body.lock().unwrap();
+            match *resp_body {
+                ResponseBody::Done(ref val) => {
+                    assert_eq!(val, &data);
+                },
+                _ => panic!(),
+            }
+        },
         None => {
-            assert_eq!(progress, Done(Err(NetworkError::Internal("invalid data uri".to_owned()))));
-        }
-        Some(dat) => {
-            assert_eq!(progress, Payload(dat));
-            assert_eq!(response.progress_port.recv().unwrap(), Done(Ok(())));
-        }
+            assert!(response.is_network_error());
+            assert_eq!(response.metadata().err(), Some(NetworkError::Internal("Decoding data URL failed".to_owned())));
+        },
     }
 }
 
@@ -73,8 +63,9 @@ fn plain() {
     assert_parse(
         "data:,hello%20world",
         Some(ContentType(Mime(TopLevel::Text, SubLevel::Plain,
-                              vec!((Attr::Charset, Value::Ext("us-ascii".to_owned())))))),
-        Some("US-ASCII".to_owned()), Some(b"hello world".iter().map(|&x| x).collect()));
+                              vec!((Attr::Charset, Value::Ext("US-ASCII".to_owned())))))),
+        Some("US-ASCII"),
+        Some(b"hello world"));
 }
 
 #[test]
@@ -83,16 +74,27 @@ fn plain_ct() {
         "data:text/plain,hello",
         Some(ContentType(Mime(TopLevel::Text, SubLevel::Plain, vec!()))),
         None,
-        Some(b"hello".iter().map(|&x| x).collect()));
+        Some(b"hello"));
+}
+
+#[test]
+fn plain_html() {
+    assert_parse(
+        "data:text/html,<p>Servo</p>",
+        Some(ContentType(Mime(TopLevel::Text, SubLevel::Html, vec!()))),
+        None,
+        Some(b"<p>Servo</p>"));
 }
 
 #[test]
 fn plain_charset() {
-    assert_parse("data:text/plain;charset=latin1,hello",
+    assert_parse(
+        "data:text/plain;charset=latin1,hello",
         Some(ContentType(Mime(TopLevel::Text,
                               SubLevel::Plain,
                               vec!((Attr::Charset, Value::Ext("latin1".to_owned())))))),
-        Some("latin1".to_owned()), Some(b"hello".iter().map(|&x| x).collect()));
+        Some("latin1"),
+        Some(b"hello"));
 }
 
 #[test]
@@ -102,7 +104,8 @@ fn plain_only_charset() {
         Some(ContentType(Mime(TopLevel::Text,
                               SubLevel::Plain,
                               vec!((Attr::Charset, Value::Utf8))))),
-        Some("utf-8".to_owned()), Some(b"hello".iter().map(|&x| x).collect()));
+        Some("utf-8"),
+        Some(b"hello"));
 }
 
 #[test]
@@ -111,23 +114,26 @@ fn base64() {
         "data:;base64,C62+7w==",
         Some(ContentType(Mime(TopLevel::Text,
                               SubLevel::Plain,
-                              vec!((Attr::Charset, Value::Ext("us-ascii".to_owned())))))),
-        Some("US-ASCII".to_owned()), Some(vec!(0x0B, 0xAD, 0xBE, 0xEF)));
+                              vec!((Attr::Charset, Value::Ext("US-ASCII".to_owned())))))),
+        Some("US-ASCII"),
+        Some(&[0x0B, 0xAD, 0xBE, 0xEF]));
 }
 
 #[test]
 fn base64_ct() {
-    assert_parse("data:application/octet-stream;base64,C62+7w==",
+    assert_parse(
+        "data:application/octet-stream;base64,C62+7w==",
         Some(ContentType(Mime(TopLevel::Application, SubLevel::Ext("octet-stream".to_owned()), vec!()))),
         None,
-        Some(vec!(0x0B, 0xAD, 0xBE, 0xEF)));
+        Some(&[0x0B, 0xAD, 0xBE, 0xEF]));
 }
 
 #[test]
 fn base64_charset() {
-    assert_parse("data:text/plain;charset=koi8-r;base64,8PLl9+XkIO3l5Pfl5A==",
+    assert_parse(
+        "data:text/plain;charset=koi8-r;base64,8PLl9+XkIO3l5Pfl5A==",
         Some(ContentType(Mime(TopLevel::Text, SubLevel::Plain,
                               vec!((Attr::Charset, Value::Ext("koi8-r".to_owned())))))),
-        Some("koi8-r".to_owned()),
-        Some(vec!(0xF0, 0xF2, 0xE5, 0xF7, 0xE5, 0xE4, 0x20, 0xED, 0xE5, 0xE4, 0xF7, 0xE5, 0xE4)));
+        Some("koi8-r"),
+        Some(&[0xF0, 0xF2, 0xE5, 0xF7, 0xE5, 0xE4, 0x20, 0xED, 0xE5, 0xE4, 0xF7, 0xE5, 0xE4]));
 }

tests/unit/net/fetch.rs

@@ -2,10 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+use {DEFAULT_USER_AGENT, FetchResponseCollector, new_fetch_context, fetch_async, fetch_sync};
 use devtools_traits::DevtoolsControlMsg;
 use devtools_traits::HttpRequest as DevtoolsHttpRequest;
 use devtools_traits::HttpResponse as DevtoolsHttpResponse;
-use filemanager_thread::{TestProvider, TEST_PROVIDER};
 use http_loader::{expect_devtools_http_request, expect_devtools_http_response};
 use hyper::LanguageTag;
 use hyper::header::{Accept, AccessControlAllowCredentials, AccessControlAllowHeaders, AccessControlAllowOrigin};
@@ -22,10 +22,7 @@ use hyper::status::StatusCode;
 use hyper::uri::RequestUri;
 use msg::constellation_msg::{ReferrerPolicy, TEST_PIPELINE_ID};
 use net::fetch::cors_cache::CORSCache;
-use net::fetch::methods::{FetchContext, fetch, fetch_with_cors_cache};
-use net::filemanager_thread::FileManager;
-use net::http_loader::HttpState;
-use net_traits::FetchTaskTarget;
+use net::fetch::methods::{fetch, fetch_with_cors_cache};
 use net_traits::request::{Origin, RedirectMode, Referrer, Request, RequestMode};
 use net_traits::response::{CacheState, Response, ResponseBody, ResponseType};
 use std::fs::File;
@@ -34,49 +31,13 @@ use std::rc::Rc;
 use std::sync::{Arc, Mutex};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::mpsc::{Sender, channel};
-use std::thread;
 use time::{self, Duration};
 use unicase::UniCase;
 use url::{Origin as UrlOrigin, Url};
 use util::resource_files::resources_dir_path;
 
-const DEFAULT_USER_AGENT: &'static str = "Such Browser. Very Layout. Wow.";
-
 // TODO write a struct that impls Handler for storing test values
 
-struct FetchResponseCollector {
-    sender: Sender<Response>,
-}
-
-fn new_fetch_context(dc: Option<Sender<DevtoolsControlMsg>>) -> FetchContext<TestProvider> {
-    FetchContext {
-        state: HttpState::new(),
-        user_agent: DEFAULT_USER_AGENT.into(),
-        devtools_chan: dc,
-        filemanager: FileManager::new(TEST_PROVIDER),
-    }
-}
-impl FetchTaskTarget for FetchResponseCollector {
-    fn process_request_body(&mut self, _: &Request) {}
-    fn process_request_eof(&mut self, _: &Request) {}
-    fn process_response(&mut self, _: &Response) {}
-    fn process_response_chunk(&mut self, _: Vec<u8>) {}
-    /// Fired when the response is fully fetched
-    fn process_response_eof(&mut self, response: &Response) {
-        let _ = self.sender.send(response.clone());
-    }
-}
-
-fn fetch_async(request: Request, target: Box<FetchTaskTarget + Send>, dc: Option<Sender<DevtoolsControlMsg>>) {
-    thread::spawn(move || {
-        fetch(Rc::new(request), &mut Some(target), new_fetch_context(dc));
-    });
-}
-
-fn fetch_sync(request: Request, dc: Option<Sender<DevtoolsControlMsg>>) -> Response {
-    fetch(Rc::new(request), &mut None, new_fetch_context(dc))
-}
-
 fn make_server<H: Handler + 'static>(handler: H) -> (Listening, Url) {
     // this is a Listening server because of handle_threads()
     let server = Server::http("0.0.0.0:0").unwrap().handle_threads(handler, 1).unwrap();
@@ -142,31 +103,6 @@ fn test_fetch_aboutblank() {
     assert!(*fetch_response.body.lock().unwrap() == ResponseBody::Done(vec![]));
 }
 
-#[test]
-fn test_fetch_data() {
-    let url = Url::parse("data:text/html,<p>Servo</p>").unwrap();
-    let origin = Origin::Origin(url.origin());
-    let request = Request::new(url, Some(origin), false, None);
-    let expected_resp_body = "<p>Servo</p>".to_owned();
-    let fetch_response = fetch_sync(request, None);
-
-    assert!(!fetch_response.is_network_error());
-    assert_eq!(fetch_response.headers.len(), 1);
-    let content_type: &ContentType = fetch_response.headers.get().unwrap();
-    assert!(**content_type == Mime(TopLevel::Text, SubLevel::Html, vec![]));
-    let resp_body = fetch_response.body.lock().unwrap();
-
-    match *resp_body {
-        ResponseBody::Done(ref val) => {
-            assert_eq!(val, &expected_resp_body.into_bytes());
-        }
-        ResponseBody::Receiving(_) => {
-            panic!();
-        },
-        ResponseBody::Empty => panic!(),
-    }
-}
-
 #[test]
 fn test_fetch_blob() {
     use ipc_channel::ipc;

tests/unit/net/lib.rs

@@ -32,3 +32,50 @@ extern crate util;
 #[cfg(test)] mod hsts;
 #[cfg(test)] mod http_loader;
 #[cfg(test)] mod filemanager_thread;
+
+use devtools_traits::DevtoolsControlMsg;
+use filemanager_thread::{TestProvider, TEST_PROVIDER};
+use net::fetch::methods::{FetchContext, fetch};
+use net::filemanager_thread::FileManager;
+use net::http_loader::HttpState;
+use net_traits::FetchTaskTarget;
+use net_traits::request::Request;
+use net_traits::response::Response;
+use std::rc::Rc;
+use std::sync::mpsc::Sender;
+use std::thread;
+
+const DEFAULT_USER_AGENT: &'static str = "Such Browser. Very Layout. Wow.";
+
+struct FetchResponseCollector {
+    sender: Sender<Response>,
+}
+
+fn new_fetch_context(dc: Option<Sender<DevtoolsControlMsg>>) -> FetchContext<TestProvider> {
+    FetchContext {
+        state: HttpState::new(),
+        user_agent: DEFAULT_USER_AGENT.into(),
+        devtools_chan: dc,
+        filemanager: FileManager::new(TEST_PROVIDER),
+    }
+}
+impl FetchTaskTarget for FetchResponseCollector {
+    fn process_request_body(&mut self, _: &Request) {}
+    fn process_request_eof(&mut self, _: &Request) {}
+    fn process_response(&mut self, _: &Response) {}
+    fn process_response_chunk(&mut self, _: Vec<u8>) {}
+    /// Fired when the response is fully fetched
+    fn process_response_eof(&mut self, response: &Response) {
+        let _ = self.sender.send(response.clone());
+    }
+}
+
+fn fetch_async(request: Request, target: Box<FetchTaskTarget + Send>, dc: Option<Sender<DevtoolsControlMsg>>) {
+    thread::spawn(move || {
+        fetch(Rc::new(request), &mut Some(target), new_fetch_context(dc));
+    });
+}
+
+fn fetch_sync(request: Request, dc: Option<Sender<DevtoolsControlMsg>>) -> Response {
+    fetch(Rc::new(request), &mut None, new_fetch_context(dc))
+}