Add file backend support for Blob and related

Changes include:
- Add BlobImpl to Blob, and related caching mechanism
- Expose ResourceThreads to document_loader, workerglobalscope, worker, and global
- Fix encode_multipart_form_data
- Other small fixes to accommodate the above changes

components/script/document_loader.rs

@@ -8,9 +8,9 @@
 use dom::bindings::js::JS;
 use dom::document::Document;
 use msg::constellation_msg::PipelineId;
-use net_traits::{PendingAsyncLoad, CoreResourceThread, LoadContext};
+use net_traits::{PendingAsyncLoad, LoadContext};
 use net_traits::{RequestSource, AsyncResponseTarget};
-use std::sync::Arc;
+use net_traits::{ResourceThreads, IpcSend};
 use std::thread;
 use url::Url;
 
@@ -93,30 +93,24 @@ impl Drop for LoadBlocker {
 
 #[derive(JSTraceable, HeapSizeOf)]
 pub struct DocumentLoader {
-    /// We use an `Arc<CoreResourceThread>` here in order to avoid file descriptor exhaustion when there
-    /// are lots of iframes.
-    #[ignore_heap_size_of = "channels are hard"]
-    pub resource_thread: Arc<CoreResourceThread>,
+    resource_threads: ResourceThreads,
     pipeline: Option<PipelineId>,
     blocking_loads: Vec<LoadType>,
     events_inhibited: bool,
 }
 
 impl DocumentLoader {
     pub fn new(existing: &DocumentLoader) -> DocumentLoader {
-        DocumentLoader::new_with_thread(existing.resource_thread.clone(), None, None)
+        DocumentLoader::new_with_threads(existing.resource_threads.clone(), None, None)
     }
 
-    /// We use an `Arc<CoreResourceThread>` here in order to avoid file descriptor exhaustion when there
-    /// are lots of iframes.
-    pub fn new_with_thread(resource_thread: Arc<CoreResourceThread>,
-                         pipeline: Option<PipelineId>,
-                         initial_load: Option<Url>)
-                         -> DocumentLoader {
+    pub fn new_with_threads(resource_threads: ResourceThreads,
+                            pipeline: Option<PipelineId>,
+                            initial_load: Option<Url>) -> DocumentLoader {
         let initial_loads = initial_load.into_iter().map(LoadType::PageSource).collect();
 
         DocumentLoader {
-            resource_thread: resource_thread,
+            resource_threads: resource_threads,
             pipeline: pipeline,
             blocking_loads: initial_loads,
             events_inhibited: false,
@@ -138,7 +132,7 @@ impl DocumentLoader {
         self.add_blocking_load(load);
         let client_chan = referrer.window().custom_message_chan();
         PendingAsyncLoad::new(context,
-                              (*self.resource_thread).clone(),
+                              self.resource_threads.sender(),
                               url,
                               self.pipeline,
                               referrer.get_referrer_policy(),
@@ -169,7 +163,12 @@ impl DocumentLoader {
     pub fn inhibit_events(&mut self) {
         self.events_inhibited = true;
     }
+
     pub fn events_inhibited(&self) -> bool {
         self.events_inhibited
     }
+
+    pub fn resource_threads(&self) -> &ResourceThreads {
+        &self.resource_threads
+    }
 }

components/script/dom/bindings/global.rs

@@ -19,7 +19,8 @@ use js::jsapi::{CurrentGlobalOrNull, GetGlobalForObjectCrossCompartment};
 use js::jsapi::{JSContext, JSObject, JS_GetClass, MutableHandleValue};
 use js::{JSCLASS_IS_DOMJSCLASS, JSCLASS_IS_GLOBAL};
 use msg::constellation_msg::{PipelineId, PanicMsg};
-use net_traits::{CoreResourceThread, RequestSource};
+use net_traits::filemanager_thread::FileManagerThreadMsg;
+use net_traits::{ResourceThreads, CoreResourceThread, RequestSource, IpcSend};
 use profile_traits::{mem, time};
 use script_runtime::{CommonScriptMsg, ScriptChan, ScriptPort};
 use script_thread::{MainThreadScriptChan, ScriptThread};
@@ -122,19 +123,29 @@ impl<'a> GlobalRef<'a> {
         }
     }
 
-    /// Get the `CoreResourceThread` for this global scope.
-    pub fn core_resource_thread(&self) -> CoreResourceThread {
+    /// Get the `ResourceThreads` for this global scope.
+    pub fn resource_threads(&self) -> ResourceThreads {
         match *self {
             GlobalRef::Window(ref window) => {
                 let doc = window.Document();
                 let doc = doc.r();
                 let loader = doc.loader();
-                (*loader.resource_thread).clone()
+                loader.resource_threads().clone()
             }
-            GlobalRef::Worker(ref worker) => worker.core_resource_thread().clone(),
+            GlobalRef::Worker(ref worker) => worker.resource_threads().clone(),
         }
     }
 
+    /// Get the `CoreResourceThread` for this global scope
+    pub fn core_resource_thread(&self) -> CoreResourceThread {
+        self.resource_threads().sender()
+    }
+
+    /// Get the port to file manager for this global scope
+    pub fn filemanager_thread(&self) -> IpcSender<FileManagerThreadMsg> {
+        self.resource_threads().sender()
+    }
+
     /// Get the worker's id.
     pub fn get_worker_id(&self) -> Option<WorkerId> {
         match *self {

components/script/dom/blob.rs

@@ -2,22 +2,26 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+use dom::bindings::cell::DOMRefCell;
 use dom::bindings::codegen::Bindings::BlobBinding;
 use dom::bindings::codegen::Bindings::BlobBinding::BlobMethods;
 use dom::bindings::codegen::UnionTypes::BlobOrString;
-use dom::bindings::error::Fallible;
+use dom::bindings::error::{Error, Fallible};
 use dom::bindings::global::GlobalRef;
 use dom::bindings::js::Root;
 use dom::bindings::reflector::{Reflectable, Reflector, reflect_dom_object};
 use dom::bindings::str::DOMString;
 use encoding::all::UTF_8;
 use encoding::types::{EncoderTrap, Encoding};
+use ipc_channel::ipc;
+use net_traits::filemanager_thread::FileManagerThreadMsg;
 use num_traits::ToPrimitive;
 use std::ascii::AsciiExt;
 use std::borrow::ToOwned;
 use std::cell::Cell;
 use std::cmp::{max, min};
 use std::sync::Arc;
+use uuid::Uuid;
 
 #[derive(Clone, JSTraceable)]
 pub struct DataSlice {
@@ -62,6 +66,11 @@ impl DataSlice {
         }
     }
 
+    /// Construct data slice from a vector of bytes
+    pub fn from_bytes(bytes: Vec<u8>) -> DataSlice {
+        DataSlice::new(Arc::new(bytes), None, None)
+    }
+
     /// Construct an empty data slice
     pub fn empty() -> DataSlice {
         DataSlice {
@@ -83,26 +92,51 @@ impl DataSlice {
 }
 
 
+#[derive(Clone, JSTraceable)]
+pub enum BlobImpl {
+    /// File-based, cached backend
+    File(Uuid, DOMRefCell<Option<DataSlice>>),
+    /// Memory-based backend
+    Memory(DataSlice),
+}
+
+impl BlobImpl {
+    /// Construct memory-backed BlobImpl from DataSlice
+    pub fn new_from_slice(slice: DataSlice) -> BlobImpl {
+        BlobImpl::Memory(slice)
+    }
+
+    /// Construct file-backed BlobImpl from File ID
+    pub fn new_from_file(file_id: Uuid) -> BlobImpl {
+        BlobImpl::File(file_id, DOMRefCell::new(None))
+    }
+
+    /// Construct empty, memory-backed BlobImpl
+    pub fn new_from_empty_slice() -> BlobImpl {
+        BlobImpl::new_from_slice(DataSlice::empty())
+    }
+}
+
 // https://w3c.github.io/FileAPI/#blob
 #[dom_struct]
 pub struct Blob {
     reflector_: Reflector,
     #[ignore_heap_size_of = "No clear owner"]
-    data: DataSlice,
+    blob_impl: BlobImpl,
     typeString: String,
     isClosed_: Cell<bool>,
 }
 
 impl Blob {
-    pub fn new(global: GlobalRef, slice: DataSlice, typeString: &str) -> Root<Blob> {
-        let boxed_blob = box Blob::new_inherited(slice, typeString);
+    pub fn new(global: GlobalRef, blob_impl: BlobImpl, typeString: &str) -> Root<Blob> {
+        let boxed_blob = box Blob::new_inherited(blob_impl, typeString);
         reflect_dom_object(boxed_blob, global, BlobBinding::Wrap)
     }
 
-    pub fn new_inherited(slice: DataSlice, typeString: &str) -> Blob {
+    pub fn new_inherited(blob_impl: BlobImpl, typeString: &str) -> Blob {
         Blob {
             reflector_: Reflector::new(),
-            data: slice,
+            blob_impl: blob_impl,
             typeString: typeString.to_owned(),
             isClosed_: Cell::new(false),
         }
@@ -116,35 +150,81 @@ impl Blob {
         // TODO: accept other blobParts types - ArrayBuffer or ArrayBufferView
         let bytes: Vec<u8> = match blobParts {
             None => Vec::new(),
-            Some(blobparts) => blob_parts_to_bytes(blobparts),
+            Some(blobparts) => match blob_parts_to_bytes(blobparts) {
+                Ok(bytes) => bytes,
+                Err(_) => return Err(Error::InvalidCharacter),
+            }
         };
 
-        let slice = DataSlice::new(Arc::new(bytes), None, None);
-        Ok(Blob::new(global, slice, &blobPropertyBag.get_typestring()))
+        let slice = DataSlice::from_bytes(bytes);
+        Ok(Blob::new(global, BlobImpl::new_from_slice(slice), &blobPropertyBag.get_typestring()))
     }
 
-    pub fn get_data(&self) -> &DataSlice {
-        &self.data
+    /// Get a slice to inner data, this might incur synchronous read and caching
+    pub fn get_slice(&self) -> Result<DataSlice, ()> {
+        match self.blob_impl {
+            BlobImpl::File(ref id, ref slice) => {
+                match *slice.borrow() {
+                    Some(ref s) => Ok(s.clone()),
+                    None => {
+                        let global = self.global();
+                        let s = read_file(global.r(), id.clone())?;
+                        *slice.borrow_mut() = Some(s.clone()); // Cached
+                        Ok(s)
+                    }
+                }
+            }
+            BlobImpl::Memory(ref s) => Ok(s.clone())
+        }
+    }
+
+    /// Try to get a slice, and if any exception happens, return the empty slice
+    pub fn get_slice_or_empty(&self) -> DataSlice {
+        self.get_slice().unwrap_or(DataSlice::empty())
     }
 }
 
-pub fn blob_parts_to_bytes(blobparts: Vec<BlobOrString>) -> Vec<u8> {
-    blobparts.iter().flat_map(|blobpart| {
-            match blobpart {
-                &BlobOrString::String(ref s) => {
-                    UTF_8.encode(s, EncoderTrap::Replace).unwrap()
-                },
-                &BlobOrString::Blob(ref b) => {
-                    b.get_data().get_bytes().to_vec()
-                },
-            }
-        }).collect::<Vec<u8>>()
+fn read_file(global: GlobalRef, id: Uuid) -> Result<DataSlice, ()> {
+    let file_manager = global.filemanager_thread();
+    let (chan, recv) = ipc::channel().map_err(|_|())?;
+    let _ = file_manager.send(FileManagerThreadMsg::ReadFile(chan, id));
+
+    let result = match recv.recv() {
+        Ok(ret) => ret,
+        Err(e) => {
+            debug!("File manager thread has problem {:?}", e);
+            return Err(())
+        }
+    };
+
+    let bytes = result.map_err(|_|())?;
+    Ok(DataSlice::from_bytes(bytes))
+}
+
+/// Extract bytes from BlobParts, used by Blob and File constructor
+/// https://w3c.github.io/FileAPI/#constructorBlob
+pub fn blob_parts_to_bytes(blobparts: Vec<BlobOrString>) -> Result<Vec<u8>, ()> {
+    let mut ret = vec![];
+
+    for blobpart in &blobparts {
+        match blobpart {
+            &BlobOrString::String(ref s) => {
+                let mut bytes = UTF_8.encode(s, EncoderTrap::Replace).map_err(|_|())?;
+                ret.append(&mut bytes);
+            },
+            &BlobOrString::Blob(ref b) => {
+                ret.append(&mut b.get_slice_or_empty().bytes.to_vec());
+            },
+        }
+    }
+
+    Ok(ret)
 }
 
 impl BlobMethods for Blob {
     // https://w3c.github.io/FileAPI/#dfn-size
     fn Size(&self) -> u64 {
-        self.data.size()
+        self.get_slice_or_empty().size()
     }
 
     // https://w3c.github.io/FileAPI/#dfn-type
@@ -169,9 +249,11 @@ impl BlobMethods for Blob {
                 }
             }
         };
+
         let global = self.global();
-        let bytes = self.data.bytes.clone();
-        Blob::new(global.r(), DataSlice::new(bytes, start, end), &relativeContentType)
+        let bytes = self.get_slice_or_empty().bytes.clone();
+        let slice = DataSlice::new(bytes, start, end);
+        Blob::new(global.r(), BlobImpl::new_from_slice(slice), &relativeContentType)
     }
 
     // https://w3c.github.io/FileAPI/#dfn-isClosed
@@ -196,6 +278,8 @@ impl BlobMethods for Blob {
 
 
 impl BlobBinding::BlobPropertyBag {
+    /// Get the normalized inner type string
+    /// https://w3c.github.io/FileAPI/#dfn-type
     pub fn get_typestring(&self) -> String {
         if is_ascii_printable(&self.type_) {
             self.type_.to_lowercase()

components/script/dom/dedicatedworkerglobalscope.rs

@@ -28,7 +28,7 @@ use js::jsapi::{JSAutoCompartment, JSContext, RootedValue};
 use js::jsval::UndefinedValue;
 use js::rust::Runtime;
 use msg::constellation_msg::PipelineId;
-use net_traits::{LoadContext, load_whole_resource, CustomResponse};
+use net_traits::{LoadContext, load_whole_resource, CustomResponse, IpcSend};
 use rand::random;
 use script_runtime::ScriptThreadEventCategory::WorkerEvent;
 use script_runtime::{CommonScriptMsg, ScriptChan, ScriptPort, StackRootTLS, get_reports, new_rt_and_cx};
@@ -226,7 +226,7 @@ impl DedicatedWorkerGlobalScope {
             let roots = RootCollection::new();
             let _stack_roots_tls = StackRootTLS::new(&roots);
             let (url, source) = match load_whole_resource(LoadContext::Script,
-                                                          &init.core_resource_thread,
+                                                          &init.resource_threads.sender(),
                                                           worker_url,
                                                           &worker_load_origin) {
                 Err(_) => {