Redesign network response decoding to avoid creating decoders before …

…some content is present.
servo · Jan 4, 2019 · 4542c80 · 4542c80
1 parent 7a64588
commit 4542c80
Show file tree

Hide file tree

Showing 9 changed files with 607 additions and 183 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -24,3 +24,4 @@ opt-level = 3
 #
 #     [patch."https://github.com/servo/<repository>"]
 #     <crate> = { path = "/path/to/local/checkout" }
+brotli-decompressor = { git = "https://github.com/servo/rust-brotli-decompressor", branch = "async" }
diff --git a/components/net/Cargo.toml b/components/net/Cargo.toml
@@ -22,6 +22,7 @@ crossbeam-channel = "0.3"
 devtools_traits = {path = "../devtools_traits"}
 embedder_traits = { path = "../embedder_traits" }
 flate2 = "1"
+futures = "0.1"
 headers-core = "0.0.1"
 headers-ext = "0.0.3"
 http = "0.1"
@@ -31,6 +32,7 @@ hyper-openssl = "0.7"
 immeta = "0.4"
 ipc-channel = "0.11"
 lazy_static = "1"
+libflate = "0.1"
 log = "0.4"
 malloc_size_of = { path = "../malloc_size_of" }
 malloc_size_of_derive = { path = "../malloc_size_of_derive" }

diff --git a/components/net/connector.rs b/components/net/connector.rs
@@ -3,19 +3,14 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 use crate::hosts::replace_host;
-use crate::http_loader::Decoder;
-use flate2::read::GzDecoder;
-use hyper::body::Payload;
 use hyper::client::connect::{Connect, Destination};
 use hyper::client::HttpConnector as HyperHttpConnector;
 use hyper::rt::Future;
 use hyper::{Body, Client};
 use hyper_openssl::HttpsConnector;
 use openssl::ssl::{SslConnector, SslConnectorBuilder, SslMethod, SslOptions};
 use openssl::x509;
-use std::io::{Cursor, Read};
 use tokio::prelude::future::Executor;
-use tokio::prelude::{Async, Stream};
 
 pub const BUF_SIZE: usize = 32768;
 
@@ -47,105 +42,6 @@ impl Connect for HttpConnector {
 }
 
 pub type Connector = HttpsConnector<HttpConnector>;
-pub struct WrappedBody {
-    pub body: Body,
-    pub decoder: Decoder,
-}
-
-impl WrappedBody {
-    pub fn new(body: Body) -> Self {
-        Self::new_with_decoder(body, Decoder::Plain)
-    }
-
-    pub fn new_with_decoder(body: Body, decoder: Decoder) -> Self {
-        WrappedBody { body, decoder }
-    }
-}
-
-impl Payload for WrappedBody {
-    type Data = <Body as Payload>::Data;
-    type Error = <Body as Payload>::Error;
-    fn poll_data(&mut self) -> Result<Async<Option<Self::Data>>, Self::Error> {
-        self.body.poll_data()
-    }
-}
-
-impl Stream for WrappedBody {
-    type Item = <Body as Stream>::Item;
-    type Error = <Body as Stream>::Error;
-    fn poll(&mut self) -> Result<Async<Option<Self::Item>>, Self::Error> {
-        self.body.poll().map(|res| {
-            res.map(|maybe_chunk| {
-                if let Some(chunk) = maybe_chunk {
-                    match self.decoder {
-                        Decoder::Plain => Some(chunk),
-                        Decoder::Gzip(Some(ref mut decoder)) => {
-                            let mut buf = vec![0; BUF_SIZE];
-                            decoder.get_mut().get_mut().extend(chunk.as_ref());
-                            let len = decoder.read(&mut buf).ok()?;
-                            buf.truncate(len);
-                            Some(buf.into())
-                        },
-                        Decoder::Gzip(None) => {
-                            let mut buf = vec![0; BUF_SIZE];
-                            let mut decoder = GzDecoder::new(Cursor::new(chunk.into_bytes()));
-                            let len = decoder.read(&mut buf).ok()?;
-                            buf.truncate(len);
-                            self.decoder = Decoder::Gzip(Some(decoder));
-                            Some(buf.into())
-                        },
-                        Decoder::Deflate(ref mut decoder) => {
-                            let mut buf = vec![0; BUF_SIZE];
-                            decoder.get_mut().get_mut().extend(chunk.as_ref());
-                            let len = decoder.read(&mut buf).ok()?;
-                            buf.truncate(len);
-                            Some(buf.into())
-                        },
-                        Decoder::Brotli(ref mut decoder) => {
-                            let mut buf = vec![0; BUF_SIZE];
-                            decoder.get_mut().get_mut().extend(chunk.as_ref());
-                            let len = decoder.read(&mut buf).ok()?;
-                            buf.truncate(len);
-                            Some(buf.into())
-                        },
-                    }
-                } else {
-                    // Hyper is done downloading but we still have uncompressed data
-                    match self.decoder {
-                        Decoder::Gzip(Some(ref mut decoder)) => {
-                            let mut buf = vec![0; BUF_SIZE];
-                            let len = decoder.read(&mut buf).ok()?;
-                            if len == 0 {
-                                return None;
-                            }
-                            buf.truncate(len);
-                            Some(buf.into())
-                        },
-                        Decoder::Deflate(ref mut decoder) => {
-                            let mut buf = vec![0; BUF_SIZE];
-                            let len = decoder.read(&mut buf).ok()?;
-                            if len == 0 {
-                                return None;
-                            }
-                            buf.truncate(len);
-                            Some(buf.into())
-                        },
-                        Decoder::Brotli(ref mut decoder) => {
-                            let mut buf = vec![0; BUF_SIZE];
-                            let len = decoder.read(&mut buf).ok()?;
-                            if len == 0 {
-                                return None;
-                            }
-                            buf.truncate(len);
-                            Some(buf.into())
-                        },
-                        _ => None,
-                    }
-                }
-            })
-        })
-    }
-}
 
 pub fn create_ssl_connector_builder(certs: &str) -> SslConnectorBuilder {
     // certs include multiple certificates. We could add all of them at once,
@@ -189,7 +85,7 @@ pub fn create_ssl_connector_builder(certs: &str) -> SslConnectorBuilder {
 pub fn create_http_client<E>(
     ssl_connector_builder: SslConnectorBuilder,
     executor: E,
-) -> Client<Connector, WrappedBody>
+) -> Client<Connector, Body>
 where
     E: Executor<Box<dyn Future<Error = (), Item = ()> + Send + 'static>> + Sync + Send + 'static,
 {