reuse io buffer between documents (#164)

Cargo.toml

@@ -16,7 +16,7 @@ rayon = "1.5.1"
 pulldown-cmark = "0.8.0"
 blake3 = "1.0.0"
 
-html5gum = "0.5.3"
+html5gum = "0.5.4"
 jwalk = "0.7.0"
 patricia_tree = "0.3.1"
 bumpalo = { version = "3.11.1", features = ["collections"] }

src/html/mod.rs

@@ -170,15 +170,27 @@ impl<'a, P> Link<'a, P> {
     }
 }
 
+const BUF_SIZE: usize = 1024 * 1024;
+
 /// This struct is initialized once per "batch of documents" that will be processed on a single
 /// worker thread (as determined by rayon). It pays off to do as much heap allocation as possible
 /// here once instead of in Document::links.
-#[derive(Default)]
 pub struct DocumentBuffers {
     arena: bumpalo::Bump,
+    html_read_buffer: Box<[u8; BUF_SIZE]>,
     parser_buffers: parser::ParserBuffers,
 }
 
+impl Default for DocumentBuffers {
+    fn default() -> Self {
+        DocumentBuffers {
+            arena: Default::default(),
+            html_read_buffer: Box::new([0; BUF_SIZE]),
+            parser_buffers: Default::default(),
+        }
+    }
+}
+
 impl DocumentBuffers {
     pub fn reset(&mut self) {
         self.arena.reset();
@@ -301,7 +313,8 @@ impl Document {
                 current_tag_is_closing: false,
                 check_anchors,
             };
-            let reader = Tokenizer::new_with_emitter(IoReader::new(read), emitter);
+            let ioreader = IoReader::new_with_buffer(read, doc_buf.html_read_buffer.as_mut());
+            let reader = Tokenizer::new_with_emitter(ioreader, emitter);
 
             for error in reader {
                 error?;