Skip to content

Commit

Permalink
Use Mmap to open the rmeta file.
Browse files Browse the repository at this point in the history
Because those files are quite large, contribute significantly to peak
memory usage, but only a small fraction of the data is ever read.
  • Loading branch information
nnethercote authored and eddyb committed Nov 14, 2018
1 parent f1d6183 commit 818257e
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 4 deletions.
2 changes: 2 additions & 0 deletions src/Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2278,12 +2278,14 @@ version = "0.0.0"
dependencies = [
"flate2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"memmap 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"proc_macro 0.0.0",
"rustc 0.0.0",
"rustc_data_structures 0.0.0",
"rustc_errors 0.0.0",
"rustc_target 0.0.0",
"serialize 0.0.0",
"stable_deref_trait 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"syntax 0.0.0",
"syntax_ext 0.0.0",
"syntax_pos 0.0.0",
Expand Down
4 changes: 3 additions & 1 deletion src/librustc_metadata/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@ crate-type = ["dylib"]
[dependencies]
flate2 = "1.0"
log = "0.4"
memmap = "0.6"
proc_macro = { path = "../libproc_macro" }
rustc = { path = "../librustc" }
rustc_target = { path = "../librustc_target" }
rustc_data_structures = { path = "../librustc_data_structures" }
rustc_errors = { path = "../librustc_errors" }
rustc_target = { path = "../librustc_target" }
serialize = { path = "../libserialize" }
stable_deref_trait = "1.0.0"
syntax = { path = "../libsyntax" }
syntax_ext = { path = "../libsyntax_ext" }
syntax_pos = { path = "../libsyntax_pos" }
2 changes: 2 additions & 0 deletions src/librustc_metadata/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
extern crate libc;
#[macro_use]
extern crate log;
extern crate memmap;
extern crate stable_deref_trait;
#[macro_use]
extern crate syntax;
extern crate syntax_pos;
Expand Down
26 changes: 23 additions & 3 deletions src/librustc_metadata/locator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -243,12 +243,14 @@ use std::cmp;
use std::fmt;
use std::fs;
use std::io::{self, Read};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::time::Instant;

use flate2::read::DeflateDecoder;

use rustc_data_structures::owning_ref::OwningRef;

pub struct CrateMismatch {
path: PathBuf,
got: String,
Expand Down Expand Up @@ -856,6 +858,19 @@ fn get_metadata_section(target: &Target,
return ret;
}

/// A trivial wrapper for `Mmap` that implements `StableDeref`.
struct StableDerefMmap(memmap::Mmap);

impl Deref for StableDerefMmap {
type Target = [u8];

fn deref(&self) -> &[u8] {
self.0.deref()
}
}

unsafe impl stable_deref_trait::StableDeref for StableDerefMmap {}

fn get_metadata_section_imp(target: &Target,
flavor: CrateFlavor,
filename: &Path,
Expand Down Expand Up @@ -892,9 +907,14 @@ fn get_metadata_section_imp(target: &Target,
}
}
CrateFlavor::Rmeta => {
let buf = fs::read(filename).map_err(|_|
format!("failed to read rmeta metadata: '{}'", filename.display()))?;
rustc_erase_owner!(OwningRef::new(buf).map_owner_box())
// mmap the file, because only a small fraction of it is read.
let file = std::fs::File::open(filename).map_err(|_|
format!("failed to open rmeta metadata: '{}'", filename.display()))?;
let mmap = unsafe { memmap::Mmap::map(&file) };
let mmap = mmap.map_err(|_|
format!("failed to mmap rmeta metadata: '{}'", filename.display()))?;

rustc_erase_owner!(OwningRef::new(StableDerefMmap(mmap)).map_owner_box())
}
};
let blob = MetadataBlob(raw_bytes);
Expand Down

0 comments on commit 818257e

Please sign in to comment.