Simple module compilation cache (#203)

* Simple module compilation cache

* Fix base64 encoding bug

* Use warn! everywhere in cache system

* Remove unused import

* Temporary workaround for long path on Windows

* Remove unused import for non-windows builds

* Add command line argument to enable cache system + apply minor review feedback

Cargo.toml

@@ -22,8 +22,8 @@ name = "wasm2obj"
 path = "src/wasm2obj.rs"
 
 [dependencies]
-cranelift-codegen = "0.33.0"
-cranelift-native = "0.33.0"
+cranelift-codegen = { version = "0.36.0", features = ["enable-serde"] }
+cranelift-native = "0.36.0"
 wasmtime-debug = { path = "wasmtime-debug" }
 wasmtime-environ = { path = "wasmtime-environ" }
 wasmtime-runtime = { path = "wasmtime-runtime" }
@@ -34,8 +34,7 @@ wasmtime-wasi = { path = "wasmtime-wasi" }
 wasmtime-wasi-c = { path = "wasmtime-wasi-c", optional = true }
 wasi-common = { git = "https://github.com/CraneStation/wasi-common", rev = "c3994bf57b5d2f1f973b0e4e37bc385695aa4ed2"}
 docopt = "1.0.1"
-serde = "1.0.75"
-serde_derive = "1.0.75"
+serde = { "version" = "1.0.94", features = ["derive"] }
 faerie = "0.10.1"
 target-lexicon = { version = "0.4.0", default-features = false }
 pretty_env_logger = "0.3.0"

fuzz/Cargo.toml

@@ -11,9 +11,9 @@ cargo-fuzz = true
 [dependencies]
 wasmtime-environ = { path = "../wasmtime-environ" }
 wasmtime-jit = { path = "../wasmtime-jit" }
-cranelift-codegen = "0.33.0"
-cranelift-wasm = "0.33.0"
-cranelift-native = "0.33.0"
+cranelift-codegen = { version = "0.36.0", features = ["enable-serde"] }
+cranelift-wasm = { version = "0.36.0", features = ["enable-serde"] }
+cranelift-native = "0.36.0"
 libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer-sys.git" }
 wasmparser = { version = "0.32.1", default-features = false }
 binaryen = "0.5.0"

src/wasm2obj.rs

@@ -29,14 +29,12 @@
     )
 )]
 
-#[macro_use]
-extern crate serde_derive;
-
 use cranelift_codegen::isa;
 use cranelift_codegen::settings;
 use cranelift_native;
 use docopt::Docopt;
 use faerie::Artifact;
+use serde::Deserialize;
 use std::error::Error;
 use std::fmt::format;
 use std::fs::File;
@@ -49,6 +47,7 @@ use std::str;
 use std::str::FromStr;
 use target_lexicon::Triple;
 use wasmtime_debug::{emit_debugsections, read_debuginfo};
+use wasmtime_environ::cache_conf;
 use wasmtime_environ::{Compiler, Cranelift, ModuleEnvironment, Tunables};
 use wasmtime_obj::emit_module;
 
@@ -63,14 +62,15 @@ The translation is dependent on the environment chosen.
 The default is a dummy environment that produces placeholder values.
 
 Usage:
-    wasm2obj [--target TARGET] [-g] <file> -o <output>
+    wasm2obj [--target TARGET] [-cdg] <file> -o <output>
     wasm2obj --help | --version
 
 Options:
     -v, --verbose       displays the module and translated functions
     -h, --help          print this help message
     --target <TARGET>   build for the target triple; default is the host machine
     -g                  generate debug information
+    -c, --cache         enable caching system
     --version           print the Cranelift version
     -d, --debug         enable debug output on stderr/stdout
 ";
@@ -82,6 +82,7 @@ struct Args {
     arg_target: Option<String>,
     flag_g: bool,
     flag_debug: bool,
+    flag_cache: bool,
 }
 
 fn read_wasm_file(path: PathBuf) -> Result<Vec<u8>, io::Error> {
@@ -107,6 +108,8 @@ fn main() {
         utils::init_file_per_thread_logger();
     }
 
+    cache_conf::init(args.flag_cache);
+
     let path = Path::new(&args.arg_file);
     match handle_module(
         path.to_path_buf(),

src/wasmtime.rs

@@ -30,14 +30,12 @@
     )
 )]
 
-#[macro_use]
-extern crate serde_derive;
-
 use cranelift_codegen::settings;
 use cranelift_codegen::settings::Configurable;
 use cranelift_native;
 use docopt::Docopt;
 use pretty_env_logger;
+use serde::Deserialize;
 use std::error::Error;
 use std::ffi::OsStr;
 use std::fs::File;
@@ -48,6 +46,7 @@ use std::path::{Path, PathBuf};
 use std::process::exit;
 use wabt;
 use wasi_common::preopen_dir;
+use wasmtime_environ::cache_conf;
 use wasmtime_jit::{ActionOutcome, Context};
 use wasmtime_wasi::instantiate_wasi;
 use wasmtime_wast::instantiate_spectest;
@@ -67,13 +66,14 @@ including calling the start function if one is present. Additional functions
 given with --invoke are then called.
 
 Usage:
-    wasmtime [-odg] [--wasi-c] [--preload=<wasm>...] [--env=<env>...] [--dir=<dir>...] [--mapdir=<mapping>...] <file> [<arg>...]
-    wasmtime [-odg] [--wasi-c] [--preload=<wasm>...] [--env=<env>...] [--dir=<dir>...] [--mapdir=<mapping>...] --invoke=<fn> <file> [<arg>...]
+    wasmtime [-ocdg] [--wasi-c] [--preload=<wasm>...] [--env=<env>...] [--dir=<dir>...] [--mapdir=<mapping>...] <file> [<arg>...]
+    wasmtime [-ocdg] [--wasi-c] [--preload=<wasm>...] [--env=<env>...] [--dir=<dir>...] [--mapdir=<mapping>...] --invoke=<fn> <file> [<arg>...]
     wasmtime --help | --version
 
 Options:
     --invoke=<fn>       name of function to run
     -o, --optimize      runs optimization passes on the translated functions
+    -c, --cache         enable caching system
     -g                  generate debug information
     -d, --debug         enable debug output on stderr/stdout
     --wasi-c            enable the wasi-c implementation of WASI
@@ -91,6 +91,7 @@ struct Args {
     arg_file: String,
     arg_arg: Vec<String>,
     flag_optimize: bool,
+    flag_cache: bool,
     flag_debug: bool,
     flag_g: bool,
     flag_invoke: Option<String>,
@@ -207,6 +208,8 @@ fn main() {
         utils::init_file_per_thread_logger();
     }
 
+    cache_conf::init(args.flag_cache);
+
     let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
         panic!("host machine is not a supported target");
     });

src/wast.rs

@@ -25,16 +25,15 @@
     )
 )]
 
-#[macro_use]
-extern crate serde_derive;
-
 use cranelift_codegen::settings;
 use cranelift_codegen::settings::Configurable;
 use cranelift_native;
 use docopt::Docopt;
 use pretty_env_logger;
+use serde::Deserialize;
 use std::path::Path;
 use std::process;
+use wasmtime_environ::cache_conf;
 use wasmtime_jit::Compiler;
 use wasmtime_wast::WastContext;
 
@@ -46,13 +45,14 @@ const USAGE: &str = "
 Wast test runner.
 
 Usage:
-    run_wast [-do] <file>...
+    run_wast [-cdo] <file>...
     run_wast --help | --version
 
 Options:
     -h, --help          print this help message
     --version           print the Cranelift version
     -o, --optimize      runs optimization passes on the translated functions
+    -c, --cache         enable caching system
     -d, --debug         enable debug output on stderr/stdout
 ";
 
@@ -62,6 +62,7 @@ struct Args {
     flag_debug: bool,
     flag_function: Option<String>,
     flag_optimize: bool,
+    flag_cache: bool,
 }
 
 fn main() {
@@ -80,6 +81,8 @@ fn main() {
         utils::init_file_per_thread_logger();
     }
 
+    cache_conf::init(args.flag_cache);
+
     let isa_builder = cranelift_native::builder().unwrap_or_else(|_| {
         panic!("host machine is not a supported target");
     });

wasmtime-debug/Cargo.toml

@@ -14,9 +14,9 @@ edition = "2018"
 [dependencies]
 gimli = "0.19.0"
 wasmparser = { version = "0.34.0" }
-cranelift-codegen = "0.33.0"
-cranelift-entity = "0.33.0"
-cranelift-wasm = "0.33.0"
+cranelift-codegen = { version = "0.36.0", features = ["enable-serde"] }
+cranelift-entity = { version = "0.36.0", features = ["enable-serde"] }
+cranelift-wasm = { version = "0.36.0", features = ["enable-serde"] }
 faerie = "0.10.1"
 wasmtime-environ = { path = "../wasmtime-environ", default-features = false }
 target-lexicon = { version = "0.4.0", default-features = false }

wasmtime-environ/Cargo.toml

@@ -12,14 +12,22 @@ readme = "README.md"
 edition = "2018"
 
 [dependencies]
-cranelift-codegen = "0.33.0"
-cranelift-entity = "0.33.0"
-cranelift-wasm = "0.33.0"
+cranelift-codegen = { version = "0.36.0", features = ["enable-serde"] }
+cranelift-entity = { version = "0.36.0", features = ["enable-serde"] }
+cranelift-wasm = { version = "0.36.0", features = ["enable-serde"] }
 lightbeam = { path = "../lightbeam", optional = true }
 failure = { version = "0.1.3", default-features = false }
 failure_derive = { version = "0.1.3", default-features = false }
 indexmap = "1.0.2"
 rayon = "1.1"
+directories = "2.0.1"
+sha2 = "0.8.0"
+base64 = "0.10.1"
+serde = { version = "1.0.94", features = ["derive"] }
+bincode = "1.1.4"
+lazy_static = "1.3.0"
+spin = "0.5.0"
+log = { version = "0.4.6", default-features = false }
 
 [features]
 default = ["std"]

wasmtime-environ/src/address_map.rs

@@ -4,10 +4,11 @@
 use cranelift_codegen::ir;
 use cranelift_entity::PrimaryMap;
 use cranelift_wasm::DefinedFuncIndex;
+use serde::{Deserialize, Serialize};
 use std::vec::Vec;
 
 /// Single source location to generated address mapping.
-#[derive(Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct InstructionAddressMap {
     /// Original source location.
     pub srcloc: ir::SourceLoc,
@@ -20,7 +21,7 @@ pub struct InstructionAddressMap {
 }
 
 /// Function and its instructions addresses mappings.
-#[derive(Debug, Clone)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct FunctionAddressMap {
     /// Instructions maps.
     /// The array is sorted by the InstructionAddressMap::code_offset field.