diff --git a/src/cmdline.rs b/src/cmdline.rs index 08529e43ad..e5973cc575 100644 --- a/src/cmdline.rs +++ b/src/cmdline.rs @@ -43,6 +43,8 @@ pub enum Command { cmdline: Vec, /// The directory in which to execute the command. cwd: PathBuf, + /// The environment variables to use for execution. + env_vars: Vec<(OsString, OsString)>, }, } @@ -146,6 +148,7 @@ pub fn parse() -> Result { exe: exe.to_owned(), cmdline: cmdline, cwd: cwd, + env_vars: env::vars_os().collect(), }) } else { bail!("No compile command"); diff --git a/src/commands.rs b/src/commands.rs index 8298a344b0..88fa924b14 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -430,13 +430,19 @@ fn print_stats(stats: CacheStats) -> Result<()> { } /// Send a `Compile` request to the server, and return the server response if successful. -fn request_compile, X: AsRef, Y: AsRef>(conn: &mut ServerConnection, exe: W, args: &Vec, cwd: Y) -> Result { +fn request_compile(conn: &mut ServerConnection, exe: W, args: &Vec, cwd: Y, + env_vars: Vec<(OsString, OsString)>) -> Result + where W: AsRef, + X: AsRef, + Y: AsRef, +{ let req = Request::Compile(Compile { exe: exe.as_ref().to_str().ok_or("bad exe")?.to_owned(), cwd: cwd.as_ref().to_str().ok_or("bad cwd")?.to_owned(), args: args.iter().map(|a| { - a.as_ref().to_str().ok_or("bad cwd".into()).map(|s| s.to_owned()) + a.as_ref().to_str().ok_or("bad arg".into()).map(|s| s.to_owned()) }).collect::>()?, + env_vars: env_vars, }); trace!("request_compile: {:?}", req); //TODO: better error mapping? @@ -567,13 +573,14 @@ pub fn do_compile(creator: T, cmdline: Vec, cwd: &Path, path: Option, + env_vars: Vec<(OsString, OsString)>, stdout: &mut Write, stderr: &mut Write) -> Result - where T : CommandCreatorSync, + where T: CommandCreatorSync, { trace!("do_compile"); let exe_path = which_in(exe, path, &cwd)?; - let res = request_compile(&mut conn, &exe_path, &cmdline, &cwd)?; + let res = request_compile(&mut conn, &exe_path, &cmdline, &cwd, env_vars)?; handle_compile_response(creator, core, &mut conn, res, &exe_path, cmdline, cwd, stdout, stderr) } @@ -620,7 +627,7 @@ pub fn run_command(cmd: Command) -> Result { let stats = request_shutdown(server)?; print_stats(stats)? } - Command::Compile { exe, cmdline, cwd } => { + Command::Compile { exe, cmdline, cwd, env_vars } => { trace!("Command::Compile {{ {:?}, {:?}, {:?} }}", exe, cmdline, cwd); let conn = connect_or_start_server(get_port())?; let mut core = Core::new()?; @@ -631,6 +638,7 @@ pub fn run_command(cmd: Command) -> Result { cmdline, &cwd, env::var_os("PATH"), + env_vars, &mut io::stdout(), &mut io::stderr()); return res.chain_err(|| { diff --git a/src/compiler/c.rs b/src/compiler/c.rs index 8328203163..42b0d78737 100644 --- a/src/compiler/c.rs +++ b/src/compiler/c.rs @@ -18,12 +18,12 @@ use futures_cpupool::CpuPool; use mock_command::CommandCreatorSync; use sha1; use std::borrow::Cow; -use std::collections::HashMap; -use std::env; +use std::collections::{HashMap, HashSet}; +use std::ffi::{OsStr, OsString}; use std::fmt; use std::path::Path; use std::process; -use util::sha1_digest; +use util::{os_str_bytes, sha1_digest}; use errors::*; @@ -106,6 +106,7 @@ pub trait CCompilerImpl: Clone + fmt::Debug + Send + 'static { executable: &str, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture where T: CommandCreatorSync; /// Run the C compiler with the specified set of arguments, using the @@ -116,6 +117,7 @@ pub trait CCompilerImpl: Clone + fmt::Debug + Send + 'static { preprocessor_output: Vec, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> where T: CommandCreatorSync; @@ -167,13 +169,15 @@ impl CompilerHasher for CCompilerHasher fn generate_hash_key(self: Box, creator: &T, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture> { let me = *self; let CCompilerHasher { parsed_args, executable, executable_digest, compiler } = me; - let result = compiler.preprocess(creator, &executable, &parsed_args, cwd, pool); + let result = compiler.preprocess(creator, &executable, &parsed_args, cwd, env_vars, pool); let out_file = parsed_args.output_file().into_owned(); + let env_vars = env_vars.to_vec(); let result = result.map_err(move |e| { debug!("[{}]: preprocessor failed: {:?}", out_file, e); e @@ -202,7 +206,7 @@ impl CompilerHasher for CCompilerHasher .filter(|a| **a != out_file) .map(|a| a.as_str()) .collect::(); - hash_key(&executable_digest, &arguments, &preprocessor_result.stdout) + hash_key(&executable_digest, &arguments, &env_vars, &preprocessor_result.stdout) }; HashResult::Ok { key: key, @@ -231,12 +235,14 @@ impl Compilation for CCompilation fn compile(self: Box, creator: &T, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> { let me = *self; let CCompilation { parsed_args, executable, preprocessor_output, compiler } = me; - compiler.compile(creator, &executable, preprocessor_output, &parsed_args, cwd, pool) + compiler.compile(creator, &executable, preprocessor_output, &parsed_args, cwd, env_vars, + pool) } fn outputs<'a>(&'a self) -> Box + 'a> @@ -255,19 +261,21 @@ pub const CACHED_ENV_VARS : &'static [&'static str] = &[ ]; /// Compute the hash key of `compiler` compiling `preprocessor_output` with `args`. -pub fn hash_key(compiler_digest: &str, arguments: &str, preprocessor_output: &[u8]) -> String { +pub fn hash_key(compiler_digest: &str, arguments: &str, env_vars: &[(OsString, OsString)], + preprocessor_output: &[u8]) -> String +{ // If you change any of the inputs to the hash, you should change `CACHE_VERSION`. let mut m = sha1::Sha1::new(); m.update(compiler_digest.as_bytes()); m.update(CACHE_VERSION); m.update(arguments.as_bytes()); - //TODO: should propogate these over from the client. - // https://github.com/glandium/sccache/issues/5 - for var in CACHED_ENV_VARS.iter() { - if let Ok(val) = env::var(var) { - m.update(var.as_bytes()); + //TODO: use lazy_static. + let cached_env_vars: HashSet = CACHED_ENV_VARS.iter().map(|v| OsStr::new(v).to_os_string()).collect(); + for &(ref var, ref val) in env_vars.iter() { + if cached_env_vars.contains(var) { + m.update(os_str_bytes(var)); m.update(&b"="[..]); - m.update(val.as_bytes()); + m.update(os_str_bytes(val)); } } m.update(preprocessor_output); @@ -277,35 +285,34 @@ pub fn hash_key(compiler_digest: &str, arguments: &str, preprocessor_output: &[u #[cfg(test)] mod test { use super::*; - use std::env; #[test] fn test_hash_key_executable_contents_differs() { let args = "a b c"; const PREPROCESSED : &'static [u8] = b"hello world"; - assert_neq!(hash_key("abcd", &args, &PREPROCESSED), - hash_key("wxyz", &args, &PREPROCESSED)); + assert_neq!(hash_key("abcd", &args, &[], &PREPROCESSED), + hash_key("wxyz", &args, &[], &PREPROCESSED)); } #[test] fn test_hash_key_args_differs() { let digest = "abcd"; const PREPROCESSED: &'static [u8] = b"hello world"; - assert_neq!(hash_key(digest, "a b c", &PREPROCESSED), - hash_key(digest, "x y z", &PREPROCESSED)); + assert_neq!(hash_key(digest, "a b c", &[], &PREPROCESSED), + hash_key(digest, "x y z", &[], &PREPROCESSED)); - assert_neq!(hash_key(digest, "a b c", &PREPROCESSED), - hash_key(digest, "a b", &PREPROCESSED)); + assert_neq!(hash_key(digest, "a b c", &[], &PREPROCESSED), + hash_key(digest, "a b", &[], &PREPROCESSED)); - assert_neq!(hash_key(digest, "a b c", &PREPROCESSED), - hash_key(digest, "a", &PREPROCESSED)); + assert_neq!(hash_key(digest, "a b c", &[], &PREPROCESSED), + hash_key(digest, "a", &[], &PREPROCESSED)); } #[test] fn test_hash_key_preprocessed_content_differs() { let args = "a b c"; - assert_neq!(hash_key("abcd", &args, &b"hello world"[..]), - hash_key("abcd", &args, &b"goodbye"[..])); + assert_neq!(hash_key("abcd", &args, &[], &b"hello world"[..]), + hash_key("abcd", &args, &[], &b"goodbye"[..])); } #[test] @@ -314,17 +321,11 @@ mod test { let digest = "abcd"; const PREPROCESSED: &'static [u8] = b"hello world"; for var in CACHED_ENV_VARS.iter() { - let old = env::var_os(var); - env::remove_var(var); - let h1 = hash_key(digest, &args, &PREPROCESSED); - env::set_var(var, "something"); - let h2 = hash_key(digest, &args, &PREPROCESSED); - env::set_var(var, "something else"); - let h3 = hash_key(digest, &args, &PREPROCESSED); - match old { - Some(val) => env::set_var(var, val), - None => env::remove_var(var), - } + let h1 = hash_key(digest, &args, &[], &PREPROCESSED); + let vars = vec![(OsString::from(var), OsString::from("something"))]; + let h2 = hash_key(digest, &args, &vars, &PREPROCESSED); + let vars = vec![(OsString::from(var), OsString::from("something else"))]; + let h3 = hash_key(digest, &args, &vars, &PREPROCESSED); assert_neq!(h1, h2); assert_neq!(h2, h3); } diff --git a/src/compiler/clang.rs b/src/compiler/clang.rs index b1b7878dcc..2d1163b7c1 100644 --- a/src/compiler/clang.rs +++ b/src/compiler/clang.rs @@ -29,6 +29,7 @@ use mock_command::{ CommandCreatorSync, RunCommand, }; +use std::ffi::OsString; use std::fs::File; use std::io::{ self, @@ -57,10 +58,11 @@ impl CCompilerImpl for Clang { executable: &str, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture where T: CommandCreatorSync { - gcc::preprocess(creator, executable, parsed_args, cwd, pool) + gcc::preprocess(creator, executable, parsed_args, cwd, env_vars, pool) } fn compile(&self, @@ -69,11 +71,12 @@ impl CCompilerImpl for Clang { preprocessor_output: Vec, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> where T: CommandCreatorSync { - compile(creator, executable, preprocessor_output, parsed_args, cwd, pool) + compile(creator, executable, preprocessor_output, parsed_args, cwd, env_vars, pool) } } @@ -90,6 +93,7 @@ fn compile(creator: &T, preprocessor_output: Vec, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> where T: CommandCreatorSync, @@ -117,6 +121,8 @@ fn compile(creator: &T, .arg("-o") .arg(&out_file) .args(&parsed_args.common_args) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(&cwd); let output = write.and_then(move |(tempdir, input)| { attempt.arg(&input); @@ -141,6 +147,8 @@ fn compile(creator: &T, .arg("-o") .arg(&out_file) .args(&parsed_args.common_args) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(&cwd); Box::new(output.and_then(move |output| -> SFuture<_> { if !output.status.success() { @@ -223,6 +231,7 @@ mod test { vec!(), &parsed_args, f.tempdir.path().to_str().unwrap(), + &[], &pool).wait().unwrap(); assert_eq!(Cacheable::Yes, cacheable); // Ensure that we ran all processes. @@ -252,6 +261,7 @@ mod test { vec!(), &parsed_args, f.tempdir.path().to_str().unwrap(), + &[], &pool).wait().unwrap(); assert_eq!(Cacheable::Yes, cacheable); assert_eq!(exit_status(0), output.status); diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index d85e754dd1..2338561301 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -90,6 +90,7 @@ pub trait CompilerHasher: fmt::Debug + Send + 'static fn generate_hash_key(self: Box, creator: &T, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture>; /// Look up a cached compile result in `storage`. If not found, run the @@ -99,6 +100,7 @@ pub trait CompilerHasher: fmt::Debug + Send + 'static storage: Arc, arguments: Vec, cwd: String, + env_vars: Vec<(OsString, OsString)>, cache_control: CacheControl, pool: CpuPool, handle: Handle) @@ -110,7 +112,7 @@ pub trait CompilerHasher: fmt::Debug + Send + 'static debug!("[{}]: get_cached_or_compile: {}", out_file, cmd_str); } let start = Instant::now(); - let result = self.generate_hash_key(&creator, &cwd, &pool); + let result = self.generate_hash_key(&creator, &cwd, &env_vars, &pool); Box::new(result.and_then(move |hash_res| -> SFuture<_> { debug!("[{}]: generate_hash_key took {}", out_file, fmt_duration_as_secs(&start.elapsed())); let (key, compilation) = match hash_res { @@ -194,7 +196,7 @@ pub trait CompilerHasher: fmt::Debug + Send + 'static // Cache miss, so compile it. let start = Instant::now(); let out_file = out_file.clone(); - let compile = compilation.compile(&creator, &cwd, &pool); + let compile = compilation.compile(&creator, &cwd, &env_vars, &pool); Box::new(compile.and_then(move |(cacheable, compiler_result)| { let duration = start.elapsed(); if !compiler_result.status.success() { @@ -276,6 +278,7 @@ pub trait Compilation fn compile(self: Box, creator: &T, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture<(Cacheable, process::Output)>; fn outputs<'a>(&'a self) -> Box + 'a>; @@ -777,6 +780,7 @@ mod test { storage.clone(), arguments.clone(), cwd.clone(), + vec![], CacheControl::Default, pool.clone(), handle.clone()).wait().unwrap(); @@ -801,6 +805,7 @@ mod test { storage.clone(), arguments, cwd, + vec![], CacheControl::Default, pool.clone(), handle).wait().unwrap(); @@ -856,6 +861,7 @@ mod test { storage.clone(), arguments.clone(), cwd.clone(), + vec![], CacheControl::Default, pool.clone(), handle.clone()).wait().unwrap(); @@ -881,6 +887,7 @@ mod test { storage, arguments, cwd, + vec![], CacheControl::Default, pool, handle).wait().unwrap(); @@ -940,6 +947,7 @@ mod test { storage.clone(), arguments.clone(), cwd.clone(), + vec![], CacheControl::Default, pool.clone(), handle.clone()).wait().unwrap(); @@ -961,6 +969,7 @@ mod test { storage, arguments, cwd, + vec![], CacheControl::ForceRecache, pool, handle).wait().unwrap(); @@ -1009,6 +1018,7 @@ mod test { storage, arguments, cwd, + vec![], CacheControl::Default, pool, handle).wait().unwrap(); diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs index ec08f154b8..656bab4a32 100644 --- a/src/compiler/gcc.rs +++ b/src/compiler/gcc.rs @@ -27,6 +27,7 @@ use mock_command::{ }; use std::collections::HashMap; use std::io::Read; +use std::ffi::OsString; use std::fs::File; use std::path::Path; use std::process; @@ -51,10 +52,11 @@ impl CCompilerImpl for GCC { executable: &str, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture where T: CommandCreatorSync { - preprocess(creator, executable, parsed_args, cwd, pool) + preprocess(creator, executable, parsed_args, cwd, env_vars, pool) } fn compile(&self, @@ -63,11 +65,12 @@ impl CCompilerImpl for GCC { preprocessor_output: Vec, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> where T: CommandCreatorSync { - compile(creator, executable, preprocessor_output, parsed_args, cwd, pool) + compile(creator, executable, preprocessor_output, parsed_args, cwd, env_vars, pool) } } @@ -232,6 +235,7 @@ pub fn preprocess(creator: &T, executable: &str, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], _pool: &CpuPool) -> SFuture where T: CommandCreatorSync @@ -242,6 +246,8 @@ pub fn preprocess(creator: &T, .arg(&parsed_args.input) .args(&parsed_args.preprocessor_args) .args(&parsed_args.common_args) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(cwd); if log_enabled!(Trace) { trace!("preprocess: {:?}", cmd); @@ -254,6 +260,7 @@ fn compile(creator: &T, preprocessor_output: Vec, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], _pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> where T: CommandCreatorSync @@ -279,6 +286,8 @@ fn compile(creator: &T, }) .args(&["-", "-o", &output.clone()]) .args(&parsed_args.common_args) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(cwd); Box::new(run_input_output(cmd, Some(preprocessor_output)).map(|output| { (Cacheable::Yes, output) diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs index e9382567c5..8ad042731e 100644 --- a/src/compiler/msvc.rs +++ b/src/compiler/msvc.rs @@ -28,7 +28,7 @@ use mock_command::{ RunCommand, }; use std::collections::{HashMap,HashSet}; -use std::ffi::OsStr; +use std::ffi::{OsStr, OsString}; use std::fs::File; use std::io::{ self, @@ -62,10 +62,11 @@ impl CCompilerImpl for MSVC { executable: &str, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture where T: CommandCreatorSync { - preprocess(creator, executable, parsed_args, cwd, &self.includes_prefix, pool) + preprocess(creator, executable, parsed_args, cwd, env_vars, &self.includes_prefix, pool) } fn compile(&self, @@ -74,11 +75,12 @@ impl CCompilerImpl for MSVC { preprocessor_output: Vec, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> where T: CommandCreatorSync { - compile(creator, executable, preprocessor_output, parsed_args, cwd, pool) + compile(creator, executable, preprocessor_output, parsed_args, cwd, env_vars, pool) } } @@ -294,6 +296,7 @@ pub fn preprocess(creator: &T, executable: &str, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], includes_prefix: &str, _pool: &CpuPool) -> SFuture @@ -304,6 +307,8 @@ pub fn preprocess(creator: &T, .arg(&parsed_args.input) .arg("-nologo") .args(&parsed_args.common_args) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(&cwd); if parsed_args.depfile.is_some() { cmd.arg("-showIncludes"); @@ -361,6 +366,7 @@ fn compile(creator: &T, preprocessor_output: Vec, parsed_args: &ParsedArguments, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> where T: CommandCreatorSync @@ -399,6 +405,8 @@ fn compile(creator: &T, cmd.arg("-c") .arg(&format!("-Fo{}", out_file)) .args(&parsed_args.common_args) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(&cwd); let output = write.and_then(move |(tempdir, input)| { cmd.arg(input); @@ -419,6 +427,8 @@ fn compile(creator: &T, .arg(&parsed_args.input) .arg(&format!("-Fo{}", out_file)) .args(&parsed_args.common_args) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(cwd); Box::new(output.and_then(move |output| -> SFuture<_> { if output.status.success() { @@ -593,6 +603,7 @@ mod test { vec!(), &parsed_args, f.tempdir.path().to_str().unwrap(), + &[], &pool).wait().unwrap(); assert_eq!(Cacheable::Yes, cacheable); // Ensure that we ran all processes. @@ -623,6 +634,7 @@ mod test { vec!(), &parsed_args, f.tempdir.path().to_str().unwrap(), + &[], &pool).wait().unwrap(); assert_eq!(Cacheable::No, cacheable); // Ensure that we ran all processes. @@ -652,6 +664,7 @@ mod test { vec!(), &parsed_args, f.tempdir.path().to_str().unwrap(), + &[], &pool).wait().unwrap(); assert_eq!(Cacheable::Yes, cacheable); // Ensure that we ran all processes. diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 2a8bbbf827..6c1810d412 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -22,6 +22,7 @@ use sha1; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::env::consts::DLL_EXTENSION; +use std::ffi::OsString; use std::fs::{self, File}; use std::io::Read; use std::iter::FromIterator; @@ -30,7 +31,7 @@ use std::process::{self, Stdio}; use std::slice; use std::time::Instant; use tempdir::TempDir; -use util::{fmt_duration_as_secs, sha1_digest}; +use util::{fmt_duration_as_secs, os_str_bytes, sha1_digest}; use errors::*; @@ -144,7 +145,9 @@ fn hash_all(files: Vec, pool: &CpuPool) -> SFuture> } /// Calculate SHA-1 digests for all source files listed in rustc's dep-info output. -fn hash_source_files(creator: &T, crate_name: &str, executable: &str, arguments: &[String], cwd: &str, pool: &CpuPool) -> SFuture> +fn hash_source_files(creator: &T, crate_name: &str, executable: &str, arguments: &[String], + cwd: &str, env_vars: &[(OsString, OsString)], pool: &CpuPool) + -> SFuture> where T: CommandCreatorSync, { let start = Instant::now(); @@ -159,6 +162,8 @@ fn hash_source_files(creator: &T, crate_name: &str, executable: &str, argumen .args(&["--emit", "dep-info"]) .arg("-o") .arg(&dep_file) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(cwd); trace!("[{}]: get dep-info: {:?}", crate_name, cmd); let dep_info = run_input_output(cmd, None); @@ -215,12 +220,15 @@ fn parse_dep_info(dep_info: &str, cwd: T) -> Vec } /// Run `rustc --print file-names` to get the outputs of compilation. -fn get_compiler_outputs(creator: &T, executable: &str, arguments: &[String], cwd: &str) -> SFuture> +fn get_compiler_outputs(creator: &T, executable: &str, arguments: &[String], cwd: &str, + env_vars: &[(OsString, OsString)]) -> SFuture> where T: CommandCreatorSync, { let mut cmd = creator.clone().new_command_sync(executable); cmd.args(&arguments) .args(&["--print", "file-names"]) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(cwd); if log_enabled!(Trace) { trace!("get_compiler_outputs: {:?}", cmd); @@ -482,6 +490,7 @@ impl CompilerHasher for RustHasher fn generate_hash_key(self: Box, creator: &T, cwd: &str, + env_vars: &[(OsString, OsString)], pool: &CpuPool) -> SFuture> { @@ -500,7 +509,7 @@ impl CompilerHasher for RustHasher .flat_map(|(arg, val)| Some(arg).into_iter().chain(val)) .map(|a| a.clone()) .collect::>(); - let source_hashes = hash_source_files(creator, &crate_name, &executable, &filtered_arguments, cwd, pool); + let source_hashes = hash_source_files(creator, &crate_name, &executable, &filtered_arguments, cwd, env_vars, pool); // Hash the contents of the externs listed on the commandline. let cwp = Path::new(cwd); trace!("[{}]: hashing {} externs", crate_name, externs.len()); @@ -510,6 +519,7 @@ impl CompilerHasher for RustHasher &pool); let creator = creator.clone(); let cwd = cwd.to_owned(); + let env_vars = env_vars.to_vec(); let hashes = source_hashes.join(extern_hashes); Box::new(hashes.and_then(move |(source_hashes, extern_hashes)| -> SFuture<_> { // If you change any of the inputs to the hash, you should change `CACHE_VERSION`. @@ -540,15 +550,28 @@ impl CompilerHasher for RustHasher for h in source_hashes.into_iter().chain(extern_hashes) { m.update(h.as_bytes()); } - // 6. TODO: Environment variables: CARGO_* at the very least. - // https://github.com/mozilla/sccache/issues/89 + // 6. Environment variables. Ideally we'd use anything referenced + // via env! in the program, but we don't have a way to determine that + // currently, and hashing all environment variables is too much, so + // we'll just hash the CARGO_ env vars and hope that's sufficient. + // Upstream Rust issue tracking getting information about env! usage: + // https://github.com/rust-lang/rust/issues/40364 + let mut env_vars = env_vars.clone(); + env_vars.sort(); + for &(ref var, ref val) in env_vars.iter() { + if var.to_str().map(|s| s.starts_with("CARGO_")).unwrap_or(false) { + m.update(os_str_bytes(var)); + m.update(b"="); + m.update(os_str_bytes(val)); + } + } // 7. TODO: native libraries being linked. // https://github.com/mozilla/sccache/issues/88 // Turn arguments into a simple Vec for compilation. let arguments = arguments.into_iter() .flat_map(|(arg, val)| Some(arg).into_iter().chain(val)) .collect::>(); - Box::new(get_compiler_outputs(&creator, &executable, &arguments, &cwd).map(move |outputs| { + Box::new(get_compiler_outputs(&creator, &executable, &arguments, &cwd, &env_vars).map(move |outputs| { let output_dir = PathBuf::from(output_dir); // Convert output files into a map of basename -> full path. let mut outputs = outputs.into_iter() @@ -589,6 +612,7 @@ impl Compilation for RustCompilation fn compile(self: Box, creator: &T, cwd: &str, + env_vars: &[(OsString, OsString)], _pool: &CpuPool) -> SFuture<(Cacheable, process::Output)> { @@ -597,6 +621,8 @@ impl Compilation for RustCompilation trace!("[{}]: compile", crate_name); let mut cmd = creator.clone().new_command_sync(&executable); cmd.args(&arguments) + .env_clear() + .envs(env_vars.iter().map(|&(ref k, ref v)| (k, v))) .current_dir(cwd); trace!("compile: {:?}", cmd); Box::new(run_input_output(cmd, None).map(|output| { @@ -723,7 +749,7 @@ mod test { fn test_get_compiler_outputs() { let creator = new_creator(); next_command(&creator, Ok(MockChild::new(exit_status(0), "foo\nbar\nbaz", ""))); - let outputs = get_compiler_outputs(&creator, "rustc", &stringvec!("a", "b"), "cwd").wait().unwrap(); + let outputs = get_compiler_outputs(&creator, "rustc", &stringvec!("a", "b"), "cwd", &[]).wait().unwrap(); assert_eq!(outputs, &["foo", "bar", "baz"]); } @@ -731,7 +757,7 @@ mod test { fn test_get_compiler_outputs_fail() { let creator = new_creator(); next_command(&creator, Ok(MockChild::new(exit_status(1), "", "error"))); - assert!(get_compiler_outputs(&creator, "rustc", &stringvec!("a", "b"), "cwd").wait().is_err()); + assert!(get_compiler_outputs(&creator, "rustc", &stringvec!("a", "b"), "cwd", &[]).wait().is_err()); } #[test] @@ -834,6 +860,9 @@ bar.rs: let pool = CpuPool::new(1); let res = hasher.generate_hash_key(&creator, &f.tempdir.path().to_string_lossy(), + &[(OsString::from("CARGO_PKG_NAME"), OsString::from("foo")), + (OsString::from("FOO"), OsString::from("bar")), + (OsString::from("CARGO_BLAH"), OsString::from("abc"))], &pool).wait().unwrap(); let mut m = sha1::Sha1::new(); // Version. @@ -848,6 +877,9 @@ bar.rs: m.update(EMPTY_DIGEST.as_bytes()); // bar.rlib (extern crate, from externs) m.update(EMPTY_DIGEST.as_bytes()); + // Env vars + m.update(b"CARGO_BLAH=abc"); + m.update(b"CARGO_PKG_NAME=foo"); let digest = m.digest().to_string(); match res { HashResult::Ok { key, compilation } => { diff --git a/src/mock_command.rs b/src/mock_command.rs index 364bd5910e..a131594c50 100644 --- a/src/mock_command.rs +++ b/src/mock_command.rs @@ -100,6 +100,15 @@ pub trait RunCommand: fmt::Debug { fn arg>(&mut self, arg: S) -> &mut Self; /// Append `args` to the process commandline. fn args>(&mut self, args: &[S]) -> &mut Self; + /// Insert or update an environment variable mapping. + fn env(&mut self, key: K, val: V) -> &mut Self + where K: AsRef, + V: AsRef; + /// Add or update multiple environment variable mappings. + fn envs(&mut self, vars: I) -> &mut Self + where I: IntoIterator, K: AsRef, V: AsRef; + /// Clears the entire environment map for the child process. + fn env_clear(&mut self) -> &mut Self; /// Set the working directory of the process to `dir`. fn current_dir>(&mut self, dir: P) -> &mut Self; /// Create the proces without a visible console on Windows. @@ -184,6 +193,27 @@ impl RunCommand for AsyncCommand { self.inner.args(args); self } + fn env(&mut self, key: K, val: V) -> &mut AsyncCommand + where K: AsRef, + V: AsRef, + { + self.inner.env(key, val); + self + } + fn envs(&mut self, vars: I) -> &mut Self + where I: IntoIterator, K: AsRef, V: AsRef + { + //TODO: when Command::envs stabilizes, use that: + // https://github.com/rust-lang/rust/issues/38526 + for (k, v) in vars { + self.inner.env(k, v); + } + self + } + fn env_clear(&mut self) -> &mut AsyncCommand { + self.inner.env_clear(); + self + } fn current_dir>(&mut self, dir: P) -> &mut AsyncCommand { self.inner.current_dir(dir); self @@ -377,6 +407,20 @@ impl RunCommand for MockCommand { self.args.extend(args.iter().map(|a| a.as_ref().to_owned())); self } + fn env(&mut self, _key: K, _val: V) -> &mut MockCommand + where K: AsRef, + V: AsRef, + { + self + } + fn envs(&mut self, _vars: I) -> &mut Self + where I: IntoIterator, K: AsRef, V: AsRef + { + self + } + fn env_clear(&mut self) -> &mut MockCommand { + self + } fn current_dir>(&mut self, _dir: P) -> &mut MockCommand { //TODO: assert value of dir self diff --git a/src/protocol.rs b/src/protocol.rs index a128665164..fd6910b7f0 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -1,16 +1,28 @@ +use std::ffi::OsString; + +/// A client request. #[derive(Serialize, Deserialize, Debug)] pub enum Request { + /// Zero the server's statistics. ZeroStats, + /// Get server statistics. GetStats, + /// Shut the server down gracefully. Shutdown, + /// Execute a compile or fetch a cached compilation result. Compile(Compile), } +/// A server response. #[derive(Serialize, Deserialize, Debug)] pub enum Response { + /// Response for `Request::Compile`. Compile(CompileResponse), + /// Response for `Request::GetStats`, containing server statistics. Stats(CacheStats), + /// Response for `Request::Shutdown`, containing server statistics. ShuttingDown(CacheStats), + /// Second response for `Request::Compile`, containing the results of the compilation. CompileFinished(CompileFinished), } @@ -23,35 +35,55 @@ pub enum CompileResponse { UnhandledCompile, } +/// Server statistics. #[derive(Serialize, Deserialize, Debug)] pub struct CacheStats { + /// A `Vec` of individual statistics. pub stats: Vec, } +/// A single server statistic. #[derive(Serialize, Deserialize, Debug)] pub struct CacheStatistic { + /// Stat name. pub name: String, + /// Stat value. pub value: CacheStat, } +/// A statistic value. #[derive(Serialize, Deserialize, Debug, PartialEq)] pub enum CacheStat { + /// A count of occurrences. Count(u64), + /// An opaque string, such as a name. String(String), + /// A size in bytes. Size(u64), } +/// Information about a finished compile, either from cache or executed locally. #[derive(Serialize, Deserialize, Debug, Default)] pub struct CompileFinished { + /// The return code of the compile process, if available. pub retcode: Option, + /// The signal that terminated the compile process, if available. pub signal: Option, + /// The compiler's stdout. pub stdout: Vec, + /// The compiler's stderr. pub stderr: Vec, } +/// The contents of a compile request from a client. #[derive(Serialize, Deserialize, Debug)] pub struct Compile { + /// The full path to the compiler executable. pub exe: String, + /// The current working directory in which to execute the compile. pub cwd: String, + /// The commandline arguments passed to the compiler. pub args: Vec, + /// The environment variables present when the compiler was executed, as (var, val). + pub env_vars: Vec<(OsString, OsString)>, } diff --git a/src/server.rs b/src/server.rs index f430478c68..e85b074e58 100644 --- a/src/server.rs +++ b/src/server.rs @@ -431,9 +431,10 @@ impl SccacheService let exe = compile.exe; let cmd = compile.args; let cwd = compile.cwd; + let env_vars = compile.env_vars; let me = self.clone(); Box::new(self.compiler_info(exe).map(move |info| { - me.check_compiler(info, cmd, cwd) + me.check_compiler(info, cmd, cwd, env_vars) })) } @@ -480,8 +481,9 @@ impl SccacheService fn check_compiler(&self, compiler: Option>>, cmd: Vec, - cwd: String) - -> SccacheResponse { + cwd: String, + env_vars: Vec<(OsString, OsString)>) -> SccacheResponse + { let mut stats = self.stats.borrow_mut(); match compiler { None => { @@ -497,7 +499,7 @@ impl SccacheService debug!("parse_arguments: Ok"); stats.requests_executed += 1; let (tx, rx) = Body::pair(); - self.start_compile_task(hasher, cmd, cwd, tx); + self.start_compile_task(hasher, cmd, cwd, env_vars, tx); let res = CompileResponse::CompileStarted; return Message::WithBody(Response::Compile(res), rx) } @@ -524,6 +526,7 @@ impl SccacheService hasher: Box>, arguments: Vec, cwd: String, + env_vars: Vec<(OsString, OsString)>, tx: mpsc::Sender>) { let cache_control = if self.force_recache { CacheControl::ForceRecache @@ -535,6 +538,7 @@ impl SccacheService self.storage.clone(), arguments, cwd, + env_vars, cache_control, self.pool.clone(), self.handle.clone()); diff --git a/src/test/tests.rs b/src/test/tests.rs index 07eb7d1719..2715e96d84 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -166,7 +166,7 @@ fn test_server_unsupported_compiler() { let mut stderr = Cursor::new(Vec::new()); let path = Some(f.paths); let mut core = Core::new().unwrap(); - assert_eq!(0, do_compile(client_creator.clone(), &mut core, conn, exe, cmdline, cwd, path, &mut stdout, &mut stderr).unwrap()); + assert_eq!(0, do_compile(client_creator.clone(), &mut core, conn, exe, cmdline, cwd, path, vec![], &mut stdout, &mut stderr).unwrap()); // Make sure we ran the mock processes. assert_eq!(0, server_creator.lock().unwrap().children.len()); assert_eq!(0, client_creator.lock().unwrap().children.len()); @@ -222,7 +222,7 @@ fn test_server_compile() { let mut stderr = Cursor::new(Vec::new()); let path = Some(f.paths); let mut core = Core::new().unwrap(); - assert_eq!(0, do_compile(client_creator.clone(), &mut core, conn, exe, cmdline, cwd, path, &mut stdout, &mut stderr).unwrap()); + assert_eq!(0, do_compile(client_creator.clone(), &mut core, conn, exe, cmdline, cwd, path, vec![], &mut stdout, &mut stderr).unwrap()); // Make sure we ran the mock processes. assert_eq!(0, server_creator.lock().unwrap().children.len()); assert_eq!(STDOUT, stdout.into_inner().as_slice()); diff --git a/src/util.rs b/src/util.rs index 3171e31ce5..3fc40986e9 100644 --- a/src/util.rs +++ b/src/util.rs @@ -14,6 +14,7 @@ use futures_cpupool::CpuPool; use sha1; +use std::ffi::OsStr; use std::fs::File; use std::io::prelude::*; use std::io::BufReader; @@ -49,3 +50,26 @@ pub fn fmt_duration_as_secs(duration: &Duration) -> String { format!("{}.{:03}s", duration.as_secs(), duration.subsec_nanos() / 1000_000) } + + +#[cfg(unix)] +pub fn os_str_bytes(s: &OsStr) -> &[u8] +{ + use std::os::unix::ffi::OsStrExt; + s.as_bytes() +} + +#[cfg(windows)] +pub fn os_str_bytes(s: &OsStr) -> &[u8] +{ + use std::mem; + unsafe { mem::transmute(s) } +} + +#[test] +fn test_os_str_bytes() { + // Just very basic sanity checks in case anyone changes the underlying + // representation of OsStr on Windows. + assert_eq!(os_str_bytes(OsStr::new("hello")), b"hello"); + assert_eq!(os_str_bytes(OsStr::new("你好")), "你好".as_bytes()); +}