From 8fe150e613bab7da891c03e234221ec450b4a2f4 Mon Sep 17 00:00:00 2001 From: Weihang Lo Date: Sat, 25 May 2024 10:44:39 -0400 Subject: [PATCH] feat: new arg `--cargo-config` for passing `--config` to cargo This is another step toward making opt-dist work in sandboxed environments. `--cargo-config` is added to the following subcommands: * `binary_stats compile` * `bench_runtime_local` * `bench_local` * `profile_local` The current rustc-perf uses `tempfile::Tempdir` as the working directory when collecting profiles from some of these packages. This "tmp" working directory usage make it impossible for Cargo to pick up the correct vendor sources setting in `.cargo/config.toml` bundled in the rustc-src tarball. We can leverage [`cargo --config`][1] to specify additional Cargo's configuration. Training data then could build with correct vendored settings via the `--cargo-config` flag. [1]: https://doc.rust-lang.org/nightly/cargo/commands/cargo.html#option-cargo---config --- collector/src/artifact_stats.rs | 3 +++ collector/src/bin/collector.rs | 14 +++++++++----- collector/src/compile/execute/mod.rs | 4 ++++ collector/src/runtime/benchmark.rs | 4 ++++ collector/src/toolchain.rs | 17 ++++++++++++----- 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/collector/src/artifact_stats.rs b/collector/src/artifact_stats.rs index ea2d7912e..9a0b12fb9 100644 --- a/collector/src/artifact_stats.rs +++ b/collector/src/artifact_stats.rs @@ -164,6 +164,9 @@ pub fn compile_and_get_stats( let mut cmd = Command::new(&toolchain.components.cargo); cmd.arg("build").arg("--target-dir").arg(tempdir.path()); + for config in &toolchain.components.cargo_config { + cmd.arg("--config").arg(config); + } match profile { CargoProfile::Debug => {} CargoProfile::Release => { diff --git a/collector/src/bin/collector.rs b/collector/src/bin/collector.rs index f0a419031..17d0b1b47 100644 --- a/collector/src/bin/collector.rs +++ b/collector/src/bin/collector.rs @@ -313,6 +313,10 @@ struct LocalOptions { #[arg(long)] cargo: Option, + /// Arguments passed to `cargo --config `. + #[arg(long)] + cargo_config: Option>, + /// Exclude all benchmarks matching a prefix in this comma-separated list #[arg(long, value_delimiter = ',')] exclude: Vec, @@ -845,7 +849,7 @@ fn main_result() -> anyhow::Result { *ToolchainConfig::default() .rustdoc(opts.rustdoc.as_deref()) .clippy(opts.clippy.as_deref()) - .cargo(local.cargo.as_deref()) + .cargo(local.cargo.as_deref(), local.cargo_config.as_deref()) .id(local.id.as_deref()), "", target_triple, @@ -1070,7 +1074,7 @@ fn main_result() -> anyhow::Result { *ToolchainConfig::default() .rustdoc(opts.rustdoc.as_deref()) .clippy(opts.clippy.as_deref()) - .cargo(local.cargo.as_deref()) + .cargo(local.cargo.as_deref(), local.cargo_config.as_deref()) .id(local.id.as_deref()), suffix, target_triple.clone(), @@ -1228,7 +1232,7 @@ fn binary_stats_compile( &[codegen_backend], &local.rustc, *ToolchainConfig::default() - .cargo(local.cargo.as_deref()) + .cargo(local.cargo.as_deref(), local.cargo_config.as_deref()) .id(local.id.as_deref()), "", target_triple.to_string(), @@ -1240,7 +1244,7 @@ fn binary_stats_compile( &[codegen_backend2], &rustc, *ToolchainConfig::default() - .cargo(local.cargo.as_deref()) + .cargo(local.cargo.as_deref(), local.cargo_config.as_deref()) .id(local.id.as_deref()), "", target_triple.to_string(), @@ -1484,7 +1488,7 @@ fn get_local_toolchain_for_runtime_benchmarks( &[CodegenBackend::Llvm], &local.rustc, *ToolchainConfig::default() - .cargo(local.cargo.as_deref()) + .cargo(local.cargo.as_deref(), local.cargo_config.as_deref()) .id(local.id.as_deref()), "", target_triple.to_string(), diff --git a/collector/src/compile/execute/mod.rs b/collector/src/compile/execute/mod.rs index 9bb71dbf3..dc6abbc7f 100644 --- a/collector/src/compile/execute/mod.rs +++ b/collector/src/compile/execute/mod.rs @@ -180,6 +180,10 @@ impl<'a> CargoProcess<'a> { if let Some(c) = &self.toolchain.components.clippy { cmd.env("CLIPPY", &*FAKE_CLIPPY).env("CLIPPY_REAL", c); } + + for config in &self.toolchain.components.cargo_config { + cmd.arg("--config").arg(config); + } cmd } diff --git a/collector/src/runtime/benchmark.rs b/collector/src/runtime/benchmark.rs index 81ce0eea5..f92505e97 100644 --- a/collector/src/runtime/benchmark.rs +++ b/collector/src/runtime/benchmark.rs @@ -337,6 +337,10 @@ fn start_cargo_build( #[cfg(feature = "precise-cachegrind")] command.arg("--features").arg("benchlib/precise-cachegrind"); + for config in &toolchain.components.cargo_config { + command.arg("--config").arg(config); + } + CargoArtifactIter::from_cargo_cmd(command) .map_err(|error| anyhow::anyhow!("Failed to start cargo: {:?}", error)) } diff --git a/collector/src/toolchain.rs b/collector/src/toolchain.rs index 249693ebb..6375ca60d 100644 --- a/collector/src/toolchain.rs +++ b/collector/src/toolchain.rs @@ -253,6 +253,7 @@ pub struct ToolchainComponents { pub rustdoc: Option, pub clippy: Option, pub cargo: PathBuf, + pub cargo_config: Vec, pub lib_rustc: Option, pub lib_std: Option, pub lib_test: Option, @@ -329,6 +330,8 @@ pub struct ToolchainConfig<'a> { rustdoc: Option<&'a Path>, clippy: Option<&'a Path>, cargo: Option<&'a Path>, + /// For `cargo --config `. + cargo_config: Option<&'a [String]>, id: Option<&'a str>, } @@ -343,8 +346,9 @@ impl<'a> ToolchainConfig<'a> { self } - pub fn cargo(&mut self, cargo: Option<&'a Path>) -> &mut Self { + pub fn cargo(&mut self, cargo: Option<&'a Path>, config: Option<&'a [String]>) -> &mut Self { self.cargo = cargo; + self.cargo_config = config; self } @@ -520,13 +524,16 @@ pub fn get_local_toolchain( debug!("found cargo: {:?}", &cargo); cargo }; - let lib_dir = get_lib_dir_from_rustc(&rustc).context("Cannot find libdir for rustc")?; + let mut components = + ToolchainComponents::from_binaries_and_libdir(rustc, rustdoc, clippy, cargo, &lib_dir)?; + components.cargo_config = toolchain_config + .cargo_config + .map(|c| c.into()) + .unwrap_or_default(); Ok(Toolchain { - components: ToolchainComponents::from_binaries_and_libdir( - rustc, rustdoc, clippy, cargo, &lib_dir, - )?, + components, id, triple: target_triple, })