From 684aa2c9d11d9e818ea65f7e4ebd194b97318ac7 Mon Sep 17 00:00:00 2001 From: David Koloski Date: Wed, 6 Dec 2023 18:25:13 +0000 Subject: [PATCH] Add support for shell argfiles --- Cargo.lock | 1 + compiler/rustc_driver_impl/Cargo.toml | 1 + compiler/rustc_driver_impl/src/args.rs | 107 +++++++++++++++--- compiler/rustc_interface/src/tests.rs | 1 + compiler/rustc_session/src/options.rs | 2 + .../src/compiler-flags/shell-argfiles.md | 11 ++ src/tools/tidy/src/deps.rs | 1 + src/tools/tidy/src/ui_tests.rs | 6 +- .../shell-argfiles-badquotes-windows.rs | 11 ++ .../shell-argfiles-badquotes-windows.stderr | 2 + .../shell-argfiles-badquotes.args | 1 + .../shell-argfiles-badquotes.rs | 12 ++ .../shell-argfiles-badquotes.stderr | 2 + .../shell-argfiles-via-argfile-shell.args | 1 + .../shell-argfiles-via-argfile.args | 1 + .../shell-argfiles-via-argfile.rs | 10 ++ tests/ui/shell-argfiles/shell-argfiles.args | 3 + tests/ui/shell-argfiles/shell-argfiles.rs | 19 ++++ 18 files changed, 175 insertions(+), 17 deletions(-) create mode 100644 src/doc/unstable-book/src/compiler-flags/shell-argfiles.md create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes.args create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes.rs create mode 100644 tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr create mode 100644 tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args create mode 100644 tests/ui/shell-argfiles/shell-argfiles-via-argfile.args create mode 100644 tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs create mode 100644 tests/ui/shell-argfiles/shell-argfiles.args create mode 100644 tests/ui/shell-argfiles/shell-argfiles.rs diff --git a/Cargo.lock b/Cargo.lock index b8192e333fe91..c7fa0b5675286 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3738,6 +3738,7 @@ dependencies = [ "rustc_trait_selection", "rustc_ty_utils", "serde_json", + "shlex", "time", "tracing", "windows", diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml index 490429845538d..97a7dfef3b395 100644 --- a/compiler/rustc_driver_impl/Cargo.toml +++ b/compiler/rustc_driver_impl/Cargo.toml @@ -50,6 +50,7 @@ rustc_target = { path = "../rustc_target" } rustc_trait_selection = { path = "../rustc_trait_selection" } rustc_ty_utils = { path = "../rustc_ty_utils" } serde_json = "1.0.59" +shlex = "1.0" time = { version = "0.3", default-features = false, features = ["alloc", "formatting"] } tracing = { version = "0.1.35" } # tidy-alphabetical-end diff --git a/compiler/rustc_driver_impl/src/args.rs b/compiler/rustc_driver_impl/src/args.rs index b7407f5a508e4..5dfd37a6da4de 100644 --- a/compiler/rustc_driver_impl/src/args.rs +++ b/compiler/rustc_driver_impl/src/args.rs @@ -5,18 +5,92 @@ use std::io; use rustc_session::EarlyDiagCtxt; -fn arg_expand(arg: String) -> Result, Error> { - if let Some(path) = arg.strip_prefix('@') { - let file = match fs::read_to_string(path) { - Ok(file) => file, - Err(ref err) if err.kind() == io::ErrorKind::InvalidData => { - return Err(Error::Utf8Error(Some(path.to_string()))); +/// Expands argfiles in command line arguments. +#[derive(Default)] +struct Expander { + shell_argfiles: bool, + next_is_unstable_option: bool, + expanded: Vec, +} + +impl Expander { + /// Handles the next argument. If the argument is an argfile, it is expanded + /// inline. + fn arg(&mut self, arg: &str) -> Result<(), Error> { + if let Some(argfile) = arg.strip_prefix('@') { + match argfile.split_once(':') { + Some(("shell", path)) if self.shell_argfiles => { + shlex::split(&Self::read_file(path)?) + .ok_or_else(|| Error::ShellParseError(path.to_string()))? + .into_iter() + .for_each(|arg| self.push(arg)); + } + _ => { + let contents = Self::read_file(argfile)?; + contents.lines().for_each(|arg| self.push(arg.to_string())); + } + } + } else { + self.push(arg.to_string()); + } + + Ok(()) + } + + /// Adds a command line argument verbatim with no argfile expansion. + fn push(&mut self, arg: String) { + // Unfortunately, we have to do some eager argparsing to handle unstable + // options which change the behavior of argfile arguments. + // + // Normally, all of the argfile arguments (e.g. `@args.txt`) are + // expanded into our arguments list *and then* the whole list of + // arguments are passed on to be parsed. However, argfile parsing + // options like `-Zshell_argfiles` need to change the behavior of that + // argument expansion. So we have to do a little parsing on our own here + // instead of leaning on the existing logic. + // + // All we care about are unstable options, so we parse those out and + // look for any that affect how we expand argfiles. This argument + // inspection is very conservative; we only change behavior when we see + // exactly the options we're looking for and everything gets passed + // through. + + if self.next_is_unstable_option { + self.inspect_unstable_option(&arg); + self.next_is_unstable_option = false; + } else if let Some(unstable_option) = arg.strip_prefix("-Z") { + if unstable_option.is_empty() { + self.next_is_unstable_option = true; + } else { + self.inspect_unstable_option(unstable_option); + } + } + + self.expanded.push(arg); + } + + /// Consumes the `Expander`, returning the expanded arguments. + fn finish(self) -> Vec { + self.expanded + } + + /// Parses any relevant unstable flags specified on the command line. + fn inspect_unstable_option(&mut self, option: &str) { + match option { + "shell-argfiles" => self.shell_argfiles = true, + _ => (), + } + } + + /// Reads the contents of a file as UTF-8. + fn read_file(path: &str) -> Result { + fs::read_to_string(path).map_err(|e| { + if e.kind() == io::ErrorKind::InvalidData { + Error::Utf8Error(Some(path.to_string())) + } else { + Error::IOError(path.to_string(), e) } - Err(err) => return Err(Error::IOError(path.to_string(), err)), - }; - Ok(file.lines().map(ToString::to_string).collect()) - } else { - Ok(vec![arg]) + }) } } @@ -24,20 +98,20 @@ fn arg_expand(arg: String) -> Result, Error> { /// If this function is intended to be used with command line arguments, /// `argv[0]` must be removed prior to calling it manually. pub fn arg_expand_all(early_dcx: &EarlyDiagCtxt, at_args: &[String]) -> Vec { - let mut args = Vec::new(); + let mut expander = Expander::default(); for arg in at_args { - match arg_expand(arg.clone()) { - Ok(arg) => args.extend(arg), - Err(err) => early_dcx.early_fatal(format!("Failed to load argument file: {err}")), + if let Err(err) = expander.arg(arg) { + early_dcx.early_fatal(format!("Failed to load argument file: {err}")); } } - args + expander.finish() } #[derive(Debug)] pub enum Error { Utf8Error(Option), IOError(String, io::Error), + ShellParseError(String), } impl fmt::Display for Error { @@ -46,6 +120,7 @@ impl fmt::Display for Error { Error::Utf8Error(None) => write!(fmt, "Utf8 error"), Error::Utf8Error(Some(path)) => write!(fmt, "Utf8 error in {path}"), Error::IOError(path, err) => write!(fmt, "IO Error: {path}: {err}"), + Error::ShellParseError(path) => write!(fmt, "Invalid shell-style arguments in {path}"), } } } diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 75410db1e364c..7b5de4cc11720 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -700,6 +700,7 @@ fn test_unstable_options_tracking_hash() { untracked!(query_dep_graph, true); untracked!(self_profile, SwitchWithOptPath::Enabled(None)); untracked!(self_profile_events, Some(vec![String::new()])); + untracked!(shell_argfiles, true); untracked!(span_debug, true); untracked!(span_free_formats, true); untracked!(temps_dir, Some(String::from("abc"))); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 0b0b67ef890b0..21113c298f04a 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1875,6 +1875,8 @@ written to standard error output)"), query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm, artifact-sizes"), share_generics: Option = (None, parse_opt_bool, [TRACKED], "make the current crate share its generic instantiations"), + shell_argfiles: bool = (false, parse_bool, [UNTRACKED], + "allow argument files to be specified with POSIX \"shell-style\" argument quoting"), show_span: Option = (None, parse_opt_string, [TRACKED], "show spans for compiler debugging (expr|pat|ty)"), simulate_remapped_rust_src_base: Option = (None, parse_opt_pathbuf, [TRACKED], diff --git a/src/doc/unstable-book/src/compiler-flags/shell-argfiles.md b/src/doc/unstable-book/src/compiler-flags/shell-argfiles.md new file mode 100644 index 0000000000000..4f3c780972de5 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/shell-argfiles.md @@ -0,0 +1,11 @@ +# `shell-argfiles` + +-------------------- + +The `-Zshell-argfiles` compiler flag allows argfiles to be parsed using POSIX +"shell-style" quoting. When enabled, the compiler will use `shlex` to parse the +arguments from argfiles specified with `@shell:`. + +Because this feature controls the parsing of input arguments, the +`-Zshell-argfiles` flag must be present before the argument specifying the +shell-style arguemnt file. diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 3c00027b9fdc9..62d48315d434e 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -325,6 +325,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "sha1", "sha2", "sharded-slab", + "shlex", "smallvec", "snap", "stable_deref_trait", diff --git a/src/tools/tidy/src/ui_tests.rs b/src/tools/tidy/src/ui_tests.rs index b4745d4883c55..ab0e647e13043 100644 --- a/src/tools/tidy/src/ui_tests.rs +++ b/src/tools/tidy/src/ui_tests.rs @@ -11,7 +11,7 @@ use std::path::{Path, PathBuf}; const ENTRY_LIMIT: usize = 900; // FIXME: The following limits should be reduced eventually. const ISSUES_ENTRY_LIMIT: usize = 1849; -const ROOT_ENTRY_LIMIT: usize = 867; +const ROOT_ENTRY_LIMIT: usize = 868; const EXPECTED_TEST_FILE_EXTENSIONS: &[&str] = &[ "rs", // test source files @@ -36,6 +36,10 @@ const EXTENSION_EXCEPTION_PATHS: &[&str] = &[ "tests/ui/unused-crate-deps/test.mk", // why would you use make "tests/ui/proc-macro/auxiliary/included-file.txt", // more include "tests/ui/invalid/foo.natvis.xml", // sample debugger visualizer + "tests/ui/shell-argfiles/shell-argfiles.args", // passing args via a file + "tests/ui/shell-argfiles/shell-argfiles-badquotes.args", // passing args via a file + "tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args", // passing args via a file + "tests/ui/shell-argfiles/shell-argfiles-via-argfile.args", // passing args via a file ]; fn check_entries(tests_path: &Path, bad: &mut bool) { diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs b/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs new file mode 100644 index 0000000000000..800735cf3a768 --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.rs @@ -0,0 +1,11 @@ +// Check to see if we can get parameters from an @argsfile file +// +// Path replacement in .stderr files (i.e. `$DIR`) doesn't handle mixed path +// separators. This test uses backslash as the path separator for the command +// line arguments and is only run on windows. +// +// only-windows +// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}\shell-argfiles\shell-argfiles-badquotes.args + +fn main() { +} diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr b/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr new file mode 100644 index 0000000000000..14adb1f740abb --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes-windows.stderr @@ -0,0 +1,2 @@ +error: Failed to load argument file: Invalid shell-style arguments in $DIR/shell-argfiles-badquotes.args + diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes.args b/tests/ui/shell-argfiles/shell-argfiles-badquotes.args new file mode 100644 index 0000000000000..c0d531adf3ffb --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes.args @@ -0,0 +1 @@ +"--cfg" "unquoted_set diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes.rs b/tests/ui/shell-argfiles/shell-argfiles-badquotes.rs new file mode 100644 index 0000000000000..f9160143a0410 --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes.rs @@ -0,0 +1,12 @@ +// Check to see if we can get parameters from an @argsfile file +// +// Path replacement in .stderr files (i.e. `$DIR`) doesn't handle mixed path +// separators. We have a duplicated version of this test that uses backslash as +// the path separator for the command line arguments that is only run on +// windows. +// +// ignore-windows +// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}/shell-argfiles/shell-argfiles-badquotes.args + +fn main() { +} diff --git a/tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr b/tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr new file mode 100644 index 0000000000000..14adb1f740abb --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles-badquotes.stderr @@ -0,0 +1,2 @@ +error: Failed to load argument file: Invalid shell-style arguments in $DIR/shell-argfiles-badquotes.args + diff --git a/tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args b/tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args new file mode 100644 index 0000000000000..4e66d5a039529 --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args @@ -0,0 +1 @@ +"--cfg" "shell_args_set" \ No newline at end of file diff --git a/tests/ui/shell-argfiles/shell-argfiles-via-argfile.args b/tests/ui/shell-argfiles/shell-argfiles-via-argfile.args new file mode 100644 index 0000000000000..d0af54e24e33c --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles-via-argfile.args @@ -0,0 +1 @@ +-Zshell-argfiles \ No newline at end of file diff --git a/tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs b/tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs new file mode 100644 index 0000000000000..d71e3218f53b8 --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles-via-argfile.rs @@ -0,0 +1,10 @@ +// Check to see if we can get parameters from an @argsfile file +// +// build-pass +// compile-flags: @{{src-base}}/shell-argfiles/shell-argfiles-via-argfile.args @shell:{{src-base}}/shell-argfiles/shell-argfiles-via-argfile-shell.args + +#[cfg(not(shell_args_set))] +compile_error!("shell_args_set not set"); + +fn main() { +} diff --git a/tests/ui/shell-argfiles/shell-argfiles.args b/tests/ui/shell-argfiles/shell-argfiles.args new file mode 100644 index 0000000000000..e5bb4b807ec4d --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles.args @@ -0,0 +1,3 @@ +--cfg unquoted_set +'--cfg' 'single_quoted_set' +"--cfg" "double_quoted_set" diff --git a/tests/ui/shell-argfiles/shell-argfiles.rs b/tests/ui/shell-argfiles/shell-argfiles.rs new file mode 100644 index 0000000000000..9bc252ea628a9 --- /dev/null +++ b/tests/ui/shell-argfiles/shell-argfiles.rs @@ -0,0 +1,19 @@ +// Check to see if we can get parameters from an @argsfile file +// +// build-pass +// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}/shell-argfiles/shell-argfiles.args + +#[cfg(not(cmdline_set))] +compile_error!("cmdline_set not set"); + +#[cfg(not(unquoted_set))] +compile_error!("unquoted_set not set"); + +#[cfg(not(single_quoted_set))] +compile_error!("single_quoted_set not set"); + +#[cfg(not(double_quoted_set))] +compile_error!("double_quoted_set not set"); + +fn main() { +}