Add ! to expect command failures

CHANGELOG.md

@@ -7,6 +7,7 @@
 
 **Improvements**
 
+* Add `!` syntax to expect command failures (panics or errors).
 * Add `Command.consume_args()` for convenient argument handling.
 * Allow `@` in unquoted strings.
 

src/command.rs

@@ -26,6 +26,9 @@ pub struct Command {
     /// Silences the output of this command. This is handled automatically, the
     /// [`Runner`](crate::Runner) does not have to take this into account.
     pub silent: bool,
+    /// If true, the command is expected to fail with a panic or error. If the
+    /// command does not fail, the test fails.
+    pub fail: bool,
     /// The command's line number position in the script.
     pub line_number: u32,
 }

src/lib.rs

@@ -187,9 +187,9 @@
 //!     ```
 //!
 //! * [**Prefix:**](Command::prefix) an optional :-terminated string prefix
-//!   before the command name. The command's output will be given the same
-//!   prefix. The prefix can be used by the test runner, e.g. to signify two
-//!   different clients.
+//!   before the command. The command's output will be given the same prefix.
+//!   The prefix can be used by the test runner, e.g. to signify two different
+//!   clients.
 //!
 //!     ```text
 //!     client1: put key=value
@@ -210,6 +210,21 @@
 //!     foo
 //!     ```
 //!
+//! * [**Failure:**](Command::fail) if `!` precedes the command, it is expected
+//!   to fail with an error or panic, and the failure message is used as output.
+//!   If the command unexpectedly succeeds, the test fails. If the line contains
+//!   other symbols before the command name (e.g. a prefix or silencing), the
+//!   `!` must be used immediately before the command name.
+//!
+//!     ```text
+//!     ! command error=foo
+//!     prefix: ! command panic=bar
+//!     (!command error=foo)
+//!     ---
+//!     Error: foo
+//!     prefix: Panic: bar
+//!     ```
+//!
 //! ## Output
 //!
 //! The command output following a `---` separator can contain any arbitrary

src/parser.rs

@@ -98,8 +98,12 @@ fn command(input: Span) -> IResult<Command> {
     let (input, maybe_silent) = opt(terminated(char('('), space0))(input)?;
     let silent = maybe_silent.is_some();
 
-    // The command itself.
+    // The prefix and fail marker.
     let (input, prefix) = opt(terminated(identifier, pair(tag(":"), space0)))(input)?;
+    let (input, maybe_fail) = opt(terminated(char('!'), space0))(input)?;
+    let fail = maybe_fail.is_some();
+
+    // The command itself.
     let line_number = input.location_line();
     let (input, name) = identifier(input)?;
     let (mut input, args) = many0(preceded(space1, argument))(input)?;
@@ -114,7 +118,7 @@ fn command(input: Span) -> IResult<Command> {
     let (input, _) = opt(comment)(input)?;
     let (input, _) = line_ending(input)?;
 
-    Ok((input, Command { name, args, prefix, silent, line_number }))
+    Ok((input, Command { name, args, prefix, silent, fail, line_number }))
 }
 
 /// Parses a single command argument, consisting of an argument value and

src/runner.rs

@@ -116,16 +116,52 @@ pub fn generate<R: Runner>(runner: &mut R, input: &str) -> std::io::Result<Strin
         ));
 
         for command in &block.commands {
-            // Execute the command.
-            let mut command_output = runner.run(command).map_err(|e| {
-                std::io::Error::new(
-                    std::io::ErrorKind::Other,
-                    format!(
-                        "command '{}' failed at line {}: {e}",
-                        command.name, command.line_number
-                    ),
-                )
-            })?;
+            // Execute the command. Handle panics and errors if requested. We
+            // assume the command is unwind-safe when handling panics, it is up
+            // to callers to manage this appropriately.
+            let run = std::panic::AssertUnwindSafe(|| runner.run(command));
+            let mut command_output = match std::panic::catch_unwind(run) {
+                // Unexpected success, error out.
+                Ok(Ok(output)) if command.fail => {
+                    return Err(std::io::Error::new(
+                        std::io::ErrorKind::Other,
+                        format!(
+                            "expected command '{}' to fail at line {}, succeeded with: {output}",
+                            command.name, command.line_number
+                        ),
+                    ))
+                }
+
+                // Expected success, output the result.
+                Ok(Ok(output)) => output,
+
+                // Expected error, output it.
+                Ok(Err(e)) if command.fail => format!("Error: {e}"),
+
+                // Unexpected error, return it.
+                Ok(Err(e)) => {
+                    return Err(std::io::Error::new(
+                        std::io::ErrorKind::Other,
+                        format!(
+                            "command '{}' failed at line {}: {e}",
+                            command.name, command.line_number
+                        ),
+                    ))
+                }
+
+                // Expected panic, output it.
+                Err(panic) if command.fail => {
+                    let message = panic
+                        .downcast_ref::<&str>()
+                        .map(|s| s.to_string())
+                        .or_else(|| panic.downcast_ref::<String>().cloned())
+                        .unwrap_or_else(|| std::panic::resume_unwind(panic));
+                    format!("Panic: {message}")
+                }
+
+                // Unexpected panic, throw it.
+                Err(panic) => std::panic::resume_unwind(panic),
+            };
 
             // Silence the output if requested.
             if command.silent {

tests/errors/fail_before_prefix.error

@@ -0,0 +1,3 @@
+parse error at line 1 column 8 for CrLf:
+!prefix: command
+       ^

tests/errors/fail_before_prefix.in

@@ -0,0 +1,2 @@
+!prefix: command
+---

tests/errors/fail_before_silence.error

@@ -0,0 +1,3 @@
+parse error at line 1 column 2 for Tag:
+!(command)
+ ^

tests/errors/fail_before_silence.in

@@ -0,0 +1,2 @@
+!(command)
+---

tests/errors/fail_error.error

@@ -0,0 +1 @@
+command '_error' failed at line 1: message

tests/errors/fail_error.in

@@ -0,0 +1,2 @@
+_error message
+---

tests/errors/fail_ok_unexpected.error

@@ -0,0 +1 @@
+expected command '_echo' to fail at line 1, succeeded with: message

tests/errors/fail_ok_unexpected.in

@@ -0,0 +1,2 @@
+! _echo message
+---

tests/errors/fail_panic.error

@@ -0,0 +1 @@
+message

tests/errors/fail_panic.in

@@ -0,0 +1,2 @@
+_panic message
+---

tests/generate/empty_output.out

@@ -2,12 +2,12 @@
 # an empty block at the end of the file.
 command
 ---
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 3 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 3 }
 
 command
 ---
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 6 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 6 }
 
 command
 ---
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 9 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 9 }

tests/scripts/commands

@@ -1,35 +1,40 @@
 # A bare command.
 command
 ---
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 2 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 2 }
 
 # Commands with arguments.
 foo arg
 bar key=value
 baz arg key=value
 ---
-Command { name: "foo", args: [Argument { key: None, value: "arg" }], prefix: None, silent: false, line_number: 7 }
-Command { name: "bar", args: [Argument { key: Some("key"), value: "value" }], prefix: None, silent: false, line_number: 8 }
-Command { name: "baz", args: [Argument { key: None, value: "arg" }, Argument { key: Some("key"), value: "value" }], prefix: None, silent: false, line_number: 9 }
+Command { name: "foo", args: [Argument { key: None, value: "arg" }], prefix: None, silent: false, fail: false, line_number: 7 }
+Command { name: "bar", args: [Argument { key: Some("key"), value: "value" }], prefix: None, silent: false, fail: false, line_number: 8 }
+Command { name: "baz", args: [Argument { key: None, value: "arg" }, Argument { key: Some("key"), value: "value" }], prefix: None, silent: false, fail: false, line_number: 9 }
 
 # Commands with prefixes.
 a: foo arg
 b: bar key=value
 ---
-a: Command { name: "foo", args: [Argument { key: None, value: "arg" }], prefix: Some("a"), silent: false, line_number: 16 }
-b: Command { name: "bar", args: [Argument { key: Some("key"), value: "value" }], prefix: Some("b"), silent: false, line_number: 17 }
+a: Command { name: "foo", args: [Argument { key: None, value: "arg" }], prefix: Some("a"), silent: false, fail: false, line_number: 16 }
+b: Command { name: "bar", args: [Argument { key: Some("key"), value: "value" }], prefix: Some("b"), silent: false, fail: false, line_number: 17 }
+
+# Failing commands.
+! foo bar
+---
+Error: Command { name: "foo", args: [Argument { key: None, value: "bar" }], prefix: None, silent: false, fail: true, line_number: 23 }
 
 # Both prefixes, commands, and argument identifiers can be whitespace, but not
 # empty (tested under errors/).
 " ": " " " "=" "
 ---
- : Command { name: " ", args: [Argument { key: Some(" "), value: " " }], prefix: Some(" "), silent: false, line_number: 24 }
+ : Command { name: " ", args: [Argument { key: Some(" "), value: " " }], prefix: Some(" "), silent: false, fail: false, line_number: 29 }
 
 # Empty argument values are fine.
 command ""
 command arg=""
 command arg=
 ---
-Command { name: "command", args: [Argument { key: None, value: "" }], prefix: None, silent: false, line_number: 29 }
-Command { name: "command", args: [Argument { key: Some("arg"), value: "" }], prefix: None, silent: false, line_number: 30 }
-Command { name: "command", args: [Argument { key: Some("arg"), value: "" }], prefix: None, silent: false, line_number: 31 }
+Command { name: "command", args: [Argument { key: None, value: "" }], prefix: None, silent: false, fail: false, line_number: 34 }
+Command { name: "command", args: [Argument { key: Some("arg"), value: "" }], prefix: None, silent: false, fail: false, line_number: 35 }
+Command { name: "command", args: [Argument { key: Some("arg"), value: "" }], prefix: None, silent: false, fail: false, line_number: 36 }

tests/scripts/comments

@@ -5,25 +5,25 @@
 # Comment before command.
 command id=1
 ---
-Command { name: "command", args: [Argument { key: Some("id"), value: "1" }], prefix: None, silent: false, line_number: 6 }
+Command { name: "command", args: [Argument { key: Some("id"), value: "1" }], prefix: None, silent: false, fail: false, line_number: 6 }
 
 command id=2 # Comment beside command.
 ---
-Command { name: "command", args: [Argument { key: Some("id"), value: "2" }], prefix: None, silent: false, line_number: 10 }
+Command { name: "command", args: [Argument { key: Some("id"), value: "2" }], prefix: None, silent: false, fail: false, line_number: 10 }
 
 command id=3
 # Comment after command.
 ---
-Command { name: "command", args: [Argument { key: Some("id"), value: "3" }], prefix: None, silent: false, line_number: 14 }
+Command { name: "command", args: [Argument { key: Some("id"), value: "3" }], prefix: None, silent: false, fail: false, line_number: 14 }
 
 # Comment between blocks.
 
 command id=4
 ---
-Command { name: "command", args: [Argument { key: Some("id"), value: "4" }], prefix: None, silent: false, line_number: 21 }
+Command { name: "command", args: [Argument { key: Some("id"), value: "4" }], prefix: None, silent: false, fail: false, line_number: 21 }
 
 command id=5 // Comment using //.
 ---
-Command { name: "command", args: [Argument { key: Some("id"), value: "5" }], prefix: None, silent: false, line_number: 25 }
+Command { name: "command", args: [Argument { key: Some("id"), value: "5" }], prefix: None, silent: false, fail: false, line_number: 25 }
 
 # Comment at end.

tests/scripts/dos_line_endings

@@ -2,25 +2,25 @@
 
 command
 ---
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 3 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 3 }
 
 // Comment.
 
 foo arg
 bar key=value
 baz arg key=value
 ---
-Command { name: "foo", args: [Argument { key: None, value: "arg" }], prefix: None, silent: false, line_number: 9 }
-Command { name: "bar", args: [Argument { key: Some("key"), value: "value" }], prefix: None, silent: false, line_number: 10 }
-Command { name: "baz", args: [Argument { key: None, value: "arg" }, Argument { key: Some("key"), value: "value" }], prefix: None, silent: false, line_number: 11 }
+Command { name: "foo", args: [Argument { key: None, value: "arg" }], prefix: None, silent: false, fail: false, line_number: 9 }
+Command { name: "bar", args: [Argument { key: Some("key"), value: "value" }], prefix: None, silent: false, fail: false, line_number: 10 }
+Command { name: "baz", args: [Argument { key: None, value: "arg" }, Argument { key: Some("key"), value: "value" }], prefix: None, silent: false, fail: false, line_number: 11 }
 
 # Comment.
 a: foo arg
 # Comment.
 b: bar key=value
 # Comment.
 ---
-a: Command { name: "foo", args: [Argument { key: None, value: "arg" }], prefix: Some("a"), silent: false, line_number: 18 }
-b: Command { name: "bar", args: [Argument { key: Some("key"), value: "value" }], prefix: Some("b"), silent: false, line_number: 20 }
+a: Command { name: "foo", args: [Argument { key: None, value: "arg" }], prefix: Some("a"), silent: false, fail: false, line_number: 18 }
+b: Command { name: "bar", args: [Argument { key: Some("key"), value: "value" }], prefix: Some("b"), silent: false, fail: false, line_number: 20 }
 
 # Comment.

tests/scripts/fail

@@ -0,0 +1,27 @@
+# ! is parsed correctly into the fail attribute. It is output as an error, since
+# the runner expects an error.
+! command arg
+---
+Error: Command { name: "command", args: [Argument { key: None, value: "arg" }], prefix: None, silent: false, fail: true, line_number: 3 }
+
+# Errors and panics are handled when ! is given.
+! _error foo
+---
+Error: foo
+
+! _panic foo
+---
+Panic: foo
+
+# () can be used to silence the output.
+(!_error foo)
+(!_panic foo)
+---
+ok
+
+# Prefixes can be used too.
+a: ! _error foo
+b:!_panic foo
+---
+a: Error: foo
+b: Panic: foo

tests/scripts/hooks

@@ -2,35 +2,35 @@
 _set start_block="start" end_block="end"
 command
 ---
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 3 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 3 }
 end
 
 command
 ---
 start
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 8 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 8 }
 end
 
 # Newlines in block hooks should be handled appropriately.
 _set start_block="start\n" end_block="end\n"
 command
 ---
 start
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 16 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 16 }
 end
 
 command
 ---
 start
-Command { name: "command", args: [], prefix: None, silent: false, line_number: 22 }
+Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 22 }
 end
 
 # As should empty lines.
 _set start_block="start\n\n" end_block="end\n\n"
 command
 ---
 > start
-> Command { name: "command", args: [], prefix: None, silent: false, line_number: 30 }
+> Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 30 }
 > end
 > 
 
@@ -39,5 +39,5 @@ command
 ---
 > start
 > 
-> Command { name: "command", args: [], prefix: None, silent: false, line_number: 38 }
+> Command { name: "command", args: [], prefix: None, silent: false, fail: false, line_number: 38 }
 > end