Add --max-total-time option

trailofbits · Dec 12, 2023 · 370a8bc · 370a8bc
1 parent 2502bc4
commit 370a8bc
Show file tree

Hide file tree

Showing 8 changed files with 81 additions and 60 deletions.
diff --git a/README.md b/README.md
@@ -252,48 +252,46 @@ Arguments:
   [ARGS]...     Arguments for the fuzzer
 
 Options:
-      --backtrace             Display backtraces
-      --consolidate           Move one target's crashes, hangs, and work queue to its corpus; to
-                              consolidate all targets, use --consolidate-all
-      --display <OBJECT>      Display concretizations, corpus, crashes, `impl` concretizations,
-                              hangs, or work queue. By default, corpus uses an uninstrumented fuzz
-                              target; the others use an instrumented fuzz target. To display the
-                              corpus with instrumentation, use --display corpus-instrumented.
-                              [possible values: concretizations, corpus, corpus-instrumented,
-                              crashes, hangs, impl-concretizations, queue]
-      --exact                 Target name is an exact name rather than a substring
-      --exit-code             Exit with 0 if the time limit was reached, 1 for other programmatic
-                              aborts, and 2 if an error occurred; implies --no-ui, does not imply
-                              --run-until-crash or -- -V <SECONDS>
-      --features <FEATURES>   Space or comma separated list of features to activate
-      --list                  List fuzz targets
-      --manifest-path <PATH>  Path to Cargo.toml
-      --no-default-features   Do not activate the `default` feature
-      --no-instrumentation    Compile without instrumentation (for testing build process)
-      --no-run                Compile, but don't fuzz
-      --no-ui                 Disable user interface
-  -p, --package <PACKAGE>     Package containing fuzz target
-      --persistent            Enable persistent mode fuzzing
-      --pretty-print          Pretty-print debug output when displaying/replaying
-      --replay <OBJECT>       Replay corpus, crashes, hangs, or work queue. By default, corpus uses
-                              an uninstrumented fuzz target; the others use an instrumented fuzz
-                              target. To replay the corpus with instrumentation, use --replay
-                              corpus-instrumented. [possible values: concretizations, corpus,
-                              corpus-instrumented, crashes, hangs, impl-concretizations, queue]
-      --reset                 Clear fuzzing data for one target, but leave corpus intact; to reset
-                              all targets, use --reset-all
-      --resume                Resume target's last fuzzing session
-      --run-until-crash       Stop fuzzing once a crash is found
-      --test <NAME>           Integration test containing fuzz target
-      --timeout <TIMEOUT>     Number of seconds to consider a hang when fuzzing or replaying
-                              (equivalent to -- -t <TIMEOUT * 1000> when fuzzing)
-      --verbose               Show build output when displaying/replaying
-  -h, --help                  Print help
-  -V, --version               Print version
-
-To fuzz at most <SECONDS> of time, use:
-
-    cargo test-fuzz ... -- -V <SECONDS>
+      --backtrace                 Display backtraces
+      --consolidate               Move one target's crashes, hangs, and work queue to its corpus; to
+                                  consolidate all targets, use --consolidate-all
+      --display <OBJECT>          Display concretizations, corpus, crashes, `impl` concretizations,
+                                  hangs, or work queue. By default, corpus uses an uninstrumented
+                                  fuzz target; the others use an instrumented fuzz target. To
+                                  display the corpus with instrumentation, use --display
+                                  corpus-instrumented. [possible values: concretizations, corpus,
+                                  corpus-instrumented, crashes, hangs, impl-concretizations, queue]
+      --exact                     Target name is an exact name rather than a substring
+      --exit-code                 Exit with 0 if the time limit was reached, 1 for other
+                                  programmatic aborts, and 2 if an error occurred; implies --no-ui,
+                                  does not imply --run-until-crash or -- --max-total-time <SECONDS>
+      --features <FEATURES>       Space or comma separated list of features to activate
+      --list                      List fuzz targets
+      --manifest-path <PATH>      Path to Cargo.toml
+      --max-total-time <SECONDS>  Fuzz at most <SECONDS> of time (equivalent to -- -V <SECONDS>)
+      --no-default-features       Do not activate the `default` feature
+      --no-instrumentation        Compile without instrumentation (for testing build process)
+      --no-run                    Compile, but don't fuzz
+      --no-ui                     Disable user interface
+  -p, --package <PACKAGE>         Package containing fuzz target
+      --persistent                Enable persistent mode fuzzing
+      --pretty-print              Pretty-print debug output when displaying/replaying
+      --replay <OBJECT>           Replay corpus, crashes, hangs, or work queue. By default, corpus
+                                  uses an uninstrumented fuzz target; the others use an instrumented
+                                  fuzz target. To replay the corpus with instrumentation, use
+                                  --replay corpus-instrumented. [possible values: concretizations,
+                                  corpus, corpus-instrumented, crashes, hangs, impl-concretizations,
+                                  queue]
+      --reset                     Clear fuzzing data for one target, but leave corpus intact; to
+                                  reset all targets, use --reset-all
+      --resume                    Resume target's last fuzzing session
+      --run-until-crash           Stop fuzzing once a crash is found
+      --test <NAME>               Integration test containing fuzz target
+      --timeout <TIMEOUT>         Number of seconds to consider a hang when fuzzing or replaying
+                                  (equivalent to -- -t <TIMEOUT * 1000> when fuzzing)
+      --verbose                   Show build output when displaying/replaying
+  -h, --help                      Print help
+  -V, --version                   Print version
 
 Try `cargo afl fuzz --help` to see additional fuzzer options.
 ```

diff --git a/cargo-test-fuzz/src/bin/cargo_test_fuzz/transition.rs b/cargo-test-fuzz/src/bin/cargo_test_fuzz/transition.rs
@@ -19,10 +19,7 @@ enum SubCommand {
 // smoelius: Wherever possible, try to reuse cargo test and libtest option names.
 #[allow(clippy::struct_excessive_bools)]
 #[derive(Clone, Debug, Deserialize, Parser, Serialize)]
-#[command(version = crate_version!(), after_help = "To fuzz at most <SECONDS> of time, use:
-
-    cargo test-fuzz ... -- -V <SECONDS>
-
+#[command(version = crate_version!(), after_help = "\
 Try `cargo afl fuzz --help` to see additional fuzzer options.
 ")]
 #[remain::sorted]
@@ -51,8 +48,8 @@ struct TestFuzzWithDeprecations {
     #[arg(
         long,
         help = "Exit with 0 if the time limit was reached, 1 for other programmatic aborts, and 2 \
-                if an error occurred; implies --no-ui, does not imply --run-until-crash or -- -V \
-                <SECONDS>"
+                if an error occurred; implies --no-ui, does not imply --run-until-crash or -- \
+                --max-total-time <SECONDS>"
     )]
     exit_code: bool,
     #[arg(
@@ -65,6 +62,12 @@ struct TestFuzzWithDeprecations {
     list: bool,
     #[arg(long, value_name = "PATH", help = "Path to Cargo.toml")]
     manifest_path: Option<String>,
+    #[arg(
+        long,
+        value_name = "SECONDS",
+        help = "Fuzz at most <SECONDS> of time (equivalent to -- -V <SECONDS>)"
+    )]
+    max_total_time: Option<u64>,
     #[arg(long, help = "Do not activate the `default` feature")]
     no_default_features: bool,
     #[arg(
@@ -137,6 +140,7 @@ impl From<TestFuzzWithDeprecations> for super::TestFuzz {
             features,
             list,
             manifest_path,
+            max_total_time,
             no_default_features,
             no_instrumentation,
             no_run,
@@ -165,6 +169,7 @@ impl From<TestFuzzWithDeprecations> for super::TestFuzz {
             features,
             list,
             manifest_path,
+            max_total_time,
             no_default_features,
             no_instrumentation,
             no_run,

diff --git a/cargo-test-fuzz/src/lib.rs b/cargo-test-fuzz/src/lib.rs
@@ -77,6 +77,7 @@ pub struct TestFuzz {
     pub features: Vec<String>,
     pub list: bool,
     pub manifest_path: Option<String>,
+    pub max_total_time: Option<u64>,
     pub no_default_features: bool,
     pub no_instrumentation: bool,
     pub no_run: bool,
@@ -936,6 +937,9 @@ fn fuzz(opts: &TestFuzz, executable: &Executable, target: &str) -> Result<()> {
         .into_iter()
         .map(String::from),
     );
+    if let Some(max_total_time) = opts.max_total_time {
+        args.extend(["-V".to_owned(), max_total_time.to_string()]);
+    }
     if let Some(timeout) = opts.timeout {
         args.extend(["-t".to_owned(), format!("{}", timeout * MILLIS_PER_SEC)]);
     }

diff --git a/cargo-test-fuzz/tests/auto_generate.rs b/cargo-test-fuzz/tests/auto_generate.rs
@@ -4,7 +4,7 @@ use predicates::prelude::*;
 use std::fs::{read_dir, remove_dir_all};
 use testing::{examples, retry, CommandExt};
 
-const TIMEOUT: &str = "60";
+const MAX_TOTAL_TIME: &str = "60";
 
 // smoelius: It would be nice if these first two tests could distinguish how many "auto_generate"
 // tests get run (0 vs. 1). But right now, I can't think of an easy way to do this.
@@ -43,7 +43,12 @@ fn auto_generate(krate: &str, target: &str, success: bool, pattern: &str, n: usi
     retry(3, || {
         let assert = examples::test_fuzz(krate, target)
             .unwrap()
-            .args(["--no-ui", "--run-until-crash", "--", "-V", TIMEOUT])
+            .args([
+                "--no-ui",
+                "--run-until-crash",
+                "--max-total-time",
+                MAX_TOTAL_TIME,
+            ])
             .logged_assert();
 
         let assert = if success {

diff --git a/cargo-test-fuzz/tests/consolidate.rs b/cargo-test-fuzz/tests/consolidate.rs
@@ -4,17 +4,21 @@ use predicates::prelude::*;
 use std::fs::{read_dir, remove_dir_all};
 use testing::{examples, retry, CommandExt};
 
-const CRASH_TIMEOUT: &str = "60";
+const CRASH_MAX_TOTAL_TIME: &str = "60";
 
-const HANG_TIMEOUT: &str = "120";
+const HANG_MAX_TOTAL_TIME: &str = "120";
 
 #[cfg_attr(dylint_lib = "general", allow(non_thread_safe_call_in_test))]
 #[test]
 fn consolidate_crashes() {
     consolidate(
         "assert",
         "target",
-        &["--run-until-crash", "--", "-V", CRASH_TIMEOUT],
+        &[
+            "--run-until-crash",
+            "--max-total-time",
+            CRASH_MAX_TOTAL_TIME,
+        ],
         "Args { x: true }",
     );
 }
@@ -25,7 +29,7 @@ fn consolidate_hangs() {
     consolidate(
         "parse_duration",
         "parse",
-        &["--persistent", "--", "-V", HANG_TIMEOUT],
+        &["--persistent", "--max-total-time", HANG_MAX_TOTAL_TIME],
         "",
     );
 }

diff --git a/cargo-test-fuzz/tests/fuzz.rs b/cargo-test-fuzz/tests/fuzz.rs
@@ -3,7 +3,7 @@ use predicates::prelude::*;
 use std::fs::remove_dir_all;
 use testing::{examples, retry, CommandExt};
 
-const TIMEOUT: &str = "60";
+const MAX_TOTAL_TIME: &str = "60";
 
 #[cfg_attr(dylint_lib = "general", allow(non_thread_safe_call_in_test))]
 #[test]
@@ -36,7 +36,7 @@ fn fuzz(krate: &str, persistent: bool) {
         if persistent {
             args.push("--persistent");
         }
-        args.extend_from_slice(&["--", "-V", TIMEOUT]);
+        args.extend_from_slice(&["--max-total-time", MAX_TOTAL_TIME]);
 
         command
             .args(&args)

diff --git a/cargo-test-fuzz/tests/fuzz_generic.rs b/cargo-test-fuzz/tests/fuzz_generic.rs
@@ -3,7 +3,7 @@ use predicates::prelude::*;
 use std::{fs::remove_dir_all, sync::Mutex};
 use testing::{examples, retry, CommandExt};
 
-const TIMEOUT: &str = "60";
+const MAX_TOTAL_TIME: &str = "60";
 
 #[cfg_attr(dylint_lib = "general", allow(non_thread_safe_call_in_test))]
 #[test]
@@ -45,7 +45,12 @@ fn fuzz(test: &str, code: i32) {
     retry(3, || {
         examples::test_fuzz("generic", "target")
             .unwrap()
-            .args(["--exit-code", "--run-until-crash", "--", "-V", TIMEOUT])
+            .args([
+                "--exit-code",
+                "--run-until-crash",
+                "--max-total-time",
+                MAX_TOTAL_TIME,
+            ])
             .logged_assert()
             .try_code(predicate::eq(code))
     })

diff --git a/cargo-test-fuzz/tests/replay.rs b/cargo-test-fuzz/tests/replay.rs
@@ -10,7 +10,7 @@ use testing::{examples, retry, CommandExt};
 // smoelius: MEMORY_LIMIT must be large enough for the build process to complete.
 const MEMORY_LIMIT: u64 = 1024 * 1024 * 1024;
 
-const TIMEOUT: &str = "240";
+const MAX_TOTAL_TIME: &str = "240";
 
 #[derive(Clone, Copy)]
 enum Object {
@@ -42,7 +42,7 @@ fn replay_hangs() {
     replay(
         "parse_duration",
         "parse",
-        &["--persistent", "--", "-V", TIMEOUT],
+        &["--persistent", "--max-total-time", MAX_TOTAL_TIME],
         Object::Hangs,
         r"(?m)\bTimeout$",
     );