Full libfuzzer shimming (for cargo-fuzz libfuzzer alternative and oth…

…er use cases) (#981)

* squash libfuzzer edits

* fixup: compat with custom mutators

* use tui flag

* add introspection support

* use libfuzzer dep now that we've merged

* force input loading

* some fixes

* begin docs, impl shrink

* make whole-archive conditional and not default

* make more copies of counters maps

* lol, remember to add the observer

* make size edge map observer an observer

* fixup: make def of run driver conditional

* add sanity checks for insertion

* revert silencing of forks

* add experimental tmin support; add default asan flags

* use default options instead of specifying our own

* implement lockless mode

* fix merge

* fixup lockless corpus

* fixup for generalisation

* remove erroneous drop_in_place

* improve error logging in the case of corpus loading failure

* ok, use lock files 😔

* fix tmin

* implement merge (again); fix rare cases with maps being too small

* implement a scheduler for removing excess

* implement a walking strategy for corpus loading for large corpora

* revert filename parameter; rename and remove duplicates

* various cleanup and clippy satisfaction

* fix no_std tests

* clang-format

* expand and satisfy the clippy gods

* fix sanitizer_ifaces bindgen for no_std

* fix wasm fuzzer

* fixup clippy script

* rename and provide a small amount of explanation for sanitizer_interfaces

* fixup: HasLastReportTime

* fix clippy oddities

* restrict clippy checks to linux-only for libafl_libfuzzer_runtime

* name the mutators

* format

* fix clippy warning

* hope docker is fixed

* fix cmin lint

* clippy pass

* more docs

* more clippy

* fix remaining clippy complaints

* fix import

* miri fixes (no constructors executed)

* exclude libafl_libfuzzer from cargo-hack

* fix clippy check for sanitizer_interfaces

* fmt

* fix CI (?)

* deduplicate sancov 8bit for improved perf on ASAN

* merge 8bit coverage regions + comment out insane deduplication

* no erroring out on free hooks

* fixup for non-forking merge

* skip the corpus dir if we use it

* fixup: recent libafl changes and feature flags

* libafl_libfuzzer: use rust-lld for whole-archive feature

* clarify cause of failure

* mark unsafe

* clippy :cursed_cowboy:

* attempt to fix wasm

* spooky unknowable bug 👻

* more clippy lints

* clippy fix for merge

* use the version pin

* add unsafe to ::register

* Serdeany autoreg fix

* make type assert actionable

* miri fixes

---------

Co-authored-by: Dominik Maier <domenukk@gmail.com>
Co-authored-by: Dominik Maier <dmnk@google.com>
Co-authored-by: Mrmaxmeier <Mrmaxmeier@gmail.com>

.github/workflows/build_and_test.yml

@@ -114,6 +114,7 @@ jobs:
       with:
         profile: minimal
         toolchain: stable
+        components: llvm-tools
     - name: Install and cache deps
       uses: awalsh128/cache-apt-pkgs-action@v1.1.0
       with:

.gitignore

@@ -33,6 +33,8 @@ perf.data.old
 .vscode
 test.dict
 
+.idea/
+
 # Ignore all built fuzzers
 fuzzer_*
 AFLplusplus

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "libafl_concolic/test/runtime_test",
     "libafl_derive",
     "libafl_frida",
+    "libafl_libfuzzer",
     "libafl_nyx",
     "libafl_qemu",
     "libafl_sugar",

Dockerfile

@@ -78,6 +78,10 @@ COPY scripts/dummy.rs libafl_nyx/src/lib.rs
 COPY libafl_tinyinst/Cargo.toml libafl_tinyinst/
 COPY scripts/dummy.rs libafl_tinyinst/src/lib.rs
 
+# avoid pulling in the runtime, as this is quite an expensive build, until later
+COPY libafl_libfuzzer/Cargo.toml libafl_libfuzzer/
+COPY scripts/dummy.rs libafl_libfuzzer/src/lib.rs
+
 COPY utils utils
 
 RUN cargo build && cargo build --release
@@ -117,6 +121,10 @@ COPY libafl_concolic/symcc_runtime libafl_concolic/symcc_runtime
 COPY libafl_concolic/test libafl_concolic/test
 COPY libafl_nyx/src libafl_nyx/src
 RUN touch libafl_nyx/src/lib.rs
+COPY libafl_libfuzzer/src libafl_libfuzzer/src
+COPY libafl_libfuzzer/libafl_libfuzzer_runtime libafl_libfuzzer/libafl_libfuzzer_runtime
+COPY libafl_libfuzzer/build.rs libafl_libfuzzer/build.rs
+RUN touch libafl_libfuzzer/src/lib.rs
 RUN cargo build && cargo build --release
 
 # Copy fuzzers over

fuzzers/baby_fuzzer_wasm/pkg/package.json

@@ -11,5 +11,7 @@
   ],
   "module": "baby_fuzzer_wasm.js",
   "types": "baby_fuzzer_wasm.d.ts",
-  "sideEffects": false
+  "sideEffects": [
+    "./snippets/*"
+  ]
 }

fuzzers/baby_fuzzer_wasm/src/lib.rs

@@ -4,7 +4,7 @@ use libafl::{
     corpus::{Corpus, InMemoryCorpus},
     events::SimpleEventManager,
     executors::{ExitKind, InProcessExecutor},
-    feedbacks::{CrashFeedback, MaxMapFeedback},
+    feedbacks::{CrashFeedback, MapFeedbackMetadata, MaxMapFeedback},
     generators::RandPrintablesGenerator,
     inputs::{BytesInput, HasTargetBytes},
     monitors::SimpleMonitor,
@@ -15,7 +15,9 @@ use libafl::{
     state::{HasSolutions, StdState},
     Fuzzer, StdFuzzer,
 };
-use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice};
+use libafl_bolts::{
+    current_nanos, rands::StdRand, serdeany::RegistryBuilder, tuples::tuple_list, AsSlice,
+};
 use wasm_bindgen::prelude::*;
 use web_sys::{Performance, Window};
 
@@ -37,6 +39,10 @@ pub extern "C" fn external_current_millis() -> u64 {
 pub fn fuzz() {
     set_panic_hook();
 
+    unsafe {
+        RegistryBuilder::register::<MapFeedbackMetadata<u8>>();
+    }
+
     let mut signals = [0u8; 64];
     let signals_ptr = signals.as_mut_ptr();
     let signals_set = |i: usize| unsafe {

fuzzers/fuzzbench/src/lib.rs

@@ -60,7 +60,7 @@ use nix::{self, unistd::dup};
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     let res = match Command::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))

fuzzers/fuzzbench_fork_qemu/src/fuzzer.rs

@@ -60,7 +60,7 @@ use nix::{self, unistd::dup};
 pub fn main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     let res = match Command::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))

fuzzers/fuzzbench_qemu/src/fuzzer.rs

@@ -69,7 +69,7 @@ pub const MAX_INPUT_SIZE: usize = 1048576; // 1MB
 pub fn main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     let res = match Command::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))

fuzzers/fuzzbench_text/src/lib.rs

@@ -67,7 +67,7 @@ use nix::{self, unistd::dup};
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     let res = match Command::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))

fuzzers/libafl_atheris/src/lib.rs

@@ -110,7 +110,7 @@ pub extern "C" fn LLVMFuzzerRunDriver(
 ) {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     assert!(harness_fn.is_some(), "No harness callback provided");
     let harness_fn = harness_fn.unwrap();

fuzzers/libfuzzer_libmozjpeg/src/lib.rs

@@ -45,7 +45,7 @@ extern "C" {
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     println!(
         "Workdir: {:?}",

fuzzers/libfuzzer_libpng/src/lib.rs

@@ -44,7 +44,7 @@ use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, EDGES_MAP,
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     println!(
         "Workdir: {:?}",

fuzzers/libfuzzer_libpng_accounting/src/lib.rs

@@ -115,7 +115,7 @@ struct Opt {
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
     let opt = Opt::parse();
 
     let broker_port = opt.broker_port;

fuzzers/libfuzzer_libpng_centralized/src/lib.rs

@@ -115,7 +115,7 @@ pub extern "C" fn libafl_main() {
 
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
     let opt = Opt::parse();
 
     let broker_port = opt.broker_port;

fuzzers/libfuzzer_libpng_cmin/src/lib.rs

@@ -47,7 +47,7 @@ use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, std_edges_m
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     println!(
         "Workdir: {:?}",

fuzzers/libfuzzer_libpng_ctx/src/lib.rs

@@ -110,7 +110,7 @@ struct Opt {
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
     let opt = Opt::parse();
 
     let broker_port = opt.broker_port;

fuzzers/libfuzzer_libpng_launcher/src/lib.rs

@@ -113,7 +113,7 @@ struct Opt {
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
     let opt = Opt::parse();
 
     let broker_port = opt.broker_port;

fuzzers/libfuzzer_libpng_norestart/src/lib.rs

@@ -131,7 +131,7 @@ struct Opt {
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
     let opt = Opt::parse();
 
     let broker_port = opt.broker_port;

fuzzers/libfuzzer_libpng_tcp_manager/src/lib.rs

@@ -43,7 +43,7 @@ use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input, EDGES_MAP,
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     println!(
         "Workdir: {:?}",

fuzzers/libfuzzer_reachability/src/lib.rs

@@ -35,7 +35,7 @@ extern "C" {
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     println!(
         "Workdir: {:?}",

fuzzers/libfuzzer_stb_image/src/main.rs

@@ -33,7 +33,7 @@ use libafl_targets::{
 pub fn main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     println!(
         "Workdir: {:?}",

fuzzers/libfuzzer_stb_image_concolic/fuzzer/src/main.rs

@@ -63,7 +63,7 @@ struct Opt {
 pub fn main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     let opt = Opt::parse();
 

fuzzers/libfuzzer_stb_image_sugar/src/main.rs

@@ -13,7 +13,7 @@ use libafl_targets::{libfuzzer_initialize, libfuzzer_test_one_input};
 pub fn main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     println!(
         "Workdir: {:?}",

fuzzers/nautilus_sync/src/lib.rs

@@ -102,7 +102,7 @@ struct Opt {
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
     let opt = Opt::parse();
 
     let broker_port = opt.broker_port;

fuzzers/tutorial/src/lib.rs

@@ -39,7 +39,7 @@ use metadata::{PacketLenFeedback, PacketLenMinimizerScheduler};
 pub extern "C" fn libafl_main() {
     // Registry the metadata types used in this fuzzer
     // Needed only on no_std
-    //RegistryBuilder::register::<Tokens>();
+    // unsafe { RegistryBuilder::register::<Tokens>(); }
 
     println!(
         "Workdir: {:?}",

libafl/src/corpus/cached.rs

@@ -1,6 +1,6 @@
 //! The [`CachedOnDiskCorpus`] stores [`Testcase`]s to disk, keeping a subset of them in memory/cache, evicting in a FIFO manner.
 
-use alloc::collections::vec_deque::VecDeque;
+use alloc::{collections::vec_deque::VecDeque, string::String};
 use core::cell::RefCell;
 use std::path::Path;
 
@@ -193,7 +193,7 @@ where
     pub fn with_meta_format<P>(
         dir_path: P,
         cache_max_len: usize,
-        meta_format: OnDiskMetadataFormat,
+        meta_format: Option<OnDiskMetadataFormat>,
     ) -> Result<Self, Error>
     where
         P: AsRef<Path>,
@@ -204,6 +204,31 @@ where
         )
     }
 
+    /// Creates the [`CachedOnDiskCorpus`] specifying the metadata format and the prefix to prepend
+    /// to each testcase.
+    ///
+    /// Will error, if [`std::fs::create_dir_all()`] failed for `dir_path`.
+    pub fn with_meta_format_and_prefix<P>(
+        dir_path: P,
+        cache_max_len: usize,
+        meta_format: Option<OnDiskMetadataFormat>,
+        prefix: Option<String>,
+        locking: bool,
+    ) -> Result<Self, Error>
+    where
+        P: AsRef<Path>,
+    {
+        Self::_new(
+            InMemoryOnDiskCorpus::with_meta_format_and_prefix(
+                dir_path,
+                meta_format,
+                prefix,
+                locking,
+            )?,
+            cache_max_len,
+        )
+    }
+
     /// Internal constructor `fn`
     fn _new(on_disk_corpus: InMemoryOnDiskCorpus<I>, cache_max_len: usize) -> Result<Self, Error> {
         if cache_max_len == 0 {
@@ -217,6 +242,11 @@ where
             cache_max_len,
         })
     }
+
+    /// Fetch the inner corpus
+    pub fn inner(&self) -> &InMemoryOnDiskCorpus<I> {
+        &self.inner
+    }
 }
 
 /// ``CachedOnDiskCorpus`` Python bindings