Merge branch 'main' into main

.github/workflows/build_and_test.yml

@@ -16,6 +16,9 @@ jobs:
           os: [ubuntu-latest, windows-latest, macOS-latest]
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Install mimetype
+        if: runner.os == 'Linux'
+        run: sudo apt-get install libfile-mimeinfo-perl
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -31,9 +34,6 @@ jobs:
           crate: mdbook-linkcheck
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
-      - name: Install mimetype
-        if: runner.os == 'Linux'
-        run: sudo apt-get install libfile-mimeinfo-perl
       - name: Check for binary blobs
         if: runner.os == 'Linux'
         run: ./scripts/check_for_blobs.sh
@@ -49,6 +49,8 @@ jobs:
         run: cargo test
       - name: Test libafl no_std
         run: cd libafl && cargo test --no-default-features
+      - name: Test libafl_bolts no_std no_alloc
+        run: cd libafl_bolts && cargo test --no-default-features
       - name: Test libafl_targets no_std
         run: cd libafl_targets && cargo test --no-default-features
 
@@ -163,7 +165,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: Swatinem/rust-cache@v2
     - name: Run a maturin build
-      run: cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install . && ./test.sh
+      run: cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh
     - name: Run python test
       run: . ./bindings/pylibafl/.env/bin/activate && cd ./fuzzers/baby_fuzzer && python3 baby_fuzzer.py 2>&1 | grep "Bye"
 
@@ -285,6 +287,8 @@ jobs:
       run: cd ./libafl && cargo test --no-default-features 
     - name: libafl armv6m-none-eabi (32 bit no_std) clippy
       run: cd ./libafl && cargo clippy --target thumbv6m-none-eabi --no-default-features
+    - name: Build no_std no_alloc bolts
+      run: cd ./libafl_bolts && cargo +nightly build -Zbuild-std=core --target aarch64-unknown-none --no-default-features -v --release && cd ../
 
   build-docker:
     runs-on: ubuntu-latest

.gitignore

@@ -56,3 +56,5 @@ __pycache__
 
 libafl_nyx/QEMU-Nyx
 libafl_nyx/packer
+
+.z3-trace

Cargo.toml

@@ -1,35 +1,38 @@
 [workspace]
+resolver = "2"
 members = [
     "libafl",
-    "libafl_derive",
+    "libafl_bolts",
     "libafl_cc",
-    "libafl_targets",
-    "libafl_frida",
-    "libafl_qemu",
-    "libafl_tinyinst",
-    "libafl_sugar",
-    "libafl_nyx",
     "libafl_concolic/symcc_runtime",
     "libafl_concolic/symcc_libafl",
     "libafl_concolic/test/dump_constraints",
     "libafl_concolic/test/runtime_test",
+    "libafl_derive",
+    "libafl_frida",
+    "libafl_nyx",
+    "libafl_qemu",
+    "libafl_sugar",
+    "libafl_targets",
+    "libafl_tinyinst",
+    "utils/build_and_test_fuzzers",
     "utils/deexit",
-    "utils/gramatron/construct_automata",
     "utils/libafl_benches",
-    "utils/build_and_test_fuzzers",
+    "utils/gramatron/construct_automata",
 ]
 default-members = [
     "libafl",
-    "libafl_derive",
+    "libafl_bolts",
     "libafl_cc",
+    "libafl_derive",
     "libafl_targets",
 ]
 exclude = [
-    "fuzzers",
     "bindings",
-    "scripts",
+    "fuzzers",
     "libafl_qemu/libafl_qemu_build",
-    "libafl_qemu/libafl_qemu_sys"
+    "libafl_qemu/libafl_qemu_sys",
+    "scripts",
 ]
 
 [workspace.package]

Dockerfile

@@ -29,9 +29,12 @@ COPY libafl_derive/Cargo.toml libafl_derive/Cargo.toml
 COPY scripts/dummy.rs libafl_derive/src/lib.rs
 
 COPY libafl/Cargo.toml libafl/build.rs libafl/
-COPY libafl/examples libafl/examples
 COPY scripts/dummy.rs libafl/src/lib.rs
 
+COPY libafl_bolts/Cargo.toml libafl_bolts/build.rs libafl_bolts/README.md libafl_bolts/
+COPY libafl_bolts/examples libafl_bolts/examples
+COPY scripts/dummy.rs libafl_bolts/src/lib.rs
+
 COPY libafl_frida/Cargo.toml libafl_frida/build.rs libafl_frida/
 COPY scripts/dummy.rs libafl_frida/src/lib.rs
 COPY libafl_frida/src/gettls.c libafl_frida/src/gettls.c
@@ -95,6 +98,8 @@ COPY libafl_cc/src libafl_cc/src
 RUN touch libafl_cc/src/lib.rs
 COPY libafl_derive/src libafl_derive/src
 RUN touch libafl_derive/src/lib.rs
+COPY libafl_bolts/src libafl_bolts/src
+RUN touch libafl_bolts/src/lib.rs
 COPY libafl/src libafl/src
 RUN touch libafl/src/lib.rs
 COPY libafl_targets/src libafl_targets/src

bindings/pylibafl/Cargo.toml

@@ -9,6 +9,7 @@ pyo3-log = "0.8.1"
 libafl_qemu = { path = "../../libafl_qemu", version = "0.10.1", features = ["python"] }
 libafl_sugar = { path = "../../libafl_sugar", version = "0.10.1", features = ["python"] }
 libafl = { path = "../../libafl", version = "0.10.1", features = ["python"] }
+libafl_bolts = { path = "../../libafl_bolts", version = "0.10.1", features = ["python"] }
 
 [build-dependencies]
 pyo3-build-config = { version = "0.17" }

bindings/pylibafl/src/lib.rs

@@ -1,4 +1,5 @@
 use libafl;
+use libafl_bolts;
 #[cfg(target_os = "linux")]
 use libafl_qemu;
 use libafl_sugar;
@@ -91,18 +92,20 @@ pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> {
     let sugar_module = PyModule::new(py, "sugar")?;
     libafl_sugar::python_module(py, sugar_module)?;
     m.add_submodule(sugar_module)?;
-
     modules.set_item("pylibafl.sugar", sugar_module)?;
 
     #[cfg(target_os = "linux")]
-    let qemu_module = PyModule::new(py, "qemu")?;
-    #[cfg(target_os = "linux")]
-    libafl_qemu::python_module(py, qemu_module)?;
-    #[cfg(target_os = "linux")]
-    m.add_submodule(qemu_module)?;
+    {
+        let qemu_module = PyModule::new(py, "qemu")?;
+        libafl_qemu::python_module(py, qemu_module)?;
+        m.add_submodule(qemu_module)?;
+        modules.set_item("pylibafl.qemu", qemu_module)?;
+    }
 
-    #[cfg(target_os = "linux")]
-    modules.set_item("pylibafl.qemu", qemu_module)?;
+    let bolts_module = PyModule::new(py, "libafl_bolts")?;
+    libafl_bolts::pybind::python_module(py, bolts_module)?;
+    m.add_submodule(bolts_module)?;
+    modules.set_item("pylibafl.libafl_bolts", bolts_module)?;
 
     let libafl_module = PyModule::new(py, "libafl")?;
     libafl::pybind::python_module(py, libafl_module)?;
@@ -116,7 +119,6 @@ pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> {
     }
 
     m.add_submodule(libafl_module)?;
-
     modules.set_item("pylibafl.libafl", libafl_module)?;
 
     Ok(())

docs/listings/baby_fuzzer/listing-02/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2018"
 
 [dependencies]
 libafl = { path = "path/to/libafl/" }
+libafl_bolts = { path = "path/to/libafl_bolts/" }
 
 [profile.dev]
 panic = "abort"

docs/listings/baby_fuzzer/listing-03/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2018"
 
 [dependencies]
 libafl = { path = "path/to/libafl/" }
+libafl_bolts = { path = "path/to/libafl_bolts/" }
 
 [profile.dev]
 panic = "abort"

docs/listings/baby_fuzzer/listing-03/src/main.rs

@@ -1,9 +1,10 @@
 extern crate libafl;
+extern crate libafl_bolts;
 use libafl::{
-    bolts::AsSlice,
     executors::ExitKind,
     inputs::{BytesInput, HasTargetBytes},
 };
+use libafl_bolts::AsSlice;
 
 fn main() {
     let mut harness = |input: &BytesInput| {

docs/listings/baby_fuzzer/listing-04/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2018"
 
 [dependencies]
 libafl = { path = "path/to/libafl/" }
+libafl_bolts = { path = "path/to/libafl_bolts/" }
 
 [profile.dev]
 panic = "abort"

docs/listings/baby_fuzzer/listing-04/src/main.rs

@@ -1,8 +1,8 @@
 /* ANCHOR: use */
 extern crate libafl;
+extern crate libafl_bolts;
 
 use libafl::{
-    bolts::{current_nanos, rands::StdRand, AsSlice},
     corpus::{InMemoryCorpus, OnDiskCorpus},
     events::SimpleEventManager,
     executors::{inprocess::InProcessExecutor, ExitKind},
@@ -13,6 +13,7 @@ use libafl::{
     schedulers::QueueScheduler,
     state::StdState,
 };
+use libafl_bolts::{current_nanos, rands::StdRand, AsSlice};
 use std::path::PathBuf;
 /* ANCHOR_END: use */
 

docs/listings/baby_fuzzer/listing-05/src/main.rs

@@ -1,8 +1,8 @@
 /* ANCHOR: use */
 extern crate libafl;
+extern crate libafl_bolts;
 
 use libafl::{
-    bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice},
     corpus::{InMemoryCorpus, OnDiskCorpus},
     events::SimpleEventManager,
     executors::{inprocess::InProcessExecutor, ExitKind},
@@ -15,6 +15,7 @@ use libafl::{
     schedulers::QueueScheduler,
     state::StdState,
 };
+use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice};
 use std::path::PathBuf;
 /* ANCHOR_END: use */
 

docs/listings/baby_fuzzer/listing-06/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2018"
 
 [dependencies]
 libafl = { path = "path/to/libafl/" }
+libafl_bolts = { path = "path/to/libafl_bolts/" }
 
 [profile.dev]
 panic = "abort"

docs/listings/baby_fuzzer/listing-06/src/main.rs

@@ -1,8 +1,8 @@
 /* ANCHOR: use */
 extern crate libafl;
+extern crate libafl_bolts;
 
 use libafl::{
-    bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice},
     corpus::{InMemoryCorpus, OnDiskCorpus},
     events::SimpleEventManager,
     executors::{inprocess::InProcessExecutor, ExitKind},
@@ -17,6 +17,7 @@ use libafl::{
     stages::mutational::StdMutationalStage,
     state::StdState,
 };
+use libafl_bolts::{current_nanos, rands::StdRand, tuples::tuple_list, AsSlice};
 use std::path::PathBuf;
 /* ANCHOR_END: use */
 

docs/src/SUMMARY.md

@@ -25,6 +25,7 @@
   - [Architecture](./design/architecture.md)
   - [Metadata](./design/metadata.md)
   - [Migrating from LibAFL <0.9 to 0.9](./design/migration-0.9.md)
+  - [Migrating from LibAFL <0.11 to 0.11](./design/migration-0.11.md)
 
 - [Message Passing](./message_passing/message_passing.md)
   - [Spawning Instances](./message_passing/spawn_instances.md)

docs/src/core_concepts/feedback.md

@@ -10,8 +10,13 @@ The concept of "interestingness" is abstract, but typically it is related to a n
 
 As an example, given an Observer that reports all the sizes of memory allocations, a maximization Feedback can be used to maximize these sizes to sport pathological inputs in terms of memory consumption.
 
-In terms of code, the library offers the [`Feedback`](https://docs.rs/libafl/0/libafl/feedbacks/trait.Feedback.html) and the [`FeedbackState`](https://docs.rs/libafl/0/libafl/feedbacks/trait.FeedbackState.html) traits.
-The first is used to implement functors that, given the state of the observers from the last execution, tells if the execution was interesting. The second is tied with `Feedback` and is the state of the data that the feedback wants to persist in the fuzzers's state, for instance the cumulative map holding all the edges seen so far in the case of a feedback based on edge coverage.
+In terms of code, the library offers the [`Feedback`](https://docs.rs/libafl/0/libafl/feedbacks/trait.Feedback.html) trait.
+It is used to implement functors that, given the state of the observers from the last execution, tells if the execution was interesting.
+So to speak, it reduces the observations to a boolean result of [`is_interesting`](https://docs.rs/libafl/0/libafl/feedbacks/trait.Feedback.html#tymethod.is_interesting) - or not.
+For this, a `Feedback` can store anything it wants to persist in the fuzzers's state.
+This might be, for instance, the cumulative map of all edges seen so far, in the case of a feedback based on edge coverage.
+This can be achieved by adding `Metadata` in [`init_state`](https://docs.rs/libafl/0/libafl/feedbacks/trait.Feedback.html#method.init_state) and accessing it later in `is_interesting`.
+`Feedback` can also add custom metadata to a newly created [`Testcase`](https://docs.rs/libafl/0/libafl/corpus/testcase/struct.Testcase.html) using [`append_metadata`](https://docs.rs/libafl/0.10.1/libafl/feedbacks/trait.Feedback.html#method.append_metadata).
 
 Multiple Feedbacks can be combined into a boolean expression, considering for instance an execution as interesting if it triggers new code paths or execute in less time compared to the average execution time using [`feedback_or`](https://docs.rs/libafl/*/libafl/macro.feedback_or.html).
 
@@ -24,3 +29,6 @@ Using `feedback_and_fast` in combination with [`ConstFeedback`](https://docs.rs/
 While feedbacks are commonly used to decide if an [`Input`](https://docs.rs/libafl/*/libafl/inputs/trait.Input.html) should be kept for future mutations, they serve a double-purpose, as so-called `Objective Feedbacks`.
 In this case, the `interestingness` of a feedback indicates if an `Objective` has been hit.
 Commonly, these objectives would be a crash or a timeout, but they can also be used to detect if specific parts of the program have been reached, for sanitization, or a differential fuzzing success.
+Objectives use the same trait as a normal [`Feedback`](https://docs.rs/libafl/0/libafl/feedbacks/trait.Feedback.html) and the implementations can be used interchangeably.
+
+The only difference is that `interesting` Objectives won't be mutated further, and are counted as `Solutions`, a successful fuzzing campaign.

docs/src/design/metadata.md

@@ -5,10 +5,10 @@ A metadata in LibAFL is a self-contained structure that holds associated data to
 In terms of code, a metadata can be defined as a Rust struct registered in the SerdeAny register.
 
 ```rust
-extern crate libafl;
-extern crate serde;
+# extern crate libafl_bolts;
+# extern crate serde;
 
-use libafl::SerdeAny;
+use libafl_bolts::SerdeAny;
 use serde::{Serialize, Deserialize};
 
 #[derive(Debug, Serialize, Deserialize, SerdeAny)]
@@ -19,7 +19,7 @@ pub struct MyMetadata {
 
 The struct must be static, so it cannot hold references to borrowed objects.
 
-As an alternative to `derive(SerdeAny)` which is a proc-macro in `libafl_derive` the user can use `libafl::impl_serdeany!(MyMetadata);`.
+As an alternative to `derive(SerdeAny)` which is a proc-macro in `libafl_derive` the user can use `libafl_bolts::impl_serdeany!(MyMetadata);`.
 
 ## Usage
 

docs/src/design/migration-0.11.md

@@ -0,0 +1,27 @@
+# Migrating from LibAFL <0.11 to 0.11
+
+We moved the old `libafl::bolts` module to its own crate called `libafl_bolts`.
+For this, imports for types in LibAFL bolts have changed in version 0.11, everything else should remain the same.
+
+## Reasons for This Change
+
+With the change we can now use a lot of low-level features of LibAFL for projects that are unrelated to fuzzing, or just completely different to LibAFL.
+Some cross-platform things in bolts include
+
+* SerdeAnyMap: a map that stores and retrieves elements by type and is serializable and deserializable
+* ShMem: A cross-platform (Windows, Linux, Android, MacOS) shared memory implementation
+* LLMP: A fast, lock-free IPC mechanism via SharedMap
+* Core_affinity: A maintained version of `core_affinity` that can be used to get core information and bind processes to cores
+* Rands: Fast random number generators for fuzzing (like [RomuRand](http://www.romu-random.org/))
+* MiniBSOD: get and print information about the current process state including important registers.
+* Tuples: Haskel-like compile-time tuple lists
+* Os: OS specific stuff like signal handling, windows exception handling, pipes, and helpers for `fork`
+
+## What changed
+
+You will need to move all `libafl::bolts::` imports to `libafl_bolts:::` and add the crate dependency in your Cargo.toml (and specify feature flags there).
+As only exception, the `libafl::bolts::launcher::Launcher` has moved to `libafl::events::launcher::Launcher` since it has fuzzer and `EventManager` specific code.
+If you are using `prelude`, you may need to also ad `libafl_bolts::prelude`.
+
+That's it.
+Enjoy using `libafl_bolts` in other projects.

docs/src/getting_started/crates.md

@@ -31,6 +31,21 @@ You can choose the features by using `features = ["feature1", "feature2", ...]`
 Out of this list, by default, `std`, `derive`, and `rand_trait` are already set.
 You can choose to disable them by setting `default-features = false` in your `Cargo.toml`.
 
+## [`libafl_bolts](https://github.com/AFLplusplus/LibAFL/tree/main/libafl)
+
+The `libafl_bolts` create is a minimal tool shed filled with useful low-level rust features, not necessarily related to fuzzers.
+In it, you'll find highlights like:
+
+- `core_affinity` to bind the current process to cores
+- `SerdeAnyMap` a map that can store typed values in a serializable fashion
+- `minibsod` to dump the current process state
+- `LLMP`, "low level message passing", a lock-free IPC mechanism
+- `Rand`, different fast (non-cryptographically secure) RNG implementations like RomuRand
+- `ShMem`, a platform independent shard memory implementation
+- `Tuples`, a compiletime tuple implementation
+
+... and much more.
+
 ### libafl_sugar
 
 The sugar crate abstracts away most of the complexity of LibAFL's API.

fuzzers/baby_fuzzer/Cargo.toml

@@ -21,3 +21,4 @@ debug = true
 
 [dependencies]
 libafl = { path = "../../libafl/" }
+libafl_bolts = { path = "../../libafl_bolts/" }