diff --git a/Cargo.lock b/Cargo.lock index 00ee74ab3..5b2ecd7dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -349,6 +349,15 @@ dependencies = [ "serde", ] +[[package]] +name = "bump-scope" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4747cadbe8ffeff2a58cb86bcb35f9ec490f2329e12a88a696bccfd3dc97bf4d" +dependencies = [ + "allocator-api2", +] + [[package]] name = "bumpalo" version = "3.19.1" @@ -1295,6 +1304,7 @@ dependencies = [ name = "fspy_preload_unix" version = "0.0.0" dependencies = [ + "allocator-api2", "anyhow", "artifact_profile", "bstr", @@ -1303,6 +1313,7 @@ dependencies = [ "fspy_shared_unix", "libc", "nix 0.31.2", + "sigsafe", "wincode", ] @@ -3428,6 +3439,15 @@ dependencies = [ "libc", ] +[[package]] +name = "sigsafe" +version = "0.0.0" +dependencies = [ + "allocator-api2", + "bump-scope", + "rustix", +] + [[package]] name = "simd-adler32" version = "0.3.8" diff --git a/Cargo.toml b/Cargo.toml index ee6abee66..457ba5cc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ multiple_crate_versions = "allow" future_not_send = "allow" [workspace.dependencies] +allocator-api2 = { version = "0.2", default-features = false } artifact_profile = { path = "crates/artifact_profile" } anstream = "1.0.0" anyhow = "1.0.103" @@ -51,6 +52,7 @@ bindgen = "0.72.1" bitflags = "2.10.0" brush-parser = "0.4.0" bstr = { version = "1.12.0", default-features = false, features = ["alloc", "std"] } +bump-scope = { version = "2", default-features = false, features = ["allocator-api2-02"] } bumpalo = { version = "3.17.0", features = ["collections"] } bytemuck = { version = "1.23.0", features = ["extern_crate_alloc", "must_cast"] } cc = "1.2.39" @@ -120,6 +122,7 @@ ref-cast = "1.0.24" regex = "1.11.3" rusqlite = "0.39.0" rustc-hash = "2.1.1" +rustix = { version = "1", default-features = false, features = ["mm"] } # SeccompAction::UserNotif (SECCOMP_RET_USER_NOTIF) was added after the latest published release (v0.5.0) seccompiler = { git = "https://github.com/rust-vmm/seccompiler", rev = "08587106340b8e3cb361c7561411510039436857" } serde = "1.0.219" @@ -127,6 +130,7 @@ serde_json = "1.0.140" serde_norway = "0.9.42" sha2 = "0.11.0" shell-escape = "0.1.5" +sigsafe = { path = "crates/sigsafe" } similar = "3.0.0" smallvec = { version = "2.0.0-alpha.12", features = ["std"] } snapshot_test = { path = "crates/snapshot_test" } diff --git a/crates/fspy_preload_unix/Cargo.toml b/crates/fspy_preload_unix/Cargo.toml index 4b89bbd85..89ffaabdc 100644 --- a/crates/fspy_preload_unix/Cargo.toml +++ b/crates/fspy_preload_unix/Cargo.toml @@ -8,6 +8,7 @@ publish = false crate-type = ["cdylib"] [target.'cfg(unix)'.dependencies] +allocator-api2 = { workspace = true, features = ["alloc"] } anyhow = { workspace = true } wincode = { workspace = true } bstr = { workspace = true, default-features = false } @@ -16,6 +17,7 @@ fspy_shared = { workspace = true } fspy_shared_unix = { workspace = true } libc = { workspace = true } nix = { workspace = true, features = ["signal", "fs", "socket", "mman", "time"] } +sigsafe = { workspace = true } [build-dependencies] artifact_profile = { workspace = true } diff --git a/crates/fspy_preload_unix/src/client/raw_exec.rs b/crates/fspy_preload_unix/src/client/raw_exec.rs index 250f9e282..a4d1a6336 100644 --- a/crates/fspy_preload_unix/src/client/raw_exec.rs +++ b/crates/fspy_preload_unix/src/client/raw_exec.rs @@ -43,10 +43,17 @@ impl RawExec { mut strs: Vec, f: impl FnOnce(*const *const libc::c_char) -> R, ) -> R { - let mut ptr_vec = Vec::<*const libc::c_char>::with_capacity(strs.len() + 1); + // The pointer array exists only for the `f` call below, and building + // it must not go through libc malloc: exec runs in the child of + // `fork()` in multithreaded programs (`posix_spawn` forks then + // execs), where malloc's lock may be held by a thread that no longer + // exists. A per-call arena has exactly this lifetime, and hands back + // the memory when the call ends. + let arena = sigsafe::alloc::arena(); + let mut ptr_vec = allocator_api2::vec::Vec::with_capacity_in(strs.len() + 1, &arena); for s in &mut strs { s.push(0); - ptr_vec.push(s.as_ptr().cast()); + ptr_vec.push(s.as_ptr().cast::()); } ptr_vec.push(null()); f(ptr_vec.as_ptr()) diff --git a/crates/sigsafe/Cargo.toml b/crates/sigsafe/Cargo.toml new file mode 100644 index 000000000..9bd1af418 --- /dev/null +++ b/crates/sigsafe/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "sigsafe" +edition = "2024" +license.workspace = true +publish = false + +[lib] +doctest = false + +[target.'cfg(unix)'.dependencies] +allocator-api2 = { workspace = true } +bump-scope = { workspace = true } +rustix = { workspace = true } + +# On Linux the page size is probed from the kernel directly (see param.rs); +# rustix's `param` is only needed where sysconf is the platform interface. +[target.'cfg(all(unix, not(target_os = "linux")))'.dependencies] +rustix = { workspace = true, features = ["param"] } + +# The compile-time backend check in lib.rs needs a `linux_raw`-gated rustix +# item to reference; `runtime` is the module that has one. +[target.'cfg(target_os = "linux")'.dependencies] +rustix = { workspace = true, features = ["runtime"] } + +# Cross-validates the page-size probe against rustix's auxv-based answer. +[target.'cfg(target_os = "linux")'.dev-dependencies] +rustix = { workspace = true, features = ["param"] } + +[target.'cfg(unix)'.dev-dependencies] +# The `alloc` feature provides `Global`, letting tests run the pool against +# the host allocator (and thus under Miri). +allocator-api2 = { workspace = true, features = ["alloc"] } + +[lints] +workspace = true diff --git a/crates/sigsafe/README.md b/crates/sigsafe/README.md new file mode 100644 index 000000000..083c4adc6 --- /dev/null +++ b/crates/sigsafe/README.md @@ -0,0 +1,60 @@ +# sigsafe + +Unix syscall wrappers that are safe to call where libc is not. + +## Why this crate exists + +The fspy preload library injects itself into traced programs and intercepts +their libc calls (`open`, `stat`, `execve`, ...). POSIX declares those +functions async-signal-safe, so programs are allowed to call them: + +- inside a signal handler, +- in the child of `fork()` of a multithreaded program, +- while the process is still starting up, before libc is fully initialized. + +Most of libc is off limits in those places. `malloc` is the classic trap: a +signal can pause a thread while it holds malloc's lock, and `fork()` copies a +locked lock into a child that has no thread left to unlock it — the next +`malloc` waits forever. Interception code runs exactly there, so anything it +calls must work without libc's machinery, or the traced program can hang. + +## The rules + +Every function in this crate follows three rules: + +1. **Syscalls only.** On Linux, nothing goes through libc — the syscall + instructions are emitted directly (rustix's raw backend). On macOS there + is no stable syscall interface, so calls go through libSystem's wrappers; + for the calls exposed here those are thin stubs with no locks and no + state. +2. **No locks, no hidden state.** Nothing a signal or a `fork()` could catch + locked or half-written. Where shared state is unavoidable it is a fixed + set of atomics, each touched by single complete operations. +3. **No global allocation.** No function touches a heap behind the caller's + back. Code that needs memory gets it from an explicit allocator — + [`alloc`](src/alloc/mod.rs) provides one built on `mmap`. + +## How rule 1 is enforced on Linux + +rustix can be built with a libc backend instead of raw syscalls, and anything +in the dependency graph — including crates outside this repository — can +select it (the `rustix/use-libc` feature, or +`RUSTFLAGS=--cfg=rustix_use_libc`). No build script can detect that reliably, +so [`lib.rs`](src/lib.rs) checks at compile time instead: it references +`rustix::runtime`, a module that exists only in rustix's raw-syscall build. +Selecting the libc backend makes this crate fail to compile, rather than +silently losing the guarantee. + +## What's inside + +Functions whose rustix implementation already meets the rules are re-exposed +as-is; being listed in a module here is what marks a call as allowed, and the +backend check above is what keeps that true. + +- `mm` — anonymous memory mappings: `mmap_anonymous`, `munmap`. +- `param` — `page_size`. +- `alloc` — allocation without malloc: `alloc::arena()` gives one + intercepted call a bump arena that draws 64 KiB chunks from a process-wide + lock-free pool and returns them when the call ends. Taking or returning a + chunk is one atomic swap; when the pool is empty, chunks come straight + from the kernel through `mm`. diff --git a/crates/sigsafe/src/alloc/mmap.rs b/crates/sigsafe/src/alloc/mmap.rs new file mode 100644 index 000000000..40791d287 --- /dev/null +++ b/crates/sigsafe/src/alloc/mmap.rs @@ -0,0 +1,158 @@ +//! Page-granularity allocator backed by anonymous memory mappings. + +use core::{ + alloc::Layout, + ptr::{self, NonNull}, +}; + +use allocator_api2::alloc::{AllocError, Allocator}; + +use crate::{ + mm::{MapFlags, ProtFlags, mmap_anonymous, munmap}, + param::page_size, +}; + +/// A stateless allocator: every allocation is a fresh anonymous mapping and +/// every deallocation an `munmap`. +/// +/// # Why it is safe in signal handlers and forked children +/// +/// It holds no state at all — no locks, no free lists, no thread-locals; +/// the kernel does all the bookkeeping. A signal or a `fork()` can never +/// catch it holding a lock or a half-written structure, because there is +/// nothing to hold. The mapping calls and the page-size read come from +/// [`crate::mm`] and [`crate::param`], which carry the same guarantee (see +/// their docs). +/// +/// # What it accepts +/// +/// The only intended caller is `bump_scope::Bump`, and `Bump` only ever +/// asks its base allocator for chunks ([single call site][site]). Chunks +/// are never zero-sized (a chunk always contains its own header), and +/// their alignment is [`max(MIN_CHUNK_ALIGN, header alignment)`][chunk-align], +/// [where `MIN_CHUNK_ALIGN` is 16][mca] — so 16 bytes in practice. +/// +/// This allocator accepts more than that, because mapped memory gives the +/// extra range away for free: any non-zero size, and any alignment up to +/// the page size (mappings are always page-aligned). It refuses only two +/// kinds of request, which would each need extra code and never happen: +/// zero-sized layouts (they would need fake dangling blocks) and alignment +/// above the page size (it would need mapping extra space and trimming the +/// misaligned edges). The [`Allocator`] contract allows refusing any +/// request; refused requests get [`AllocError`]. +/// +/// # Cost +/// +/// Every allocation takes whole pages (4 KiB at least, 16 KiB on Apple +/// silicon) and one syscall, and so does every deallocation. This +/// allocator is meant to sit below a chunk pool and bump arenas, not to +/// serve small allocations directly. +/// +/// `allocate` returns the whole page-rounded block, and `bump_scope` [uses +/// the full returned length][fit], so none of the page is wasted. +/// +/// [mca]: https://docs.rs/bump-scope/2.3.3/src/bump_scope/chunk/size_config.rs.html#8 +/// [chunk-align]: https://docs.rs/bump-scope/2.3.3/src/bump_scope/chunk/size_config.rs.html#56-58 +/// [site]: https://docs.rs/bump-scope/2.3.3/src/bump_scope/raw_bump.rs.html#865 +/// [fit]: https://docs.rs/bump-scope/2.3.3/src/bump_scope/raw_bump.rs.html#873-884 +#[derive(Clone, Copy, Debug, Default)] +pub struct MmapAllocator; + +// SAFETY: returned blocks are non-null, page-aligned (at least +// `layout.align()` for every served layout), at least `layout.size()` bytes +// large (the returned length reports the exact mapped size), stay valid +// until deallocated, and distinct allocations never overlap. +unsafe impl Allocator for MmapAllocator { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + let page = page_size(); + // Outside the served request profile (see the type docs): refuse + // rather than carry an over-alignment or dangling-block code path. + if layout.size() == 0 || layout.align() > page { + return Err(AllocError); + } + // `checked_next_multiple_of` is total: `None` on overflow (already + // impossible — `Layout` caps sizes at `isize::MAX`) or a zero page + // size, with no power-of-two assumption to uphold. + let size = layout.size().checked_next_multiple_of(page).ok_or(AllocError)?; + // SAFETY: a fresh anonymous private mapping at no particular + // address has no memory-safety preconditions. + let ptr = unsafe { + mmap_anonymous( + ptr::null_mut(), + size, + ProtFlags::READ | ProtFlags::WRITE, + MapFlags::PRIVATE, + ) + } + .map_err(|_| AllocError)?; + // Mapping results are page-aligned, which covers every served + // layout. + NonNull::new(ptr.cast::()) + .map(|ptr| NonNull::slice_from_raw_parts(ptr, size)) + .ok_or(AllocError) + } + + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + // Fresh anonymous mappings are already zero-filled by the kernel. + self.allocate(layout) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + // The contract lets callers pass any size that fits the block, i.e. + // anything in `[requested, mapped]`; every such size rounds up to + // the mapped length. Zero-sized blocks are never allocated, so no + // dangling pointers can arrive here. Rounding cannot fail for a + // layout that fits a block we mapped; leaking the region is the + // safe response if it somehow did. + let Some(size) = layout.size().checked_next_multiple_of(page_size()) else { return }; + // SAFETY: caller contract — `ptr` was returned by `allocate` with a + // fitting layout and the block is no longer in use. Failure is + // impossible for a region we own. + let _ = unsafe { munmap(ptr.as_ptr().cast(), size) }; + } +} + +#[cfg(all(test, not(miri)))] +mod tests { + use super::*; + + #[test] + fn blocks_are_aligned_zeroed_and_page_rounded() { + for (size, align) in [(1, 1), (100, 64), (4096, 4096), (5 << 20, 8)] { + let layout = Layout::from_size_align(size, align).unwrap(); + let block = MmapAllocator.allocate(layout).unwrap(); + assert!(block.len() >= size, "size {size}"); + assert_eq!(block.len() % page_size(), 0); + assert_eq!(block.cast::().as_ptr().addr() % align, 0, "align {align}"); + for i in 0..block.len() { + // SAFETY: fresh exclusive block of `block.len()` bytes. + assert_eq!(unsafe { block.cast::().as_ptr().add(i).read() }, 0); + } + // SAFETY: fresh exclusive block of at least `size` bytes. + unsafe { block.cast::().as_ptr().write_bytes(0x5A, size) }; + // SAFETY: allocated above; the layout fits the block. + unsafe { MmapAllocator.deallocate(block.cast(), layout) }; + } + } + + #[test] + fn deallocate_accepts_any_fitting_size() { + let requested = Layout::from_size_align(100, 8).unwrap(); + let block = MmapAllocator.allocate(requested).unwrap(); + // Deallocate with the *returned* size instead of the requested one — + // both are within the fit range the contract allows. + let fitting = Layout::from_size_align(block.len(), 8).unwrap(); + // SAFETY: allocated above; `fitting` is within the block's fit range. + unsafe { MmapAllocator.deallocate(block.cast(), fitting) }; + } + + #[test] + fn out_of_profile_requests_are_refused() { + // Zero-sized and over-page-aligned layouts are outside the served + // request profile and must fail cleanly, not misbehave. + let zero = Layout::from_size_align(0, 16).unwrap(); + assert!(MmapAllocator.allocate(zero).is_err()); + let over_aligned = Layout::from_size_align(64, 1 << 24).unwrap(); + assert!(MmapAllocator.allocate(over_aligned).is_err()); + } +} diff --git a/crates/sigsafe/src/alloc/mod.rs b/crates/sigsafe/src/alloc/mod.rs new file mode 100644 index 000000000..1a815f36b --- /dev/null +++ b/crates/sigsafe/src/alloc/mod.rs @@ -0,0 +1,163 @@ +//! Allocation that never touches libc malloc. +//! +//! Taking malloc's lock is the classic way for interposed code to deadlock a +//! traced program (see the crate docs), so the preload library allocates +//! through this module instead. It stacks three layers and exposes only the +//! top one, [`arena`]. `MmapAllocator` is the bottom: a stateless allocator +//! where every allocation is a fresh anonymous mapping from [`crate::mm`]. +//! `ChunkPool` sits on top of it and caches fixed-size chunks, so that +//! frequent short tracing calls can reuse memory instead of paying two +//! syscalls per call. [`arena`] creates one `bump_scope::Bump` per +//! intercepted call, drawing its chunks from the process-wide pool and +//! returning them on drop. + +mod mmap; +mod pool; + +use allocator_api2::alloc::Allocator; +use bump_scope::{ + Bump, + alloc::compat::AllocatorApi2V02Compat, + settings::{BumpAllocatorSettings, BumpSettings}, +}; +use mmap::MmapAllocator; +use pool::ChunkPool; + +/// Every cached chunk is 64 KiB: a whole multiple of the page size on all +/// supported targets, and big enough that most intercepted calls fit their +/// allocations into a single chunk. [`ArenaSettings`] pins the arenas' own +/// chunk sizing to this same value. +const CHUNK_SIZE: usize = 64 * 1024; +/// The alignment chunks are allocated with. Must be at least the alignment +/// `bump_scope::Bump` uses for its chunk requests — 16 (see +/// [`MmapAllocator`]'s docs for the links). bump-scope does not export that +/// constant, so the `bump_chunk_requests_fit_the_pool_gates` test pins the +/// fit instead: it fails if a bump-scope upgrade ever requests chunks the +/// pool would refuse. +const CHUNK_ALIGN: usize = 16; +/// At most this many chunks stay cached, capping retained memory at +/// `SLOTS * CHUNK_SIZE` = 4 MiB. +const SLOTS: usize = 64; + +/// The process-wide chunk pool. `const`-initialized, so it works from the +/// first allocation on — even before any constructor has run. +static CHUNK_POOL: ChunkPool = ChunkPool::new(); + +/// The `Bump` settings the arenas use — the defaults, with two changes: +/// +/// - `WithGuaranteedAllocated`: an arena starts life without a chunk, +/// so creating one allocates nothing. +/// - `WithMinimumChunkSize`: the arena's first chunk request is +/// sized to the pool's chunks, making the coupling explicit — rather than +/// relying on the pool rounding the default 512-byte first request up to a +/// whole chunk. (Both end up serving the same memory: the pool answers any +/// request up to `CHUNK_SIZE` with a whole chunk, and `Bump` uses the full +/// returned length.) The request comes out slightly *under* `CHUNK_SIZE` — +/// bump-scope deducts an assumed allocator-header overhead from its +/// minimum — which is exactly what keeps a minimum-sized request within +/// the pool's `size <= CHUNK_SIZE` gate; the +/// `bump_chunk_requests_fit_the_pool_gates` test pins that fit. +type ArenaSettings = <::WithGuaranteedAllocated as BumpAllocatorSettings>::WithMinimumChunkSize; + +/// `Bump::unallocated` requires its base allocator to implement `Default` +/// (an arena without chunks has nowhere to store an allocator value, so it +/// conjures one on first use). Point defaulted references at the +/// process-wide pool. As an allocator, `&ChunkPool` already works through +/// allocator-api2's blanket `impl Allocator for &A`. +impl Default for &'static ChunkPool { + fn default() -> Self { + &CHUNK_POOL + } +} + +/// Creates a fresh bump arena for one intercepted call, backed by the +/// process-wide chunk pool. +/// +/// Creating the arena allocates nothing; the first allocation grabs a whole +/// chunk — usually a recycled one, so most calls touch no syscalls at all. +/// Deallocation only takes back the most recent allocation (bump-arena +/// semantics); everything is freed at once when the arena is dropped, and +/// its chunks go back to the pool. +/// +/// The arena itself is single-owner — use one per call, do not share it +/// across threads. Creating one is safe anywhere, any time: the pool +/// underneath works in signal handlers and in the child of `fork()` (see +/// `ChunkPool` and `MmapAllocator` in this crate's source for why). +#[must_use] +pub fn arena() -> impl Allocator { + Bump::< + AllocatorApi2V02Compat<&'static ChunkPool>, + ArenaSettings, + >::unallocated() +} + +#[cfg(test)] +mod tests { + use core::alloc::Layout; + + use allocator_api2::alloc::Global; + + use super::*; + + /// Pins the fit between `Bump`'s chunk requests and the pool's gates + /// (alignment at most [`CHUNK_ALIGN`], size at most [`CHUNK_SIZE`]), + /// under the same [`ArenaSettings`] the arenas use — including that a + /// request under `WithMinimumChunkSize` still fits the + /// `size <= CHUNK_SIZE` gate. bump-scope keeps its request parameters + /// private, so this test is the enforcement: it fails if an upgrade + /// ever changes them. + /// [`ArenaSettings`] with `GuaranteedAllocated` flipped back on: without + /// it, `Bump` demands a `Default` base allocator, and a reference to the + /// test's stack-local pool cannot provide one. Chunk request sizing — + /// what the test pins — is unaffected by that flag. + type TestSettings = ::WithGuaranteedAllocated; + + #[test] + fn bump_chunk_requests_fit_the_pool_gates() { + let pool = ChunkPool::::new_in(Global); + // `try_new_in` requests the first chunk right away, at the settings' + // minimum chunk size; if that request were over-aligned or + // oversized, the pool would refuse and this would be an error. + let bump: Bump< + AllocatorApi2V02Compat<&ChunkPool>, + TestSettings, + > = Bump::try_new_in(AllocatorApi2V02Compat(&pool)).unwrap(); + let block = bump.allocate(Layout::from_size_align(100, 8).unwrap()).unwrap(); + let block_addr = block.cast::().as_ptr().addr(); + drop(bump); + + // The chunk went back into the pool's cache (proving it was served + // as a chunk, not passed through): the next chunk-sized request + // returns the block's surroundings. + let recycled = pool.allocate(Layout::from_size_align(100, 8).unwrap()).unwrap(); + assert_eq!(recycled.len(), CHUNK_SIZE); + let base = recycled.cast::().as_ptr().addr(); + assert!((base..base + CHUNK_SIZE).contains(&block_addr)); + // SAFETY: allocated above with a fitting layout. + unsafe { pool.deallocate(recycled.cast(), Layout::from_size_align(100, 8).unwrap()) }; + } + + #[test] + #[cfg(not(miri))] + fn arena_allocates_and_returns_chunks_to_the_pool() { + let layout = Layout::from_size_align(100, 8).unwrap(); + + let first_arena = arena(); + let first = first_arena.allocate(layout).unwrap(); + assert!(first.len() >= 100); + // SAFETY: fresh exclusive block of at least 100 bytes. + unsafe { first.cast::().as_ptr().write_bytes(0x5A, 100) }; + let second = first_arena.allocate(layout).unwrap(); + assert_ne!(first.cast::().as_ptr().addr(), second.cast::().as_ptr().addr()); + let first_addr = first.cast::().as_ptr().addr(); + // Everything dies at once; the chunk goes back to the pool. + drop(first_arena); + + // No other test touches the process-wide pool, so a new arena draws + // the same chunk back and its first allocation lands at the same + // address. + let second_arena = arena(); + let again = second_arena.allocate(layout).unwrap(); + assert_eq!(again.cast::().as_ptr().addr(), first_addr); + } +} diff --git a/crates/sigsafe/src/alloc/pool.rs b/crates/sigsafe/src/alloc/pool.rs new file mode 100644 index 000000000..3dcf31573 --- /dev/null +++ b/crates/sigsafe/src/alloc/pool.rs @@ -0,0 +1,363 @@ +//! A small cache of memory chunks between the bump arenas and the kernel. + +use core::{ + alloc::Layout, + ptr::{self, NonNull}, + sync::atomic::{AtomicPtr, Ordering}, +}; + +use allocator_api2::alloc::{AllocError, Allocator}; + +use super::mmap::MmapAllocator; + +/// A fixed-size cache of memory chunks. +/// +/// Sits between the bump arenas and the underlying allocator `A` — +/// [`MmapAllocator`] in real use, any allocator in tests — so the chunks +/// that a finished arena gives back reach the next arena without going back +/// to the kernel. +/// +/// The parameters are chosen by the layer above (the arena layer): every +/// cached chunk is exactly `CHUNK_SIZE` bytes and allocated with +/// `CHUNK_ALIGN`, which is also the strictest alignment the pool accepts +/// (`bump_scope::Bump` requests 16 — see [`MmapAllocator`]'s docs for the +/// links). The cache holds at most `SLOTS` chunks, so at most +/// `SLOTS * CHUNK_SIZE` bytes stay retained. +/// +/// Requests of up to `CHUNK_SIZE` bytes are served with a whole chunk — +/// a cached one, or a fresh one when the cache is empty. Bigger requests go +/// straight to the underlying allocator. Deallocation mirrors that: chunks +/// go back into the cache (or to the underlying allocator when it is full), +/// bigger blocks go to the underlying allocator directly. The size passed +/// to `deallocate` tells the two apart exactly: for a block served as a +/// chunk, every size the caller may legally pass is at most `CHUNK_SIZE`; +/// for a pass-through block, every legal size is bigger. +/// +/// Zero-sized requests and alignments above `CHUNK_ALIGN` are refused — +/// its only intended caller, `bump_scope::Bump`, never sends either. +/// +/// # Why it is safe in signal handlers and forked children +/// +/// The only state of the pool itself is a fixed array of atomic pointers, +/// one slot per cached chunk. Taking a chunk swaps a slot to null; +/// returning one swaps a null slot back. Every operation visits each slot +/// at most once and never waits — there are no locks to leave locked and no +/// lists to leave half-linked, so a thread that disappears mid-operation +/// (`fork`, a signal) can strand at most the one chunk it was holding, +/// never the pool. (The underlying allocator must give the same guarantee; +/// [`MmapAllocator`] does.) +/// +/// The pool is `const`-constructible, so it can live in a `static` and +/// work before anything else has run. +pub struct ChunkPool< + A: Allocator, + const CHUNK_SIZE: usize, + const CHUNK_ALIGN: usize, + const SLOTS: usize, +> { + slots: [AtomicPtr; SLOTS], + allocator: A, +} + +impl + ChunkPool +{ + #[must_use] + pub const fn new() -> Self { + Self::new_in(MmapAllocator) + } +} + +impl + ChunkPool +{ + pub const fn new_in(allocator: A) -> Self { + const { + assert!(CHUNK_SIZE > 0, "CHUNK_SIZE must not be zero"); + assert!(CHUNK_ALIGN.is_power_of_two(), "CHUNK_ALIGN must be a power of two"); + } + Self { slots: [const { AtomicPtr::new(ptr::null_mut()) }; SLOTS], allocator } + } + + fn chunk_layout() -> Result { + Layout::from_size_align(CHUNK_SIZE, CHUNK_ALIGN).map_err(|_| AllocError) + } +} + +impl + Default for ChunkPool +{ + fn default() -> Self { + Self::new_in(A::default()) + } +} + +// SAFETY: returned blocks are non-null, aligned to at least the accepted +// `layout.align()` (chunks are allocated with CHUNK_ALIGN, and stricter +// requests are refused), and at least `layout.size()` bytes large (chunks +// are exactly CHUNK_SIZE bytes and only serve requests up to that size; +// everything else comes from the underlying allocator with its own +// guarantees). A chunk leaves its slot before it is handed out and returns +// only on deallocation, so every block has exactly one owner and stays +// valid until deallocated. +unsafe impl + Allocator for ChunkPool +{ + fn allocate(&self, layout: Layout) -> Result, AllocError> { + if layout.size() == 0 || layout.align() > CHUNK_ALIGN { + return Err(AllocError); + } + if layout.size() > CHUNK_SIZE { + return self.allocator.allocate(layout); + } + for slot in &self.slots { + if slot.load(Ordering::Relaxed).is_null() { + continue; + } + // Acquire pairs with the Release in `deallocate`; the swap makes + // this thread the chunk's only owner. A racing taker may empty + // the slot between the load and the swap — then the swap returns + // null and the scan moves on. + let chunk = slot.swap(ptr::null_mut(), Ordering::Acquire); + if let Some(chunk) = NonNull::new(chunk) { + return Ok(NonNull::slice_from_raw_parts(chunk, CHUNK_SIZE)); + } + } + // Cache empty: get a fresh chunk. + self.allocator.allocate(Self::chunk_layout()?) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + if layout.size() > CHUNK_SIZE { + // A pass-through block: only requests bigger than CHUNK_SIZE get + // one, and all their legal deallocation sizes are bigger too. + // SAFETY: caller contract — the block came from `allocate` + // (which forwarded to the underlying allocator) with a fitting + // layout. + return unsafe { self.allocator.deallocate(ptr, layout) }; + } + // A chunk: put it back into an empty slot, or give it back to the + // underlying allocator if the cache is full. Release pairs with the + // Acquire in `allocate`. A failed exchange means another thread just + // filled the slot; move on. + for slot in &self.slots { + if !slot.load(Ordering::Relaxed).is_null() { + continue; + } + if slot + .compare_exchange( + ptr::null_mut(), + ptr.as_ptr(), + Ordering::Release, + Ordering::Relaxed, + ) + .is_ok() + { + return; + } + } + let Ok(chunk_layout) = Self::chunk_layout() else { return }; + // SAFETY: every block in this branch is a CHUNK_SIZE chunk that came + // from the underlying allocator with `chunk_layout` (caller contract + // plus the size routing above), and it no longer has an owner. + unsafe { self.allocator.deallocate(ptr, chunk_layout) }; + } +} + +impl Drop + for ChunkPool +{ + fn drop(&mut self) { + // Statics never drop; this runs for pools created in tests. + for slot in &mut self.slots { + let chunk = core::mem::replace(slot.get_mut(), ptr::null_mut()); + if let Some(chunk) = NonNull::new(chunk) { + let Ok(chunk_layout) = Self::chunk_layout() else { return }; + // SAFETY: cached chunks came from the underlying allocator + // with `chunk_layout`, and `&mut self` means no owner exists. + unsafe { self.allocator.deallocate(chunk, chunk_layout) }; + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{thread, vec::Vec}; + + use allocator_api2::alloc::Global; + + use super::*; + + const CHUNK_SIZE: usize = 64 * 1024; + const CHUNK_ALIGN: usize = 16; + + fn pool() -> ChunkPool { + ChunkPool::new_in(Global) + } + + fn layout(size: usize) -> Layout { + Layout::from_size_align(size, 8).unwrap() + } + + #[test] + fn small_requests_get_whole_recycled_chunks() { + let pool = pool(); + let first = pool.allocate(layout(100)).unwrap(); + assert_eq!(first.len(), CHUNK_SIZE); + let first_addr = first.cast::().as_ptr().addr(); + // SAFETY: fresh exclusive block of at least 100 bytes. + unsafe { first.cast::().as_ptr().write_bytes(0xAB, 100) }; + // SAFETY: allocated above; the full returned size is a legal size to + // pass back. + unsafe { pool.deallocate(first.cast(), layout(CHUNK_SIZE)) }; + + // The chunk is cached now; the next small request must get it back. + let second = pool.allocate(layout(5000)).unwrap(); + assert_eq!(second.cast::().as_ptr().addr(), first_addr); + assert_eq!(second.len(), CHUNK_SIZE); + // SAFETY: allocated above with a fitting layout. + unsafe { pool.deallocate(second.cast(), layout(5000)) }; + } + + #[test] + fn deallocating_with_the_requested_size_also_recycles() { + let pool = pool(); + let block = pool.allocate(layout(100)).unwrap(); + let addr = block.cast::().as_ptr().addr(); + // SAFETY: allocated above; the requested size is a legal size to + // pass back, and must still be recognized as a chunk. + unsafe { pool.deallocate(block.cast(), layout(100)) }; + let again = pool.allocate(layout(100)).unwrap(); + assert_eq!(again.cast::().as_ptr().addr(), addr); + // SAFETY: allocated above with the same layout. + unsafe { pool.deallocate(again.cast(), layout(100)) }; + } + + #[test] + fn big_requests_bypass_the_cache() { + let pool = pool(); + let big = pool.allocate(layout(CHUNK_SIZE + 1)).unwrap(); + assert!(big.len() > CHUNK_SIZE); + // SAFETY: fresh exclusive block of at least CHUNK_SIZE + 1 + // bytes. + unsafe { big.cast::().as_ptr().write_bytes(0x5A, CHUNK_SIZE + 1) }; + // SAFETY: allocated above with the same layout. + unsafe { pool.deallocate(big.cast(), layout(CHUNK_SIZE + 1)) }; + + // A small request afterwards gets a chunk, not the big block. + let small = pool.allocate(layout(100)).unwrap(); + assert_eq!(small.len(), CHUNK_SIZE); + // SAFETY: allocated above with the same layout. + unsafe { pool.deallocate(small.cast(), layout(100)) }; + } + + #[test] + fn refuses_zero_size_and_over_chunk_alignment() { + let pool = pool(); + let zero = Layout::from_size_align(0, 16).unwrap(); + assert!(pool.allocate(zero).is_err()); + let over_aligned = Layout::from_size_align(64, CHUNK_ALIGN * 2).unwrap(); + assert!(pool.allocate(over_aligned).is_err()); + } + + #[test] + fn allocate_zeroed_scrubs_recycled_chunks() { + let pool = pool(); + let block = pool.allocate(layout(256)).unwrap(); + // SAFETY: fresh exclusive block of at least 256 bytes. + unsafe { block.cast::().as_ptr().write_bytes(0xFF, 256) }; + // SAFETY: allocated above with a fitting layout. + unsafe { pool.deallocate(block.cast(), layout(256)) }; + + // Recycled chunks are dirty, so the default `allocate_zeroed` (which + // we deliberately do not override here) must scrub them. + let zeroed = pool.allocate_zeroed(layout(256)).unwrap(); + for i in 0..256 { + // SAFETY: fresh exclusive block of at least 256 bytes. + assert_eq!(unsafe { zeroed.cast::().as_ptr().add(i).read() }, 0); + } + // SAFETY: allocated above with a fitting layout. + unsafe { pool.deallocate(zeroed.cast(), layout(256)) }; + } + + #[test] + fn overflowing_the_cache_is_fine() { + // A tiny 4-slot pool so the full-cache path is genuinely exercised. + let pool = ChunkPool::::new_in(Global); + for _ in 0..2 { + let blocks: Vec<_> = (0..8).map(|_| pool.allocate(layout(1000)).unwrap()).collect(); + for block in blocks { + // SAFETY: allocated above with a fitting layout. The first 4 + // chunks fill the cache; the rest go back to the underlying + // allocator. + unsafe { pool.deallocate(block.cast(), layout(1000)) }; + } + } + } + + #[test] + fn custom_chunk_size_and_alignment_work() { + let pool = ChunkPool::::new_in(Global); + let block = pool.allocate(layout(10)).unwrap(); + assert_eq!(block.len(), 1024); + assert_eq!(block.cast::().as_ptr().addr() % 64, 0); + // SAFETY: allocated above with a fitting layout. + unsafe { pool.deallocate(block.cast(), layout(10)) }; + + // Bigger than this pool's chunk size: passes through. + let big = pool.allocate(layout(2000)).unwrap(); + assert!(big.len() >= 2000); + // SAFETY: allocated above with the same layout. + unsafe { pool.deallocate(big.cast(), layout(2000)) }; + } + + #[test] + fn concurrent_use_smoke() { + const OPS: usize = if cfg!(miri) { 40 } else { 500 }; + + let pool = pool(); + thread::scope(|scope| { + for t in 0..4_usize { + let pool = &pool; + scope.spawn(move || { + for i in 0..OPS { + let size = 1 + (i * 37 + t * 101) % (2 * CHUNK_SIZE); + let l = Layout::from_size_align(size, 8).unwrap(); + let block = pool.allocate(l).unwrap(); + assert!(block.len() >= size); + let marker = u8::try_from((t * 50 + i) % 251).unwrap(); + // SAFETY: exclusive fresh block of at least `size` + // bytes, allocated above with layout `l`. + unsafe { + block.cast::().as_ptr().write(marker); + block.cast::().as_ptr().add(size - 1).write(marker); + assert_eq!(block.cast::().as_ptr().read(), marker); + assert_eq!(block.cast::().as_ptr().add(size - 1).read(), marker); + pool.deallocate(block.cast(), l); + } + } + }); + } + }); + } + + /// The one test against the real kernel-backed default; everything else + /// runs against `Global` so Miri can check it. + #[test] + #[cfg(not(miri))] + fn mmap_backed_pool_smoke() { + let pool = ChunkPool::::new(); + let block = pool.allocate(layout(100)).unwrap(); + assert_eq!(block.len(), CHUNK_SIZE); + let addr = block.cast::().as_ptr().addr(); + // SAFETY: fresh exclusive block of at least 100 bytes. + unsafe { block.cast::().as_ptr().write_bytes(0xCD, 100) }; + // SAFETY: allocated above with a fitting layout. + unsafe { pool.deallocate(block.cast(), layout(100)) }; + let again = pool.allocate(layout(200)).unwrap(); + assert_eq!(again.cast::().as_ptr().addr(), addr); + // SAFETY: allocated above with a fitting layout. + unsafe { pool.deallocate(again.cast(), layout(200)) }; + } +} diff --git a/crates/sigsafe/src/lib.rs b/crates/sigsafe/src/lib.rs new file mode 100644 index 000000000..9ee868345 --- /dev/null +++ b/crates/sigsafe/src/lib.rs @@ -0,0 +1,40 @@ +//! Unix syscall wrappers that are safe to call where libc is not: in signal +//! handlers, in the child of `fork()` in a multithreaded process, and before +//! libc has finished initializing. +//! +//! The fspy preload library interposes libc functions that POSIX declares +//! async-signal-safe (`open`, `stat`, `execve`, ...), so its code runs in all +//! of those places, where libc's own machinery — locks, lazy initialization, +//! malloc — is off limits. Everything this crate exposes follows three rules: +//! syscalls only (never through libc on Linux), no locks and no hidden state, +//! and no global allocation. See README.md for the full approach. + +// Compile as an empty crate on non-unix targets: the crate backs the unix +// preload library. +#![cfg(unix)] +#![cfg_attr(not(test), no_std)] + +pub mod alloc; +pub mod mm; +pub mod param; + +pub use rustix::io::Errno; + +// Compile-time proof that rustix uses its raw-syscall backend (`linux_raw`) +// on Linux — and with it, that no call in this crate goes through libc there. +// `rustix::runtime` is gated on that backend (`#[cfg(linux_raw)]`), so this +// reference fails to resolve, failing the whole build, whenever anything +// selects the libc backend instead: the `rustix/use-libc` feature (which any +// crate in the dependency graph can enable through feature unification, where +// no build script could ever see it), `RUSTFLAGS=--cfg=rustix_use_libc`, or a +// target rustix has no raw backend for. Miri also forces the libc backend, so +// it is exempted: it type-checks rather than ships code. +// +// The module's contents are not covered by rustix's stability promise, so a +// rustix upgrade may break this line. If that happens, re-point it at any +// other `rustix::runtime` item — do not delete it: it is the only enforcement +// of the no-libc rule above. +#[cfg(all(target_os = "linux", not(miri)))] +const _: () = { + let _ = rustix::runtime::exit_group; +}; diff --git a/crates/sigsafe/src/mm.rs b/crates/sigsafe/src/mm.rs new file mode 100644 index 000000000..c7cdecd05 --- /dev/null +++ b/crates/sigsafe/src/mm.rs @@ -0,0 +1,11 @@ +//! Anonymous memory mappings. +//! +//! Re-exposed from rustix as-is: these are single syscalls against the +//! kernel's own address-space bookkeeping — no libc state, no locks, no +//! allocation — so they already meet this crate's rules everywhere it +//! promises to work. What this module adds is the curation (being listed +//! here is what marks them safe for signal handlers, fork children, and +//! pre-libc startup) and the crate-level backend check, which guarantees +//! they cannot silently turn into libc calls on Linux. + +pub use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect, munmap}; diff --git a/crates/sigsafe/src/param.rs b/crates/sigsafe/src/param.rs new file mode 100644 index 000000000..0e907689a --- /dev/null +++ b/crates/sigsafe/src/param.rs @@ -0,0 +1,147 @@ +//! Process parameters. +//! +//! On Linux, `page_size` asks the kernel itself — this crate stands in for +//! libc, so it cannot lean on libc's startup knowledge. The kernel hands a +//! process its auxiliary vector exactly once, above the stack pointer at +//! entry, and only the code owning the entry point (a libc, or ld.so) sees +//! it; the retroactive interfaces (`prctl(PR_GET_AUXV)`, `/proc/self/auxv`) +//! need a 6.4 kernel or a mounted `/proc`. So instead of the auxiliary +//! vector, the value is learned from syscall behavior that any kernel +//! version guarantees: see [`linux::page_size`]. +//! +//! On other unix platforms (macOS, where every syscall goes through +//! libSystem by platform contract anyway) it is rustix's `sysconf`, a +//! lock-free read of startup data. + +#[cfg(target_os = "linux")] +pub use linux::page_size; +#[cfg(not(target_os = "linux"))] +pub use rustix::param::page_size; + +#[cfg(target_os = "linux")] +mod linux { + use core::{ + ptr, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use crate::mm::{self, MapFlags, MprotectFlags, ProtFlags}; + + /// Smallest page size of any Linux configuration; the probe starts + /// here. + const MIN_PROBE_PAGE: usize = 4096; + /// Generous upper bound for the probe; no supported configuration uses + /// larger pages. + const MAX_PROBE_PAGE: usize = 1 << 20; + + /// Returns the size of a memory page, or 0 if it cannot be determined. + /// + /// Discovered from the kernel with a handful of raw syscalls, once per + /// process, and cached in an atomic: no libc, no auxiliary vector, no + /// `/proc`, no minimum kernel version, no allocation, and no init to + /// call first — safe from the first instruction on, in signal handlers, + /// and in the child of `fork()`. + /// + /// Zero — the kernel refusing a scratch mapping, i.e. out of memory — + /// is not cached, so a later call retries; callers treat it as "fail + /// this operation" (an allocator refuses the allocation). + #[must_use] + pub fn page_size() -> usize { + static CACHE: AtomicUsize = AtomicUsize::new(0); + let cached = CACHE.load(Ordering::Relaxed); + if cached != 0 { + return cached; + } + let page = probe_page_size(); + if page != 0 { + // Every probe returns the same value; the cache only avoids + // repeated probing. + CACHE.store(page, Ordering::Relaxed); + } + page + } + + /// Discovers the page size by probing, using nothing but raw syscalls. + /// + /// `mprotect` fails with `EINVAL` unless its address is a multiple of + /// the page size, and `mmap` returns page-aligned bases — both + /// kernel-guaranteed on every version. So re-protecting a scratch + /// mapping at increasing power-of-two offsets identifies the page size + /// as the first offset the kernel accepts: for a power-of-two page + /// size P, the first power of two P divides is P itself. + /// + /// Why not `rustix::param::page_size()` (fine on macOS, used there)? + /// Its Linux sources are exactly the ones this crate must not assume: + /// with `use-libc-auxv` it is libc's `sysconf`; without, its lazy init + /// needs `prctl(PR_GET_AUXV)` (kernel 6.4+) or `/proc/self/auxv`, it + /// panics when both are unavailable, and with rustix's `alloc` feature + /// enabled — which Cargo feature unification lets any other rustix + /// user in the build graph turn on for our copy — the `/proc` path + /// heap-allocates, which is forbidden in the contexts this crate + /// serves. The probe has none of those modes: no allocation, no panic, + /// no minimum kernel, and its worst case is 0. + #[cold] + fn probe_page_size() -> usize { + // Large enough that every probe below stays inside the mapping + // even after the kernel rounds the one-byte length up to a full + // page. + const SCRATCH_LEN: usize = 2 * MAX_PROBE_PAGE; + + // SAFETY: a fresh anonymous private mapping at no particular + // address has no memory-safety preconditions. + let Ok(base) = (unsafe { + mm::mmap_anonymous( + ptr::null_mut(), + SCRATCH_LEN, + ProtFlags::READ | ProtFlags::WRITE, + MapFlags::PRIVATE, + ) + }) else { + return 0; + }; + let mut page = 0; + let mut offset = MIN_PROBE_PAGE; + while offset <= MAX_PROBE_PAGE { + // SAFETY: `base + offset` (plus the page-rounded single byte) + // lies within the scratch mapping, which we exclusively own; + // the protection flags match the mapping's existing ones. + let accepted = unsafe { + mm::mprotect( + base.cast::().add(offset).cast(), + 1, + MprotectFlags::READ | MprotectFlags::WRITE, + ) + } + .is_ok(); + if accepted { + page = offset; + break; + } + offset *= 2; + } + // SAFETY: releasing the scratch mapping created above; nothing + // uses it afterwards. + let _ = unsafe { mm::munmap(base, SCRATCH_LEN) }; + page + } + + #[cfg(test)] + mod tests { + use super::*; + + /// Cross-validates the probe against rustix's auxv-based answer + /// (a dev-dependency here; CI Linux runners always have `/proc`). + #[test] + fn probe_matches_rustix() { + assert_eq!(page_size(), rustix::param::page_size()); + #[cfg(target_arch = "x86_64")] + assert_eq!(page_size(), 4096); + } + + #[test] + fn probe_is_stable_and_cached() { + assert_eq!(probe_page_size(), page_size()); + assert_eq!(page_size(), page_size()); + } + } +}