From f2c776511a332aa6b8ad552cb80c3e00c13cb63a Mon Sep 17 00:00:00 2001 From: Sangho Lee Date: Fri, 31 Jul 2026 22:17:10 +0000 Subject: [PATCH] Support code rewriting for AArch64 Linux guests Redirects every `SVC` and `TPIDR_EL0` access through compact self-describing slots (64 bytes for `SVC`, 48 for `MSR`, 16 for `MRS`) carrying typed metadata in their final word, so a signal handler can recognize one by template alone. `SVC` gating is guest-neutral, but `TPIDR_EL0` is the Linux guest ABI's thread pointer, so this covers ELF guests only. Per-site outbound stubs preserve `x16` on syscall return, and guest thread-pointer offsets are patched in only after strict slot validation. x86-64 rewriting is unchanged. Adds the crate to the AArch64 CI job. --- .github/workflows/ci.yml | 15 +- litebox_syscall_rewriter/src/arm64.rs | 2869 ++++++++++++++--- litebox_syscall_rewriter/src/lib.rs | 1024 +++++- .../tests/aarch64_tests.rs | 25 +- .../tests/snapshot_tests.rs | 49 +- .../snapshot_tests__hello-aarch64-diff.snap | 8 +- 6 files changed, 3388 insertions(+), 602 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 156f92697b..b00b88e59d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,15 @@ jobs: runs-on: ubuntu-24.04-arm env: RUSTFLAGS: -Dwarnings + # Crates that build and run on AArch64. Linux-on-Linux userland is the + # only configuration AArch64 supports, so the other runners (LVBS, SNP) + # and shims (Windows, OP-TEE) are left to the x86-64 job. This list grows + # as each crate gains AArch64 support, so that the PR adding that support + # is the PR this job starts covering it. + AARCH64_CRATES: >- + -p litebox + -p litebox_common_linux + -p litebox_syscall_rewriter steps: - name: Check out repo uses: actions/checkout@v6 @@ -102,9 +111,9 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: ./.github/tools/github_actions_run_cargo fmt - run: | - ./.github/tools/github_actions_run_cargo clippy --all-targets --all-features -p litebox -p litebox_common_linux - ./.github/tools/github_actions_run_cargo build -p litebox -p litebox_common_linux - ./.github/tools/github_actions_run_cargo nextest -p litebox -p litebox_common_linux + ./.github/tools/github_actions_run_cargo clippy --all-targets --all-features $AARCH64_CRATES + ./.github/tools/github_actions_run_cargo build $AARCH64_CRATES + ./.github/tools/github_actions_run_cargo nextest $AARCH64_CRATES build_and_test_lvbs: name: Build and Test LVBS diff --git a/litebox_syscall_rewriter/src/arm64.rs b/litebox_syscall_rewriter/src/arm64.rs index 1dbd32e397..b8b1fc56c9 100644 --- a/litebox_syscall_rewriter/src/arm64.rs +++ b/litebox_syscall_rewriter/src/arm64.rs @@ -3,132 +3,97 @@ //! AArch64 (ARM64) syscall rewriting support for Linux ELF binaries. //! -//! Every AArch64 instruction is 4 bytes including a direct branch (`B imm26`) -//! with a ±128MB range. This lets us replace a single instruction with -//! a branch into the trampoline without instruction borrowing. +//! Instructions are a fixed 4 bytes and `B imm26` reaches ±128MB, so a site is +//! replaced in place by a branch into its trampoline gate. A site out of that +//! reach becomes `BRK #TRAP_BRK_IMM` and is reported as trapped, which makes +//! the ELF-level caller reject the binary with `Error::UnpatchableSyscalls`. +//! Executing the `BRK` faults the guest instead of letting an unpatched +//! instruction reach the host kernel. //! -//! The trampoline is placed just past the highest mapped segment, so every -//! site-to-gate branch points forward. A site farther than the `B imm26` -//! ±128MB reach from its gate cannot redirect; it is replaced with a sentinel -//! `BRK #TRAP_BRK_IMM` and reported as a trapped site. Any trapped site makes -//! the rewrite incomplete, so the ELF-level caller rejects the binary with -//! `Error::UnpatchableSyscalls`, mirroring the x86-64 unpatchable-syscall path. -//! Executing the `BRK` raises a synchronous debug exception, so a trapped site -//! faults the guest rather than letting the unpatched instruction escape to the -//! host kernel. Recognizing the `TRAP_BRK_IMM` immediate in the runtime — to -//! attribute the trap to the rewriter rather than a guest breakpoint — is -//! planned but not yet implemented. +//! Gated forms: //! -//! ### Assumption: executable sections contain only instructions -//! -//! The patch scan walks each executable section word-by-word and treats every -//! 4-byte word that matches the `SVC`/`MSR TPIDR_EL0`/`MRS TPIDR_EL0` bit -//! patterns as that instruction. It does **not** distinguish inline data — literal -//! pools or jump tables embedded in `.text` — from code, because a fixed-width -//! decode cannot tell a data word from an instruction with the same bits. In -//! practice this is safe: default AArch64 codegen places constants in `.rodata`, -//! not `.text`, and the odds of an unrelated data word colliding with these -//! patterns are tiny. A binary that stores such a word inside an executable section -//! would have it rewritten; bounding the scan to symbol-defined function ranges -//! (via `STT_FUNC` extents) would remove the assumption (TODO). +//! * `SVC #imm` — syscall. The gate records the return address and tail-jumps +//! through the trampoline header's callback pointer. +//! * `MSR TPIDR_EL0, Xn` — thread-pointer write, stored to the guest slot. +//! * `MRS Xd, TPIDR_EL0` — thread-pointer read, loaded from it. `MRS XZR, +//! TPIDR_EL0` is a discarded read and is left native. //! -//! Two kinds of access are involved, three forms of instruction gated: +//! ### Assumption: executable sections contain only instructions //! -//! * `SVC #imm` — the syscall instruction (any immediate; Linux ignores it). -//! Replaced with a branch to a per-site *SVC gate* that records the return -//! address and falls through to the shared SVC handler, a thin shim that -//! tail-jumps to the syscall callback. -//! * `MSR TPIDR_EL0, Xn` — a write to the thread pointer. Replaced with a branch -//! to a per-site *MSR gate* that stores the guest value into the guest -//! thread-pointer slot at `[TPIDR_EL0 + GUEST_TPIDR_OFFSET]`. -//! * `MRS Xd, TPIDR_EL0` — a read of the thread pointer. Replaced with a branch -//! to a per-site *MRS gate* that loads the guest value from the same slot. -//! `MRS XZR, TPIDR_EL0` is a discarded read and is left native. +//! The scan walks each executable section word-by-word and cannot tell inline +//! data — literal pools, jump tables — from code, so a data word matching a +//! gated encoding inside `.text` is rewritten. Default AArch64 codegen puts +//! constants in `.rodata`. TODO: bound the scan to `STT_FUNC` extents. //! //! ## Thread-pointer virtualization //! -//! The host owns the hardware `TPIDR_EL0` as a per-thread anchor; the guest's -//! logical thread pointer is a host-managed memory slot at `[TPIDR_EL0 + -//! GUEST_TPIDR_OFFSET]`. Every gated guest read/write of the thread pointer -//! addresses that slot with a scaled `LDR`/`STR` off the anchor: +//! The guest's thread pointer is a host-managed slot at +//! `[anchor + guest_tpidr_offset]` that the MSR and MRS gates store to and load +//! from. Two registers are involved: +//! +//! * The one the *guest* uses: `TPIDR_EL0`, per the Linux ABI. This is an ELF +//! rewriter, so it is the only one gated; a PE guest's `x18` TEB pointer and +//! a Mach-O guest's `TPIDRRO_EL0` would need different gates. +//! * The one anchoring the *host*'s per-thread block, selected by `Host` — +//! also `TPIDR_EL0` on a Linux host. //! -//! * the MSR gate reads the anchor (`MRS X16, TPIDR_EL0`) and stores the guest -//! value into the slot; -//! * the MRS gate reads the anchor and loads the guest value from the slot. +//! `guest_tpidr_offset` is fixed by the host binary's link and one rewritten +//! guest must run under any host build, so gates carry a placeholder offset. +//! **A loader must call [`patch_guest_tpidr_offset`], then prove with +//! [`find_guest_tpidr_placeholder`] that no placeholder survives, before +//! mapping the trampoline executable** — an unpatched gate does not fault. See +//! `GUEST_TPIDR_OFFSET_PLACEHOLDER`. //! -//! This mirrors the x64 model: the host keeps the native thread-pointer anchor, -//! the guest is statically relegated off it, and the gates emit nothing -//! TLS-related to the callback. +//! ## Gate scratch storage //! -//! ## Gate scratch storage and the stack invariant +//! `SVC` and `MSR TPIDR_EL0` clobber no registers, so their gates spill to a +//! frame carved from the guest stack with `SUB`/`ADD SP` and **require `SP` to +//! address a valid, writable, 16-byte-aligned stack at the patched site** — the +//! same condition the kernel relies on to write a signal frame. A site reached +//! with `SP` unmapped faults where the native instruction would not. The MRS +//! gate uses its destination as scratch and needs no frame. //! -//! `SVC` and `MSR TPIDR_EL0` clobber no general-purpose registers, so a gate has -//! no free scratch register on entry. The SVC and MSR gates therefore spill their -//! scratch registers (and, for SVC, the computed return address the callback -//! reads back) to a frame carved out of the guest stack with `SUB SP, SP, #frame` -//! / `ADD SP, SP, #frame`. The MRS gate needs no frame: it reuses its own -//! destination register as scratch and never touches the stack. +//! The gates' extra memory accesses clear the local exclusive monitor, so a +//! gated instruction between `LDXR` and `STXR` would livelock. No real codegen +//! emits that. //! -//! Consequently the SVC and MSR gates **require `SP` to hold a valid, writable, -//! 16-byte-aligned stack at the patched site** — the same condition the kernel -//! relies on when it writes a signal frame below `SP`, and which every conforming -//! AArch64 caller already satisfies at a syscall boundary. The gate decrements -//! `SP` before storing, so nothing (signal delivery included) writes into the -//! frame while it is live; there is no red-zone hazard. A site reached with `SP` -//! pointing at unmapped or guard memory would fault where the native instruction -//! would not. AArch64 offers no cheaper alternative: with no segment-relative -//! store (unlike x86's `gs:`-relative spill) and no free register, reaching any -//! runtime-owned scratch area would itself require first clobbering an unsaved -//! guest register to materialize a base pointer. +//! ## `X16` is preserved across an `SVC` //! -//! ## Trampoline layout (Linux) +//! Linux preserves every register but `x0` across an `SVC`, and `X16` is the +//! SVC gate's only scratch. The gate spills guest `X16` to `[SP, #0]`, records +//! the return address and this site's *outbound stub* in the frame, then enters +//! the callback. The runtime returns through the stub: //! //! ```text -//! Offset 0: [8 bytes] syscall callback address (filled at load time) -//! Offset 8: [8 bytes] shared SVC handler (LDR X16,; BR X16) -//! Offset 16: per-site gates (SVC: 24 bytes, MSR: 36 bytes, MRS: 12 bytes) +//! outbound_N: +//! ldr x16, [sp, #0] // restore guest X16 +//! add sp, sp, #32 // pop the gate frame +//! b site+4 // static target; needs no scratch register //! ``` //! -//! A binary with **no** patch sites gets no trampoline at all: the rewriter -//! appends only a size-0 sentinel header (matching the x86-64 path), recording -//! that the image was checked and needs no redirection. Signal returns are -//! handled by the runtime (see "Signal returns" below). +//! AArch64 has no memory-indirect branch, so a runtime-side branch back into +//! the guest would burn a register on its target; a static branch in +//! guest-adjacent code does not. The runtime rewrites `[SP, #0]` from +//! `PtRegs::regs[16]` before branching rather than relying on the frame +//! surviving the round trip. //! -//! The offset-0 callback address is **filled in by the loader/runtime, not by -//! this crate.** The emitted trampolines are therefore *not runnable as-is*: a -//! loader must write the syscall-callback address at offset 0 before any guest -//! `SVC` reaches a gate. (`callback` may be passed to [`hook_syscalls_aarch64`] -//! to prefill offset 0.) The callback reads host TLS from `TPIDR_EL0` and the -//! guest thread pointer from `[TPIDR_EL0 + GUEST_TPIDR_OFFSET]` itself. +//! The stub only resumes at the original site. A redirected `PC` (signals, +//! `execve`) or an asynchronous resume is instead handled by synthesizing an +//! `rt_sigreturn` frame, restoring all 31 GPRs, `PC` and `PSTATE` at once. //! -//! ## Signal returns +//! ## Trampoline layout //! -//! This crate emits no sigreturn gate; `rt_sigreturn` is handled by the runtime. -//! The runtime installs its own sigreturn trampoline address into the signal -//! frame's return slot; because that is an absolute address (not a `B`), a -//! single runtime-owned gate is reachable from any guest regardless of the -//! ±128MB branch range, so no per-binary gate is required. +//! A callback address slot, then gate-aligned per-site slots each ending in a +//! metadata word; `HEADER_CALLBACK_OFFSET`, `GATES_START_OFFSET` and the +//! `*_SLOT_BYTES` constants define the offsets and sizes. +//! [`crate::hook_syscalls_in_elf`] writes zero into the callback slot when its +//! caller supplies no address, leaving the loader to fill it in before the +//! trampoline is executable. A binary with no patch sites gets no trampoline, +//! only a size-0 sentinel header, matching the x86-64 path. //! -//! ## Runtime contract -//! -//! Per thread, the runtime sets the hardware `TPIDR_EL0` to the host anchor and -//! reserves the guest thread-pointer slot at `[TPIDR_EL0 + GUEST_TPIDR_OFFSET]`. -//! No new callback ABI is introduced: the callback reaches host TLS through -//! `TPIDR_EL0` directly. Multi-threaded correctness depends only on the runtime -//! keeping the anchor valid and the slot reachable for every thread it starts; -//! no process-global table is involved, so concurrent threads never contend. -//! -//! ## Host-OS scope -//! -//! This module fully virtualizes the guest thread pointer against a stable -//! per-thread host anchor register. The model is host-OS-agnostic; only the -//! choice of anchor register varies per host, selected by [`Host`] (a gate names -//! its anchor through [`Host::anchor_read`]). On a Linux host the anchor is -//! `TPIDR_EL0` itself: the kernel preserves it across host execution, so the host -//! can keep its own value there as the anchor while the guest thread pointer -//! lives in the slot beside it. The instruction encoders and gate framing here -//! are host-agnostic; see [`Host`] for the per-host anchor registers and what -//! each additional host requires. +//! `rt_sigreturn` needs no gate: the runtime installs its own trampoline +//! address into the signal frame, and an absolute address is reachable +//! regardless of branch range. use alloc::format; use alloc::vec::Vec; @@ -158,16 +123,134 @@ const MSR_TPIDR_EL0_BITS: u32 = 0xD51B_D040; const MRS_TPIDR_EL0_MASK: u32 = 0xFFFF_FFE0; const MRS_TPIDR_EL0_BITS: u32 = 0xD53B_D040; -/// `BRK` immediate planted at a patch site whose gate lies outside the `B` -/// instruction's ±128MB reach. Executing the site raises a synchronous debug -/// exception (`SIGTRAP`) carrying this immediate, faulting the guest rather than -/// letting the unpatched instruction escape to the host kernel; the site is also -/// reported as a trapped site so the ELF-level caller can reject the binary. +/// `BRK` immediate planted at a patch site whose gate is out of `B` reach. +/// Executing it raises a synchronous debug exception (`SIGTRAP`) carrying this +/// immediate instead of letting the unpatched instruction escape to the host +/// kernel. /// -/// Recognizing this immediate in the runtime — to attribute the trap to the -/// rewriter rather than a guest breakpoint — is planned but not yet implemented. +/// TODO: recognize this immediate in the runtime, to tell a rewriter trap from +/// a guest breakpoint. const TRAP_BRK_IMM: u16 = 0xB10B; +/// Alignment every emitted gate slot starts on. +pub const GATE_ALIGNMENT: usize = 16; +/// Byte size of an emitted `MRS TPIDR_EL0` gate slot. +pub const MRS_SLOT_BYTES: usize = 16; +/// Byte size of an emitted `MSR TPIDR_EL0` gate slot. +pub const MSR_SLOT_BYTES: usize = 48; +/// Byte size of an emitted `SVC` gate slot. +pub const SVC_SLOT_BYTES: usize = 64; +const NOP: u32 = 0xD503_201F; + +const GATE_METADATA_MAGIC: u32 = 0xB807; +const GATE_METADATA_VERSION: u32 = 1; +const GATE_METADATA_MAGIC_MASK: u32 = 0xffff; +const GATE_METADATA_VERSION_SHIFT: u32 = 16; +const GATE_METADATA_VERSION_MASK: u32 = 0xf << GATE_METADATA_VERSION_SHIFT; +const GATE_METADATA_KIND_SHIFT: u32 = 20; +const GATE_METADATA_KIND_MASK: u32 = 0xf << GATE_METADATA_KIND_SHIFT; +const GATE_METADATA_REGISTER_SHIFT: u32 = 24; +const GATE_METADATA_REGISTER_MASK: u32 = 0x3f << GATE_METADATA_REGISTER_SHIFT; + +/// The metadata word's `kind` field. +/// +/// A persisted format: these discriminants are baked into every rewritten +/// binary, so an existing one may never be renumbered. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +enum GateKind { + Svc = 0, + MrsTpidr = 1, + MsrTpidr = 2, +} + +impl GateKind { + const fn bits(self) -> u32 { + self as u32 + } + + fn from_bits(bits: u32) -> Option { + [Self::Svc, Self::MrsTpidr, Self::MsrTpidr] + .into_iter() + .find(|kind| kind.bits() == bits) + } +} + +/// Highest register number a 5-bit register field can name. +const MAX_REGISTER: u8 = 31; +const GATE_METADATA_USED_MASK: u32 = GATE_METADATA_MAGIC_MASK + | GATE_METADATA_VERSION_MASK + | GATE_METADATA_KIND_MASK + | GATE_METADATA_REGISTER_MASK; + +/// Which kind of gate a compact slot holds, and the register it acts on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GateMetadata { + /// A trapped `SVC`: the shim performs the syscall. + Svc, + /// A trapped `MRS , TPIDR_EL0`, reading the guest thread pointer. + MrsTpidr { + /// Register the guest thread pointer is read into. + destination: u8, + }, + /// A trapped `MSR TPIDR_EL0, `, writing the guest thread pointer. + MsrTpidr { + /// Register holding the value to write. + source: u8, + }, +} + +#[repr(transparent)] +#[derive(Clone, Copy)] +pub(crate) struct EncodedGateMetadata(u32); + +impl EncodedGateMetadata { + pub(crate) fn encode(metadata: GateMetadata) -> Option { + if matches!(metadata, GateMetadata::MrsTpidr { destination: XZR }) { + return None; + } + let (kind, register) = match metadata { + GateMetadata::Svc => (GateKind::Svc, 0), + GateMetadata::MrsTpidr { destination } => (GateKind::MrsTpidr, destination), + GateMetadata::MsrTpidr { source } => (GateKind::MsrTpidr, source), + }; + if register > MAX_REGISTER { + return None; + } + Some(Self( + GATE_METADATA_MAGIC + | (GATE_METADATA_VERSION << GATE_METADATA_VERSION_SHIFT) + | (kind.bits() << GATE_METADATA_KIND_SHIFT) + | (u32::from(register) << GATE_METADATA_REGISTER_SHIFT), + )) + } + + pub(crate) fn decode(self) -> Option { + let word = self.0; + if word & GATE_METADATA_MAGIC_MASK != GATE_METADATA_MAGIC + || (word & GATE_METADATA_VERSION_MASK) >> GATE_METADATA_VERSION_SHIFT + != GATE_METADATA_VERSION + || word & !GATE_METADATA_USED_MASK != 0 + { + return None; + } + let kind = + GateKind::from_bits((word & GATE_METADATA_KIND_MASK) >> GATE_METADATA_KIND_SHIFT)?; + let register = ((word & GATE_METADATA_REGISTER_MASK) >> GATE_METADATA_REGISTER_SHIFT) as u8; + if register > MAX_REGISTER { + return None; + } + match kind { + GateKind::Svc if register == 0 => Some(GateMetadata::Svc), + GateKind::MrsTpidr if register != XZR => Some(GateMetadata::MrsTpidr { + destination: register, + }), + GateKind::MsrTpidr => Some(GateMetadata::MsrTpidr { source: register }), + _ => None, + } + } +} + // --- Register operands used by the emitted gates/handlers --- // // X16/X17 are the intra-procedure scratch registers (IP0/IP1), and register @@ -185,35 +268,101 @@ const XZR: u8 = 31; // --- Guest thread-pointer virtualization --- // -// The host owns the hardware `TPIDR_EL0` as a per-thread anchor; the guest's -// logical thread pointer is a memory slot the runtime reserves at a fixed byte -// offset from that anchor. Every gated guest read/write of the thread pointer -// addresses the slot with a scaled `LDR`/`STR` off `TPIDR_EL0`. - -/// Byte offset from the host anchor in `TPIDR_EL0` at which the runtime reserves -/// this thread's guest thread-pointer slot. Every guest read/write of the thread -/// pointer is virtualized to `[TPIDR_EL0 + GUEST_TPIDR_OFFSET]` via a scaled -/// `LDR`/`STR`. +// The host reaches its per-thread block through an anchor register its OS +// fixes, which `Host` names; the guest's logical thread pointer is a memory +// slot the runtime reserves at some byte offset from that anchor. Every gated +// guest read/write addresses the slot with a scaled `LDR`/`STR` off the +// anchor. The rewriter does not know the offset -- it is a property of the +// *host* runtime's link, not of the guest binary -- so it emits a placeholder +// the loader overwrites. + +/// Largest value the `imm12` field of an unsigned-offset `LDR`/`STR` can hold. +/// The field is 12 bits and unsigned, counting `0..=0xFFF` *scaled units*. +const LDST_UIMM12_IMM_MAX: u16 = (1 << 12) - 1; + +/// Scale the 64-bit form applies to that `imm12`, i.e. the operand width in +/// bytes. Any offset the gates address must therefore be 8-aligned. +const LDST_UIMM12_SCALE_64BIT: u16 = 8; + +/// Largest byte offset a 64-bit unsigned-offset `LDR`/`STR` can encode +/// (`0xFFF * 8 = 32760`), derived from the encoding rather than written out. +const LDR_UIMM12_MAX_BYTE_OFFSET: u16 = LDST_UIMM12_IMM_MAX * LDST_UIMM12_SCALE_64BIT; + +/// Alignment a runtime's guest thread-pointer slot must satisfy: the scale of +/// the 64-bit unsigned-offset `LDR`/`STR` the gates address it with. +pub const GUEST_TPIDR_OFFSET_ALIGN: u16 = LDST_UIMM12_SCALE_64BIT; + +/// Largest byte offset from the host anchor at which a runtime may place the +/// guest thread-pointer slot. +/// +/// Not an independent policy choice: the gates reach the slot with one 64-bit +/// unsigned-offset `LDR`/`STR`, so the bound *is* `LDR_UIMM12_MAX_BYTE_OFFSET`. +pub const MAX_GUEST_TPIDR_OFFSET: u16 = LDR_UIMM12_MAX_BYTE_OFFSET; + +/// The smallest guest thread-pointer offset a gate may be patched with. +/// +/// A host keeps its own per-thread bookkeeping at the base of the block its +/// anchor points at, so a runtime places the guest slot past that and no +/// legitimate offset is ever this low. The bound matters because the +/// placeholder only catches a loader that forgets to patch at all: one that +/// patches with a defaulted or zeroed offset leaves nothing behind to detect, +/// and every rewritten thread-pointer write would then land on host state. +pub(crate) const MIN_GUEST_TPIDR_OFFSET: u16 = 16; + +/// Placeholder byte offset baked into every emitted gate's guest thread-pointer +/// access, replaced at load time by [`patch_guest_tpidr_offset`]. +/// +/// It is `MAX_GUEST_TPIDR_OFFSET`, the largest value the field can hold, so it +/// cannot collide with a real runtime offset. That makes scanning for it exact, +/// which is what lets [`patch_guest_tpidr_offset`] and +/// [`find_guest_tpidr_placeholder`] work off the emitted words alone with no +/// side table of patch sites. /// -/// Fixed ABI offset: the runtime points `TPIDR_EL0` at a per-thread block whose -/// `guest_tp` field sits just past the AArch64 variant-1 16-byte TCB header, so a -/// stray "deref `TPIDR_EL0` as a TCB" cannot mistake the guest pointer for the -/// dtv slot. Because the scaled immediate is baked into statically rewritten -/// binaries, this value is part of the rewriter/runtime ABI and must match the -/// runtime's block layout. -const GUEST_TPIDR_OFFSET: u16 = 16; +/// It buys **no** run-time safety. An unpatched gate does not fault: it reads +/// and writes one self-consistent address 32KB past the host thread pointer, +/// quietly corrupting eight bytes of whatever is mapped there. A loader must +/// prove no placeholder survives — see [`find_guest_tpidr_placeholder`] — +/// before making a trampoline executable. +pub(crate) const GUEST_TPIDR_OFFSET_PLACEHOLDER: u16 = MAX_GUEST_TPIDR_OFFSET; // --- SVC gate stack frame --- // -// The SVC gate touches only X16, so it needs a minimal 16-byte frame: one slot -// for the saved guest X16 and one for the computed post-SVC return address. +// The SVC gate touches only X16, so the frame holds three words: the saved +// guest X16, the post-SVC return address, and this site's outbound stub +// address. Rounded up to 16-byte stack alignment, that is 32 bytes. /// SVC gate frame size (`SUB/ADD SP, SP, #SVC_FRAME_BYTES`). 16-byte aligned. -const SVC_FRAME_BYTES: u16 = 16; -/// Saved guest X16. -const SVC_FRAME_OFF_X16: u16 = 0; -/// Computed post-SVC return address. -const SVC_FRAME_OFF_RETADDR: u16 = 8; +/// +/// ABI: `switch_to_guest` sets `SP` to `PtRegs::sp - SVC_FRAME_BYTES` before +/// entering an outbound stub, because the stub pops this frame. +pub const SVC_FRAME_BYTES: u16 = 32; +/// Saved guest X16. ABI: the outbound stub reloads `X16` from here. +pub const SVC_FRAME_OFF_X16: u16 = 0; +/// Post-SVC return address. ABI: the runtime's syscall callback reads it and +/// publishes it as the guest resume PC. +pub const SVC_FRAME_OFF_RETADDR: u16 = 8; +/// Address of this site's outbound stub. ABI: the runtime's syscall callback +/// branches here to resume at the original syscall site. +pub const SVC_FRAME_OFF_STUB: u16 = 16; + +// The gate carves the frame out of the guest stack with `SUB SP`, so the frame +// must keep `SP` 16-byte aligned, and it must be large enough for the three +// words the gate writes into it. +const _: () = assert!( + SVC_FRAME_BYTES.is_multiple_of(16), + "the SVC gate frame must keep SP 16-byte aligned" +); +const _: () = assert!( + SVC_FRAME_OFF_STUB + 8 <= SVC_FRAME_BYTES, + "the SVC gate frame must hold the saved X16, the return address and the stub address" +); + +/// Size of the SVC gate proper, i.e. the distance from the gate's first +/// instruction to its outbound stub. The gate is `SUB SP / STR X16 / ADRP / +/// ADD / STR / ADR / STR / LDR X16 / BR` = 9 instructions. +pub const SVC_GATE_BYTES: usize = 9 * 4; +/// Size of the per-site outbound stub (`LDR X16 / ADD SP / B`). +const SVC_OUTBOUND_STUB_BYTES: usize = 3 * 4; // --- MSR gate stack frame --- // @@ -221,7 +370,7 @@ const SVC_FRAME_OFF_RETADDR: u16 = 8; // guest value so the source register needs no special-casing. /// MSR gate frame size (`SUB/ADD SP, SP, #MSR_FRAME_BYTES`). 16-byte aligned. -const MSR_FRAME_BYTES: u16 = 32; +pub const MSR_FRAME_BYTES: u16 = 32; /// Saved X16 (and, +8, X17 via the `STP`/`LDP` pair). const MSR_FRAME_OFF_X16: u16 = 0; /// Captured guest thread-pointer value, staged while all guest registers are @@ -233,10 +382,17 @@ const MSR_FRAME_OFF_VALUE: u16 = 16; /// Callback address slot. const HEADER_CALLBACK_OFFSET: usize = 0; -/// Shared SVC handler, placed just past the 8-byte callback slot. Per-site gates -/// follow it and are each appended dynamically, so this shared prologue is the -/// only fixed-offset region the emitters reference. -const SHARED_SVC_HANDLER_OFFSET: usize = HEADER_CALLBACK_OFFSET + 8; +/// First byte past the 8-byte callback slot, and so the first that can be read +/// as an instruction word. Scans start here: the slot holds an address, which +/// could bit-for-bit resemble any instruction they look for. +const FIRST_SCANNABLE_OFFSET: usize = HEADER_CALLBACK_OFFSET + 8; + +/// First byte of the per-site gates; the callback header is padded with NOPs to +/// the 16-byte slot alignment. Everything from [`FIRST_SCANNABLE_OFFSET`] on is +/// instructions this module emitted, which is what lets +/// [`patch_guest_tpidr_offset`] scan for its patch sites instead of carrying a +/// side table of them. +const GATES_START_OFFSET: usize = GATE_ALIGNMENT; // ============================================================ // Instruction encoders @@ -251,9 +407,74 @@ const SHARED_SVC_HANDLER_OFFSET: usize = HEADER_CALLBACK_OFFSET + 8; /// 26-bit `imm26` branch-offset field (`B`/`BL`), bits \[25:0]. const IMM26_MASK: u32 = 0x03FF_FFFF; +const OPCODE_TOP6_MASK: u32 = 0xFC00_0000; /// 19-bit `imm19` offset field (`B.cond`/`LDR`-literal/`ADRP` immhi), bits \[18:0]. const IMM19_MASK: u32 = 0x0007_FFFF; +/// A PC-relative immediate counts instructions, so it scales by this many bits +/// to reach a byte displacement. +const INSN_BYTES_LOG2: u32 = 2; + +/// Bytes in one AArch64 instruction. Every patch site, gate slot and scan +/// stride is a whole number of these. +const INSN_BYTES: usize = 1 << INSN_BYTES_LOG2; + +/// [`INSN_BYTES`] where a virtual address is being measured. +const INSN_BYTES_U64: u64 = 1 << INSN_BYTES_LOG2; + +/// The metadata word closing every compact gate slot. +const GATE_METADATA_BYTES: usize = 4; + +// Field positions within an instruction word. +/// `Rn`, and the low bit of a PC-relative `imm19`. +const RN_SHIFT: u32 = 5; +/// `Rt2` of `STP`/`LDP`, and `imm12` of the add/sub and load/store forms. +const RT2_SHIFT: u32 = 10; +/// Signed `imm7` of `STP`/`LDP`. +const IMM7_SHIFT: u32 = 15; +/// `immlo` of `ADR`/`ADRP`; `immhi` sits at [`RN_SHIFT`]. +const ADR_IMMLO_SHIFT: u32 = 29; +/// Bits an `ADRP` immediate is scaled by: it addresses 4KiB pages. +const ADRP_PAGE_SHIFT: u32 = 12; + +// Widths of the signed immediate fields, used to sign-extend them into an +// `i64` by shifting left and back. +const IMM26_BITS: u32 = 26; +const IMM21_BITS: u32 = 21; +const IMM19_BITS: u32 = 19; + +/// Sign-extends the low `bits` of `value`. +const fn sign_extend(value: i64, bits: u32) -> i64 { + (value << (i64::BITS - bits)) >> (i64::BITS - bits) +} + +/// Sign-extends a PC-relative immediate and scales it to a byte displacement. +const fn pcrel_bytes(imm: i64, bits: u32) -> i64 { + sign_extend(imm, bits) << INSN_BYTES_LOG2 +} + +/// Register field, bits \[4:0] — `Rd`, `Rt` or the `Rn`/`Rt2` fields once +/// shifted into place. +const REG_MASK: u32 = 0x1F; + +/// `immlo` of an `ADR`/`ADRP` pair, bits \[1:0] of the 21-bit immediate. +const ADR_IMMLO_MASK: u32 = 0x3; + +/// Signed 7-bit scaled immediate of `STP`/`LDP`. +const IMM7_MASK: u16 = 0x7F; + +/// Byte offset within a 4KiB page, the part an `ADRP` does not carry. +const PAGE_OFFSET_MASK: u64 = 0xFFF; + +/// `ADRP` opcode bits plus `Rd`, ignoring the immediate. +const ADRP_SHAPE_MASK: u32 = 0x9F00_001F; + +/// `ADD (immediate)` opcode bits plus `Rn` and `Rd`, ignoring the immediate. +const ADD_IMM_SHAPE_MASK: u32 = 0xFFC0_03FF; + +/// `LDR (literal)` opcode bits plus `Rt`, ignoring the immediate. +const LDR_LITERAL_SHAPE_MASK: u32 = 0xFF00_001F; + /// Base opcode of an emitted instruction: every fixed bit set with all operand /// fields zeroed. An encoder selects a variant and ORs in its operands via /// [`Opcode::bits`]. (`MRS TPIDR_EL0` is encoded from [`Opcode::MrsTpidrEl0`], @@ -264,6 +485,7 @@ const IMM19_MASK: u32 = 0x0007_FFFF; enum Opcode { B = 0x1400_0000, LdrLiteral = 0x5800_0000, + Adr = 0x1000_0000, Adrp = 0x9000_0000, Br = 0xD61F_0000, SubImm = 0xD100_0000, @@ -313,13 +535,21 @@ fn pcrel_imm19(op: Opcode, offset: i64, low: u32) -> Option { if !(-(1 << 18)..(1 << 18)).contains(&imm19) { return None; } - Some(op.bits() | ((imm19.cast_unsigned() & IMM19_MASK) << 5) | low) + Some(op.bits() | ((imm19.cast_unsigned() & IMM19_MASK) << RN_SHIFT) | low) } -/// `op | rn<<5` — instruction whose only operand is a register in the `Rn` field -/// (`BR`/`RET`). -fn reg_in_rn(op: Opcode, rn: u8) -> u32 { - op.bits() | (u32::from(rn) << 5) +/// `op | immlo<<29 | immhi<<5 | rd` — 21-bit-signed PC-relative address form +/// (`ADR`/`ADRP`). The units of `imm` are the instruction's own: bytes for +/// `ADR` (±1MB), 4KB pages for `ADRP` (±4GB). +fn pcrel_imm21(op: Opcode, rd: u8, imm: i64) -> Option { + let imm = i32::try_from(imm).ok()?; + if !(-(1 << 20)..(1 << 20)).contains(&imm) { + return None; + } + let imm = imm.cast_unsigned(); + let immlo = (imm & ADR_IMMLO_MASK) << ADR_IMMLO_SHIFT; + let immhi = ((imm >> 2) & IMM19_MASK) << RN_SHIFT; + Some(op.bits() | immlo | immhi | u32::from(rd)) } /// `op | imm12<<10 | rn<<5 | rd` — 12-bit-immediate add/sub form @@ -328,20 +558,24 @@ fn data_imm12(op: Opcode, rd: u8, rn: u8, imm12: u16) -> Option { if imm12 >= (1 << 12) { return None; } - Some(op.bits() | (u32::from(imm12) << 10) | (u32::from(rn) << 5) | u32::from(rd)) + Some(op.bits() | (u32::from(imm12) << RT2_SHIFT) | (u32::from(rn) << RN_SHIFT) | u32::from(rd)) } /// `op | imm12<<10 | rn<<5 | rt` — unsigned scaled (×8) 64-bit load/store -/// (`STR`/`LDR [Xn, #imm]`). `imm_bytes` must be a multiple of 8. +/// (`STR`/`LDR [Xn, #imm]`). `imm_bytes` must be a multiple of +/// `LDST_UIMM12_SCALE_64BIT` and at most `LDR_UIMM12_MAX_BYTE_OFFSET`. fn ldst_uimm12(op: Opcode, rt: u8, rn: u8, imm_bytes: u16) -> Option { - if !imm_bytes.is_multiple_of(8) { - return None; - } - let imm12 = imm_bytes / 8; - if imm12 >= (1 << 12) { + if !imm_bytes.is_multiple_of(LDST_UIMM12_SCALE_64BIT) || imm_bytes > LDR_UIMM12_MAX_BYTE_OFFSET + { return None; } - Some(op.bits() | (u32::from(imm12) << 10) | (u32::from(rn) << 5) | u32::from(rt)) + let imm12 = imm_bytes / LDST_UIMM12_SCALE_64BIT; + Some( + op.bits() + | (u32::from(imm12) << LDST_UIMM12_IMM_SHIFT) + | (u32::from(rn) << 5) + | u32::from(rt), + ) } /// `op | imm7<<15 | rt2<<10 | rn<<5 | rt` — signed scaled (×8) 64-bit load/store @@ -354,14 +588,14 @@ fn ldst_pair(op: Opcode, rt: u8, rt2: u8, rn: u8, imm_bytes: i16) -> Option if !(-64..=63).contains(&imm7) { return None; } - let imm7_u = u32::from(imm7.cast_unsigned() & 0x7F); - Some(op.bits() | (imm7_u << 15) | (u32::from(rt2) << 10) | (u32::from(rn) << 5) | u32::from(rt)) -} - -/// `base | rt` — system-register move (`MRS`/`MSR`); `base` already encodes the -/// system register and transfer direction. -fn sysreg_move(base: u32, rt: u8) -> u32 { - base | u32::from(rt) + let imm7_u = u32::from(imm7.cast_unsigned() & IMM7_MASK); + Some( + op.bits() + | (imm7_u << IMM7_SHIFT) + | (u32::from(rt2) << RT2_SHIFT) + | (u32::from(rn) << RN_SHIFT) + | u32::from(rt), + ) } /// A single AArch64 instruction emitted into a trampoline, described by its @@ -375,6 +609,8 @@ fn sysreg_move(base: u32, rt: u8) -> u32 { enum Insn { /// `B` (unconditional branch), PC-relative, ±128MB, 4-byte aligned. B(i64), + /// `ADR Xd, #byte_off` — PC-relative address, ±1MB (byte granularity). + Adr { rd: u8, byte_off: i64 }, /// `ADRP Xd, #page_off` — page-relative address, ±4GB (in 4KB pages). Adrp { rd: u8, page_off: i64 }, /// `LDR Xt, ` (PC-relative literal load), ±1MB, 4-byte aligned. @@ -417,18 +653,10 @@ impl Insn { fn encode(self) -> Option { match self { Insn::B(off) => branch_imm26(Opcode::B, off), - Insn::Adrp { rd, page_off } => { - let imm = i32::try_from(page_off).ok()?; - if !(-(1 << 20)..(1 << 20)).contains(&imm) { - return None; - } - let imm = imm.cast_unsigned(); - let immlo = (imm & 0x3) << 29; - let immhi = ((imm >> 2) & IMM19_MASK) << 5; - Some(Opcode::Adrp.bits() | immlo | immhi | u32::from(rd)) - } + Insn::Adr { rd, byte_off } => pcrel_imm21(Opcode::Adr, rd, byte_off), + Insn::Adrp { rd, page_off } => pcrel_imm21(Opcode::Adrp, rd, page_off), Insn::LdrLiteral { rt, off } => pcrel_imm19(Opcode::LdrLiteral, off, u32::from(rt)), - Insn::Br(rn) => Some(reg_in_rn(Opcode::Br, rn)), + Insn::Br(rn) => Some(Opcode::Br.bits() | (u32::from(rn) << RN_SHIFT)), Insn::SubSp(imm12) => data_imm12(Opcode::SubImm, SP, SP, imm12), Insn::AddSp(imm12) => data_imm12(Opcode::AddImm, SP, SP, imm12), Insn::AddImm { rd, rn, imm12 } => data_imm12(Opcode::AddImm, rd, rn, imm12), @@ -446,43 +674,29 @@ impl Insn { rn, imm_bytes, } => ldst_pair(Opcode::Ldp, rt, rt2, rn, imm_bytes), - Insn::MrsTpidrEl0(rt) => Some(sysreg_move(Opcode::MrsTpidrEl0.bits(), rt)), - Insn::Brk(imm) => Some(Opcode::Brk.bits() | (u32::from(imm) << 5)), + Insn::MrsTpidrEl0(rt) => Some(Opcode::MrsTpidrEl0.bits() | u32::from(rt)), + Insn::Brk(imm) => Some(Opcode::Brk.bits() | (u32::from(imm) << RN_SHIFT)), } } } // ============================================================ -// Host anchor selection +// Host anchor register // ============================================================ -/// The host OS the rewritten guest runs under. -/// -/// The guest thread pointer is virtualized the same way on every host; only the -/// *anchor register* a gate reads to reach the host's per-thread block varies. -/// [`Host`] selects that register, so a gate names the anchor through -/// [`Host::anchor_read`] rather than hardcoding a system register. Adding a host -/// is a new variant plus its anchor-read arm. -/// -/// Other host OSes need a different stable anchor register (a future variant -/// supplying its own [`Host::anchor_read`]), and beyond that a host-specific -/// shared SVC handler — not just a different trampoline base address: +/// The host OS the rewritten guest runs under. Its ABI fixes the *anchor +/// register* a gate reads to reach the host's per-thread block, and this names +/// which one; gates read it through [`Host::anchor_read`], so adding a host is +/// a new variant plus its arm there. /// -/// * **Linux-on-macOS** (Apple Silicon): XNU clobbers `TPIDR_EL0` on -/// signals/preemption and zeroes `x18` on every exception entry, so neither -/// register survives a host transition. The stable anchor becomes the -/// read-only `TPIDRRO_EL0` (which XNU keeps per-pthread), and *both* -/// `TPIDR_EL0` and `x18` must be fully virtualized — `x18` via per-site gates. -/// * **Linux-on-Windows** (Windows on ARM64): Windows does not preserve -/// `TPIDR_EL0` across context switches and reserves `x18` as the TEB pointer -/// (always valid). The TEB is the stable anchor: the per-thread TLS state is -/// reached through a TEB TLS slot, and `TPIDR_EL0` (plus guest `x18`, where the -/// guest uses it) is virtualized against that. +/// Only [`Host::Linux`] exists today. macOS would anchor on `TPIDRRO_EL0` and +/// Windows on the `x18` TEB pointer, since neither preserves `TPIDR_EL0` across +/// a host transition; both would also need host-specific `SVC` callback +/// dispatch, not just a new anchor. #[derive(Clone, Copy)] pub(crate) enum Host { - /// Linux host. The kernel preserves `TPIDR_EL0` across host execution, so the - /// host keeps its anchor there and the guest thread-pointer slot lives beside - /// it; the anchor read is `MRS Xd, TPIDR_EL0`. + /// Linux host: the kernel preserves `TPIDR_EL0` across host execution, so + /// the anchor lives there and the anchor read is `MRS Xd, TPIDR_EL0`. Linux, } @@ -541,19 +755,30 @@ fn find_patch_sites(sections: &[TextSectionInfo], buf: &[u8]) -> Result section_data.len() { + for i in (0..section_data.len()).step_by(INSN_BYTES) { + if i + INSN_BYTES > section_data.len() { break; } - let insn = u32::from_le_bytes(section_data[i..i + 4].try_into().unwrap()); + let insn = u32::from_le_bytes(section_data[i..i + INSN_BYTES].try_into().unwrap()); let kind = if (insn & SVC_OPCODE_MASK) == SVC_OPCODE_BITS { PatchKind::Svc } else if (insn & MSR_TPIDR_EL0_MASK) == MSR_TPIDR_EL0_BITS { - PatchKind::MsrTpidr((insn & 0x1F) as u8) + PatchKind::MsrTpidr((insn & REG_MASK) as u8) } else if (insn & MRS_TPIDR_EL0_MASK) == MRS_TPIDR_EL0_BITS { - let rd = (insn & 0x1F) as u8; + let rd = (insn & REG_MASK) as u8; // `MRS XZR, TPIDR_EL0` discards its result (a no-op read); gating // it would mean using register 31 as an `LDR` base (= SP), so // leave it native. @@ -583,11 +808,10 @@ fn find_patch_sites(sections: &[TextSectionInfo], buf: &[u8]) -> Result, - /// Virtual addresses of patch sites that could not be redirected to their - /// gate — the inbound `B` or one of the gate's own branches fell outside the - /// branch's ±128MB range — and were replaced with a trap instead of a - /// redirect. A non-empty list means the rewrite is incomplete: those sites - /// fault at runtime rather than entering the trampoline. + /// Virtual addresses of patch sites replaced with a trap instead of a + /// redirect, because the inbound `B` or one of the gate's own branches fell + /// outside ±128MB. A non-empty list means the rewrite is incomplete: those + /// sites fault at runtime rather than entering the trampoline. pub trapped_sites: Vec, } @@ -595,20 +819,17 @@ pub(crate) struct HookOutcome { /// AArch64 ELF image. (`MRS XZR, TPIDR_EL0` is a discarded read and is left /// native — see the module docs.) /// -/// `buf` is patched in place; the returned [`HookOutcome::trampoline`] is the -/// blob that the caller appends after the ELF (page-aligned). -/// `trampoline_base_addr` is the virtual address the trampoline will be mapped -/// at; `callback` is the absolute address stored in the callback slot (0 if the -/// loader fills it in later). +/// `buf` is patched in place. `trampoline_base_addr` is the virtual address the +/// trampoline will be mapped at; `callback` is the absolute address stored in +/// the callback slot (0 if the loader fills it in later). /// -/// Returns `Ok(None)` when the image contains no patch sites: no trampoline is -/// needed and the caller emits a size-0 sentinel header instead (matching the -/// x86-64 path). Signal returns are handled by the runtime — not a per-binary -/// gate — so a syscall-free binary needs no trampoline at all. +/// Returns `Ok(None)` when the image contains no patch sites, so the caller +/// emits a size-0 sentinel header instead (matching the x86-64 path). Signal +/// returns are handled by the runtime rather than a per-binary gate, so a +/// syscall-free binary needs no trampoline at all. /// -/// Otherwise returns `Ok(Some(outcome))`. A site whose inbound `B` cannot reach -/// its gate, or whose gate cannot branch back within the `B` instruction's -/// ±128MB reach, cannot be redirected; it is replaced with a trap and listed in +/// Otherwise returns `Ok(Some(outcome))`. A site that cannot reach its gate, or +/// whose gate cannot branch back, is replaced with a trap and listed in /// [`HookOutcome::trapped_sites`] so the caller can reject the incomplete /// rewrite, mirroring the x86-64 unpatchable-syscall path. pub(crate) fn hook_syscalls_aarch64( @@ -618,6 +839,11 @@ pub(crate) fn hook_syscalls_aarch64( callback: u64, host: Host, ) -> Result> { + if !trampoline_base_addr.is_multiple_of(GATE_ALIGNMENT as u64) { + return Err(Error::AddressOverflow(format!( + "AArch64 trampoline base {trampoline_base_addr:#x} is not {GATE_ALIGNMENT}-byte aligned" + ))); + } let sites = find_patch_sites(text_sections, buf)?; if sites.is_empty() { @@ -627,7 +853,7 @@ pub(crate) fn hook_syscalls_aarch64( } let mut trampoline_data: Vec = Vec::new(); - emit_shared_prologue(&mut trampoline_data, trampoline_base_addr, callback)?; + emit_shared_prologue(&mut trampoline_data, callback); let mut trapped_sites: Vec = Vec::new(); @@ -636,15 +862,12 @@ pub(crate) fn hook_syscalls_aarch64( let gate_vaddr = checked_add_u64(trampoline_base_addr, gate_offset as u64, "trampoline gate")?; - // A site is redirected to its gate with a single in-place `B` (±128MB - // forward reach), and each gate branches back to `site + 4` (the SVC gate - // also reaches its shared handler). The gate's return branch spans a wider - // displacement than the inbound one, so the inbound branch encoding is - // necessary but not sufficient: the gate is built only when the inbound - // branch fits, and a gate whose own branches are out of range reports - // `GateBuild::Unreachable` and appends nothing. If either the inbound - // branch or the gate is unreachable, replace the site with the sentinel - // trap, record it as unpatchable, and emit no gate. + // The gate's return branch spans a wider displacement than the inbound + // one, so an encodable inbound branch is necessary but not sufficient: + // the gate is built only once the inbound branch fits, and a gate whose + // own branches are out of range reports `GateBuild::Unreachable` and + // appends nothing. If either is unreachable the site is trapped and no + // gate is emitted. let b_offset = gate_vaddr .cast_signed() .saturating_sub(site.vaddr.cast_signed()); @@ -681,12 +904,10 @@ pub(crate) fn hook_syscalls_aarch64( if let (Some(b_insn), GateBuild::Emitted) = (inbound, build) { // Replace the original instruction with `B `. - buf[site.file_offset..site.file_offset + 4].copy_from_slice(&b_insn.to_le_bytes()); + buf[site.file_offset..site.file_offset + INSN_BYTES] + .copy_from_slice(&b_insn.to_le_bytes()); } else { - let brk = Insn::Brk(TRAP_BRK_IMM) - .encode() - .expect("BRK always encodes"); - buf[site.file_offset..site.file_offset + 4].copy_from_slice(&brk.to_le_bytes()); + trap_site(buf, site.file_offset); trapped_sites.push(site.vaddr); } } @@ -697,50 +918,85 @@ pub(crate) fn hook_syscalls_aarch64( })) } -/// Emit the header slot and the shared SVC handler — the fixed-size shared -/// prologue that per-site gates follow. -fn emit_shared_prologue( - trampoline_data: &mut Vec, - trampoline_base_addr: u64, - callback: u64, -) -> Result<()> { +/// Replace the four bytes at `file_offset` with `BRK #TRAP_BRK_IMM`. +/// +/// A patch site left native escapes the virtualization: an `SVC` reaches the +/// host kernel directly, and an `MSR TPIDR_EL0` writes the hardware register +/// instead of the guest's slot -- on a Linux host, over the host's own anchor. +/// Both are silent, which is worse than a fault. +fn trap_site(buf: &mut [u8], file_offset: usize) { + let brk = Insn::Brk(TRAP_BRK_IMM) + .encode() + .expect("BRK always encodes"); + buf[file_offset..file_offset + INSN_BYTES].copy_from_slice(&brk.to_le_bytes()); +} + +/// Replace every patch site in `buf` with `BRK #TRAP_BRK_IMM`, returning how +/// many were trapped. +/// +/// The fail-safe behind [`crate::trap_all_syscalls_in_code`], for a segment +/// that could not be patched at all. [`hook_syscalls_aarch64`] traps its own +/// unreachable sites as it goes and does not come through here. +/// +/// The `cfg` tracks reachability, not capability: the scan and the rewrite are +/// host-agnostic, but nothing calls this in an x86-64 build. +#[cfg(any(test, target_arch = "aarch64"))] +pub(crate) fn trap_all_patch_sites( + buf: &mut [u8], + text_sections: &[TextSectionInfo], +) -> Result { + let sites = find_patch_sites(text_sections, buf)?; + for site in &sites { + trap_site(buf, site.file_offset); + } + Ok(sites.len()) +} + +/// Emit the callback header and deterministic alignment padding. +fn emit_shared_prologue(trampoline_data: &mut Vec, callback: u64) { // Offset 0: callback address. trampoline_data.extend_from_slice(&callback.to_le_bytes()); - emit_shared_svc_handler( - trampoline_data, - SHARED_SVC_HANDLER_OFFSET, - trampoline_base_addr, - )?; - - Ok(()) + while trampoline_data.len() < GATES_START_OFFSET { + trampoline_data.extend_from_slice(&NOP.to_le_bytes()); + } } // ============================================================ -// SVC gate + shared SVC handler +// SVC gate // ============================================================ /// Whether a gate was fully emitted or could not be placed within reach. /// -/// A gate redirects back to the guest (and, for the SVC gate, out to the shared -/// handler) with PC-relative branches. When any of those branches is out of -/// range the gate emits nothing and reports [`GateBuild::Unreachable`], leaving -/// the trampoline blob untouched so the caller can trap the originating site. +/// A gate branches back to the guest, and the SVC gate also names its own +/// outbound stub. When one of those PC-relative references is out of range the +/// gate emits nothing and reports [`GateBuild::Unreachable`], leaving the blob +/// untouched so the caller can trap the originating site. +/// +/// The SVC gate's callback literal is not one of them: it addresses the +/// trampoline header, so exceeding `LDR`-literal range means the trampoline +/// itself has outgrown that reach, and `Asm::ldr_literal` fails the whole +/// rewrite rather than trapping one site. enum GateBuild { Emitted, Unreachable, } -/// Per-site SVC gate (6 instructions, 24 bytes, 16-byte frame). +/// Per-site 64-byte SVC slot including callback dispatch and outbound stub. /// -/// Saves only X16 (already a scratch register), computes the post-SVC return -/// address into X16, records it on the frame, then branches to the shared SVC -/// handler. Guest X17/X18/LR and NZCV are untouched; the callback finds the -/// post-SVC return address at `[SP, #8]` and restores X16 from `[SP, #0]`. +/// The gate saves only X16, computes the post-SVC return address into it and +/// records that on the frame, records this site's outbound stub address +/// alongside, then loads and branches through the callback pointer in the +/// trampoline header. Guest X17/X18/LR and NZCV are untouched. /// -/// Frame layout (relative to the decremented SP): `[0]=X16 [8]=return_addr`. -/// Requires `SP` to address a valid writable stack at the site (see the module -/// docs, "Gate scratch storage and the stack invariant"). +/// Frame layout, relative to the decremented SP: +/// `[0]=X16 [8]=return_addr [16]=outbound_stub [24]=pad`. Requires `SP` to +/// address a valid writable stack at the site; see the module docs, "Gate +/// scratch storage". +/// +/// The outbound stub is emitted immediately after the gate and is the runtime's +/// normal way back into the guest; see the module docs, "`X16` is preserved +/// across an `SVC`". fn emit_svc_gate( trampoline_data: &mut Vec, gate_offset: usize, @@ -750,7 +1006,7 @@ fn emit_svc_gate( let gate_vaddr = checked_add_u64(trampoline_base_addr, gate_offset as u64, "SVC gate")?; let mut asm = Asm::new(gate_vaddr); - // SUB SP, SP, #16 ; STR X16, [SP] — save the guest X16. + // SUB SP, SP, #32 ; STR X16, [SP] — save the guest X16. asm.emit(Insn::SubSp(SVC_FRAME_BYTES)); asm.emit(Insn::StrUimm { rt: X16, @@ -760,11 +1016,11 @@ fn emit_svc_gate( // ADRP X16, ; ADD X16, X16, # — post-SVC return // address. - let return_addr = checked_add_u64(site.vaddr, 4, "SVC return")?; + let return_addr = checked_add_u64(site.vaddr, INSN_BYTES_U64, "SVC return")?; if !asm.adrp(X16, return_addr)? { return Ok(GateBuild::Unreachable); } - let page_lo = u16::try_from(return_addr & 0xFFF).expect("masked to 12 bits"); + let page_lo = u16::try_from(return_addr & PAGE_OFFSET_MASK).expect("masked to 12 bits"); asm.emit(Insn::AddImm { rd: X16, rn: X16, @@ -778,81 +1034,84 @@ fn emit_svc_gate( imm_bytes: SVC_FRAME_OFF_RETADDR, }); - // B . - let handler_vaddr = checked_add_u64( - trampoline_base_addr, - SHARED_SVC_HANDLER_OFFSET as u64, - "SVC handler", - )?; - if !asm.branch_to(handler_vaddr)? { + // ADR X16, ; STR X16, [SP, #16] — record the stub. The stub + // starts right after this gate's last instruction, well within ADR's ±1MB + // reach, so a single ADR suffices (no ADRP/ADD pair). + let stub_vaddr = checked_add_u64(gate_vaddr, SVC_GATE_BYTES as u64, "SVC outbound stub")?; + if !asm.adr(X16, stub_vaddr)? { return Ok(GateBuild::Unreachable); } + asm.emit(Insn::StrUimm { + rt: X16, + rn: SP, + imm_bytes: SVC_FRAME_OFF_STUB, + }); - trampoline_data.extend_from_slice(&asm.finish()); - Ok(GateBuild::Emitted) -} - -/// Shared SVC handler (2 instructions, 8 bytes). -/// -/// A thin shim that conveys nothing TLS-related: it loads the syscall-callback -/// pointer from the trampoline header and tail-jumps to it. The callback reads -/// host TLS from `TPIDR_EL0` (the host anchor) and the guest thread pointer from -/// `[TPIDR_EL0 + GUEST_TPIDR_OFFSET]` itself, so the handler carries no TLS state. -/// -/// Nothing in the handler clobbers NZCV, so the guest's pre-svc flags reach the -/// callback unchanged with no save/restore. -fn emit_shared_svc_handler( - trampoline_data: &mut Vec, - handler_offset: usize, - trampoline_base_addr: u64, -) -> Result<()> { - let handler_vaddr = - checked_add_u64(trampoline_base_addr, handler_offset as u64, "SVC handler")?; + // LDR X16, =callback ; BR X16. The literal reaches back to the header, so + // the last SVC gate has to sit within LDR-literal's ±1MiB of offset 0. + // That caps one object at about 16K SVC slots; beyond it `ldr_literal` + // reports `AddressOverflow` rather than encoding a wrapped offset. let callback_vaddr = checked_add_u64( trampoline_base_addr, HEADER_CALLBACK_OFFSET as u64, "callback slot", )?; - let mut asm = Asm::new(handler_vaddr); - - // LDR X16, =callback ; BR X16. Nothing here clobbers NZCV, so the guest's - // pre-svc flags reach the callback unchanged with no save/restore. asm.ldr_literal(X16, callback_vaddr)?; asm.emit(Insn::Br(X16)); - trampoline_data.extend_from_slice(&asm.finish()); - Ok(()) + debug_assert_eq!( + asm.here()?, + stub_vaddr, + "the outbound stub must start immediately after the gate" + ); + + // The outbound stub: LDR X16, [SP] ; ADD SP, SP, #32 ; B . + asm.emit(Insn::LdrUimm { + rt: X16, + rn: SP, + imm_bytes: SVC_FRAME_OFF_X16, + }); + asm.emit(Insn::AddSp(SVC_FRAME_BYTES)); + if !asm.branch_to(return_addr)? { + return Ok(GateBuild::Unreachable); + } + + debug_assert_eq!( + asm.here()?, + checked_add_u64(stub_vaddr, SVC_OUTBOUND_STUB_BYTES as u64, "SVC stub end")?, + "the outbound stub must be SVC_OUTBOUND_STUB_BYTES long" + ); + + append_gate_slot( + trampoline_data, + asm.finish(), + trampoline_base_addr, + gate_vaddr, + GateMetadata::Svc, + SVC_SLOT_BYTES, + )?; + Ok(GateBuild::Emitted) } // ============================================================ // MSR + MRS gates // ============================================================ -/// Per-site MSR gate (9 instructions, 36 bytes, 32-byte frame). +/// Per-site MSR gate with a 32-byte frame, padded into one 48-byte slot. /// -/// Virtualizes a guest `MSR TPIDR_EL0, Xn` write. The hardware register holds the -/// host anchor, so the gate stores the guest value into the guest thread-pointer -/// slot at `[TPIDR_EL0 + GUEST_TPIDR_OFFSET]`: +/// Virtualizes a guest `MSR TPIDR_EL0, Xn` write by storing the guest value +/// into the guest thread-pointer slot at `[anchor + guest_tpidr_offset]`: /// -/// 1. spill X16/X17 and capture the guest value `Xn` to the frame while all guest -/// registers are still pristine (so `Xn` needs no special-casing, even when it -/// is one of the scratch registers just spilled (X16/X17) or XZR); -/// 2. `MRS X16, TPIDR_EL0` reads the host anchor; -/// 3. reload the captured value into X17 and `STR X17, [X16, #GUEST_TPIDR_OFFSET]` -/// stores it into the guest thread-pointer slot; +/// 1. spill X16/X17 and capture `Xn` to the frame while all guest registers are +/// still pristine, so `Xn` needs no special-casing even when it is one of the +/// scratch registers just spilled or XZR; +/// 2. read the host anchor into X16; +/// 3. reload the captured value into X17 and store it to the slot; /// 4. restore X16/X17 and branch back to the instruction after the original MSR. /// -/// The slot is always reachable because `TPIDR_EL0` is the host anchor the -/// runtime keeps valid, so a guest value of `0` (XZR) is an ordinary store — never -/// a fault. -/// -/// The X16/X17 spill and the captured value use a guest-stack frame, so this gate -/// requires `SP` to address a valid writable stack at the site (see the module -/// docs, "Gate scratch storage and the stack invariant"). -/// -/// `MSR TPIDR_EL0` does not touch the condition flags and the gate uses only -/// plain loads/stores and `B` (never `BL`), so NZCV and X30 reach the guest -/// unchanged with no save/restore. +/// The slot is always reachable, so a guest value of `0` (XZR) is an ordinary +/// store, never a fault. Requires `SP` to address a valid writable stack at the +/// site (see the module docs). NZCV and X30 reach the guest unchanged. fn emit_msr_gate( trampoline_data: &mut Vec, gate_offset: usize, @@ -885,7 +1144,7 @@ fn emit_msr_gate( // MRS X16, — read the host anchor. asm.emit(host.anchor_read(X16)); - // LDR X17, [SP, #16] ; STR X17, [X16, #GUEST_TPIDR_OFFSET] — store the guest + // LDR X17, [SP, #16] ; STR X17, [X16, #] — store the guest // value into its slot off the host anchor. asm.emit(Insn::LdrUimm { rt: X17, @@ -895,7 +1154,7 @@ fn emit_msr_gate( asm.emit(Insn::StrUimm { rt: X17, rn: X16, - imm_bytes: GUEST_TPIDR_OFFSET, + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER, }); // Restore: LDP X16, X17, [SP] ; ADD SP, SP, #32. @@ -907,21 +1166,31 @@ fn emit_msr_gate( }); asm.emit(Insn::AddSp(MSR_FRAME_BYTES)); - // B . - if !asm.branch_to(checked_add_u64(site.vaddr, 4, "MSR return")?)? { + // Two branches to the same return address. Only the first executes; the + // pair pins the slot's absolute position, so the original PC need not be + // stored in metadata. `validate_gate_slot` requires both to decode to one + // target, so a gate that can encode only the first is unclassifiable and + // must not be emitted -- which costs the second branch's 4 bytes of reach. + let return_addr = checked_add_u64(site.vaddr, INSN_BYTES_U64, "MSR return")?; + if !asm.branch_to(return_addr)? || !asm.branch_to(return_addr)? { return Ok(GateBuild::Unreachable); } - trampoline_data.extend_from_slice(&asm.finish()); + append_gate_slot( + trampoline_data, + asm.finish(), + trampoline_base_addr, + gate_vaddr, + GateMetadata::MsrTpidr { source: rt }, + MSR_SLOT_BYTES, + )?; Ok(GateBuild::Emitted) } -/// Per-site MRS gate (3 instructions, 12 bytes). -/// -/// Virtualizes a guest `MRS Xd, TPIDR_EL0` read. The hardware register holds the -/// host anchor, so the gate reads the anchor and then loads the guest thread -/// pointer from its slot, reusing `Xd` as scratch (no frame needed): -/// `MRS Xd, TPIDR_EL0 ; LDR Xd, [Xd, #GUEST_TPIDR_OFFSET] ; B `. +/// Virtualizes a guest `MRS Xd, TPIDR_EL0` read. The compact gate uses `Xd` +/// itself as scratch, so once its first `MRS` executes the old destination is +/// gone; canonicalization completes the logical MRS and reports the +/// post-instruction state rather than rewinding. fn emit_mrs_gate( trampoline_data: &mut Vec, gate_offset: usize, @@ -936,187 +1205,897 @@ fn emit_mrs_gate( asm.emit(Insn::LdrUimm { rt: rd, rn: rd, - imm_bytes: GUEST_TPIDR_OFFSET, + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER, }); - if !asm.branch_to(checked_add_u64(site.vaddr, 4, "MRS return")?)? { + let return_addr = checked_add_u64(site.vaddr, INSN_BYTES_U64, "MRS return")?; + if !asm.branch_to(return_addr)? { return Ok(GateBuild::Unreachable); } - trampoline_data.extend_from_slice(&asm.finish()); + append_gate_slot( + trampoline_data, + asm.finish(), + trampoline_base_addr, + gate_vaddr, + GateMetadata::MrsTpidr { destination: rd }, + MRS_SLOT_BYTES, + )?; Ok(GateBuild::Emitted) } -// ============================================================ -// Small helpers -// ============================================================ +fn append_gate_slot( + trampoline_data: &mut Vec, + mut code: Vec, + trampoline_base: u64, + slot_vaddr: u64, + metadata: GateMetadata, + slot_size: usize, +) -> Result<()> { + let metadata_offset = slot_size - GATE_METADATA_BYTES; + if code.len() > metadata_offset { + return Err(Error::AddressOverflow(format!( + "AArch64 gate is {} bytes and does not fit before slot metadata", + code.len() + ))); + } + while code.len() < metadata_offset { + code.extend_from_slice(&NOP.to_le_bytes()); + } + let encoded = EncodedGateMetadata::encode(metadata) + .ok_or_else(|| Error::AddressOverflow("invalid AArch64 gate metadata".into()))?; + code.extend_from_slice(&encoded.0.to_le_bytes()); + debug_assert_eq!(code.len(), slot_size); + let metadata_bytes: [u8; 4] = code[metadata_offset..] + .try_into() + .map_err(|_| Error::AddressOverflow("AArch64 metadata word length".into()))?; + let decoded = EncodedGateMetadata(u32::from_le_bytes(metadata_bytes)) + .decode() + .ok_or_else(|| Error::AddressOverflow("emitter produced invalid metadata".into()))?; + debug_assert!(validate_gate_slot( + &code, + trampoline_base, + slot_vaddr, + decoded + )); + trampoline_data.extend_from_slice(&code); + Ok(()) +} -/// A position-tracking assembler for one trampoline fragment (a gate or a shared -/// handler). It owns the emitted words and the base virtual address of the first -/// word, so the current vaddr — [`Asm::here`] — is always known without manual -/// instruction counting. -/// -/// Branches and loads to an absolute target ([`Asm::branch_to`], -/// [`Asm::ldr_literal`], [`Asm::adrp`]) resolve immediately against -/// [`Asm::here`]. The per-site branches ([`Asm::branch_to`], [`Asm::adrp`]) -/// report an out-of-range target by emitting nothing and returning `false`, so -/// the caller can trap the site; [`Asm::ldr_literal`] (used only by the fixed -/// prologue) instead errors, since a prologue that cannot be placed is fatal. -struct Asm { - base_vaddr: u64, - code: Vec, +/// Validate the complete instruction template and metadata word of one slot. +/// This is structural recognition, not authentication: the gate-signal +/// canonicalization path must additionally fault-safely validate that the +/// recovered original site branches into this slot before canonicalizing an +/// interrupted context. +pub(crate) fn validate_gate_slot( + slot: &[u8], + trampoline_base: u64, + slot_vaddr: u64, + metadata: GateMetadata, +) -> bool { + validate_gate_slot_inner( + slot, + SlotAddressing::Placed { + trampoline_base, + slot_vaddr, + }, + metadata, + ) } -impl Asm { - fn new(base_vaddr: u64) -> Self { - Asm { - base_vaddr, - code: Vec::new(), - } - } +/// Where a slot being validated lives, which decides how exactly its branch and +/// literal targets can be checked. +#[derive(Clone, Copy, Debug)] +enum SlotAddressing { + /// The trampoline has been placed at its final address, so every target + /// resolves to an absolute address and can be compared exactly. + Placed { + /// Address the trampoline starts at. + trampoline_base: u64, + /// Address the slot itself starts at. + slot_vaddr: u64, + }, + /// The trampoline is still a position-independent blob, so targets can only + /// be checked structurally: the right opcodes, self-consistent with each + /// other. + Unplaced { + /// Byte offset of the slot within the blob. + slot_offset: u64, + }, +} - /// Virtual address of the next instruction to be emitted. - fn here(&self) -> Result { - checked_add_u64( - self.base_vaddr, - self.code.len() as u64, - "trampoline gate next-instruction", - ) +fn validate_gate_slot_inner( + slot: &[u8], + addressing: SlotAddressing, + metadata: GateMetadata, +) -> bool { + let slot_size = metadata.slot_size(); + // Both variants' positions are 16-byte aligned by construction, because the + // blob is laid out and mapped at gate alignment. + let anchor = match addressing { + SlotAddressing::Placed { slot_vaddr, .. } => slot_vaddr, + SlotAddressing::Unplaced { slot_offset } => slot_offset, + }; + if slot.len() != slot_size || !anchor.is_multiple_of(GATE_ALIGNMENT as u64) { + return false; } - - /// Append a raw little-endian word. - fn push_word(&mut self, word: u32) { - self.code.extend_from_slice(&word.to_le_bytes()); + let Some(encoded) = EncodedGateMetadata::encode(metadata) else { + return false; + }; + let metadata_offset = slot_size - GATE_METADATA_BYTES; + if slot[metadata_offset..] != encoded.0.to_le_bytes() { + return false; } - - /// Append a fixed-operand instruction. Every operand at the call sites is a - /// compile-time-known register or frame offset, so encoding cannot fail; a - /// `None` would be a rewriter bug rather than an unencodable program. - fn emit(&mut self, insn: Insn) { - let word = insn.encode().expect("statically valid instruction"); - self.push_word(word); + let word = |offset: usize| { + u32::from_le_bytes( + slot[offset..offset + INSN_BYTES] + .try_into() + .expect("word-sized slice"), + ) + }; + let exact = |offset: usize, insn: Insn| word(offset) == insn.encode().unwrap(); + let padding_is_nops = |start: usize| { + (start..metadata_offset) + .step_by(INSN_BYTES) + .all(|offset| word(offset) == NOP) + }; + + match metadata { + GateMetadata::Svc => { + let adrp = word(8); + let add = word(12); + exact(0, Insn::SubSp(SVC_FRAME_BYTES)) + && exact( + 4, + Insn::StrUimm { + rt: X16, + rn: SP, + imm_bytes: SVC_FRAME_OFF_X16, + }, + ) + && exact( + 16, + Insn::StrUimm { + rt: X16, + rn: SP, + imm_bytes: SVC_FRAME_OFF_RETADDR, + }, + ) + && exact( + 20, + Insn::Adr { + rd: X16, + byte_off: 16, + }, + ) + && exact( + 24, + Insn::StrUimm { + rt: X16, + rn: SP, + imm_bytes: SVC_FRAME_OFF_STUB, + }, + ) + && match addressing { + // Unplaced, the literal load is still relative to the blob, + // so it resolves to the header slot's own offset. + SlotAddressing::Unplaced { slot_offset } => { + decode_ldr_literal_target(word(28), slot_offset + 28) + == Some(HEADER_CALLBACK_OFFSET as u64) + } + SlotAddressing::Placed { + trampoline_base, + slot_vaddr, + } => { + decode_ldr_literal_target(word(28), slot_vaddr + 28) + == Some(trampoline_base + HEADER_CALLBACK_OFFSET as u64) + } + } + && exact(32, Insn::Br(X16)) + && exact( + 36, + Insn::LdrUimm { + rt: X16, + rn: SP, + imm_bytes: SVC_FRAME_OFF_X16, + }, + ) + && exact(40, Insn::AddSp(SVC_FRAME_BYTES)) + && match addressing { + SlotAddressing::Unplaced { .. } => { + adrp & ADRP_SHAPE_MASK == Opcode::Adrp.bits() | u32::from(X16) + && add & ADD_IMM_SHAPE_MASK + == Opcode::AddImm.bits() + | (u32::from(X16) << RN_SHIFT) + | u32::from(X16) + && word(44) & OPCODE_TOP6_MASK == Opcode::B.bits() + } + SlotAddressing::Placed { slot_vaddr, .. } => { + let return_from_adrp = decode_adrp_add_target(adrp, add, slot_vaddr + 8); + let return_from_branch = decode_branch_target(word(44), slot_vaddr + 44); + return_from_adrp.is_some() && return_from_adrp == return_from_branch + } + } + && padding_is_nops(48) + } + GateMetadata::MrsTpidr { destination } => { + exact(0, Insn::MrsTpidrEl0(destination)) + && is_tpidr_access(word(4), Opcode::LdrUimm, destination, destination) + && match addressing { + SlotAddressing::Unplaced { .. } => { + word(8) & OPCODE_TOP6_MASK == Opcode::B.bits() + } + SlotAddressing::Placed { slot_vaddr, .. } => { + decode_branch_target(word(8), slot_vaddr + 8).is_some() + } + } + } + GateMetadata::MsrTpidr { source } => { + exact(0, Insn::SubSp(MSR_FRAME_BYTES)) + && exact( + 4, + Insn::Stp { + rt: X16, + rt2: X17, + rn: SP, + imm_bytes: 0, + }, + ) + && exact( + 8, + Insn::StrUimm { + rt: source, + rn: SP, + imm_bytes: MSR_FRAME_OFF_VALUE, + }, + ) + && exact(12, Insn::MrsTpidrEl0(X16)) + && exact( + 16, + Insn::LdrUimm { + rt: X17, + rn: SP, + imm_bytes: MSR_FRAME_OFF_VALUE, + }, + ) + && is_tpidr_access(word(20), Opcode::StrUimm, X17, X16) + && exact( + 24, + Insn::Ldp { + rt: X16, + rt2: X17, + rn: SP, + imm_bytes: 0, + }, + ) + && exact(28, Insn::AddSp(MSR_FRAME_BYTES)) + && match addressing { + SlotAddressing::Unplaced { .. } => { + word(32) & OPCODE_TOP6_MASK == Opcode::B.bits() + && word(36) & OPCODE_TOP6_MASK == Opcode::B.bits() + && branch_local_target(word(32), 32) + == branch_local_target(word(36), 36) + } + SlotAddressing::Placed { slot_vaddr, .. } => { + decode_branch_target(word(32), slot_vaddr + 32) + == decode_branch_target(word(36), slot_vaddr + 36) + && decode_branch_target(word(32), slot_vaddr + 32).is_some() + } + } + && padding_is_nops(40) + } } +} - /// `B ` — unconditional branch to an absolute address. Returns - /// whether the target was within the branch's ±128MB reach: an out-of-range - /// target emits nothing and yields `false`, so the caller can trap the - /// originating site instead of failing the whole rewrite. - fn branch_to(&mut self, target_vaddr: u64) -> Result { - let offset = self.delta_to(target_vaddr)?; - let Some(word) = Insn::B(offset).encode() else { - return Ok(false); - }; - self.push_word(word); - Ok(true) +impl GateMetadata { + /// Fixed byte size of this metadata version's compact slot. + pub const fn slot_size(self) -> usize { + match self { + GateMetadata::Svc => SVC_SLOT_BYTES, + GateMetadata::MrsTpidr { .. } => MRS_SLOT_BYTES, + GateMetadata::MsrTpidr { .. } => MSR_SLOT_BYTES, + } } - /// `LDR Xt, =target` — PC-relative literal load of an absolute address. - fn ldr_literal(&mut self, rt: u8, target_vaddr: u64) -> Result<()> { - let offset = self.delta_to(target_vaddr)?; - let word = Insn::LdrLiteral { rt, off: offset } - .encode() - .ok_or_else(|| { - Error::AddressOverflow(format!("LDR literal offset {offset:#x} out of ±1MB range")) - })?; - self.push_word(word); - Ok(()) + /// First byte offset past the slot's executable body. + /// + /// Everything from here to the trailing metadata word is `NOP` padding, so + /// a PC at or past it is not inside the gate proper and must not be + /// classified as one. + pub(crate) const fn executable_end(self) -> usize { + match self { + // The `B` back to the original site at 44 is the last instruction. + GateMetadata::Svc => 48, + // `MRS`, the guest-TLS `LDR`, then the `B` back. + GateMetadata::MrsTpidr { .. } => 12, + // Frame teardown ends at 28, then the `B` back at 32. The second + // `B` at 36 never executes, so a PC there is not a live gate PC. + GateMetadata::MsrTpidr { .. } => 36, + } } - /// `ADRP Xd, ` — page-relative address of an absolute target. - /// Returns whether the target's page was within ADRP's ±4GB reach (see - /// [`Asm::branch_to`] for the out-of-range contract). - fn adrp(&mut self, rd: u8, target_vaddr: u64) -> Result { - let here = self.here()?; - let page_off = (target_vaddr & !0xFFF) - .cast_signed() - .saturating_sub((here & !0xFFF).cast_signed()) - >> 12; - let Some(word) = Insn::Adrp { rd, page_off }.encode() else { - return Ok(false); - }; - self.push_word(word); - Ok(true) + /// Byte offset of the branch that returns to the instruction after the + /// original site. + pub(crate) const fn return_offset(self) -> usize { + match self { + GateMetadata::Svc => 44, + GateMetadata::MrsTpidr { .. } => 8, + GateMetadata::MsrTpidr { .. } => 32, + } } - /// Signed byte distance from [`Asm::here`] to `target_vaddr`. The subtraction - /// saturates so a pathological address can't overflow it; a distance the - /// branch can't encode is rejected by the encoder's range check at the call - /// site, with the saturated value reported for diagnostics. - fn delta_to(&self, target_vaddr: u64) -> Result { - Ok(target_vaddr - .cast_signed() - .saturating_sub(self.here()?.cast_signed())) + /// Byte offset of the instruction that commits the gate's architectural + /// effect. A saved PC names the instruction about to execute, so past this + /// offset the effect has happened. + /// + /// This says when the effect lands, not that the context is rewindable + /// before it. Only [`GateMetadata::MsrTpidr`] is, from its spill frame. + /// [`GateMetadata::MrsTpidr`] destroys the register a rewind would restore, + /// so every PC inside it is carried forward instead. + pub const fn commit_offset(self) -> usize { + match self { + // The shim performs the syscall, so any PC in an `SVC` slot is + // still pre-commit — though at 4 or beyond `SP` and `X16` still + // need undoing. + GateMetadata::Svc => 0, + // The guest-TLS load. + GateMetadata::MrsTpidr { .. } => 4, + // The guest-TLS store. + GateMetadata::MsrTpidr { .. } => 20, + } } +} - /// Return the emitted bytes. - fn finish(self) -> Vec { - self.code - } +/// Decode one little-endian metadata word copied from a candidate slot. +pub fn decode_gate_metadata_word(word: u32) -> Option { + EncodedGateMetadata(word).decode() } -#[cfg(test)] -mod tests { - use super::*; - use alloc::vec; +/// A validated gate slot containing some PC, with what a signal handler needs +/// to canonicalize the interrupted context: where the slot starts, how far +/// into it the guest-visible effect commits, and which guest instruction it +/// replaced. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ClassifiedGate { + slot_offset: usize, + slot_size: u8, + commit_offset: u8, + original_site: u64, + metadata: GateMetadata, +} - // Gate and shared-handler sizes. The emitters append each gate dynamically - // (`gate_offset = trampoline_data.len()`), so these sizes drive no emission; - // the tests use them to slice individual gates out of the trampoline blob and - // to assert its total length. `GATES_START_OFFSET` is the fixed shared-prologue - // size that the per-site gates follow. - const SVC_GATE_INSNS: usize = 6; - const SVC_GATE_SIZE: usize = SVC_GATE_INSNS * 4; - const SHARED_SVC_HANDLER_INSNS: usize = 2; - const SHARED_SVC_HANDLER_SIZE: usize = SHARED_SVC_HANDLER_INSNS * 4; - const MSR_GATE_INSNS: usize = 9; - const MSR_GATE_SIZE: usize = MSR_GATE_INSNS * 4; - const MRS_GATE_INSNS: usize = 3; - const MRS_GATE_SIZE: usize = MRS_GATE_INSNS * 4; - const GATES_START_OFFSET: usize = SHARED_SVC_HANDLER_OFFSET + SHARED_SVC_HANDLER_SIZE; +impl ClassifiedGate { + /// Byte offset of the validated slot within the trampoline. + pub fn slot_offset(self) -> usize { + self.slot_offset + } - /// Top-6 opcode bits, isolating the `B`/`BL` major opcode for read-back checks. - const OPCODE_TOP6_MASK: u32 = 0xFC00_0000; + /// Kind-derived byte size of the validated slot. + pub fn slot_size(self) -> u8 { + self.slot_size + } - fn word_at(data: &[u8], byte_off: usize) -> u32 { - u32::from_le_bytes(data[byte_off..byte_off + 4].try_into().unwrap()) + /// Template-derived architectural commit offset. + pub fn commit_offset(self) -> u8 { + self.commit_offset } - /// `MSR TPIDR_EL0, Xrt` guest instruction word (the low 5 bits select Xrt). - /// The rewriter only matches/scans this form; it never emits it, so the - /// encoder lives only here for building test inputs. - fn msr_tpidr_el0(rt: u8) -> u32 { - MSR_TPIDR_EL0_BITS | u32::from(rt) + /// Original guest instruction address recovered from the return branch. + pub fn original_site(self) -> u64 { + self.original_site } - /// Helper: emit just the shared SVC handler and return its instruction words. - fn shared_svc_handler_words() -> vec::Vec { - let mut buf = vec::Vec::new(); - emit_shared_svc_handler(&mut buf, 0, 0x1000).unwrap(); - buf.chunks_exact(4) - .map(|w| u32::from_le_bytes(w.try_into().unwrap())) - .collect() + /// Decoded semantic gate metadata. + pub fn metadata(self) -> GateMetadata { + self.metadata } +} - #[test] - fn svc_handler_jumps_to_callback_without_tls() { - let words = shared_svc_handler_words(); - assert_eq!(words.len(), SHARED_SVC_HANDLER_INSNS); - // The handler conveys nothing TLS-related: it loads the callback pointer - // and tail-jumps. The callback reads host TLS from TPIDR_EL0 itself. - assert_eq!( - words[1], - Insn::Br(X16).encode().unwrap(), - "handler ends in BR X16" - ); - // No MRS TPIDR_EL0 anywhere in the handler. - assert!( - !words - .iter() - .any(|&w| w & MRS_TPIDR_EL0_MASK == MRS_TPIDR_EL0_BITS) - ); +/// Classify an AArch64 trampoline PC by testing at most four aligned candidate +/// slot starts and requiring exactly one exact-template match. +pub fn classify_gate_pc( + trampoline: &[u8], + trampoline_base: u64, + pc: u64, +) -> Option { + if !pc.is_multiple_of(INSN_BYTES_U64) { + return None; } + let relative = usize::try_from(pc.checked_sub(trampoline_base)?).ok()?; + let aligned = relative & !(GATE_ALIGNMENT - 1); + let candidates = core::array::from_fn(|index| aligned.saturating_sub(index * GATE_ALIGNMENT)); + classify_gate_pc_with_candidates(trampoline, trampoline_base, pc, candidates) +} - #[test] - fn encoders_match_known_words() { - // `B #0`. +/// Classifies one fault-safely copied compact slot containing `pc`. +/// +/// The caller is responsible for selecting candidate slot starts and for +/// requiring exactly one match. Keeping that policy outside this pure helper +/// lets a signal handler copy each candidate before inspecting it. +pub fn classify_copied_gate_slot(slot: &[u8], slot_vaddr: u64, pc: u64) -> Option { + let offset = usize::try_from(pc.checked_sub(slot_vaddr)?).ok()?; + if !pc.is_multiple_of(INSN_BYTES_U64) || !slot_vaddr.is_multiple_of(GATE_ALIGNMENT as u64) { + return None; + } + let metadata_word = + u32::from_le_bytes(slot.get(slot.len().checked_sub(4)?..)?.try_into().ok()?); + let metadata = EncodedGateMetadata(metadata_word).decode()?; + if metadata.slot_size() != slot.len() || offset >= metadata.executable_end() { + return None; + } + let trampoline_base = match metadata { + GateMetadata::Svc => decode_ldr_literal_target( + u32::from_le_bytes(slot.get(28..32)?.try_into().ok()?), + slot_vaddr + 28, + )?, + GateMetadata::MrsTpidr { .. } | GateMetadata::MsrTpidr { .. } => 0, + }; + // For `Svc` the base was just recovered from the slot's own literal load, + // so `validate_gate_slot`'s callback check is a tautology here. The other + // template checks still do real work; this is structural recognition, not + // authentication. + if !validate_gate_slot(slot, trampoline_base, slot_vaddr, metadata) { + return None; + } + let return_offset = metadata.return_offset(); + let return_target = decode_branch_target( + u32::from_le_bytes( + slot.get(return_offset..return_offset + INSN_BYTES)? + .try_into() + .ok()?, + ), + slot_vaddr + return_offset as u64, + )?; + Some(ClassifiedGate { + slot_offset: 0, + slot_size: u8::try_from(slot.len()).ok()?, + commit_offset: u8::try_from(metadata.commit_offset()).ok()?, + original_site: return_target.checked_sub(4)?, + metadata, + }) +} + +fn classify_gate_pc_with_candidates( + trampoline: &[u8], + trampoline_base: u64, + pc: u64, + candidates: [usize; 4], +) -> Option { + let relative = usize::try_from(pc.checked_sub(trampoline_base)?).ok()?; + let mut match_found = None; + for start in candidates { + if start < GATES_START_OFFSET || !start.is_multiple_of(GATE_ALIGNMENT) { + continue; + } + for slot_size in [MRS_SLOT_BYTES, MSR_SLOT_BYTES, SVC_SLOT_BYTES] { + let end = start.checked_add(slot_size)?; + if relative < start || relative >= end || end > trampoline.len() { + continue; + } + let metadata_word = + u32::from_le_bytes(trampoline[end - GATE_METADATA_BYTES..end].try_into().ok()?); + let Some(metadata) = EncodedGateMetadata(metadata_word).decode() else { + continue; + }; + let slot = &trampoline[start..end]; + let slot_vaddr = trampoline_base + start as u64; + // The size agreement has to come first: it is what makes the + // decoded metadata's layout accessors apply to these bytes. + if metadata.slot_size() != slot_size + || relative >= start + metadata.executable_end() + || !validate_gate_slot(slot, trampoline_base, slot_vaddr, metadata) + { + continue; + } + if match_found.is_some() { + return None; + } + let return_offset = metadata.return_offset(); + let return_target = decode_branch_target( + u32::from_le_bytes( + slot[return_offset..return_offset + INSN_BYTES] + .try_into() + .ok()?, + ), + slot_vaddr + return_offset as u64, + )?; + match_found = Some(ClassifiedGate { + slot_offset: start, + slot_size: u8::try_from(slot_size).ok()?, + commit_offset: u8::try_from(metadata.commit_offset()).ok()?, + original_site: return_target.checked_sub(4)?, + metadata, + }); + } + } + match_found +} + +fn is_tpidr_access(word: u32, opcode: Opcode, rt: u8, rn: u8) -> bool { + if word & !LDST_UIMM12_IMM_MASK != opcode.bits() | (u32::from(rn) << RN_SHIFT) | u32::from(rt) { + return false; + } + let encoded = (word & LDST_UIMM12_IMM_MASK) >> LDST_UIMM12_IMM_SHIFT; + valid_emitted_tpidr_offset(encoded * u32::from(GUEST_TPIDR_OFFSET_ALIGN)) +} + +fn valid_emitted_tpidr_offset(offset: u32) -> bool { + offset == u32::from(GUEST_TPIDR_OFFSET_PLACEHOLDER) + || (offset.is_multiple_of(u32::from(GUEST_TPIDR_OFFSET_ALIGN)) + && (u32::from(MIN_GUEST_TPIDR_OFFSET)..u32::from(GUEST_TPIDR_OFFSET_PLACEHOLDER)) + .contains(&offset)) +} + +fn decode_branch_target(word: u32, pc: u64) -> Option { + if word & OPCODE_TOP6_MASK != Opcode::B.bits() { + return None; + } + let imm26 = i64::from(word & IMM26_MASK); + let displacement = pcrel_bytes(imm26, IMM26_BITS); + pc.checked_add_signed(displacement) +} + +fn branch_local_target(word: u32, pc_offset: i64) -> Option { + if word & OPCODE_TOP6_MASK != Opcode::B.bits() { + return None; + } + let imm26 = i64::from(word & IMM26_MASK); + Some(pc_offset + pcrel_bytes(imm26, IMM26_BITS)) +} + +fn decode_ldr_literal_target(word: u32, pc: u64) -> Option { + if word & LDR_LITERAL_SHAPE_MASK != Opcode::LdrLiteral.bits() | u32::from(X16) { + return None; + } + let imm19 = i64::from((word >> RN_SHIFT) & IMM19_MASK); + let displacement = pcrel_bytes(imm19, IMM19_BITS); + pc.checked_add_signed(displacement) +} + +fn decode_adrp_add_target(adrp: u32, add: u32, pc: u64) -> Option { + if adrp & ADRP_SHAPE_MASK != Opcode::Adrp.bits() | u32::from(X16) + || add & ADD_IMM_SHAPE_MASK + != Opcode::AddImm.bits() | (u32::from(X16) << RN_SHIFT) | u32::from(X16) + { + return None; + } + let immlo = i64::from((adrp >> ADR_IMMLO_SHIFT) & ADR_IMMLO_MASK); + let immhi = i64::from((adrp >> RN_SHIFT) & IMM19_MASK); + let imm21 = (immhi << 2) | immlo; + let page_delta = sign_extend(imm21, IMM21_BITS) << ADRP_PAGE_SHIFT; + let page = (pc & !PAGE_OFFSET_MASK).checked_add_signed(page_delta)?; + page.checked_add(u64::from(add >> RT2_SHIFT) & PAGE_OFFSET_MASK) +} + +// ============================================================ +// Load-time guest thread-pointer offset patching +// ============================================================ + +/// Bits \[31:10] of an unsigned-offset load/store: everything except `Rn` +/// (\[9:5]) and `Rt` (\[4:0]), i.e. the opcode *and* the scaled `imm12`. +const LDST_UIMM12_OPCODE_AND_IMM_MASK: u32 = 0xFFFF_FC00; +/// The scaled 12-bit immediate field of an unsigned-offset load/store, \[21:10]. +const LDST_UIMM12_IMM_MASK: u32 = 0x003F_FC00; +/// Bit position of that immediate field. +const LDST_UIMM12_IMM_SHIFT: u32 = 10; + +/// Rewrites the guest thread-pointer offset in every gate of one emitted +/// trampoline, replacing the emitted placeholder with `offset`. The loader must +/// call this before making the trampoline executable. +/// +/// Returns the number of instructions patched. Zero is normal: a binary whose +/// only patch sites are `SVC` has no thread-pointer gate. +/// +/// # Errors +/// +/// Fails if `offset` is not a legitimate gate target — a multiple of +/// [`GUEST_TPIDR_OFFSET_ALIGN`], at least `MIN_GUEST_TPIDR_OFFSET` so it clears +/// the host's own per-thread state, and strictly below +/// [`MAX_GUEST_TPIDR_OFFSET`], which is reserved as the placeholder. Also fails +/// if the blob is not a well-formed trampoline: too short for the shared +/// prologue, not a whole number of instruction words, or holding a placeholder +/// in a shape no gate emits. +/// +/// # Panics +/// +/// Panics if a validated slot is shorter than its own metadata word, which the +/// slot templates make impossible. +pub fn patch_guest_tpidr_offset(trampoline: &mut [u8], offset: u16) -> Result { + if !offset.is_multiple_of(GUEST_TPIDR_OFFSET_ALIGN) + || !(MIN_GUEST_TPIDR_OFFSET..GUEST_TPIDR_OFFSET_PLACEHOLDER).contains(&offset) + { + return Err(Error::TrampolinePatchFailure(format!( + "guest thread-pointer offset {offset} is not a legitimate gate target: it must be a \ + multiple of {GUEST_TPIDR_OFFSET_ALIGN}, at least {MIN_GUEST_TPIDR_OFFSET} so it \ + cannot land on the host's own per-thread state, and below \ + {GUEST_TPIDR_OFFSET_PLACEHOLDER} so a patched gate is never mistaken for an \ + unpatched one" + ))); + } + + let patch_offsets = validate_trampoline_and_collect_placeholders(trampoline, offset)?; + let new_imm = u32::from(offset / GUEST_TPIDR_OFFSET_ALIGN) << LDST_UIMM12_IMM_SHIFT; + for offset in &patch_offsets { + let word = &mut trampoline[*offset..*offset + INSN_BYTES]; + let insn = u32::from_le_bytes(word.try_into().expect("four-byte patch offset")); + let patched_insn = (insn & !LDST_UIMM12_IMM_MASK) | new_imm; + word.copy_from_slice(&patched_insn.to_le_bytes()); + } + Ok(patch_offsets.len()) +} + +/// Validates every slot and returns the offsets of the thread-pointer +/// instructions still holding the placeholder. +/// +/// A trampoline arrives from the guest's own file and the slot templates accept +/// any encodable offset, so a thread-pointer access holding neither the +/// placeholder nor `expected_offset` is one this rewriter did not put there and +/// is rejected rather than left alone. +fn validate_trampoline_and_collect_placeholders( + trampoline: &[u8], + expected_offset: u16, +) -> Result> { + if trampoline.len() < GATES_START_OFFSET || !trampoline.len().is_multiple_of(INSN_BYTES) { + return Err(Error::TrampolinePatchFailure( + "malformed AArch64 trampoline length".into(), + )); + } + if trampoline[8..GATES_START_OFFSET] + .chunks_exact(4) + .any(|word| u32::from_le_bytes(word.try_into().unwrap()) != NOP) + { + return Err(Error::TrampolinePatchFailure( + "malformed AArch64 trampoline header padding".into(), + )); + } + + let mut cursor = GATES_START_OFFSET; + let mut patch_offsets = Vec::new(); + while cursor < trampoline.len() { + let mut matched = None; + for slot_size in [MRS_SLOT_BYTES, MSR_SLOT_BYTES, SVC_SLOT_BYTES] { + let Some(end) = cursor + .checked_add(slot_size) + .filter(|&end| end <= trampoline.len()) + else { + continue; + }; + let metadata_word = u32::from_le_bytes( + trampoline[end - GATE_METADATA_BYTES..end] + .try_into() + .unwrap(), + ); + let Some(metadata) = EncodedGateMetadata(metadata_word).decode() else { + continue; + }; + if metadata.slot_size() != slot_size + || !validate_gate_slot_inner( + &trampoline[cursor..end], + SlotAddressing::Unplaced { + slot_offset: cursor as u64, + }, + metadata, + ) + { + continue; + } + if matched.is_some() { + return Err(Error::TrampolinePatchFailure( + "ambiguous AArch64 compact slot".into(), + )); + } + matched = Some((end, metadata)); + } + let Some((end, metadata)) = matched else { + return Err(Error::TrampolinePatchFailure(format!( + "malformed AArch64 compact slot at byte {cursor}" + ))); + }; + let instruction_offset = match metadata { + GateMetadata::MrsTpidr { .. } => Some(cursor + 4), + GateMetadata::MsrTpidr { .. } => Some(cursor + 20), + GateMetadata::Svc => None, + }; + if let Some(offset) = instruction_offset { + let insn = + u32::from_le_bytes(trampoline[offset..offset + INSN_BYTES].try_into().unwrap()); + let encoded = (insn & LDST_UIMM12_IMM_MASK) >> LDST_UIMM12_IMM_SHIFT; + let gate_offset = encoded * u32::from(GUEST_TPIDR_OFFSET_ALIGN); + if gate_offset == u32::from(GUEST_TPIDR_OFFSET_PLACEHOLDER) { + patch_offsets.push(offset); + } else if gate_offset != u32::from(expected_offset) { + return Err(Error::TrampolinePatchFailure(format!( + "AArch64 gate at byte {offset} addresses the host thread pointer at \ + {gate_offset}, which is neither the placeholder nor the offset being \ + patched in ({expected_offset})" + ))); + } + } + cursor = end; + } + Ok(patch_offsets) +} + +/// The `imm12` field, already shifted into place, that an unpatched gate's +/// `LDR`/`STR` carries. +fn placeholder_imm_field() -> u32 { + u32::from(GUEST_TPIDR_OFFSET_PLACEHOLDER / GUEST_TPIDR_OFFSET_ALIGN) << LDST_UIMM12_IMM_SHIFT +} + +/// Byte offset of the first instruction in `trampoline` that still carries the +/// emitted placeholder, or `None` if no gate is left unpatched. This is the +/// loader's proof obligation: an unpatched gate does not fault, so its absence +/// has to be checked rather than assumed. +/// +/// Skips the header's callback slot, which is a 64-bit address and could +/// bit-for-bit resemble a gate instruction. Matches the shape-independent +/// opcode-and-immediate pattern, so a placeholder-bearing word in a shape no +/// gate emits is still reported. +/// +/// # Panics +/// +/// Panics if a four-byte window fails to convert to an array, which the +/// chunked iteration makes impossible. +pub fn find_guest_tpidr_placeholder(trampoline: &[u8]) -> Option { + let placeholder_imm = placeholder_imm_field(); + let ldr_pattern = Opcode::LdrUimm.bits() | placeholder_imm; + let str_pattern = Opcode::StrUimm.bits() | placeholder_imm; + + trampoline + .get(FIRST_SCANNABLE_OFFSET..)? + .chunks_exact(4) + .position(|word| { + let insn = u32::from_le_bytes(word.try_into().expect("chunks_exact(4) yields 4 bytes")); + let opcode_and_imm = insn & LDST_UIMM12_OPCODE_AND_IMM_MASK; + opcode_and_imm == ldr_pattern || opcode_and_imm == str_pattern + }) + .map(|index| FIRST_SCANNABLE_OFFSET + index * 4) +} + +// ============================================================ +// Small helpers +// ============================================================ + +/// A position-tracking assembler for one gate. It owns the emitted words and +/// the base virtual address of the first, so [`Asm::here`] is always known +/// without manual instruction counting. +/// +/// Absolute-target forms resolve immediately against [`Asm::here`]. +/// [`Asm::branch_to`] and [`Asm::adrp`] report an out-of-range target by +/// emitting nothing and returning `false`, so the caller can trap the site; +/// [`Asm::ldr_literal`] instead errors, since a callback literal that cannot be +/// placed is fatal. +struct Asm { + base_vaddr: u64, + code: Vec, +} + +impl Asm { + fn new(base_vaddr: u64) -> Self { + Asm { + base_vaddr, + code: Vec::new(), + } + } + + /// Virtual address of the next instruction to be emitted. + fn here(&self) -> Result { + checked_add_u64( + self.base_vaddr, + self.code.len() as u64, + "trampoline gate next-instruction", + ) + } + + /// Append a raw little-endian word. + fn push_word(&mut self, word: u32) { + self.code.extend_from_slice(&word.to_le_bytes()); + } + + /// Append a fixed-operand instruction. Every operand at the call sites is a + /// compile-time-known register or frame offset, so encoding cannot fail; a + /// `None` would be a rewriter bug rather than an unencodable program. + fn emit(&mut self, insn: Insn) { + let word = insn.encode().expect("statically valid instruction"); + self.push_word(word); + } + + /// `B ` — unconditional branch to an absolute address. Returns + /// whether the target was within the branch's ±128MB reach: an out-of-range + /// target emits nothing and yields `false`, so the caller can trap the + /// originating site instead of failing the whole rewrite. + fn branch_to(&mut self, target_vaddr: u64) -> Result { + let offset = self.delta_to(target_vaddr)?; + let Some(word) = Insn::B(offset).encode() else { + return Ok(false); + }; + self.push_word(word); + Ok(true) + } + + /// `LDR Xt, =target` — PC-relative literal load of an absolute address. + fn ldr_literal(&mut self, rt: u8, target_vaddr: u64) -> Result<()> { + let offset = self.delta_to(target_vaddr)?; + let word = Insn::LdrLiteral { rt, off: offset } + .encode() + .ok_or_else(|| { + Error::AddressOverflow(format!("LDR literal offset {offset:#x} out of ±1MB range")) + })?; + self.push_word(word); + Ok(()) + } + + /// `ADR Xd, ` — byte-granular PC-relative address of an absolute + /// target, ±1MB. Returns whether the target was in reach (see + /// [`Asm::branch_to`] for the out-of-range contract). + fn adr(&mut self, rd: u8, target_vaddr: u64) -> Result { + let byte_off = self.delta_to(target_vaddr)?; + let Some(word) = Insn::Adr { rd, byte_off }.encode() else { + return Ok(false); + }; + self.push_word(word); + Ok(true) + } + + /// `ADRP Xd, ` — page-relative address of an absolute target. + /// Returns whether the target's page was within ADRP's ±4GB reach (see + /// [`Asm::branch_to`] for the out-of-range contract). + fn adrp(&mut self, rd: u8, target_vaddr: u64) -> Result { + let here = self.here()?; + let page_off = (target_vaddr & !PAGE_OFFSET_MASK) + .cast_signed() + .saturating_sub((here & !PAGE_OFFSET_MASK).cast_signed()) + >> 12; + let Some(word) = Insn::Adrp { rd, page_off }.encode() else { + return Ok(false); + }; + self.push_word(word); + Ok(true) + } + + /// Signed byte distance from [`Asm::here`] to `target_vaddr`. The subtraction + /// saturates so a pathological address can't overflow it; a distance the + /// branch can't encode is rejected by the encoder's range check at the call + /// site, with the saturated value reported for diagnostics. + fn delta_to(&self, target_vaddr: u64) -> Result { + Ok(target_vaddr + .cast_signed() + .saturating_sub(self.here()?.cast_signed())) + } + + /// Return the emitted bytes. + fn finish(self) -> Vec { + self.code + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::vec; + + // The emitters append each gate at `trampoline_data.len()`, so these sizes + // drive no emission; the tests use them to slice individual gates out of + // the blob and to assert its total length. + /// Bytes emitted per SVC site: the gate proper plus its outbound stub. + const SVC_GATE_SIZE: usize = SVC_SLOT_BYTES; + /// MSR gate instructions up to and including the first return branch. The + /// slot holds one more `B`, which never executes. + const MSR_GATE_INSNS: usize = 9; + const MSR_GATE_SIZE: usize = MSR_SLOT_BYTES; + const MRS_GATE_SIZE: usize = MRS_SLOT_BYTES; + + fn word_at(data: &[u8], byte_off: usize) -> u32 { + u32::from_le_bytes(data[byte_off..byte_off + 4].try_into().unwrap()) + } + + /// `MSR TPIDR_EL0, Xrt` guest instruction word. The rewriter only scans for + /// this form and never emits it, so the encoder lives here. + fn msr_tpidr_el0(rt: u8) -> u32 { + MSR_TPIDR_EL0_BITS | u32::from(rt) + } + + #[test] + fn encoders_match_known_words() { + // `B #0`. assert_eq!(Insn::B(0).encode().unwrap(), 0x1400_0000); // `B #4` advances one instruction. assert_eq!(Insn::B(4).encode().unwrap(), 0x1400_0001); @@ -1124,6 +2103,16 @@ mod tests { assert_eq!(Insn::B(-4).encode().unwrap(), 0x17FF_FFFF); // `BR X16`. assert_eq!(Insn::Br(16).encode().unwrap(), 0xD61F_0200); + // `ADR X16, .+12` — how an SVC gate names its outbound stub. + assert_eq!( + Insn::Adr { + rd: 16, + byte_off: 12 + } + .encode() + .unwrap(), + 0x1000_0070 + ); // TPIDR_EL0 accessor. assert_eq!(Insn::MrsTpidrEl0(9).encode().unwrap(), 0xD53B_D049); // `MSR TPIDR_EL0, X9` guest word (scanned, never emitted). @@ -1149,61 +2138,36 @@ mod tests { .unwrap(), 0xF900_0A11 ); - // The guest thread-pointer slot lives at the fixed ABI offset - // GuestThreadBlock::guest_tp; pin both the value and the emitted word. - assert_eq!(GUEST_TPIDR_OFFSET, 16); - // Slot access at GUEST_TPIDR_OFFSET: `ldr x9,[x9,#16]` / `str x17,[x16,#16]`. + // The guest thread-pointer slot is emitted with the placeholder offset + // that `patch_guest_tpidr_offset` overwrites at load time; pin both the + // value and the emitted words. The placeholder saturates the scaled + // 12-bit immediate (`imm12 = 0xFFF`). + assert_eq!(GUEST_TPIDR_OFFSET_PLACEHOLDER, 32760); + // Slot access: `ldr x9,[x9,#32760]` / `str x17,[x16,#32760]`. assert_eq!( Insn::LdrUimm { rt: 9, rn: 9, - imm_bytes: GUEST_TPIDR_OFFSET + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER } .encode() .unwrap(), - 0xF940_0929 + 0xF97F_FD29 ); assert_eq!( Insn::StrUimm { rt: 17, rn: 16, - imm_bytes: GUEST_TPIDR_OFFSET + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER } .encode() .unwrap(), - 0xF900_0A11 + 0xF93F_FE11 ); // `BRK #0xB10B`, the trap that replaces an out-of-range patch site. assert_eq!(Insn::Brk(TRAP_BRK_IMM).encode().unwrap(), 0xD436_2160); } - #[test] - fn encoder_range_checks() { - assert!(Insn::B(2).encode().is_none()); // not 4-aligned - assert!(Insn::B(1 << 27).encode().is_none()); // out of ±128MB - assert!( - Insn::StrUimm { - rt: 0, - rn: 0, - imm_bytes: 4 - } - .encode() - .is_none() - ); // not 8-scaled - } - - #[test] - fn asm_ldr_literal_computes_pc_relative_offset() { - let mut asm = Asm::new(0x1000); - asm.emit(Insn::Br(30)); // [0] at 0x1000 - asm.ldr_literal(16, 0x1010).unwrap(); // [1] at 0x1004, target 0x1010 => +0xC - let code = asm.finish(); - assert_eq!( - word_at(&code, 4), - Insn::LdrLiteral { rt: 16, off: 0xC }.encode().unwrap() - ); - } - /// Build a one-section image whose section data == the supplied words and /// run the hooker. Returns `(patched_section, trampoline)`. Panics if the /// input has no patch sites (use [`hook_words_opt`] for that case). @@ -1217,6 +2181,29 @@ mod tests { ) } + /// Like [`hook_words`] but prefills the trampoline's callback slot, so a + /// test can plant arbitrary data there. + fn hook_words_with_callback( + words: &[u32], + base: u64, + tramp_base: u64, + callback: u64, + ) -> Vec { + let mut buf = Vec::new(); + for w in words { + buf.extend_from_slice(&w.to_le_bytes()); + } + let sections = vec![TextSectionInfo { + vaddr: base, + file_offset: 0, + size: buf.len() as u64, + }]; + hook_syscalls_aarch64(&mut buf, §ions, tramp_base, callback, Host::Linux) + .unwrap() + .expect("expected a trampoline (input has patch sites)") + .trampoline + } + /// Like [`hook_words`] but returns the raw `Option` outcome so callers can /// assert the "no patch sites" (`None`) sentinel and trapped-site cases. fn hook_words_opt(words: &[u32], base: u64, tramp_base: u64) -> (Vec, Option) { @@ -1286,10 +2273,9 @@ mod tests { #[test] fn site_beyond_branch_range_is_trapped() { - // The trampoline sits 256MB above the section, past the `B` instruction's - // ±128MB reach, so the site cannot branch into its gate. It is replaced - // with the sentinel `BRK`, surfaced as a trapped site, and no gate is - // emitted for it, leaving the trampoline at the prologue-only size. + // The trampoline sits 256MB above the section, past `B`'s ±128MB reach. + // The site becomes `BRK`, is surfaced as trapped, and gets no gate, + // leaving the trampoline at its prologue-only size. let (patched, outcome) = hook_words_opt(&[SVC_0], 0x1000, 0x1000_0000); let outcome = outcome.expect("expected a trampoline (input has patch sites)"); assert_eq!( @@ -1303,14 +2289,13 @@ mod tests { #[test] fn msr_gate_return_branch_out_of_range_is_trapped() { // Boundary window where the site can reach its gate but the gate cannot - // reach back. The MSR gate's return `B` is its last instruction, at - // `gate + 32`, branching to `site + 4`; its displacement magnitude is - // `b_offset + 28`, larger than the inbound `B`'s `b_offset`. Placing the - // gate at the maximum encodable forward offset (`2^27 - 4`) makes the - // inbound branch encode while the return needs `-(2^27 + 24)`, just past - // the `-2^27` reach. The site must still be trapped gracefully — replaced - // with `BRK` and surfaced through `trapped_sites` — not error out. - let base = 0x1000u64; + // reach back. The MSR gate's return `B` sits at `gate + 32` and targets + // `site + 4`, so its displacement magnitude is `b_offset + 28`. Placing + // the gate at the maximum encodable forward offset (`2^27 - 4`) lets + // the inbound branch encode while the return needs `-(2^27 + 24)`, just + // past reach. The site must be trapped gracefully, not error out. The + // site is at +4 so the trampoline base stays 16-byte aligned. + let base = 0x1004u64; let max_fwd = (1u64 << 27) - 4; // largest 4-aligned forward `B` offset let tramp_base = base + max_fwd - GATES_START_OFFSET as u64; let (patched, outcome) = hook_words_opt(&[msr_tpidr_el0(5)], base, tramp_base); @@ -1385,11 +2370,11 @@ mod tests { let anc_i = (0..MSR_GATE_INSNS) .find(|&i| word_at(gate, i * 4) == anchor) .expect("MSR gate must read the host anchor"); - // Store to the slot: STR X17, [X16, #GUEST_TPIDR_OFFSET]. + // Store to the slot: STR X17, [X16, #]. let store = Insn::StrUimm { rt: X17, rn: X16, - imm_bytes: GUEST_TPIDR_OFFSET, + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER, } .encode() .unwrap(); @@ -1420,7 +2405,7 @@ mod tests { Insn::LdrUimm { rt: d, rn: d, - imm_bytes: GUEST_TPIDR_OFFSET + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER } .encode() .unwrap() @@ -1435,7 +2420,8 @@ mod tests { let tramp_base = 0x600000; let (_p, tramp) = hook_words(&[SVC_0], base, tramp_base); let gate = &tramp[GATES_START_OFFSET..GATES_START_OFFSET + SVC_GATE_SIZE]; - // SUB SP,#16 ; STR X16,[SP] ; ADRP X16,.. ; ADD X16,X16,#.. ; STR X16,[SP,#8] ; B + // SUB SP,#32 ; STR X16,[SP] ; ADRP X16,.. ; ADD X16,X16,#.. ; + // STR X16,[SP,#8] ; ADR X16,stub ; STR X16,[SP,#16] ; LDR callback ; BR. assert_eq!( word_at(gate, 0), Insn::SubSp(SVC_FRAME_BYTES).encode().unwrap() @@ -1460,8 +2446,831 @@ mod tests { .encode() .unwrap() ); - // Self-contained tail branch to the shared handler (B, never BL). - assert_eq!(word_at(gate, 20) & OPCODE_TOP6_MASK, Opcode::B.bits()); + // ADR X16, . Inline callback dispatch puts the stub 16 bytes past + // this instruction. + assert_eq!( + word_at(gate, 20), + Insn::Adr { + rd: X16, + byte_off: 16 + } + .encode() + .unwrap() + ); + assert_eq!( + word_at(gate, 24), + Insn::StrUimm { + rt: X16, + rn: SP, + imm_bytes: SVC_FRAME_OFF_STUB + } + .encode() + .unwrap() + ); + assert_eq!(word_at(gate, 32), Insn::Br(X16).encode().unwrap()); assert_eq!(tramp.len(), GATES_START_OFFSET + SVC_GATE_SIZE); } + + #[test] + fn svc_outbound_stub_restores_x16_pops_the_frame_and_returns_to_the_site() { + // The stub is what makes `X16` survive an `SVC`: it reloads the guest + // value the runtime staged at `[SP, #0]`, pops the gate frame so `SP` + // becomes the true guest `SP` again, and branches to `site + 4` with a + // static direct branch that needs no scratch register. + let base = 0x1000; + let tramp_base = 0x600000; + let (_p, tramp) = hook_words(&[SVC_0], base, tramp_base); + let stub_off = GATES_START_OFFSET + SVC_GATE_BYTES; + let stub = &tramp[stub_off..stub_off + SVC_OUTBOUND_STUB_BYTES]; + + assert_eq!( + word_at(stub, 0), + Insn::LdrUimm { + rt: X16, + rn: SP, + imm_bytes: SVC_FRAME_OFF_X16 + } + .encode() + .unwrap() + ); + assert_eq!( + word_at(stub, 4), + Insn::AddSp(SVC_FRAME_BYTES).encode().unwrap() + ); + + // The final `B` targets `site + 4`. + let branch = word_at(stub, 8); + assert_eq!(branch & OPCODE_TOP6_MASK, Opcode::B.bits()); + let imm26 = i64::from(branch & IMM26_MASK); + // Sign-extend the 26-bit field, then scale by 4. + let disp = ((imm26 << 38) >> 38) * 4; + let branch_vaddr = tramp_base + (stub_off + 8) as u64; + assert_eq!( + branch_vaddr.cast_signed() + disp, + (base + 4).cast_signed(), + "outbound stub must return to site + 4" + ); + } + + // --- Load-time guest thread-pointer offset patching --- + + /// An offset a real host runtime might measure: not the placeholder. + const MEASURED_OFFSET: u16 = 96; + + #[test] + fn patch_rewrites_both_gate_shapes_and_leaves_everything_else_alone() { + // One MSR site and one MRS site, so both patchable shapes are present. + let (_p, mut tramp) = hook_words( + &[msr_tpidr_el0(3), Insn::MrsTpidrEl0(9).encode().unwrap()], + 0x1000, + 0x400000, + ); + let before = tramp.clone(); + + let patched = patch_guest_tpidr_offset(&mut tramp, MEASURED_OFFSET).unwrap(); + assert_eq!(patched, 2, "one MSR store and one MRS load"); + + // Exactly two words changed, and each became the same instruction with + // the measured offset in place of the placeholder. + let changed: alloc::vec::Vec = (0..tramp.len() / 4) + .filter(|&i| word_at(&tramp, i * 4) != word_at(&before, i * 4)) + .collect(); + assert_eq!(changed.len(), 2); + for i in changed { + let old = word_at(&before, i * 4); + let new = word_at(&tramp, i * 4); + assert_eq!( + old & !LDST_UIMM12_IMM_MASK, + new & !LDST_UIMM12_IMM_MASK, + "only the immediate field may change" + ); + assert_eq!( + (new & LDST_UIMM12_IMM_MASK) >> LDST_UIMM12_IMM_SHIFT, + u32::from(MEASURED_OFFSET / GUEST_TPIDR_OFFSET_ALIGN) + ); + } + + // Spot-check the exact encodings the gates must now hold. + let msr_store = Insn::StrUimm { + rt: X17, + rn: X16, + imm_bytes: MEASURED_OFFSET, + } + .encode() + .unwrap(); + let mrs_load = Insn::LdrUimm { + rt: 9, + rn: 9, + imm_bytes: MEASURED_OFFSET, + } + .encode() + .unwrap(); + let words: alloc::vec::Vec = (0..tramp.len() / 4) + .map(|i| word_at(&tramp, i * 4)) + .collect(); + assert!(words.contains(&msr_store), "MSR gate store must be patched"); + assert!(words.contains(&mrs_load), "MRS gate load must be patched"); + + // Patching is idempotent in the sense that a second pass finds nothing: + // the placeholder is gone. + assert_eq!( + patch_guest_tpidr_offset(&mut tramp, MEASURED_OFFSET).unwrap(), + 0 + ); + } + + #[test] + fn patch_leaves_an_svc_only_trampoline_untouched() { + // The SVC gate and its outbound stub are full of `LDR`/`STR` words with + // an `SP` base. None of them may be mistaken for a thread-pointer + // access, and the header's callback slot is data that must be skipped. + let (_p, mut tramp) = hook_words(&[SVC_0], 0x1000, 0x400000); + let before = tramp.clone(); + assert_eq!( + patch_guest_tpidr_offset(&mut tramp, MEASURED_OFFSET).unwrap(), + 0 + ); + assert_eq!(tramp, before); + } + + #[test] + fn patch_skips_a_callback_slot_that_looks_like_a_gate_instruction() { + // The callback address is arbitrary data. Prefill it with two copies of + // a word that *is* a placeholder-bearing gate load, and check the patch + // pass does not touch the header. + let decoy = Insn::LdrUimm { + rt: 9, + rn: 9, + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER, + } + .encode() + .unwrap(); + let callback = (u64::from(decoy) << 32) | u64::from(decoy); + let mut tramp = hook_words_with_callback(&[SVC_0], 0x1000, 0x400000, callback); + assert_eq!( + patch_guest_tpidr_offset(&mut tramp, MEASURED_OFFSET).unwrap(), + 0 + ); + assert_eq!( + u64::from_le_bytes(tramp[..8].try_into().unwrap()), + callback, + "the callback slot is data and must survive verbatim" + ); + } + + /// The check a loader needs: an unpatched gate does not fault when + /// executed, so scanning is the only way to detect one. See + /// `GUEST_TPIDR_OFFSET_PLACEHOLDER`. + #[test] + fn find_placeholder_reports_gates_before_patching_and_none_after() { + let (_p, mut tramp) = hook_words( + &[msr_tpidr_el0(3), Insn::MrsTpidrEl0(9).encode().unwrap()], + 0x1000, + 0x400000, + ); + + let at = find_guest_tpidr_placeholder(&tramp).expect("an unpatched gate must be found"); + assert!(at >= FIRST_SCANNABLE_OFFSET && at.is_multiple_of(4)); + + assert_eq!( + patch_guest_tpidr_offset(&mut tramp, MEASURED_OFFSET).unwrap(), + 2 + ); + assert_eq!( + find_guest_tpidr_placeholder(&tramp), + None, + "patching must leave no gate on the placeholder" + ); + } + + #[test] + fn find_placeholder_ignores_svc_gates_and_the_callback_slot() { + // An SVC-only trampoline has no thread-pointer gate at all, and its + // `LDR`/`STR` words off `SP` must not be mistaken for one. + let (_p, tramp) = hook_words(&[SVC_0], 0x1000, 0x400000); + assert_eq!(find_guest_tpidr_placeholder(&tramp), None); + + // The callback slot is a 64-bit address, i.e. data. Even when it + // happens to spell a placeholder-bearing gate load, it is not one. + let decoy = Insn::LdrUimm { + rt: 9, + rn: 9, + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER, + } + .encode() + .unwrap(); + let callback = (u64::from(decoy) << 32) | u64::from(decoy); + let tramp = hook_words_with_callback(&[SVC_0], 0x1000, 0x400000, callback); + assert_eq!(find_guest_tpidr_placeholder(&tramp), None); + } + + #[test] + fn patch_rejects_an_offset_a_gate_cannot_address() { + let (_p, mut tramp) = + hook_words(&[Insn::MrsTpidrEl0(9).encode().unwrap()], 0x1000, 0x400000); + for bad in [ + // Not a multiple of the LDR scale, so not encodable at all. + 4u16, + 12, + // Past the top of the imm12 field. + MAX_GUEST_TPIDR_OFFSET + 8, + // Inside the host's own per-thread state: patching with one of + // these would aim every rewritten thread-pointer write at it, and + // would leave no placeholder behind for anything downstream to + // object to. + 0, + 8, + MIN_GUEST_TPIDR_OFFSET - GUEST_TPIDR_OFFSET_ALIGN, + // The placeholder itself: patching with it is indistinguishable + // from not having patched. + GUEST_TPIDR_OFFSET_PLACEHOLDER, + ] { + assert!( + matches!( + patch_guest_tpidr_offset(&mut tramp, bad), + Err(Error::TrampolinePatchFailure(_)) + ), + "offset {bad} must be rejected" + ); + } + // The legitimate bounds are accepted, each on its own trampoline: + // patching is single-shot, since an already-patched gate is + // indistinguishable from one the guest shipped that way. + for good in [ + MIN_GUEST_TPIDR_OFFSET, + GUEST_TPIDR_OFFSET_PLACEHOLDER - GUEST_TPIDR_OFFSET_ALIGN, + ] { + let (_p, mut fresh) = + hook_words(&[Insn::MrsTpidrEl0(9).encode().unwrap()], 0x1000, 0x400000); + assert!( + patch_guest_tpidr_offset(&mut fresh, good).is_ok(), + "offset {good} must be accepted" + ); + } + } + + #[test] + fn patch_rejects_a_blob_that_is_not_a_trampoline() { + // Shorter than the shared prologue. + let mut short = vec![0u8; GATES_START_OFFSET - 4]; + assert!(matches!( + patch_guest_tpidr_offset(&mut short, MEASURED_OFFSET), + Err(Error::TrampolinePatchFailure(_)) + )); + + // Not a whole number of instructions. + let mut ragged = vec![0u8; GATES_START_OFFSET + 2]; + assert!(matches!( + patch_guest_tpidr_offset(&mut ragged, MEASURED_OFFSET), + Err(Error::TrampolinePatchFailure(_)) + )); + + // A placeholder-bearing instruction in a shape no gate emits. + let mut foreign = vec![0u8; GATES_START_OFFSET + 4]; + let bogus = Insn::LdrUimm { + rt: 1, + rn: 2, + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER, + } + .encode() + .unwrap(); + foreign[GATES_START_OFFSET..].copy_from_slice(&bogus.to_le_bytes()); + assert!(matches!( + patch_guest_tpidr_offset(&mut foreign, MEASURED_OFFSET), + Err(Error::TrampolinePatchFailure(_)) + )); + } + + #[test] + fn classifier_rejects_mrs_xzr_metadata() { + let base = 0x400000; + let (_patched, mut trampoline) = + hook_words(&[Insn::MrsTpidrEl0(9).encode().unwrap()], 0x1000, base); + let xzr = EncodedGateMetadata::encode(GateMetadata::MrsTpidr { destination: 30 }) + .unwrap() + .0 + | (1 << GATE_METADATA_REGISTER_SHIFT); + trampoline[28..32].copy_from_slice(&xzr.to_le_bytes()); + trampoline[16..20].copy_from_slice(&Insn::MrsTpidrEl0(31).encode().unwrap().to_le_bytes()); + trampoline[20..24].copy_from_slice( + &Insn::LdrUimm { + rt: 31, + rn: 31, + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER, + } + .encode() + .unwrap() + .to_le_bytes(), + ); + + assert_eq!(EncodedGateMetadata(xzr).decode(), None); + assert_eq!(classify_gate_pc(&trampoline, base, base + 16), None); + assert_eq!( + classify_copied_gate_slot(&trampoline[16..32], base + 16, base + 16), + None + ); + } + + /// A trampoline arrives from the guest's own file, so a gate may hold an + /// offset this rewriter never emitted. It carries no placeholder, so + /// nothing downstream would object, and the runtime would execute a gate + /// addressing a slot it did not choose. Not a containment boundary — the + /// userland platform shares one address space with the guest — but the + /// invariant has to hold for thread-pointer virtualization to mean + /// anything. + #[test] + fn patch_rejects_a_gate_holding_a_guest_chosen_thread_pointer_offset() { + const TAMPERED: u16 = 4096; + const REQUESTED: u16 = 96; + + let (_p, mut tramp) = + hook_words(&[Insn::MrsTpidrEl0(9).encode().unwrap()], 0x1000, 0x400000); + let site = GATES_START_OFFSET + INSN_BYTES; + let insn = u32::from_le_bytes(tramp[site..site + INSN_BYTES].try_into().unwrap()); + let tampered_imm = u32::from(TAMPERED / GUEST_TPIDR_OFFSET_ALIGN) << LDST_UIMM12_IMM_SHIFT; + tramp[site..site + INSN_BYTES] + .copy_from_slice(&((insn & !LDST_UIMM12_IMM_MASK) | tampered_imm).to_le_bytes()); + + // The tampered gate carries no placeholder, so the loader's proof + // obligation is satisfied and only this check stands between it and + // an executable mapping. + assert_eq!(find_guest_tpidr_placeholder(&tramp), None); + assert!(matches!( + patch_guest_tpidr_offset(&mut tramp, REQUESTED), + Err(Error::TrampolinePatchFailure(_)) + )); + } + + /// The metadata word is a persisted format, baked into every rewritten + /// binary. A round-trip test cannot catch a renumbering — encode and decode + /// move together — so pin the words. + #[test] + fn gate_metadata_words_are_a_stable_persisted_format() { + let word = |m| EncodedGateMetadata::encode(m).unwrap().0; + assert_eq!(word(GateMetadata::Svc), 0x0001b807); + assert_eq!(word(GateMetadata::MrsTpidr { destination: 9 }), 0x0911b807); + assert_eq!(word(GateMetadata::MsrTpidr { source: 9 }), 0x0921b807); + } + + #[test] + fn gate_metadata_rejects_invalid_fields_and_reserved_bits() { + let valid = EncodedGateMetadata::encode(GateMetadata::MrsTpidr { destination: 9 }) + .unwrap() + .0; + + for magic in 0..=u16::MAX { + if u32::from(magic) != GATE_METADATA_MAGIC { + let invalid = (valid & !GATE_METADATA_MAGIC_MASK) | u32::from(magic); + assert_eq!( + EncodedGateMetadata(invalid).decode(), + None, + "magic {magic:#x}" + ); + } + } + for version in 0..16 { + if version != GATE_METADATA_VERSION { + let invalid = (valid & !GATE_METADATA_VERSION_MASK) + | (version << GATE_METADATA_VERSION_SHIFT); + assert_eq!( + EncodedGateMetadata(invalid).decode(), + None, + "version {version}" + ); + } + } + for kind in 3..16 { + let invalid = (valid & !GATE_METADATA_KIND_MASK) | (kind << GATE_METADATA_KIND_SHIFT); + assert_eq!(EncodedGateMetadata(invalid).decode(), None, "kind {kind}"); + } + for register in 32..64 { + let invalid = + (valid & !GATE_METADATA_REGISTER_MASK) | (register << GATE_METADATA_REGISTER_SHIFT); + assert_eq!( + EncodedGateMetadata(invalid).decode(), + None, + "register {register}" + ); + } + for bit in 30..32 { + let invalid = valid | (1 << bit); + assert_eq!( + EncodedGateMetadata(invalid).decode(), + None, + "reserved bit {bit}" + ); + } + for register in 1..64 { + let invalid = EncodedGateMetadata::encode(GateMetadata::Svc).unwrap().0 + | (register << GATE_METADATA_REGISTER_SHIFT); + assert_eq!( + EncodedGateMetadata(invalid).decode(), + None, + "SVC register {register}" + ); + } + for register in 32..=u8::MAX { + assert!( + EncodedGateMetadata::encode(GateMetadata::MrsTpidr { + destination: register, + }) + .is_none() + ); + } + } + + #[test] + fn unaligned_trampoline_base_is_rejected() { + let mut code = SVC_0.to_le_bytes(); + let section = TextSectionInfo { + vaddr: 0x1000, + file_offset: 0, + size: 4, + }; + assert!(matches!( + hook_syscalls_aarch64(&mut code, &[section], 0x400004, 0, Host::Linux), + Err(Error::AddressOverflow(_)) + )); + } + + #[test] + fn slot_padding_and_metadata_do_not_false_match_placeholder_scan() { + let (_patched, mut tramp) = hook_words(&[SVC_0], 0x1000, 0x400000); + assert_eq!(find_guest_tpidr_placeholder(&tramp), None); + let before = tramp.clone(); + assert_eq!( + patch_guest_tpidr_offset(&mut tramp, MEASURED_OFFSET).unwrap(), + 0 + ); + assert_eq!(tramp, before); + } + + #[test] + fn libc_scale_compact_slots_fit_in_64k() { + let mut words = vec![Insn::MrsTpidrEl0(9).encode().unwrap(); 1_522]; + words.extend(core::iter::repeat_n(SVC_0, 503)); + let (_patched, trampoline) = hook_words(&words, 0x1000, 0x400000); + + assert_eq!(trampoline.len(), 16 + 1_522 * 16 + 503 * 64); + assert!(trampoline.len() <= 64 * 1024); + } + + #[test] + fn compact_mixed_slots_have_exact_strides_and_trailing_metadata() { + let words = [ + Insn::MrsTpidrEl0(9).encode().unwrap(), + msr_tpidr_el0(5), + SVC_0, + Insn::MrsTpidrEl0(3).encode().unwrap(), + ]; + let (_patched, trampoline) = hook_words(&words, 0x1000, 0x400000); + let expected = [ + (16, 16, GateMetadata::MrsTpidr { destination: 9 }), + (32, 48, GateMetadata::MsrTpidr { source: 5 }), + (80, 64, GateMetadata::Svc), + (144, 16, GateMetadata::MrsTpidr { destination: 3 }), + ]; + + assert_eq!(trampoline.len(), 160); + for (start, size, metadata) in expected { + assert_eq!((0x400000 + start as u64) % 16, 0); + let encoded = EncodedGateMetadata::encode(metadata) + .unwrap() + .0 + .to_le_bytes(); + assert_eq!(&trampoline[start + size - 4..start + size], &encoded); + } + } + + #[test] + fn compact_metadata_exhaustively_rejects_invalid_encodings() { + let valid = EncodedGateMetadata::encode(GateMetadata::MrsTpidr { destination: 9 }) + .unwrap() + .0; + for magic in 0..=u16::MAX { + if u32::from(magic) != GATE_METADATA_MAGIC { + let word = (valid & !GATE_METADATA_MAGIC_MASK) | u32::from(magic); + assert_eq!(EncodedGateMetadata(word).decode(), None, "magic {magic:#x}"); + } + } + for version in 0..16 { + if version != GATE_METADATA_VERSION { + let word = (valid & !GATE_METADATA_VERSION_MASK) + | (version << GATE_METADATA_VERSION_SHIFT); + assert_eq!( + EncodedGateMetadata(word).decode(), + None, + "version {version}" + ); + } + } + for kind in 3..16 { + let word = (valid & !GATE_METADATA_KIND_MASK) | (kind << GATE_METADATA_KIND_SHIFT); + assert_eq!(EncodedGateMetadata(word).decode(), None, "kind {kind}"); + } + for register in 32..64 { + let word = + (valid & !GATE_METADATA_REGISTER_MASK) | (register << GATE_METADATA_REGISTER_SHIFT); + assert_eq!( + EncodedGateMetadata(word).decode(), + None, + "register {register}" + ); + } + for bit in 30..32 { + assert_eq!( + EncodedGateMetadata(valid | (1 << bit)).decode(), + None, + "reserved bit {bit}" + ); + } + } + + #[test] + fn classifier_finds_every_instruction_boundary_and_rejects_non_slots() { + let words = [ + Insn::MrsTpidrEl0(9).encode().unwrap(), + msr_tpidr_el0(5), + SVC_0, + ]; + let base = 0x400000; + let (_patched, trampoline) = hook_words(&words, 0x1000, base); + for (start, size, executable_end, metadata) in [ + ( + 16usize, + 16usize, + 12usize, + GateMetadata::MrsTpidr { destination: 9 }, + ), + (32, 48, 36, GateMetadata::MsrTpidr { source: 5 }), + (80, 64, 48, GateMetadata::Svc), + ] { + for offset in (0..executable_end).step_by(INSN_BYTES) { + assert_eq!( + classify_gate_pc(&trampoline, base, base + (start + offset) as u64), + Some(ClassifiedGate { + slot_offset: start, + slot_size: u8::try_from(size).unwrap(), + commit_offset: match metadata { + GateMetadata::Svc => 0, + GateMetadata::MrsTpidr { .. } => 4, + GateMetadata::MsrTpidr { .. } => 20, + }, + original_site: 0x1000 + + match metadata { + GateMetadata::MrsTpidr { .. } => 0, + GateMetadata::MsrTpidr { .. } => 4, + GateMetadata::Svc => 8, + }, + metadata, + }) + ); + } + } + for offset in (0..16).step_by(INSN_BYTES) { + assert_eq!(classify_gate_pc(&trampoline, base, base + offset), None); + } + assert_eq!(classify_gate_pc(&trampoline, base, base + 144), None); + assert_eq!(classify_gate_pc(&trampoline, base, base - 4), None); + } + + #[test] + fn copied_slot_classifier_finds_every_instruction_boundary() { + let words = [ + Insn::MrsTpidrEl0(9).encode().unwrap(), + msr_tpidr_el0(5), + SVC_0, + ]; + let base = 0x400000; + let (_patched, trampoline) = hook_words(&words, 0x1000, base); + for (start, size, executable_end, metadata) in [ + ( + 16usize, + 16usize, + 12usize, + GateMetadata::MrsTpidr { destination: 9 }, + ), + (32, 48, 36, GateMetadata::MsrTpidr { source: 5 }), + (80, 64, 48, GateMetadata::Svc), + ] { + let slot = &trampoline[start..start + size]; + for offset in (0..executable_end).step_by(INSN_BYTES) { + let classified = classify_copied_gate_slot( + slot, + base + start as u64, + base + (start + offset) as u64, + ) + .expect("emitted slot must classify"); + assert_eq!(classified.slot_offset(), 0); + assert_eq!(classified.slot_size(), u8::try_from(size).unwrap()); + assert_eq!(classified.metadata(), metadata); + } + } + } + + #[test] + fn classifier_rejects_forged_metadata_mutations_and_ambiguity() { + let base = 0x400000; + let (_patched, trampoline) = hook_words( + &[ + Insn::MrsTpidrEl0(9).encode().unwrap(), + msr_tpidr_el0(5), + SVC_0, + ], + 0x1000, + base, + ); + for (start, size) in [(16usize, 16usize), (32, 48), (80, 64)] { + for word_offset in (0..size - 4).step_by(INSN_BYTES) { + let mut mutated = trampoline.clone(); + let replacement = if word_at(&mutated, start + word_offset) == NOP { + 0 + } else { + NOP + }; + mutated[start + word_offset..start + word_offset + 4] + .copy_from_slice(&replacement.to_le_bytes()); + assert_eq!( + classify_gate_pc(&mutated, base, base + start as u64), + None, + "accepted mutation at slot {start} + {word_offset}" + ); + } + } + + let mut forged = trampoline.clone(); + forged[12..16].copy_from_slice( + &EncodedGateMetadata::encode(GateMetadata::MrsTpidr { destination: 9 }) + .unwrap() + .0 + .to_le_bytes(), + ); + assert_eq!(classify_gate_pc(&forged, base, base), None); + + assert_eq!( + classify_gate_pc_with_candidates(&trampoline, base, base + 32, [32, 32, 16, 0]), + None, + "more than one validated candidate must be rejected" + ); + } + + #[test] + fn classifier_validates_tpidr_immediate_policy() { + let base = 0x400000; + let (_patched, trampoline) = hook_words( + &[Insn::MrsTpidrEl0(9).encode().unwrap(), msr_tpidr_el0(5)], + 0x1000, + base, + ); + for (slot_start, insn_offset) in [(16usize, 4usize), (32, 20)] { + for valid in [ + GUEST_TPIDR_OFFSET_PLACEHOLDER, + MIN_GUEST_TPIDR_OFFSET, + 96, + GUEST_TPIDR_OFFSET_PLACEHOLDER - GUEST_TPIDR_OFFSET_ALIGN, + ] { + let mut candidate = trampoline.clone(); + set_tpidr_immediate(&mut candidate, slot_start + insn_offset, valid); + assert!( + classify_gate_pc(&candidate, base, base + slot_start as u64).is_some(), + "rejected valid offset {valid}" + ); + } + for invalid in [0, 8] { + let mut candidate = trampoline.clone(); + set_tpidr_immediate(&mut candidate, slot_start + insn_offset, invalid); + assert_eq!( + classify_gate_pc(&candidate, base, base + slot_start as u64), + None, + "accepted host-header offset {invalid}" + ); + } + + // Raw imm12 encodings are scaled by eight. A byte offset that is + // not 8-aligned cannot be represented; pin rejection by mutating + // the instruction to an impossible policy value below the minimum. + let mut non_aligned = trampoline.clone(); + set_tpidr_immediate(&mut non_aligned, slot_start + insn_offset, 8); + assert_eq!( + classify_gate_pc(&non_aligned, base, base + slot_start as u64), + None + ); + } + + for invalid in [ + 4, + 12, + u32::from(GUEST_TPIDR_OFFSET_PLACEHOLDER) + 8, + u32::MAX, + ] { + assert!( + !valid_emitted_tpidr_offset(invalid), + "accepted impossible byte offset {invalid}" + ); + } + } + + #[test] + fn classifier_accepts_only_aligned_executable_instruction_pcs() { + let base = 0x400000; + let (_patched, trampoline) = hook_words( + &[ + Insn::MrsTpidrEl0(9).encode().unwrap(), + msr_tpidr_el0(5), + SVC_0, + ], + 0x1000, + base, + ); + for (start, executable_end, slot_size) in + [(16usize, 12usize, 16usize), (32, 36, 48), (80, 48, 64)] + { + for offset in (0..executable_end).step_by(INSN_BYTES) { + assert!( + classify_gate_pc(&trampoline, base, base + (start + offset) as u64).is_some(), + "rejected executable PC at slot {start} + {offset}" + ); + for byte in 1..4 { + assert_eq!( + classify_gate_pc(&trampoline, base, base + (start + offset + byte) as u64,), + None, + "accepted unaligned PC at slot {start} + {}", + offset + byte + ); + } + } + for offset in (executable_end..slot_size - 4).step_by(INSN_BYTES) { + assert_eq!( + classify_gate_pc(&trampoline, base, base + (start + offset) as u64), + None, + "accepted padding PC at slot {start} + {offset}" + ); + } + for byte in 0..4 { + assert_eq!( + classify_gate_pc( + &trampoline, + base, + base + (start + slot_size - 4 + byte) as u64, + ), + None, + "accepted metadata PC at slot {start} + {}", + slot_size - 4 + byte + ); + } + } + } + + #[test] + fn patch_is_transactional_when_a_later_slot_is_malformed() { + let (_patched, mut trampoline) = hook_words( + &[Insn::MrsTpidrEl0(9).encode().unwrap(), msr_tpidr_el0(5)], + 0x1000, + 0x400000, + ); + trampoline[32] ^= 1; // Corrupt the later MSR template. + let before = trampoline.clone(); + + assert!(matches!( + patch_guest_tpidr_offset(&mut trampoline, MEASURED_OFFSET), + Err(Error::TrampolinePatchFailure(_)) + )); + assert_eq!( + trampoline, before, + "failed patching must not mutate earlier slots" + ); + } + + #[test] + fn patch_rejects_trailing_unknown_and_placeholder_like_content() { + let (_patched, trampoline) = hook_words(&[SVC_0], 0x1000, 0x400000); + for suffix in [ + NOP.to_le_bytes().to_vec(), + Insn::LdrUimm { + rt: 9, + rn: 9, + imm_bytes: GUEST_TPIDR_OFFSET_PLACEHOLDER, + } + .encode() + .unwrap() + .to_le_bytes() + .to_vec(), + ] { + let mut malformed = trampoline.clone(); + malformed.extend_from_slice(&suffix); + let before = malformed.clone(); + assert!(matches!( + patch_guest_tpidr_offset(&mut malformed, MEASURED_OFFSET), + Err(Error::TrampolinePatchFailure(_)) + )); + assert_eq!(malformed, before); + } + } + + fn set_tpidr_immediate(trampoline: &mut [u8], offset: usize, byte_offset: u16) { + let word = word_at(trampoline, offset); + let immediate = u32::from(byte_offset / GUEST_TPIDR_OFFSET_ALIGN) << LDST_UIMM12_IMM_SHIFT; + trampoline[offset..offset + 4] + .copy_from_slice(&((word & !LDST_UIMM12_IMM_MASK) | immediate).to_le_bytes()); + } } diff --git a/litebox_syscall_rewriter/src/lib.rs b/litebox_syscall_rewriter/src/lib.rs index ae906b5032..2c0785a8ad 100644 --- a/litebox_syscall_rewriter/src/lib.rs +++ b/litebox_syscall_rewriter/src/lib.rs @@ -25,7 +25,14 @@ #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; -mod arm64; +// Only the runtime entry points dispatch on `target_arch`; the ahead-of-time +// ones select the architecture from the input object and build anywhere. +#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +compile_error!( + "litebox_syscall_rewriter's runtime patching entry points support only x86-64 and AArch64 hosts" +); + +pub mod arm64; use alloc::collections::{BTreeMap, BTreeSet}; use alloc::format; @@ -35,9 +42,8 @@ use alloc::vec::Vec; use litebox_common_windows::NtSysno; use object::pe::{IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE}; -use object::read::elf::{ElfFile, ProgramHeader as _}; use object::read::pe::{ImageNtHeaders as _, ImageOptionalHeader as _, PeFile64}; -use object::read::{Object as _, ObjectSection as _}; +use object::read::{Object as _, ObjectSection as _, ObjectSegment as _}; use thiserror::Error; use zerocopy::{FromBytes, Immutable, IntoBytes}; @@ -55,6 +61,10 @@ pub enum Error { AddressOverflow(String), #[error("unpatchable syscall instruction(s): {0}")] UnpatchableSyscalls(String), + #[error("failed to patch trampoline: {0}")] + TrampolinePatchFailure(String), + #[error("trampoline needs {needed:#x} bytes but only {available:#x} are available")] + TrampolineTooLarge { needed: u64, available: u64 }, } /// Internal-only error variants used for control flow within the crate. @@ -80,6 +90,16 @@ impl From for InternalError { type Result = core::result::Result; +/// Offset of `e_ident[EI_DATA]`, the byte selecting the header's endianness. +/// +/// `e_ident` and `e_type` are laid out identically in ELF32 and ELF64, so the +/// ELF64 header's offsets locate these fields in either. +const ELF_EI_DATA_OFFSET: usize = core::mem::offset_of!(object::elf::Ident, data); + +/// Offset of `e_machine`. +const ELF_E_MACHINE_OFFSET: usize = + core::mem::offset_of!(object::elf::FileHeader64, e_machine); + const BUN_FOOTER_MARKER: &[u8] = b"\n---- Bun! ----\n"; /// The magic bytes used to identify the trampoline data. @@ -175,6 +195,19 @@ pub fn hook_syscalls_in_elf(input_binary: &[u8], trampoline: Option) -> Res )); } + // The AArch64 emitter reads and writes instruction words in little-endian + // order, so reject a big-endian AArch64 object before parsing or mutating it. + let data = input_binary.get(ELF_EI_DATA_OFFSET); + let machine = input_binary + .get(ELF_E_MACHINE_OFFSET..ELF_E_MACHINE_OFFSET + size_of::()) + .and_then(|bytes| <[u8; 2]>::try_from(bytes).ok()) + .map(u16::from_be_bytes); + if data == Some(&object::elf::ELFDATA2MSB) && machine == Some(object::elf::EM_AARCH64) { + return Err(Error::UnsupportedExecutable( + "big-endian AArch64 ELF".into(), + )); + } + // Relocatable object files (.o) must not be patched: they are linker // input, not executable code. Rewriting instructions or appending // trampoline data would corrupt the object file for the linker. @@ -207,7 +240,7 @@ pub fn hook_syscalls_in_elf(input_binary: &[u8], trampoline: Option) -> Res fixup_phdr_alignment(buf); // Parse the ELF and extract all metadata we need, then drop the borrow so we can mutate buf. - let (arch, text_sections, trampoline_base_addr) = { + let (arch, text_sections, placement) = { let file = object::File::parse(&*buf).map_err(|e| Error::ParseError(e.to_string()))?; let arch = match file { @@ -230,9 +263,9 @@ pub fn hook_syscalls_in_elf(input_binary: &[u8], trampoline: Option) -> Res return Ok(input_binary.to_vec()); } - let trampoline_base_addr = find_addr_for_trampoline_code(&file)?; + let placement = find_addr_for_trampoline_code(&file)?; - (arch, text_sections, trampoline_base_addr) + (arch, text_sections, placement) }; // AArch64 uses a fully separate rewriting strategy (single-instruction @@ -244,11 +277,12 @@ pub fn hook_syscalls_in_elf(input_binary: &[u8], trampoline: Option) -> Res input_binary, buf, &text_sections, - trampoline_base_addr, + placement, trampoline.unwrap_or(0), ); } + let trampoline_base_addr = placement.addr(); let control_transfer_targets = get_control_transfer_targets(arch, &*buf, &text_sections)?; let mut trampoline_data = Vec::from(trampoline.unwrap_or(0).to_le_bytes()); let patch_result = patch_syscalls_in_sections( @@ -762,22 +796,73 @@ fn append_trampoline_footer( out.extend_from_slice(header.as_bytes()); } -/// Rewrite an AArch64 ELF, appending the trampoline and trailing header. +/// Rewrites an AArch64 ELF, honoring `placement`. +/// +/// The address is baked into every rewritten site, so it has to be chosen +/// before the trampoline's size is known. The rewrite therefore runs at the +/// preferred address and, if the trampoline outgrew the object's inter-segment +/// hole, runs again at the unreserved fallback address. +fn hook_aarch64_elf( + input_binary: &[u8], + buf: &mut [u8], + text_sections: &[TextSectionInfo], + placement: TrampolinePlacement, + callback: u64, +) -> Result> { + if let TrampolinePlacement::InsideLoadSpan { addr, limit, .. } = placement { + let mut attempt = buf.to_vec(); + let out = hook_aarch64_elf_at( + input_binary, + &mut attempt, + text_sections, + addr, + Some(limit), + callback, + ); + // A gap that is too small, or too far from the text for a gate's + // branch to reach back, is a property of this address rather than of + // the binary, so both are worth retrying elsewhere. + if !matches!( + out, + Err(Error::TrampolineTooLarge { .. } | Error::UnpatchableSyscalls(_)) + ) { + buf.copy_from_slice(&attempt); + return out; + } + // Fall through to retry at the fallback address. `buf` is still + // pristine: only `attempt` was patched, and a rescan of already-patched + // bytes would find no sites. + } + hook_aarch64_elf_at( + input_binary, + buf, + text_sections, + placement.fallback_addr(), + None, + callback, + ) +} + +/// Rewrites an AArch64 ELF at a fixed trampoline address, appending the +/// trampoline and trailing header. /// /// `input_binary` is the original, unmodified ELF; `buf` is the mutable copy /// (patched in place by the arm64 module). `callback` is the absolute address /// stored in the trampoline's callback slot (0 when the loader fills it in -/// later). +/// later). `trampoline_limit` bounds how many bytes the trampoline may occupy, +/// or is `None` where nothing bounds it; overshooting a bound is reported as +/// [`Error::TrampolineTooLarge`]. /// /// Like the x86-64 path, a binary with no patch sites is emitted as the /// original bytes followed by a size-0 trampoline sentinel header (the arm64 /// module signals this by returning `None`). Otherwise the output layout is /// `[patched ELF][padding to page boundary][trampoline code][header]`. -fn hook_aarch64_elf( +fn hook_aarch64_elf_at( input_binary: &[u8], buf: &mut [u8], text_sections: &[TextSectionInfo], trampoline_base_addr: u64, + trampoline_limit: Option, callback: u64, ) -> Result> { let Some(outcome) = arm64::hook_syscalls_aarch64( @@ -803,9 +888,15 @@ fn hook_aarch64_elf( // Build output: [patched ELF][padding to page boundary][trampoline][header]. let mut trampoline_data = outcome.trampoline; - let mut out = buf.to_vec(); - append_trampoline_footer(&mut out, &mut trampoline_data, trampoline_base_addr, false); - + let needed = trampoline_data.len() as u64; + if let Some(limit) = trampoline_limit + && needed > limit + { + return Err(Error::TrampolineTooLarge { + needed, + available: limit, + }); + } if !outcome.trapped_sites.is_empty() { return Err(Error::UnpatchableSyscalls(format!( "{} unpatchable instruction(s) (SVC / MSR / MRS TPIDR_EL0) at {trapped:?}", @@ -813,6 +904,9 @@ fn hook_aarch64_elf( trapped = outcome.trapped_sites, ))); } + let mut out = buf.to_vec(); + append_trampoline_footer(&mut out, &mut trampoline_data, trampoline_base_addr, false); + Ok(out) } @@ -1357,21 +1451,54 @@ fn rel32_bytes(target: u64, base: u64, context: &'static str) -> Result<[u8; 4]> Ok(disp.to_le_bytes()) } -/// This is the runtime counterpart to [`hook_syscalls_in_elf`]. Instead of -/// processing a whole ELF file, it operates on a single already-mapped code -/// region — the caller is responsible for making the region writable before -/// calling and restoring permissions afterwards. +/// Runtime counterpart to [`hook_syscalls_in_elf`], operating on one +/// already-mapped code region. The caller makes the region writable before +/// calling and restores permissions afterwards. +/// +/// The region is rewritten for the architecture this crate is running on, and +/// `syscall_entry_addr` is interpreted per that architecture's stub ABI. /// /// # Returns /// -/// `(trampoline_stubs, skipped_addrs)`. The caller must copy the stubs to -/// `trampoline_write_vaddr`. Returns empty vecs if no syscall instructions -/// are found in `code`. +/// `(trampoline_stubs, unredirected_addrs)`, both empty if `code` has no +/// patchable instructions. The caller must copy the stubs to +/// `trampoline_write_vaddr`. On x86-64 the addresses were left native; on +/// AArch64 they were overwritten with `BRK`. +/// +/// # Caller obligations on AArch64 +/// +/// The I-cache is not coherent with the D-cache, so the caller **must** +/// synchronize the instruction stream over the patched `code` and the written +/// stubs before either is fetched, and must patch the emitted gates' guest +/// thread-pointer placeholder. Neither omission fails cleanly. See the +/// [`arm64`] module docs. x86-64 needs neither. pub fn patch_code_segment( code: &mut [u8], code_vaddr: u64, trampoline_write_vaddr: u64, syscall_entry_addr: u64, +) -> Result<(Vec, Vec)> { + #[cfg(target_arch = "x86_64")] + { + patch_x86_64_code_segment(code, code_vaddr, trampoline_write_vaddr, syscall_entry_addr) + } + #[cfg(target_arch = "aarch64")] + { + patch_aarch64_code_segment(code, code_vaddr, trampoline_write_vaddr, syscall_entry_addr) + } +} + +/// [`patch_code_segment`] for an x86-64 host. +/// +/// The emitted stubs jump *indirectly* through a shared 8-byte slot the caller +/// places once per trampoline allocation, so `syscall_entry_addr` is the +/// address of that slot. +#[cfg(target_arch = "x86_64")] +fn patch_x86_64_code_segment( + code: &mut [u8], + code_vaddr: u64, + trampoline_write_vaddr: u64, + syscall_entry_addr: u64, ) -> Result<(Vec, Vec)> { // Build control-transfer targets for this segment. let instructions = decode_section_instructions(Arch::X86_64, code, code_vaddr)?; @@ -1400,13 +1527,66 @@ pub fn patch_code_segment( } } -/// Replace all `syscall` instructions in `code` with trap sequences (`ICEBP; HLT`). +/// [`patch_code_segment`] for an AArch64 host, where `syscall_entry_addr` is +/// the callback address itself, not a slot holding it. /// -/// This is the fallback when trampoline-based patching cannot be performed -/// (e.g. trampoline allocation failed or is too far away). +/// The gates are identical to the ahead-of-time path's and carry the same guest +/// thread-pointer placeholder, so the caller must run +/// [`arm64::patch_guest_tpidr_offset`] over the returned stubs and prove with +/// [`arm64::find_guest_tpidr_placeholder`] that none survives before writing +/// them anywhere the guest can execute. +#[cfg(any(test, target_arch = "aarch64"))] +fn patch_aarch64_code_segment( + code: &mut [u8], + code_vaddr: u64, + trampoline_write_vaddr: u64, + syscall_entry_addr: u64, +) -> Result<(Vec, Vec)> { + let section = TextSectionInfo { + vaddr: code_vaddr, + file_offset: 0, + size: code.len() as u64, + }; + let Some(outcome) = arm64::hook_syscalls_aarch64( + code, + &[section], + trampoline_write_vaddr, + syscall_entry_addr, + arm64::Host::Linux, + )? + else { + return Ok((Vec::new(), Vec::new())); + }; + + Ok((outcome.trampoline, outcome.trapped_sites)) +} + +/// Replace every syscall patch site in `code` with a trap instruction, so that +/// reaching one faults instead of escaping to the host kernel. Returns how many +/// were trapped. +/// +/// The fail-safe when trampoline-based patching cannot be performed (allocation +/// failed, or the trampoline is out of branch range). It has to cover exactly +/// the sites the architecture's rewriter would have redirected, or it silently +/// fails safe on nothing. /// -/// Returns the number of syscall instructions that were patched. +/// On AArch64 the caller must synchronize the instruction stream over `code` +/// before it is fetched again; see [`patch_code_segment`]. pub fn trap_all_syscalls_in_code(code: &mut [u8], code_vaddr: u64) -> Result { + #[cfg(target_arch = "x86_64")] + { + trap_all_x86_64_syscalls(code, code_vaddr) + } + #[cfg(target_arch = "aarch64")] + { + trap_all_aarch64_patch_sites(code, code_vaddr) + } +} + +/// [`trap_all_syscalls_in_code`] for an x86-64 host, where `syscall` +/// instructions become `ICEBP; HLT`. +#[cfg(target_arch = "x86_64")] +fn trap_all_x86_64_syscalls(code: &mut [u8], code_vaddr: u64) -> Result { let instructions = decode_section_instructions(Arch::X86_64, code, code_vaddr)?; let mut count = 0; for inst in &instructions { @@ -1418,33 +1598,219 @@ pub fn trap_all_syscalls_in_code(code: &mut [u8], code_vaddr: u64) -> Result) -> Result { - // Find the highest virtual address among all PT_LOAD segments - let max_virtual_addr = match file { - object::File::Elf64(elf) => max_load_segment_end(elf), - _ => unreachable!(), +/// [`trap_all_syscalls_in_code`] for an AArch64 host, where every site the +/// scanner recognizes — `SVC` and the `MSR`/`MRS TPIDR_EL0` accesses it +/// virtualizes — becomes `BRK`. +#[cfg(any(test, target_arch = "aarch64"))] +fn trap_all_aarch64_patch_sites(code: &mut [u8], code_vaddr: u64) -> Result { + let section = TextSectionInfo { + vaddr: code_vaddr, + file_offset: 0, + size: code.len() as u64, + }; + arm64::trap_all_patch_sites(code, &[section]) +} + +/// The guest page size assumed when laying out the appended trampoline. +pub(crate) const TRAMPOLINE_PAGE_SIZE: u64 = 0x1000; + +/// The address past the object's last `PT_LOAD` where an appended trampoline +/// goes. `max_load_end` is the highest `p_vaddr + p_memsz`, `max_align` the +/// largest `p_align`. +/// +/// AArch64 objects skip one further `max_align`, because the guest loader +/// reserves `maplength + p_align` and trims the tail. Every other architecture +/// keeps the page-granular rule, so its placement cannot move. +/// +/// **Nothing reserves the returned address**, and glibc packs objects +/// adjacently, so with several shared objects there may be no free gap here at +/// all. `trampoline_placement_for` reaches this only after preferring a hole +/// in the object's own load span; mapping here requires validating the range +/// first (`litebox_shim_linux`'s `trampoline_range_is_safe_to_map`). +pub fn trampoline_addr_for(max_load_end: u64, max_align: u64, e_machine: u16) -> Result { + // Guard against a bogus `p_align`: the masking below requires a power of two. + // Non-AArch64 objects keep the historical page-granular rule verbatim, so + // their placement cannot move. + let align = if e_machine == object::elf::EM_AARCH64 && max_align.is_power_of_two() { + max_align.max(TRAMPOLINE_PAGE_SIZE) + } else { + TRAMPOLINE_PAGE_SIZE + }; + let aligned_end = checked_add_u64(max_load_end, align - 1, "trampoline base")? & !(align - 1); + if align <= TRAMPOLINE_PAGE_SIZE { + Ok(aligned_end) + } else { + checked_add_u64(aligned_end, align, "trampoline base") + } +} + +/// A `PT_LOAD` segment, reduced to the fields trampoline placement needs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct LoadSegment { + /// `p_vaddr`. + pub vaddr: u64, + /// `p_filesz`. + pub filesz: u64, + /// `p_memsz`. + pub memsz: u64, + /// `p_align`. + pub align: u64, +} + +/// Where an object's appended trampoline goes, and how large it may grow. +/// +/// TODO: one trampoline per object bounds every gate twice over -- by the hole +/// it is placed in, and by the callback literal's +-1MiB reach from the single +/// header. Both hold for `SVC` and thread-pointer sites, which are sparse, and +/// both fail if a host ever needs a gate per guest `x18` access: those run to +/// ~2% of instructions, several times the current site count. Placing several +/// smaller trampolines, each with its own header, near the sites they serve +/// would lift both limits and fit the page-sized holes real objects leave. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TrampolinePlacement { + /// A hole between two `PT_LOAD` segments, inside the object's own load + /// span. The safe case: the loader reserved the whole span for this object, + /// so nothing else can be placed there. + InsideLoadSpan { + /// Page-aligned virtual address (object-relative for `ET_DYN`). + addr: u64, + /// Maximum number of bytes that may be written at `addr` before the + /// trampoline would run into the next segment. + limit: u64, + /// [`trampoline_addr_for`]'s address past the last segment, used when + /// the trampoline outgrows `limit`. + fallback_addr: u64, + }, + /// Past the object's last segment, because its segments leave no big enough + /// hole. Nothing reserves the range, so the shim validates it before + /// mapping. + PastLastSegment { + /// Page-aligned virtual address (object-relative for `ET_DYN`). + addr: u64, + }, +} + +impl TrampolinePlacement { + /// The address the trampoline is placed at, preferring the reserved one. + pub(crate) fn addr(self) -> u64 { + match self { + Self::InsideLoadSpan { addr, .. } | Self::PastLastSegment { addr } => addr, + } } - .ok_or_else(|| Error::ParseError("no PT_LOAD segments found".into()))?; - // Round up to the nearest page (assume 0x1000 page size) - checked_add_u64(max_virtual_addr, 0xFFF, "trampoline base").map(|addr| addr & !0xFFF) + /// The unreserved address past the last segment, which is the address + /// itself once placement has already fallen back to it. + pub(crate) fn fallback_addr(self) -> u64 { + match self { + Self::InsideLoadSpan { fallback_addr, .. } => fallback_addr, + Self::PastLastSegment { addr } => addr, + } + } } -/// Returns the highest `p_vaddr + p_memsz` among all `PT_LOAD` segments. -fn max_load_segment_end(elf: &ElfFile<'_, Elf>) -> Option -where - Elf::Word: Into, -{ - let endian = elf.endian(); - elf.elf_program_headers() +/// Chooses where to put the trampoline appended to `segments`' object. +/// +/// Prefers an inter-segment gap, reported as +/// [`TrampolinePlacement::InsideLoadSpan`]. No program header covers the +/// trampoline, so it must land in space the dynamic loader already reserved for +/// *this* object: glibc reserves the whole first-`mapstart` to last-`allocend` +/// span, so the gaps AArch64 objects leave (linked for 64 KiB pages, run with a +/// 4 KiB `AT_PAGESZ`) belong to the object for its lifetime. Addresses past the +/// last segment are unreserved and routinely owned by a neighboring object. The +/// gap is exact because LiteBox pins the guest's `AT_PAGESZ` to +/// [`TRAMPOLINE_PAGE_SIZE`]. +/// +/// An object with no usable gap falls back to [`trampoline_addr_for`], reported +/// as [`TrampolinePlacement::PastLastSegment`] so the caller knows the address +/// is unreserved and must be validated before mapping. +pub(crate) fn trampoline_placement_for( + segments: &[LoadSegment], + e_machine: u16, +) -> Result { + let max_load_end = segments .iter() - .filter(|ph| ph.p_type(endian) == object::elf::PT_LOAD) - .filter_map(|ph| { - ph.p_vaddr(endian) - .into() - .checked_add(ph.p_memsz(endian).into()) - }) + .filter_map(|s| s.vaddr.checked_add(s.memsz)) .max() + .ok_or_else(|| Error::ParseError("no PT_LOAD segments found".into()))?; + let max_align = segments.iter().map(|s| s.align).max().unwrap_or(0); + let fallback_addr = trampoline_addr_for(max_load_end, max_align, e_machine)?; + let fallback = TrampolinePlacement::PastLastSegment { + addr: fallback_addr, + }; + + // Only AArch64 objects are placed in a hole: changing x86-64 placement is + // held back, as on `trampoline_addr_for`. + if e_machine != object::elf::EM_AARCH64 { + return Ok(fallback); + } + + Ok( + largest_inter_segment_hole(segments).map_or(fallback, |(start, end)| { + TrampolinePlacement::InsideLoadSpan { + addr: start, + limit: end - start, + fallback_addr, + } + }), + ) +} + +/// Returns the largest page-granular gap between consecutive `PT_LOAD` +/// segments, as `(start, end)`, or `None` when the segments are contiguous. +/// +/// The bounds mirror glibc's `mapend` / `mapstart`, except that a segment is +/// treated as occupying `max(p_filesz, p_memsz)` rather than `p_filesz`: +/// `_dl_map_segments` maps anonymous pages over the difference for any segment +/// whose `p_memsz` exceeds its `p_filesz`, not only the last one, so counting +/// only `p_filesz` would open a gap that is actually backed. +fn largest_inter_segment_hole(segments: &[LoadSegment]) -> Option<(u64, u64)> { + let page = TRAMPOLINE_PAGE_SIZE; + let mut sorted: Vec<&LoadSegment> = segments.iter().collect(); + sorted.sort_unstable_by_key(|s| s.vaddr); + + let mut best: Option<(u64, u64)> = None; + // `mapend` must account for every earlier segment, not just the previous + // one, so that overlapping or out-of-order segments cannot open a fake gap. + let mut covered_to = 0u64; + for s in sorted { + let start = s.vaddr & !(page - 1); + // A segment whose memsz exceeds its filesz has anonymous pages mapped + // over the difference, so treat the whole memsz as occupied. + let end = s + .vaddr + .checked_add(s.filesz.max(s.memsz)) + .and_then(|e| e.checked_next_multiple_of(page))?; + if start > covered_to + && covered_to != 0 + && best.is_none_or(|(b0, b1)| start - covered_to > b1 - b0) + { + best = Some((covered_to, start)); + } + covered_to = covered_to.max(end); + } + best +} + +fn find_addr_for_trampoline_code(file: &object::File<'_>) -> Result { + let object::File::Elf64(elf) = file else { + unreachable!() + }; + trampoline_placement_for( + &elf_load_segments(file), + elf.elf_header().e_machine.get(elf.endian()), + ) +} + +/// Collects the `PT_LOAD` segments of `file` in program-header order. +fn elf_load_segments(file: &object::File<'_>) -> Vec { + file.segments() + .map(|seg| LoadSegment { + vaddr: seg.address(), + filesz: seg.file_range().1, + memsz: seg.size(), + align: seg.align(), + }) + .collect() } fn get_control_transfer_targets( @@ -1775,6 +2141,416 @@ fn hook_syscall_and_after( mod tests { use super::*; + fn seg(vaddr: u64, filesz: u64, memsz: u64, align: u64) -> LoadSegment { + LoadSegment { + vaddr, + filesz, + memsz, + align, + } + } + + /// One shared object as the guest's `ld.so` laid it out. + struct Obj { + name: &'static str, + /// Address the object's first `PT_LOAD` was mapped at. + base: u64, + /// Size of the trampoline the rewriter appended to it. + tramp_size: u64, + segs: Vec, + } + + impl Obj { + /// The address ranges glibc actually populates for this object: each + /// `PT_LOAD` from `align_down(p_vaddr)` to `align_up(p_vaddr + p_memsz)`. + fn mapped_ranges(&self) -> Vec<(u64, u64)> { + let page = TRAMPOLINE_PAGE_SIZE; + self.segs + .iter() + .map(|s| { + ( + self.base + (s.vaddr & !(page - 1)), + self.base + (s.vaddr + s.memsz).next_multiple_of(page), + ) + }) + .collect() + } + } + + /// The four objects `python3 --version` loads, with the load addresses and + /// trampoline sizes captured from a live LiteBox run on aarch64. + fn python_objects() -> Vec { + alloc::vec![ + Obj { + name: "ld-linux-aarch64.so.1", + base: 0xfffffff90000, + tramp_size: 0x99c, + segs: alloc::vec![ + seg(0x0, 0x25b64, 0x25b64, 0x10000), + seg(0x3ec18, 0x2588, 0x2730, 0x10000), + ], + }, + Obj { + name: "libpython3.12.so.1.0", + base: 0xfffffef50000, + tramp_size: 0x2200, + segs: alloc::vec![ + seg(0x0, 0x45ae40, 0x45ae40, 0x10000), + seg(0x466f10, 0x1d6170, 0x1d7520, 0x10000), + ], + }, + Obj { + name: "libc.so.6", + base: 0xfffffed40000, + tramp_size: 0x9ddc, + segs: alloc::vec![ + seg(0x0, 0x18215c, 0x18215c, 0x10000), + seg(0x19d2b0, 0x64398, 0x70d20, 0x10000), + ], + }, + Obj { + name: "libm.so.6", + base: 0xfffffec90000, + tramp_size: 0xb50, + segs: alloc::vec![ + seg(0x0, 0x8a136, 0x8a136, 0x10000), + seg(0x9fc80, 0x398, 0x4d0, 0x10000), + ], + }, + ] + } + + /// No object's trampoline may land on a page another object has mapped. + /// + /// The shim maps the trampoline with `MAP_FIXED`, which over a fully + /// covered range does not fail — it silently replaces the victim's pages, + /// and the corruption surfaces much later somewhere unrelated. + /// + /// The layout is a recording of a real `python3 --version` run, in which + /// `libc`'s trampoline overwrote 40 KiB of `libpython`'s text and `libm`'s + /// overwrote 4 KiB of `libc`'s. + #[test] + fn no_trampoline_overlaps_another_objects_mapping() { + let objects = python_objects(); + let mut collisions = Vec::new(); + for obj in &objects { + let placement = + trampoline_placement_for(&obj.segs, object::elf::EM_AARCH64).expect("placement"); + let tramp = ( + obj.base + placement.addr(), + obj.base + + (placement.addr() + obj.tramp_size).next_multiple_of(TRAMPOLINE_PAGE_SIZE), + ); + for victim in &objects { + if core::ptr::eq(obj, victim) { + continue; + } + for (lo, hi) in victim.mapped_ranges() { + let start = tramp.0.max(lo); + let end = tramp.1.min(hi); + if start < end { + collisions.push(format!( + "{}'s trampoline [{:#x},{:#x}) overwrites {:#x} bytes of {} \ + [{:#x},{:#x})", + obj.name, + tramp.0, + tramp.1, + end - start, + victim.name, + lo, + hi, + )); + } + } + } + } + assert!( + collisions.is_empty(), + "trampolines silently overwrote other objects:\n {}", + collisions.join("\n ") + ); + } + + /// Placement must land inside the object's own load span — the only region + /// the dynamic loader reserves on its behalf — with room for the trampoline. + #[test] + fn placement_is_inside_the_objects_own_reservation() { + for obj in &python_objects() { + let placement = + trampoline_placement_for(&obj.segs, object::elf::EM_AARCH64).expect("placement"); + let span_end = obj + .segs + .iter() + .map(|s| (s.vaddr + s.memsz).next_multiple_of(TRAMPOLINE_PAGE_SIZE)) + .max() + .unwrap(); + let TrampolinePlacement::InsideLoadSpan { addr, limit, .. } = placement else { + panic!( + "{}: placement {:#x} is not reserved by anything", + obj.name, + placement.addr() + ); + }; + assert!( + addr.saturating_add(limit) <= span_end, + "{}: placement [{:#x},{:#x}) escapes the load span (ends {span_end:#x})", + obj.name, + addr, + addr.saturating_add(limit), + ); + assert!( + limit >= obj.tramp_size, + "{}: {:#x}-byte hole cannot hold a {:#x}-byte trampoline", + obj.name, + limit, + obj.tramp_size, + ); + } + } + + /// The chosen gap must be one the guest's loader leaves alone: glibc + /// `mprotect`s exactly `[first.mapend, last.mapstart)` to `PROT_NONE` and + /// never maps over it. These are the ranges observed in the live trace. + #[test] + fn placement_matches_the_gap_glibc_protects() { + let expected = [ + ("libpython3.12.so.1.0", 0x45b000u64, 0x466000u64), + ("libc.so.6", 0x183000, 0x19d000), + ("libm.so.6", 0x8b000, 0x9f000), + ]; + for (name, lo, hi) in expected { + let obj = python_objects() + .into_iter() + .find(|o| o.name == name) + .unwrap(); + let placement = + trampoline_placement_for(&obj.segs, object::elf::EM_AARCH64).expect("placement"); + let TrampolinePlacement::InsideLoadSpan { addr, limit, .. } = placement else { + panic!("{name}: expected a hole inside the load span"); + }; + assert_eq!( + (addr, addr.saturating_add(limit)), + (lo, hi), + "{name}: gap does not match the range glibc leaves PROT_NONE" + ); + } + } + + /// An object with no gap has nowhere reserved to go, so placement falls + /// back to the address past the last segment and says so, which is what + /// makes the shim validate it before mapping. + #[test] + fn contiguous_segments_fall_back_and_are_marked_unreserved() { + let segs = [ + seg(0x0, 0x1000, 0x1000, 0x10000), + seg(0x1000, 0x100, 0x100, 0x10000), + ]; + let placement = trampoline_placement_for(&segs, object::elf::EM_AARCH64).unwrap(); + assert!(matches!( + placement, + TrampolinePlacement::PastLastSegment { .. } + )); + } + + #[test] + fn aarch64_fallback_keeps_program_headers_unchanged() { + const ELF_HEADER_BYTES: usize = 64; + const PROGRAM_HEADER_BYTES: usize = 56; + let mut elf = vec![0u8; ELF_HEADER_BYTES + 2 * PROGRAM_HEADER_BYTES]; + elf[..4].copy_from_slice(b"\x7fELF"); + elf[4] = object::elf::ELFCLASS64; + elf[5] = object::elf::ELFDATA2LSB; + elf[18..20].copy_from_slice(&object::elf::EM_AARCH64.to_le_bytes()); + elf[32..40].copy_from_slice(&(ELF_HEADER_BYTES as u64).to_le_bytes()); + elf[54..56].copy_from_slice(&u16::try_from(PROGRAM_HEADER_BYTES).unwrap().to_le_bytes()); + elf[56..58].copy_from_slice(&2u16.to_le_bytes()); + + let text = ELF_HEADER_BYTES; + elf[text..text + 4].copy_from_slice(&object::elf::PT_LOAD.to_le_bytes()); + elf[text + 4..text + 8] + .copy_from_slice(&(object::elf::PF_R | object::elf::PF_X).to_le_bytes()); + elf[text + 32..text + 40].copy_from_slice(&0x180000u64.to_le_bytes()); + elf[text + 40..text + 48].copy_from_slice(&0x180000u64.to_le_bytes()); + elf[text + 48..text + 56].copy_from_slice(&0x10000u64.to_le_bytes()); + + let data = text + PROGRAM_HEADER_BYTES; + elf[data..data + 4].copy_from_slice(&object::elf::PT_LOAD.to_le_bytes()); + elf[data + 4..data + 8] + .copy_from_slice(&(object::elf::PF_R | object::elf::PF_W).to_le_bytes()); + elf[data + 8..data + 16].copy_from_slice(&0x18d2b0u64.to_le_bytes()); + elf[data + 16..data + 24].copy_from_slice(&0x19d2b0u64.to_le_bytes()); + elf[data + 40..data + 48].copy_from_slice(&0x70d20u64.to_le_bytes()); + elf[data + 48..data + 56].copy_from_slice(&0x10000u64.to_le_bytes()); + let phdrs_before = elf[ELF_HEADER_BYTES..].to_vec(); + let code_offset = elf.len(); + elf.extend((0..1025).flat_map(|_| 0xD400_0001u32.to_le_bytes())); + let input = elf.clone(); + let section = TextSectionInfo { + vaddr: 0x1000, + file_offset: code_offset as u64, + size: (elf.len() - code_offset) as u64, + }; + let out = hook_aarch64_elf_at(&input, &mut elf, &[section], 0x220000, None, 0).unwrap(); + + assert_eq!( + &out[ELF_HEADER_BYTES..ELF_HEADER_BYTES + phdrs_before.len()], + phdrs_before, + "fixed slots must not modify program headers" + ); + } + + /// A minimal AArch64 object whose text is a single `SVC #0` at `0x1000`. + fn aarch64_elf_with_one_svc() -> Vec { + const ELF_HEADER_BYTES: usize = 64; + const PROGRAM_HEADER_BYTES: usize = 56; + let mut elf = vec![0u8; ELF_HEADER_BYTES + PROGRAM_HEADER_BYTES]; + elf[..4].copy_from_slice(b"\x7fELF"); + elf[4] = object::elf::ELFCLASS64; + elf[5] = object::elf::ELFDATA2LSB; + elf[18..20].copy_from_slice(&object::elf::EM_AARCH64.to_le_bytes()); + elf[32..40].copy_from_slice(&(ELF_HEADER_BYTES as u64).to_le_bytes()); + elf[54..56].copy_from_slice(&u16::try_from(PROGRAM_HEADER_BYTES).unwrap().to_le_bytes()); + elf[56..58].copy_from_slice(&1u16.to_le_bytes()); + + let text = ELF_HEADER_BYTES; + elf[text..text + 4].copy_from_slice(&object::elf::PT_LOAD.to_le_bytes()); + elf[text + 4..text + 8] + .copy_from_slice(&(object::elf::PF_R | object::elf::PF_X).to_le_bytes()); + elf[text + 16..text + 24].copy_from_slice(&0x1000u64.to_le_bytes()); + elf[text + 32..text + 40].copy_from_slice(&4u64.to_le_bytes()); + elf[text + 40..text + 48].copy_from_slice(&4u64.to_le_bytes()); + elf[text + 48..text + 56].copy_from_slice(&0x10000u64.to_le_bytes()); + + elf.extend(0xD400_0001u32.to_le_bytes()); + elf + } + + /// A gap the gates cannot branch back from is retried at the fallback + /// address rather than failing the whole binary. + #[test] + fn an_out_of_branch_range_gap_falls_back_instead_of_rejecting() { + /// Comfortably past a `B`'s +-128MiB reach from the text at 0x1000. + const UNREACHABLE_GAP: u64 = 0x2000_0000; + + let mut elf = aarch64_elf_with_one_svc(); + let input = elf.clone(); + let text_file_offset = (elf.len() - 4) as u64; + let section = || TextSectionInfo { + vaddr: 0x1000, + file_offset: text_file_offset, + size: 4, + }; + + // The gap alone cannot work: every site is out of range and trapped. + let mut direct = elf.clone(); + assert!( + matches!( + hook_aarch64_elf_at(&input, &mut direct, &[section()], UNREACHABLE_GAP, None, 0), + Err(Error::UnpatchableSyscalls(_)) + ), + "the gap has to be genuinely unreachable for this test to mean anything" + ); + + // Offered the same gap plus a reachable fallback, rewriting succeeds. + let placement = TrampolinePlacement::InsideLoadSpan { + addr: UNREACHABLE_GAP, + limit: 0x10000, + fallback_addr: 0x20000, + }; + hook_aarch64_elf(&input, &mut elf, &[section()], placement, 0) + .expect("the reachable fallback address must be retried"); + } + + /// x86-64 placement is deliberately unchanged; see `trampoline_addr_for`. + #[test] + fn placement_leaves_x86_64_alone() { + for obj in &python_objects() { + let placement = + trampoline_placement_for(&obj.segs, object::elf::EM_X86_64).expect("placement"); + assert!(matches!( + placement, + TrampolinePlacement::PastLastSegment { .. } + )); + } + } + + /// With page-sized segment alignment (the x86-64 case) the trampoline goes + /// in the first page past the last `PT_LOAD`, unchanged from before. + #[test] + fn trampoline_addr_page_aligned_segments_use_next_page() { + assert_eq!( + trampoline_addr_for(0x20d_fd0, 0x1000, object::elf::EM_AARCH64).unwrap(), + 0x20e_000 + ); + assert_eq!( + trampoline_addr_for(0x20e_000, 0x1000, object::elf::EM_AARCH64).unwrap(), + 0x20e_000 + ); + // A `p_align` below the page size must not pull the address down. + assert_eq!( + trampoline_addr_for(0x20d_fd0, 0x1, object::elf::EM_AARCH64).unwrap(), + 0x20e_000 + ); + assert_eq!( + trampoline_addr_for(0x20d_fd0, 0, object::elf::EM_AARCH64).unwrap(), + 0x20e_000 + ); + } + + /// With 64 KiB segment alignment (the aarch64 case) the trampoline must + /// clear the whole `maplength + p_align` region glibc's `_dl_map_segment` + /// reserves while mapping the object, otherwise the shim's `MAP_FIXED` + /// straddles the reservation boundary and the load fails. + #[test] + fn trampoline_addr_skips_loader_alignment_slack() { + // Real values from aarch64 `libc.so.6`: last PT_LOAD ends at 0x20dfd0. + assert_eq!( + trampoline_addr_for(0x20d_fd0, 0x10000, object::elf::EM_AARCH64).unwrap(), + 0x220_000 + ); + // Exactly on an alignment boundary still skips a full unit, because the + // loader's reservation runs to `end + p_align`. + assert_eq!( + trampoline_addr_for(0x210_000, 0x10000, object::elf::EM_AARCH64).unwrap(), + 0x220_000 + ); + } + + /// A non-power-of-two `p_align` is bogus; fall back to the page size rather + /// than corrupting the mask arithmetic. + #[test] + fn trampoline_addr_rejects_non_power_of_two_align() { + assert_eq!( + trampoline_addr_for(0x20d_fd0, 0x3000, object::elf::EM_AARCH64).unwrap(), + 0x20e_000 + ); + } + + /// The slow-path rule is gated on `EM_AARCH64`, not on the host arch. GNU + /// ld's default max-page-size on x86-64 is 0x200000, so x86-64 objects + /// routinely have a `p_align` above the page size, but their placement must + /// not move: any change to this address changes which programs collide. + /// This also covers cross-rewriting an x86-64 binary from an aarch64 host, + /// where `cfg(target_arch)` would be the wrong test. + #[test] + fn trampoline_addr_slow_path_rule_is_aarch64_only() { + // 2 MiB alignment, the x86-64 default: keeps the old next-page rule. + assert_eq!( + trampoline_addr_for(0x20d_fd0, 0x200000, object::elf::EM_X86_64).unwrap(), + 0x20e_000 + ); + // The identical object as aarch64 does skip an alignment unit. + assert_eq!( + trampoline_addr_for(0x20d_fd0, 0x200000, object::elf::EM_AARCH64).unwrap(), + 0x600_000 + ); + } + + #[test] + fn trampoline_addr_reports_overflow() { + assert!(trampoline_addr_for(u64::MAX - 1, 0x10000, object::elf::EM_AARCH64).is_err()); + } + #[test] fn aarch64_out_of_range_site_is_rejected_as_unpatchable() { // A trampoline mapped 256MB above the text is outside the site's ±128MB @@ -1787,13 +2563,169 @@ mod tests { file_offset: 0, size: buf.len() as u64, }]; - let err = hook_aarch64_elf(&input, &mut buf, §ions, 0x1000_0000, 0).unwrap_err(); + let placement = TrampolinePlacement::PastLastSegment { addr: 0x1000_0000 }; + let err = hook_aarch64_elf(&input, &mut buf, §ions, placement, 0).unwrap_err(); assert!( matches!(err, Error::UnpatchableSyscalls(_)), "expected UnpatchableSyscalls, got {err:?}" ); } + /// The runtime (mmap-time) AArch64 entry point: a bare code region with no + /// ELF around it still gets its `SVC` redirected into a gate, and the blob + /// is self-describing — its own callback slot at offset 0. + #[test] + fn aarch64_runtime_patch_redirects_svc_into_an_emitted_gate() { + let mut code = 0xD400_0001u32.to_le_bytes().to_vec(); // SVC #0 + let code_vaddr = 0x1000_0000; + let trampoline_vaddr = 0x1000_1000; + let syscall_entry_addr = 0x1000_0000_0000; + + let (trampoline, trapped) = + patch_aarch64_code_segment(&mut code, code_vaddr, trampoline_vaddr, syscall_entry_addr) + .unwrap(); + + assert!( + trapped.is_empty(), + "a nearby trampoline should be reachable" + ); + assert!(!trampoline.is_empty(), "runtime patching emits a blob"); + assert_eq!( + u64::from_le_bytes(trampoline[..8].try_into().unwrap()), + syscall_entry_addr, + "the blob's callback slot holds the runtime's syscall entry directly" + ); + + let branch = u32::from_le_bytes(code[..4].try_into().unwrap()); + assert_eq!(branch & 0xFC00_0000, 0x1400_0000, "the SVC becomes a B"); + let imm26 = i64::from(branch & 0x03FF_FFFF); + let disp = ((imm26 << 38) >> 38) << 2; + assert_eq!( + code_vaddr.wrapping_add(disp.cast_unsigned()), + trampoline_vaddr + 16, + "the B targets the first aligned gate slot" + ); + } + + /// A region with no patch sites must not produce a trampoline: the caller + /// would otherwise map and charge a page for nothing, and — on the shim's + /// runtime path — advance its trampoline cursor past a blob no code + /// branches to. + #[test] + fn aarch64_runtime_patch_of_syscall_free_code_emits_nothing() { + let mut code = 0xD503_201Fu32.to_le_bytes().to_vec(); // NOP + let before = code.clone(); + let (trampoline, trapped) = + patch_aarch64_code_segment(&mut code, 0x1000, 0x2000, 0x3000).unwrap(); + assert!(trampoline.is_empty()); + assert!(trapped.is_empty()); + assert_eq!(code, before, "syscall-free code is left untouched"); + } + + /// The runtime path emits the same thread-pointer placeholder the + /// ahead-of-time path does, so the shim's loader-side finalization applies + /// unchanged. Asserted rather than assumed, because if it stopped holding + /// the gates would redirect the guest's thread pointer into host memory + /// *without faulting*. + #[test] + fn aarch64_runtime_gates_carry_the_thread_pointer_placeholder() { + // MRS X0, TPIDR_EL0 — a thread-pointer read, which is gated. + let mut code = 0xD53B_D040u32.to_le_bytes().to_vec(); + let (mut trampoline, trapped) = + patch_aarch64_code_segment(&mut code, 0x1000, 0x2000, 0x3000).unwrap(); + assert!(trapped.is_empty()); + assert!( + arm64::find_guest_tpidr_placeholder(&trampoline).is_some(), + "a runtime-emitted thread-pointer gate must arrive unpatched" + ); + assert_eq!( + arm64::patch_guest_tpidr_offset(&mut trampoline, 96).unwrap(), + 1 + ); + assert!(arm64::find_guest_tpidr_placeholder(&trampoline).is_none()); + } + + #[test] + fn aarch64_runtime_and_aot_emit_identical_slots() { + let words = [0xD400_0001, 0xD53B_D040 | 9, 0xD51B_D040 | 5]; + let mut runtime_code = words + .into_iter() + .flat_map(u32::to_le_bytes) + .collect::>(); + let mut aot_code = runtime_code.clone(); + let code_vaddr = 0x1000; + let trampoline_vaddr = 0x400000; + + let (runtime, skipped) = + patch_aarch64_code_segment(&mut runtime_code, code_vaddr, trampoline_vaddr, 0x1234) + .unwrap(); + assert!(skipped.is_empty()); + let section = TextSectionInfo { + vaddr: code_vaddr, + file_offset: 0, + size: aot_code.len() as u64, + }; + let aot = arm64::hook_syscalls_aarch64( + &mut aot_code, + &[section], + trampoline_vaddr, + 0x1234, + arm64::Host::Linux, + ) + .unwrap() + .unwrap(); + + assert_eq!(runtime_code, aot_code); + assert_eq!(runtime, aot.trampoline); + + // Exercise the compact classifier independently on both products, not + // only equality of their bytes. The three sites emit SVC, MRS, then MSR + // slots at fixed aligned starts after the 16-byte trampoline header. + let boundaries = [ + (16usize, 12usize), // 9 inbound SVC + 3 outbound-stub instructions. + (80, 3), + (96, 9), + ]; + let mut classified = 0; + for product in [&runtime, &aot.trampoline] { + for (slot_offset, count) in boundaries { + for instruction in 0..count { + let pc = trampoline_vaddr + (slot_offset + instruction * 4) as u64; + let gate = arm64::classify_gate_pc(product, trampoline_vaddr, pc) + .unwrap_or_else(|| panic!("slot {slot_offset} boundary {instruction}")); + assert_eq!(gate.slot_offset(), slot_offset); + classified += 1; + } + } + } + assert_eq!(classified, 48, "24 boundaries in each emission path"); + } + + /// The fail-safe: when no trampoline can be placed, every site the hooking + /// pass would have redirected becomes a `BRK`. A site left native escapes + /// directly to the host kernel, so "found nothing, trapped nothing" is not + /// an acceptable outcome. + #[test] + fn aarch64_trap_fallback_traps_every_patch_site() { + let mut code = Vec::new(); + code.extend_from_slice(&0xD400_0001u32.to_le_bytes()); // SVC #0 + code.extend_from_slice(&0xD503_201Fu32.to_le_bytes()); // NOP + code.extend_from_slice(&0xD51B_D040u32.to_le_bytes()); // MSR TPIDR_EL0, X0 + code.extend_from_slice(&0xD53B_D041u32.to_le_bytes()); // MRS X1, TPIDR_EL0 + + let count = trap_all_aarch64_patch_sites(&mut code, 0x1000).unwrap(); + + assert_eq!(count, 3, "SVC and both thread-pointer accesses are sites"); + for (index, word) in code.chunks_exact(4).enumerate() { + let insn = u32::from_le_bytes(word.try_into().unwrap()); + if index == 1 { + assert_eq!(insn, 0xD503_201F, "the NOP is not a patch site"); + } else { + assert_eq!(insn & 0xFFE0_001F, 0xD420_0000, "site {index} becomes BRK"); + } + } + } + const NT_STUB_BUILD_SYSNO: u32 = 0x1234; fn nt_stub_bytes() -> [u8; 24] { diff --git a/litebox_syscall_rewriter/tests/aarch64_tests.rs b/litebox_syscall_rewriter/tests/aarch64_tests.rs index f663db09ca..321c171023 100644 --- a/litebox_syscall_rewriter/tests/aarch64_tests.rs +++ b/litebox_syscall_rewriter/tests/aarch64_tests.rs @@ -12,7 +12,7 @@ // Deliberate, range-checked casts on a 64-bit host throughout this test. #![allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] -use litebox_syscall_rewriter::{TRAMPOLINE_MAGIC, hook_syscalls_in_elf}; +use litebox_syscall_rewriter::{Error, TRAMPOLINE_MAGIC, hook_syscalls_in_elf}; const HELLO_AARCH64: &[u8] = include_bytes!("hello-aarch64"); @@ -25,6 +25,18 @@ const TPIDR_REG_MASK: u32 = 0xFFFF_FFE0; const MSR_TPIDR_BITS: u32 = 0xD51B_D040; const MRS_TPIDR_BITS: u32 = 0xD53B_D040; +#[test] +fn big_endian_aarch64_is_rejected() { + let mut elf = HELLO_AARCH64.to_vec(); + elf[5] = object::elf::ELFDATA2MSB; + elf[18..20].copy_from_slice(&object::elf::EM_AARCH64.to_be_bytes()); + + assert!(matches!( + hook_syscalls_in_elf(&elf, Some(0)), + Err(Error::UnsupportedExecutable(reason)) if reason.contains("big-endian AArch64") + )); +} + fn read_u16(data: &[u8], off: usize) -> u16 { u16::from_le_bytes(data[off..off + 2].try_into().unwrap()) } @@ -132,13 +144,10 @@ fn aarch64_hello_world_is_hooked() { let tramp = &out[file_offset as usize..(file_offset + size) as usize]; // Offset 0: callback slot holds the value we passed in. assert_eq!(read_u64(tramp, 0), callback, "callback slot"); - // Offset 8: the shared SVC handler — LDR X16,; BR X16. - assert_eq!( - read_u32(tramp, 8), - 0x58FF_FFD0, - "LDR X16, (pcrel -8)" - ); - assert_eq!(read_u32(tramp, 12), 0xD61F_0200, "BR X16"); + // Offset 8: deterministic NOP padding; each SVC slot dispatches through + // the single callback pointer directly. + assert_eq!(read_u32(tramp, 8), 0xD503_201F, "header NOP padding"); + assert_eq!(read_u32(tramp, 12), 0xD503_201F, "header NOP padding"); // --- Every SVC became a branch into the trampoline region --- let tramp_range = vaddr..(vaddr + size); diff --git a/litebox_syscall_rewriter/tests/snapshot_tests.rs b/litebox_syscall_rewriter/tests/snapshot_tests.rs index 2506c39de6..cf9378f0dd 100644 --- a/litebox_syscall_rewriter/tests/snapshot_tests.rs +++ b/litebox_syscall_rewriter/tests/snapshot_tests.rs @@ -30,18 +30,31 @@ fn objdump(objdump_cmd: &str, binary: &[u8]) -> String { lines.join("\n") } -/// Return the first objdump-like command that exists on the host from -/// `candidates`, or `None` if none are available. -fn find_objdump(candidates: &[&str]) -> Option { +/// Whether `cmd` can actually disassemble `arch_token`. +/// +/// Existence is not enough: a native GNU `objdump` disassembles only the +/// architectures its BFD was built for, so on an AArch64 host `objdump` runs +/// happily but reports no `i386:x86-64` support. `--info` lists them. +/// `llvm-objdump` carries every target, and has no comparable `--info`. +fn objdump_supports(cmd: &str, arch_token: &str) -> bool { use std::process::Command; + if cmd.contains("llvm-objdump") { + return Command::new(cmd) + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()); + } + Command::new(cmd).arg("--info").output().is_ok_and(|o| { + o.status.success() && String::from_utf8_lossy(&o.stdout).contains(arch_token) + }) +} + +/// Return the first command from `candidates` that can disassemble +/// `arch_token`, or `None` if the host has none. +fn find_objdump(candidates: &[&str], arch_token: &str) -> Option { candidates .iter() - .find(|cmd| { - Command::new(cmd) - .arg("--version") - .output() - .is_ok_and(|o| o.status.success()) - }) + .find(|cmd| objdump_supports(cmd, arch_token)) .map(|cmd| (*cmd).to_owned()) } @@ -142,7 +155,20 @@ fn run_snapshot_test(objdump_cmd: &str, input: &[u8], snapshot: &str) { #[test] fn snapshot_test_hello_world_x86_64() { - run_snapshot_test("objdump", HELLO_INPUT_64, "hello-diff"); + // Only GNU objdumps are candidates: the stored snapshot records their + // output format, so falling back to `llvm-objdump` would report a diff + // that is purely a change of disassembler. Skip (rather than fail) when + // the host has none, so an AArch64 dev environment still passes -- the + // mirror of what the AArch64 test does on an x86-only host. + let Some(objdump_cmd) = find_objdump(&["x86_64-linux-gnu-objdump", "objdump"], "i386:x86-64") + else { + eprintln!( + "skipping snapshot_test_hello_world_x86_64: no x86-64-capable GNU objdump \ + (install binutils-x86-64-linux-gnu)" + ); + return; + }; + run_snapshot_test(&objdump_cmd, HELLO_INPUT_64, "hello-diff"); } #[test] @@ -157,7 +183,8 @@ fn snapshot_test_hello_world_aarch64() { // The host objdump usually cannot disassemble AArch64; prefer a cross or // LLVM objdump. Skip (rather than fail) when no capable tool is installed, // so x86-only dev environments still pass. - let Some(objdump_cmd) = find_objdump(&["aarch64-linux-gnu-objdump", "llvm-objdump"]) else { + let Some(objdump_cmd) = find_objdump(&["aarch64-linux-gnu-objdump", "llvm-objdump"], "aarch64") + else { eprintln!( "skipping snapshot_test_hello_world_aarch64: no AArch64-capable objdump \ (install binutils-aarch64-linux-gnu or llvm)" diff --git a/litebox_syscall_rewriter/tests/snapshots/snapshot_tests__hello-aarch64-diff.snap b/litebox_syscall_rewriter/tests/snapshots/snapshot_tests__hello-aarch64-diff.snap index fe098d0a27..53d0b24354 100644 --- a/litebox_syscall_rewriter/tests/snapshots/snapshot_tests__hello-aarch64-diff.snap +++ b/litebox_syscall_rewriter/tests/snapshots/snapshot_tests__hello-aarch64-diff.snap @@ -11,19 +11,19 @@ expression: diff - 400110: d51bd045 msr tpidr_el0,x5 - 400114: d53bd049 mrs x9,tpidr_el0 + 400110: -+ 400114: ++ 400114: 400118: d2800808 mov x8,#0x40 40011c: d2800020 mov x0,#0x1 400120: 910003e1 mov x1,sp 400124: d28001c2 mov x2,#0xe - 400128: d4000001 svc #0x0 -+ 400128: ++ 400128: 40012c: d2801588 mov x8,#0xac - 400130: d4000001 svc #0x0 -+ 400130: ++ 400130: 400134: d2800ba8 mov x8,#0x5d 400138: d2800000 mov x0,#0x0 - 40013c: d4000001 svc #0x0 \ No newline at end of file -+ 40013c: ++ 40013c: \ No newline at end of file