diff --git a/kernel/src/arch_impl/aarch64/boot.S b/kernel/src/arch_impl/aarch64/boot.S index 1378e9c3..3407ead4 100644 --- a/kernel/src/arch_impl/aarch64/boot.S +++ b/kernel/src/arch_impl/aarch64/boot.S @@ -18,6 +18,9 @@ // High-half kernel base (must match linker.ld) .equ KERNEL_VIRT_BASE, 0xFFFF000000000000 +.equ PERCPU_ERET_GUARD_ELR, 120 +.equ PERCPU_ERET_GUARD_SPSR, 128 +.equ PERCPU_ERET_GUARD_SOURCE, 136 // Descriptor bits .equ DESC_VALID, (1 << 0) @@ -514,12 +517,25 @@ sync_exception_handler: // Read ELR from exception frame (modified by Rust handler if redirected) ldr x1, [sp, #248] // x1 = frame.elr - // DIAGNOSTIC: Check for corrupted ELR before ERET (sync handler path) - cmp x1, #0x1000 - b.hs 10f - // ELR < 0x1000 — corrupted. Skip UART diagnostic (QEMU UART at - // 0x09000000 doesn't exist on Parallels). Redirect to idle. - // Redirect to idle_loop_arm64 instead of crashing + // Guard the EL1 return invariant: an EL1 frame may only resume in the + // kernel VA range. EL0 returns legitimately use lower addresses. + ldr x2, [sp, #256] // x2 = frame.spsr + and x3, x2, #0xF // x3 = M[3:0] + cbz x3, 10f // EL0 return + ldr x3, =KERNEL_VIRT_BASE + cmp x1, x3 + b.hs 10f // valid EL1 kernel return + + // Branch-only record, published by writing source last. Never perform + // UART or other I/O in this return corridor. + mrs x3, tpidr_el1 + cbz x3, 11f + str x1, [x3, #PERCPU_ERET_GUARD_ELR] + str x2, [x3, #PERCPU_ERET_GUARD_SPSR] + mov x4, #1 // source 1 = sync epilogue + str x4, [x3, #PERCPU_ERET_GUARD_SOURCE] +11: + // Redirect to idle_loop_arm64 instead of returning to EL1 at a user VA. adrp x1, idle_loop_arm64 add x1, x1, :lo12:idle_loop_arm64 // Write safe values to frame @@ -658,12 +674,25 @@ irq_handler: // CPU, so no other CPU can dispatch it while we still read from the frame. ldr x16, [sp, #248] // x16 = frame.elr - // DIAGNOSTIC: Check for corrupted ELR before ERET. - // If ELR < 0x1000, the saved context is corrupted (null return address). - // Fix it by redirecting to idle_loop_arm64 instead of crashing. - cmp x16, #0x1000 - b.hs 1f - // ELR < 0x1000 — corrupted context. Redirect to idle_loop_arm64. + // Guard the EL1 return invariant: an EL1 frame may only resume in the + // kernel VA range. EL0 returns legitimately use lower addresses. + ldr x1, [sp, #256] // x1 = frame.spsr + and x2, x1, #0xF // x2 = M[3:0] + cbz x2, 1f // EL0 return + ldr x2, =KERNEL_VIRT_BASE + cmp x16, x2 + b.hs 1f // valid EL1 kernel return + + // Branch-only record, published by writing source last. Never perform + // UART or other I/O in this return corridor. + mrs x2, tpidr_el1 + cbz x2, 2f + str x16, [x2, #PERCPU_ERET_GUARD_ELR] + str x1, [x2, #PERCPU_ERET_GUARD_SPSR] + mov x3, #2 // source 2 = IRQ epilogue + str x3, [x2, #PERCPU_ERET_GUARD_SOURCE] +2: + // Redirect to idle_loop_arm64 instead of returning to EL1 at a user VA. adrp x16, idle_loop_arm64 add x16, x16, :lo12:idle_loop_arm64 str x16, [sp, #248] // frame.elr = idle_loop_arm64 diff --git a/kernel/src/arch_impl/aarch64/constants.rs b/kernel/src/arch_impl/aarch64/constants.rs index 751fb3a6..afd82d62 100644 --- a/kernel/src/arch_impl/aarch64/constants.rs +++ b/kernel/src/arch_impl/aarch64/constants.rs @@ -159,6 +159,15 @@ pub const PERCPU_DISPATCH_ELR_OFFSET: usize = 104; /// Immune to cross-CPU frame overwrite race (per-CPU, not on shared stack). pub const PERCPU_DISPATCH_SPSR_OFFSET: usize = 112; +/// Offset of the ELR captured by an assembly ERET invariant redirect. +pub const PERCPU_ERET_GUARD_ELR_OFFSET: usize = 120; + +/// Offset of the SPSR captured by an assembly ERET invariant redirect. +pub const PERCPU_ERET_GUARD_SPSR_OFFSET: usize = 128; + +/// Offset of the ERET guard source tag, published after ELR/SPSR. +pub const PERCPU_ERET_GUARD_SOURCE_OFFSET: usize = 136; + // ============================================================================ // Preempt Count Bit Layout (Linux-compatible) // ============================================================================ diff --git a/kernel/src/arch_impl/aarch64/context_switch.rs b/kernel/src/arch_impl/aarch64/context_switch.rs index 69abf847..b9f083ca 100644 --- a/kernel/src/arch_impl/aarch64/context_switch.rs +++ b/kernel/src/arch_impl/aarch64/context_switch.rs @@ -1502,6 +1502,33 @@ static ERET_ANOMALY_SPSR: [AtomicU64; crate::arch_impl::aarch64::constants::MAX_ static LAST_DISPATCHED_TID: [AtomicU64; crate::arch_impl::aarch64::constants::MAX_CPUS] = [const { AtomicU64::new(0) }; crate::arch_impl::aarch64::constants::MAX_CPUS]; +const LAST_DISPATCHED_SLOT_BITS: u32 = 9; +const LAST_DISPATCHED_SLOT_MASK: u64 = (1 << LAST_DISPATCHED_SLOT_BITS) - 1; +const _: () = assert!( + crate::memory::kernel_stack::ARM64_MAX_KERNEL_STACKS + <= LAST_DISPATCHED_SLOT_MASK as usize +); + +#[inline(always)] +fn reusable_kstack_slot_for_address(address: u64) -> Option { + use crate::memory::kernel_stack::{ + ARM64_KERNEL_STACK_BASE, ARM64_KERNEL_STACK_END, ARM64_STACK_SLOT_SIZE, + }; + + if address < ARM64_KERNEL_STACK_BASE || address >= ARM64_KERNEL_STACK_END { + return None; + } + Some(((address - ARM64_KERNEL_STACK_BASE) / ARM64_STACK_SLOT_SIZE) as usize) +} + +#[inline(always)] +fn decode_last_dispatched(encoded: u64) -> (u64, Option) { + let tid = encoded >> LAST_DISPATCHED_SLOT_BITS; + let slot_code = encoded & LAST_DISPATCHED_SLOT_MASK; + let slot = (slot_code != 0).then(|| (slot_code - 1) as usize); + (tid, slot) +} + /// Stamp the OWNER-TID CANARY for this CPU. Called from `dispatch_thread_locked` /// at its two frame-finalize points. Lock-free; safe to call from inside the /// scheduler lock hold. @@ -1510,7 +1537,73 @@ fn stamp_last_dispatched_tid(cpu_id: usize, tid: u64) { if cpu_id >= crate::arch_impl::aarch64::constants::MAX_CPUS { return; } - LAST_DISPATCHED_TID[cpu_id].store(tid, Ordering::Release); + debug_assert!(tid <= (u64::MAX >> LAST_DISPATCHED_SLOT_BITS)); + let slot_code = reusable_kstack_slot_for_address( + Aarch64PerCpu::kernel_stack_top().saturating_sub(1), + ) + .map(|slot| slot as u64 + 1) + .unwrap_or(0); + let encoded = (tid << LAST_DISPATCHED_SLOT_BITS) | slot_code; + LAST_DISPATCHED_TID[cpu_id].store(encoded, Ordering::Release); +} + +/// Return the reusable kernel-stack slot containing `address` and the tid most +/// recently finalized for dispatch on that slot. A zero tid means the slot has +/// not been stamped yet. +pub fn last_dispatched_tid_for_stack_address(address: u64) -> Option<(usize, u64)> { + let slot = reusable_kstack_slot_for_address(address)?; + let tid = LAST_DISPATCHED_TID + .iter() + .find_map(|record| { + let (tid, recorded_slot) = decode_last_dispatched(record.load(Ordering::Acquire)); + (recorded_slot == Some(slot)).then_some(tid) + }) + .unwrap_or(0); + Some((slot, tid)) +} + +/// Return the tid most recently finalized for dispatch on `cpu_id`. +pub fn last_dispatched_tid(cpu_id: usize) -> Option { + if cpu_id >= crate::arch_impl::aarch64::constants::MAX_CPUS { + return None; + } + let (tid, _) = decode_last_dispatched(LAST_DISPATCHED_TID[cpu_id].load(Ordering::Acquire)); + (tid != 0).then_some(tid) +} + +/// Dump the owner-TID canary and its reusable stack slot for every CPU. +pub fn dump_all_last_dispatched_tids() { + for cpu_id in 0..crate::arch_impl::aarch64::constants::MAX_CPUS { + let (tid, slot) = + decode_last_dispatched(LAST_DISPATCHED_TID[cpu_id].load(Ordering::Acquire)); + raw_uart_str("[LAST_DISPATCHED_TID] cpu="); + raw_uart_dec(cpu_id as u64); + raw_uart_str(" tid="); + raw_uart_dec(tid); + if let Some(slot) = slot { + raw_uart_str(" kstack_slot="); + raw_uart_dec(slot as u64); + } + raw_uart_str("\n"); + } +} + +/// Dump the last branch-only ERET invariant redirect captured on each CPU. +pub fn dump_all_eret_guard_records() { + for cpu_id in 0..crate::arch_impl::aarch64::constants::MAX_CPUS { + let Some((source, elr, spsr)) = crate::per_cpu_aarch64::eret_guard_record(cpu_id) else { + continue; + }; + raw_uart_str("[ERET_GUARD_REDIRECT] cpu="); + raw_uart_dec(cpu_id as u64); + raw_uart_str(" source="); + raw_uart_dec(source); + raw_uart_str(" elr="); + raw_uart_hex(elr); + raw_uart_str(" spsr="); + raw_uart_hex(spsr); + raw_uart_str("\n"); + } } /// Record an ERET-consumer-site frame anomaly into this CPU's last-wins @@ -1649,6 +1742,19 @@ static INLINE_SCHEDULE_STATE: [InlineScheduleState; }, ]; +struct ExitScheduleState { + scheduler_ptr: AtomicUsize, + thread_id: AtomicU64, +} + +static EXIT_SCHEDULE_STATE: [ExitScheduleState; crate::arch_impl::aarch64::constants::MAX_CPUS] = + [const { + ExitScheduleState { + scheduler_ptr: AtomicUsize::new(0), + thread_id: AtomicU64::new(0), + } + }; crate::arch_impl::aarch64::constants::MAX_CPUS]; + const INLINE_SCHEDULE_BREADCRUMB_CPUS: usize = crate::arch_impl::aarch64::constants::MAX_CPUS; const INLINE_SCHEDULE_BREADCRUMB_SLOTS: usize = 16; const INLINE_BC_TRAMPOLINE_ENTRY: u8 = 0x30; @@ -2049,6 +2155,7 @@ fn log_idle_thread_context(tag: &str, thread: &Thread, sp: u64, elr: u64, x30: u #[inline(always)] fn clear_inline_schedule_state(thread: &mut Thread) { thread.saved_by_inline_schedule = false; + thread.inline_schedule_spsr = 0; thread.inline_schedule_saved_sp = 0; thread.inline_schedule_caller_lr = 0; } @@ -2682,7 +2789,7 @@ fn restore_userspace_context_inline( // Restore program counter and status frame.elr = thread.context.elr_el1; - frame.spsr = dispatch_spsr(thread.context.spsr_el1); + frame.spsr = dispatch_spsr(thread.context.spsr_el1) & !SPSR_MODE_MASK; // Restore SP_EL0 (user stack pointer) unsafe { @@ -2853,6 +2960,12 @@ fn setup_idle_return_locked( unsafe { Aarch64PerCpu::set_user_rsp_scratch(idle_stack); Aarch64PerCpu::set_kernel_stack_top(idle_stack); + let mut kernel_ttbr0 = Aarch64PerCpu::kernel_cr3(); + if kernel_ttbr0 == 0 { + kernel_ttbr0 = 0x4200_0000; + } + Aarch64PerCpu::set_next_cr3(kernel_ttbr0); + Aarch64PerCpu::set_saved_process_cr3(0); Aarch64PerCpu::set_current_thread_ptr(core::ptr::null_mut()); Aarch64PerCpu::clear_preempt_active(); } @@ -2917,6 +3030,7 @@ fn reset_idle_continuation_locked( let idle_addr = idle_loop_arm64 as *const () as u64; thread.saved_by_inline_schedule = false; + thread.inline_schedule_spsr = 0; thread.inline_schedule_saved_sp = 0; thread.inline_schedule_caller_lr = 0; thread.context.sp = idle_sp; @@ -3337,6 +3451,11 @@ pub extern "C" fn check_need_resched_and_switch_arm64( return; } + // This entry proves an earlier handoff completed, but the current handoff + // may still be using its old stack. Reclamation therefore requires two + // bumps: this one plus a subsequent exception's scheduling entry. + crate::task::scheduler::note_scheduling_epoch(cpu_id_early); + // Read deferred requeue atomically (lock-free). // CRITICAL: This must happen BEFORE the preempt_count early return below. // When IRQs are enabled during syscalls (daifclr #3 in syscall_entry.S), @@ -3910,6 +4029,95 @@ pub extern "C" fn check_need_resched_and_switch_arm64( } } +/// Final scheduler handoff for an exiting AArch64 thread. +/// +/// The inline switch saves the outgoing context and changes SP to the per-CPU +/// scheduler stack before entering `exit_schedule_trampoline`. Only that +/// neutral-stack trampoline is allowed to publish `Terminated`. +pub fn schedule_terminated_from_exit(thread_id: u64) -> ! { + unsafe { + crate::arch_impl::aarch64::cpu::disable_interrupts(); + } + // Balance rust_syscall_handler_aarch64's preempt_disable only after IRQs + // are masked, leaving no interrupt window between enabling preemption and + // the final stack pivot. + Aarch64PerCpu::preempt_enable(); + + let cpu_id = Aarch64PerCpu::cpu_id() as usize; + let mut guard = crate::task::scheduler::lock_for_context_switch(); + let sched = guard + .as_mut() + .expect("scheduler unavailable during AArch64 exit handoff"); + let old_context_ptr = sched + .get_thread_mut(thread_id) + .map(|thread| &mut thread.context as *mut CpuContext) + .expect("exiting AArch64 thread missing from scheduler"); + + EXIT_SCHEDULE_STATE[cpu_id] + .scheduler_ptr + .store(sched as *mut Scheduler as usize, Ordering::Relaxed); + EXIT_SCHEDULE_STATE[cpu_id] + .thread_id + .store(thread_id, Ordering::Release); + + let _ = spin::MutexGuard::leak(guard); + let scheduler_top = scheduler_stack_top(cpu_id); + assert_pivot_free(cpu_id, scheduler_top, read_current_sp(), 4); + + unsafe { + aarch64_inline_schedule_switch(old_context_ptr, scheduler_top, exit_schedule_trampoline); + } + + panic!("terminated AArch64 thread was dispatched after final exit handoff"); +} + +extern "C" fn exit_schedule_trampoline() -> ! { + let cpu_id = Aarch64PerCpu::cpu_id() as usize; + let state = &EXIT_SCHEDULE_STATE[cpu_id]; + let sched_ptr = state.scheduler_ptr.swap(0, Ordering::Acquire) as *mut Scheduler; + let thread_id = state.thread_id.swap(0, Ordering::Relaxed); + assert!(!sched_ptr.is_null(), "missing AArch64 exit scheduler state"); + + // The assembly caller has already executed `mov sp, scheduler_stack_top`. + // Publishing Terminated here makes "terminated while still on its own + // kernel stack" unrepresentable on the syscall-exit path. + let sched = unsafe { &mut *sched_ptr }; + sched + .get_thread_mut(thread_id) + .expect("exiting AArch64 thread disappeared before stack pivot") + .set_terminated(); + sched.remove_from_ready_queue(thread_id); + + let (old_id, new_id, should_requeue_old) = sched + .schedule_deferred_requeue() + .expect("AArch64 exit handoff found no idle or runnable successor"); + assert_eq!( + old_id, thread_id, + "AArch64 exit handoff saved the wrong thread" + ); + assert_ne!( + new_id, thread_id, + "terminated AArch64 thread selected again" + ); + debug_assert!(!should_requeue_old); + + INLINE_SCHEDULE_STATE[cpu_id] + .scheduler_ptr + .store(sched_ptr as usize, Ordering::Relaxed); + INLINE_SCHEDULE_STATE[cpu_id] + .old_thread_id + .store(old_id, Ordering::Relaxed); + INLINE_SCHEDULE_STATE[cpu_id] + .new_thread_id + .store(new_id, Ordering::Relaxed); + INLINE_SCHEDULE_STATE[cpu_id] + .should_requeue_old + .store(false, Ordering::Relaxed); + crate::task::scheduler::increment_context_switch_count(); + + inline_schedule_trampoline() +} + extern "C" fn inline_schedule_trampoline() -> ! { let cpu_id = Aarch64PerCpu::cpu_id() as usize; inline_schedule_breadcrumb(cpu_id, INLINE_BC_TRAMPOLINE_ENTRY, 0); @@ -4379,7 +4587,7 @@ pub fn schedule_from_kernel() { } old_thread.context.sp_el0 = read_sp_el0(); old_thread.context.tpidr_el0 = read_tpidr_el0(); - old_thread.context.spsr_el1 = kernel_dispatch_spsr(saved_daif & 0x3C0); + old_thread.inline_schedule_spsr = kernel_dispatch_spsr(saved_daif & 0x3C0); old_thread.inline_schedule_caller_lr = unsafe { core::ptr::read_volatile((schedule_sp + 0x20) as *const u64) }; old_thread.inline_schedule_saved_sp = schedule_sp; diff --git a/kernel/src/arch_impl/aarch64/exception.rs b/kernel/src/arch_impl/aarch64/exception.rs index bb2a49c7..8d04c7a8 100644 --- a/kernel/src/arch_impl/aarch64/exception.rs +++ b/kernel/src/arch_impl/aarch64/exception.rs @@ -29,6 +29,8 @@ pub static CPU0_LAST_SYNC_FAR: AtomicU64 = AtomicU64::new(0); pub static CPU0_LAST_SYNC_ELR: AtomicU64 = AtomicU64::new(0); static PC_ALIGN_VERBOSE_CAPTURED: AtomicBool = AtomicBool::new(false); static FATAL_POSTMORTEM_CAPTURED: [AtomicBool; 8] = [const { AtomicBool::new(false) }; 8]; +static FATAL_POSTMORTEM_SECTIONS_CLAIMED: [AtomicU64; 8] = + [const { AtomicU64::new(0) }; 8]; static EL1_UNHANDLED_FAULT_LATCHED: [AtomicBool; 8] = [const { AtomicBool::new(false) }; 8]; static FATAL_POSTMORTEM_UART_LOCK: AtomicBool = AtomicBool::new(false); @@ -287,36 +289,14 @@ fn set_idle_stack_for_eret() { unsafe { Aarch64PerCpu::set_user_rsp_scratch(idle_stack); Aarch64PerCpu::set_kernel_stack_top(idle_stack); + let kernel_ttbr0 = super::kernel_ttbr0(); + Aarch64PerCpu::set_next_cr3(kernel_ttbr0); + Aarch64PerCpu::set_saved_process_cr3(0); Aarch64PerCpu::set_current_thread_ptr(core::ptr::null_mut()); Aarch64PerCpu::clear_preempt_active(); } } -/// Switch TTBR0 to the kernel page table and flush the TLB. -/// -/// This ensures we don't return to userspace with a stale/terminated address space. -#[inline(always)] -fn switch_ttbr0_to_kernel() { - let mut kernel_ttbr0 = crate::per_cpu_aarch64::get_kernel_cr3(); - if kernel_ttbr0 == 0 { - // Fallback to boot TTBR0 table if per-CPU kernel TTBR0 is unavailable. - kernel_ttbr0 = 0x4200_0000; - } - - unsafe { - core::arch::asm!( - "dsb ishst", - "msr ttbr0_el1, {}", - "isb", - "tlbi vmalle1is", - "dsb ish", - "isb", - in(reg) kernel_ttbr0, - options(nomem, nostack) - ); - } -} - /// Mark the current thread as Terminated in the scheduler and remove from ready queue. /// /// Called from exception handlers after `pm.exit_process()` to prevent the @@ -337,10 +317,25 @@ fn terminate_current_scheduler_thread() { } } -fn defer_current_user_thread_sigsegv_exit(label: &str) { +fn defer_current_user_thread_sigsegv_exit(label: &str, frame_addr: u64) { use crate::arch_impl::aarch64::context_switch::{raw_uart_dec, raw_uart_str}; - if let Some(tid) = crate::task::scheduler::current_thread_id() { + // Publish the deferred exit only after this CPU has left the retiring root + // and cleared both assembly return shadows. A peer CPU may drain the queue + // immediately after publication. + super::quiesce_ttbr0_for_exit(); + + let stack_owner = + crate::arch_impl::aarch64::context_switch::last_dispatched_tid_for_stack_address( + frame_addr, + ) + .and_then(|(_, tid)| (tid != 0).then_some(tid)); + let cpu_id = crate::arch_impl::aarch64::percpu::Aarch64PerCpu::cpu_id() as usize; + let victim_tid = stack_owner.or_else(|| { + crate::arch_impl::aarch64::context_switch::last_dispatched_tid(cpu_id) + }); + + if let Some(tid) = victim_tid { let queued = crate::task::process_task::defer_fault_sigsegv_exit(tid); raw_uart_str(label); raw_uart_str(" deferred_tid="); @@ -354,38 +349,75 @@ fn defer_current_user_thread_sigsegv_exit(label: &str) { } } +#[cold] +#[inline(never)] +fn dump_fatal_postmortem_section(cpu_id: usize, section: usize, heading: &str, dump: F) +where + F: FnOnce(), +{ + let section_bit = 1u64 << section; + if FATAL_POSTMORTEM_SECTIONS_CLAIMED[cpu_id].fetch_or(section_bit, Ordering::AcqRel) + & section_bit + != 0 + { + return; + } + + crate::arch_impl::aarch64::context_switch::raw_uart_str(heading); + dump(); +} + fn dump_fatal_postmortem_once(label: &str) { use crate::arch_impl::aarch64::context_switch::{raw_uart_dec, raw_uart_str}; let cpu_id = crate::arch_impl::aarch64::percpu::Aarch64PerCpu::cpu_id() as usize; - if cpu_id >= FATAL_POSTMORTEM_CAPTURED.len() - || FATAL_POSTMORTEM_CAPTURED[cpu_id].swap(true, Ordering::AcqRel) - { + if cpu_id >= FATAL_POSTMORTEM_CAPTURED.len() { return; } - raw_uart_str("[FATAL_POSTMORTEM] cpu="); - raw_uart_dec(cpu_id as u64); - raw_uart_str(" label="); - raw_uart_str(label); - raw_uart_str("\n Deferred requeue snapshots:\n"); - crate::arch_impl::aarch64::context_switch::dump_defer_requeue_snapshots(); - raw_uart_str("\n Trace buffers:\n"); - crate::tracing::dump_all_buffers(); - raw_uart_str("\n Idle redirect histories:\n"); - crate::arch_impl::aarch64::context_switch::dump_all_idle_redirect_histories(); - raw_uart_str("\n Stack pivot alias histories:\n"); - crate::arch_impl::aarch64::context_switch::dump_stack_pivot_alias_history(); - raw_uart_str("\n Stack-half boundary canaries:\n"); - for canary_cpu in 0..super::constants::MAX_CPUS { - raw_uart_str(" cpu="); - raw_uart_dec(canary_cpu as u64); - raw_uart_str(" intact="); - raw_uart_dec( - super::constants::percpu_stack_boundary_canary_is_intact(canary_cpu) as u64, - ); - raw_uart_str("\n"); + if !FATAL_POSTMORTEM_CAPTURED[cpu_id].swap(true, Ordering::AcqRel) { + raw_uart_str("[FATAL_POSTMORTEM] cpu="); + raw_uart_dec(cpu_id as u64); + raw_uart_str(" label="); + raw_uart_str(label); } + + // Claim each section before entering it. If the section itself faults, a + // nested postmortem skips that in-progress section and continues with the + // remaining evidence instead of losing the rest of the dump. + dump_fatal_postmortem_section(cpu_id, 0, "\n Deferred requeue snapshots:\n", || { + crate::arch_impl::aarch64::context_switch::dump_defer_requeue_snapshots(); + }); + dump_fatal_postmortem_section(cpu_id, 1, "\n Idle redirect histories:\n", || { + crate::arch_impl::aarch64::context_switch::dump_all_idle_redirect_histories(); + }); + dump_fatal_postmortem_section(cpu_id, 2, "\n Stack pivot alias histories:\n", || { + crate::arch_impl::aarch64::context_switch::dump_stack_pivot_alias_history(); + }); + dump_fatal_postmortem_section(cpu_id, 3, "\n Save-skew slots:\n", || { + crate::arch_impl::aarch64::context_switch::dump_all_save_skew_snapshots(); + }); + dump_fatal_postmortem_section(cpu_id, 4, "\n Inline save-skew slots:\n", || { + crate::arch_impl::aarch64::context_switch::dump_all_inline_save_skew_snapshots(); + }); + dump_fatal_postmortem_section(cpu_id, 5, "\n Stack-half boundary canaries:\n", || { + for canary_cpu in 0..super::constants::MAX_CPUS { + raw_uart_str(" cpu="); + raw_uart_dec(canary_cpu as u64); + raw_uart_str(" intact="); + raw_uart_dec( + super::constants::percpu_stack_boundary_canary_is_intact(canary_cpu) as u64, + ); + raw_uart_str("\n"); + } + }); + dump_fatal_postmortem_section(cpu_id, 6, "\n Last-dispatched tids:\n", || { + crate::arch_impl::aarch64::context_switch::dump_all_last_dispatched_tids(); + crate::arch_impl::aarch64::context_switch::dump_all_eret_guard_records(); + }); + dump_fatal_postmortem_section(cpu_id, 7, "\n Trace buffers:\n", || { + crate::tracing::dump_all_buffers(); + }); } #[inline(never)] @@ -394,9 +426,7 @@ fn dump_stack_classification(frame_addr: u64) { let stack_base = super::constants::percpu_stack_region_base(); let stack_end = stack_base + super::constants::PERCPU_STACK_REGION_SIZE as u64; - const HHDM_BASE_DIAG: u64 = 0xFFFF_0000_0000_0000; - const KSTACK_BASE: u64 = HHDM_BASE_DIAG + 0x5200_0000; - const KSTACK_END: u64 = HHDM_BASE_DIAG + 0x5400_0000; + use crate::memory::kernel_stack::{ARM64_KERNEL_STACK_BASE, ARM64_KERNEL_STACK_END}; if frame_addr >= stack_base && frame_addr < stack_end { let offset_from_base = frame_addr - stack_base; @@ -408,8 +438,18 @@ fn dump_stack_classification(frame_addr: u64) { raw_uart_str("\n STACK=boot_cpu"); } raw_uart_dec(cpu_id); - } else if frame_addr >= KSTACK_BASE && frame_addr < KSTACK_END { + } else if frame_addr >= ARM64_KERNEL_STACK_BASE && frame_addr < ARM64_KERNEL_STACK_END { raw_uart_str("\n STACK=alloc_kstack"); + if let Some((slot, tid)) = + crate::arch_impl::aarch64::context_switch::last_dispatched_tid_for_stack_address( + frame_addr, + ) + { + raw_uart_str(" slot="); + raw_uart_dec(slot as u64); + raw_uart_str(" last_dispatched_tid="); + raw_uart_dec(tid); + } } else { raw_uart_str("\n STACK=unknown"); } @@ -710,6 +750,10 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: // Get current TTBR0 to find the process let page_table_phys = ttbr0 & !0xFFFF_0000_0000_0FFF; + // The process exit below can retire this root. Leave it before + // acquiring the process manager and freeing any resources. + super::switch_ttbr0_to_kernel(); + // Find and terminate the process let mut terminated = false; let mut already_terminated = false; @@ -736,7 +780,6 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: // the thread is Ready/Running and will re-dispatch it on // another CPU, causing ERET to a freed address space. terminate_current_scheduler_thread(); - switch_ttbr0_to_kernel(); crate::task::scheduler::set_need_resched(); // CRITICAL: Set frame values BEFORE switch_to_idle() — @@ -757,13 +800,12 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: use crate::arch_impl::aarch64::context_switch::raw_uart_str; raw_uart_str("[DATA_ABORT] kernel-mode fault, deferring process cleanup\n"); } - defer_current_user_thread_sigsegv_exit("[DATA_ABORT]"); + defer_current_user_thread_sigsegv_exit("[DATA_ABORT]", frame as u64); dump_fatal_postmortem_once("DATA_ABORT"); drop(fatal_uart_guard); // Mark scheduler thread as terminated (best effort) terminate_current_scheduler_thread(); - switch_ttbr0_to_kernel(); crate::task::scheduler::set_need_resched(); // CRITICAL: Set frame values BEFORE switch_to_idle_best_effort() — @@ -1030,9 +1072,9 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: let boot_stack_base = super::constants::percpu_stack_region_base(); let boot_stack_end = boot_stack_base + super::constants::PERCPU_STACK_REGION_SIZE as u64; - const HHDM_BASE_DIAG: u64 = 0xFFFF_0000_0000_0000; - const KSTACK_BASE: u64 = HHDM_BASE_DIAG + 0x5200_0000; - const KSTACK_END: u64 = HHDM_BASE_DIAG + 0x5400_0000; + use crate::memory::kernel_stack::{ + ARM64_KERNEL_STACK_BASE, ARM64_KERNEL_STACK_END, + }; dump_stack_classification(frame_addr); // DISPATCH TRACE: last 8 dispatches on this CPU @@ -1044,7 +1086,8 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: // OUTER FRAME: Read the frame 272 bytes above (if on a valid stack) let outer_frame_addr = frame_addr + 272; if outer_frame_addr + 272 <= boot_stack_end - || (outer_frame_addr >= KSTACK_BASE && outer_frame_addr + 272 <= KSTACK_END) + || (outer_frame_addr >= ARM64_KERNEL_STACK_BASE + && outer_frame_addr + 272 <= ARM64_KERNEL_STACK_END) { let outer = outer_frame_addr as *const u64; unsafe { @@ -1068,6 +1111,10 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: // From userspace - terminate the process with SIGSEGV let page_table_phys = ttbr0 & !0xFFFF_0000_0000_0FFF; + // Install the kernel root before pm.exit_process can retire + // the faulting userspace root. + super::switch_ttbr0_to_kernel(); + let mut terminated = false; let mut already_terminated = false; let mut killed_pid: u64 = 0; @@ -1096,7 +1143,6 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: if terminated || already_terminated { terminate_current_scheduler_thread(); - switch_ttbr0_to_kernel(); crate::task::scheduler::set_need_resched(); // CRITICAL: Set frame values BEFORE switch_to_idle() frame_ref.elr = crate::arch_impl::aarch64::idle_loop_arm64 as *const () as u64; @@ -1117,9 +1163,8 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: use crate::arch_impl::aarch64::context_switch::raw_uart_str; raw_uart_str("[INSTRUCTION_ABORT] deferring process cleanup\n"); } - defer_current_user_thread_sigsegv_exit("[INSTRUCTION_ABORT]"); + defer_current_user_thread_sigsegv_exit("[INSTRUCTION_ABORT]", frame as u64); terminate_current_scheduler_thread(); - switch_ttbr0_to_kernel(); crate::task::scheduler::set_need_resched(); frame_ref.elr = crate::arch_impl::aarch64::idle_loop_arm64 as *const () as u64; frame_ref.spsr = 0x5; // EL1h, DAIF clear (interrupts enabled) @@ -1159,6 +1204,7 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: core::arch::asm!("mrs {}, ttbr0_el1", out(reg) ttbr0, options(nomem, nostack)); } let page_table_phys = ttbr0 & !0xFFFF_0000_0000_0FFF; + super::switch_ttbr0_to_kernel(); crate::process::with_process_manager(|pm| { if let Some((pid, process)) = pm.find_process_by_cr3_mut(page_table_phys) { if !process.is_terminated() { @@ -1167,7 +1213,6 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: } }); terminate_current_scheduler_thread(); - switch_ttbr0_to_kernel(); } // CRITICAL: Set frame values BEFORE switch_to_idle_best_effort() frame_ref.elr = crate::arch_impl::aarch64::idle_loop_arm64 as *const () as u64; @@ -1256,6 +1301,7 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: core::arch::asm!("mrs {}, ttbr0_el1", out(reg) ttbr0, options(nomem, nostack)); } let page_table_phys = ttbr0 & !0xFFFF_0000_0000_0FFF; + super::switch_ttbr0_to_kernel(); crate::process::with_process_manager(|pm| { if let Some((pid, process)) = pm.find_process_by_cr3_mut(page_table_phys) { if !process.is_terminated() { @@ -1264,7 +1310,6 @@ pub extern "C" fn handle_sync_exception(frame: *mut Aarch64ExceptionFrame, esr: } }); terminate_current_scheduler_thread(); - switch_ttbr0_to_kernel(); } // CRITICAL: Set frame values BEFORE switch_to_idle_best_effort() frame_ref.elr = crate::arch_impl::aarch64::idle_loop_arm64 as *const () as u64; diff --git a/kernel/src/arch_impl/aarch64/mod.rs b/kernel/src/arch_impl/aarch64/mod.rs index 938f27bb..f0b36acd 100644 --- a/kernel/src/arch_impl/aarch64/mod.rs +++ b/kernel/src/arch_impl/aarch64/mod.rs @@ -28,6 +28,7 @@ pub mod syscall_entry; pub mod timer; pub mod timer_interrupt; pub mod trace; +pub mod ttbr0; // Re-export commonly used items // These re-exports are part of the complete HAL API @@ -53,6 +54,9 @@ pub use privilege::Aarch64PrivilegeLevel; pub use syscall_entry::{is_el0_confirmed, syscall_return_to_userspace_aarch64}; #[allow(unused_imports)] pub use timer::Aarch64Timer; +pub use ttbr0::{ + is_ttbr0_root_live, kernel_ttbr0, quiesce_ttbr0_for_exit, switch_ttbr0_to_kernel, +}; // Re-export interrupt control functions for convenient access // These provide the ARM64 equivalent of x86_64::instructions::interrupts::* diff --git a/kernel/src/arch_impl/aarch64/syscall_entry.S b/kernel/src/arch_impl/aarch64/syscall_entry.S index b9a6b5d1..db32e45e 100644 --- a/kernel/src/arch_impl/aarch64/syscall_entry.S +++ b/kernel/src/arch_impl/aarch64/syscall_entry.S @@ -37,6 +37,11 @@ .extern check_need_resched_and_switch_arm64 .extern trace_eret_to_el0 +.equ KERNEL_VIRT_BASE, 0xFFFF000000000000 +.equ PERCPU_ERET_GUARD_ELR, 120 +.equ PERCPU_ERET_GUARD_SPSR, 128 +.equ PERCPU_ERET_GUARD_SOURCE, 136 + /* * Syscall entry point from EL0 (userspace) * @@ -269,12 +274,23 @@ syscall_entry_from_el0: /* Read ELR from exception frame */ ldr x10, [sp, #248] /* x10 = frame.elr */ - /* DIAGNOSTIC: Check for corrupted ELR before ERET (was missing in SVC path) */ - cmp x10, #0x1000 - b.hs .Lsvc_elr_ok - /* ELR < 0x1000 — corrupted. Skip UART diagnostic (QEMU UART at - 0x09000000 doesn't exist on Parallels). Redirect to idle. */ - /* Redirect to idle_loop_arm64 */ + /* Guard the EL1 return invariant. EL0 legitimately uses lower VAs. */ + ldr x1, [sp, #256] /* x1 = frame.spsr */ + and x9, x1, #0xF /* x9 = M[3:0] */ + cbz x9, .Lsvc_elr_ok /* EL0 return */ + ldr x9, =KERNEL_VIRT_BASE + cmp x10, x9 + b.hs .Lsvc_elr_ok /* valid EL1 kernel return */ + + /* Branch-only record, published by writing source last. No UART/I/O. */ + mrs x9, tpidr_el1 + cbz x9, .Lsvc_eret_guard_recorded + str x10, [x9, #PERCPU_ERET_GUARD_ELR] + str x1, [x9, #PERCPU_ERET_GUARD_SPSR] + mov x1, #3 /* source 3 = syscall epilogue */ + str x1, [x9, #PERCPU_ERET_GUARD_SOURCE] +.Lsvc_eret_guard_recorded: + /* Redirect to idle_loop_arm64 instead of returning to EL1 at a user VA. */ adrp x10, idle_loop_arm64 add x10, x10, :lo12:idle_loop_arm64 /* Force SPSR to EL1h - write to frame */ diff --git a/kernel/src/arch_impl/aarch64/syscall_entry.rs b/kernel/src/arch_impl/aarch64/syscall_entry.rs index 7c038a34..e5dafa6d 100644 --- a/kernel/src/arch_impl/aarch64/syscall_entry.rs +++ b/kernel/src/arch_impl/aarch64/syscall_entry.rs @@ -315,13 +315,12 @@ fn result_to_u64(result: crate::syscall::SyscallResult) -> u64 { /// ARM64-specific exit implementation. /// -/// This is separate from the shared dispatcher because ARM64 needs to: -/// 1. Use `wfi` (not `hlt`) when no more userspace threads remain -/// 2. Inline the exit logic since `handlers::sys_exit` is x86_64-only +/// This is separate from the shared dispatcher because ARM64 needs to pivot +/// through its per-CPU scheduler stack before publishing `Terminated`. /// -/// CRITICAL: This function must NEVER return. After terminating the thread, -/// it enters a WFI loop. The timer interrupt will fire and context-switch -/// to another thread; the terminated thread will never be re-scheduled. +/// CRITICAL: This function must NEVER return. The final inline scheduler +/// handoff switches away explicitly; the terminated thread is never eligible +/// to be re-scheduled. /// If this function returned, the userspace exit() caller (e.g., musl's /// `for(;;) __syscall(SYS_exit, ec)` loop) would re-enter exit, causing /// double-terminate and double-decrement of COW page refcounts. @@ -366,17 +365,22 @@ fn sys_exit_aarch64(exit_code: i32) -> u64 { crate::serial_println!("[syscall] exit({}) thread={}", exit_code, thread_id); } - crate::task::process_task::ProcessScheduler::handle_thread_exit(thread_id, exit_code); + // Leave the retiring userspace address space before process teardown + // drops its page-table root. Clear both assembly return shadows so no + // later return path can reinstall that retired root. + super::switch_ttbr0_to_kernel(); + unsafe { + Aarch64PerCpu::set_saved_process_cr3(0); + Aarch64PerCpu::set_next_cr3(0); + } - crate::task::scheduler::with_scheduler(|scheduler| { - if let Some(thread) = scheduler.current_thread_mut() { - thread.set_terminated(); - } - }); + crate::task::process_task::ProcessScheduler::handle_thread_exit(thread_id, exit_code); let has_other_userspace_threads = - crate::task::scheduler::with_scheduler(|sched| sched.has_userspace_threads()) - .unwrap_or(false); + crate::task::scheduler::with_scheduler(|sched| { + sched.has_userspace_threads_other_than(thread_id) + }) + .unwrap_or(false); if !has_other_userspace_threads { crate::serial_println!(); @@ -386,31 +390,14 @@ fn sys_exit_aarch64(exit_code: i32) -> u64 { crate::serial_println!("========================================"); crate::serial_println!(); } - } - // Re-enable preemption (balances the preempt_disable in rust_syscall_handler_aarch64) - // so timer interrupts can trigger context-switch to another thread. - Aarch64PerCpu::preempt_enable(); - - // NEVER return to userspace. The thread is terminated; wait for the timer - // interrupt to context-switch away. The scheduler will not re-schedule a - // terminated thread, so this loop runs at most until the next timer tick. - // - // CRITICAL: Must unmask IRQ before WFI. The syscall entry assembly masks IRQ - // (daifset #0x2) and we never return to the assembly epilogue (which would - // call check_need_resched_and_switch_arm64 and restore interrupt state via - // ERET). Without unmasking IRQ here, the timer interrupt is pending but never - // handled — this CPU becomes permanently stuck, unable to process deferred - // thread requeues or context-switch to other threads. - loop { - unsafe { - core::arch::asm!( - "msr daifclr, #3", // Unmask IRQ+FIQ so timer interrupt can fire - "wfi", // Wait for interrupt — timer will context-switch us away - options(nomem, nostack) - ); - } + // This call first pivots to the neutral per-CPU scheduler stack. Its + // trampoline marks this thread Terminated only after that pivot and + // immediately dispatches a successor. + super::context_switch::schedule_terminated_from_exit(thread_id); } + + panic!("AArch64 sys_exit invoked without a current scheduler thread"); } /// Dispatch a syscall to the appropriate handler using the resolved SyscallNumber. @@ -940,8 +927,9 @@ fn sys_fork_aarch64(frame: &Aarch64ExceptionFrame) -> u64 { }; // PM lock dropped, interrupts restored crate::serial_aarch64::raw_serial_char(b'2'); // Fork phase 1 done, PM lock dropped - // Reclaim scheduler-owned kernel stacks from fully retired fork children before - // consuming another slot from the finite ARM64 kernel stack pool. + // Reclaim quiesced process frames and scheduler-owned kernel stacks before + // consuming more of either finite allocator pool. + crate::task::process_task::reclaim_deferred_process_resources(); crate::task::scheduler::reclaim_terminated_threads(); // Create child page table OUTSIDE PM lock (heap allocation safe — interrupts enabled) @@ -1194,6 +1182,13 @@ fn sys_exec_aarch64( // Trace: calling exec_process_with_argv (process manager) super::trace::trace_exec(b'M'); + // The manager may take the old process root after its final + // fallible setup step. Install the shared kernel TTBR0 first so + // exec cannot retire the root currently active on this CPU. On an + // error, the unchanged saved_process_cr3 restores the old root in + // the normal syscall epilogue. + super::switch_ttbr0_to_kernel(); + match manager.exec_process_with_argv( current_pid, elf_data, diff --git a/kernel/src/arch_impl/aarch64/ttbr0.rs b/kernel/src/arch_impl/aarch64/ttbr0.rs new file mode 100644 index 00000000..12435289 --- /dev/null +++ b/kernel/src/arch_impl/aarch64/ttbr0.rs @@ -0,0 +1,72 @@ +//! Shared TTBR0 transition helpers for AArch64 teardown paths. + +const TTBR0_ROOT_MASK: u64 = !0xFFFF_0000_0000_0FFF; + +/// Return the kernel TTBR0 root, falling back to the boot identity table before +/// per-CPU state has been populated. +#[inline(always)] +pub fn kernel_ttbr0() -> u64 { + let ttbr0 = crate::per_cpu_aarch64::get_kernel_cr3(); + if ttbr0 == 0 { + 0x4200_0000 + } else { + ttbr0 + } +} + +/// Switch TTBR0 to the kernel page table and invalidate stale translations. +/// +/// Exit, exec, and fault cleanup all use this implementation so none of them +/// can retire a process page-table root while the CPU still has it installed. +#[inline(always)] +pub fn switch_ttbr0_to_kernel() { + let ttbr0 = kernel_ttbr0(); + + unsafe { + core::arch::asm!( + "dsb ishst", + "msr ttbr0_el1, {ttbr0}", + "isb", + "tlbi vmalle1is", + "dsb ish", + "isb", + ttbr0 = in(reg) ttbr0, + options(nomem, nostack) + ); + } +} + +/// Leave the current userspace root and prevent an exception-return path from +/// reinstalling it. This must complete before publishing deferred exit work. +#[inline(always)] +pub fn quiesce_ttbr0_for_exit() { + switch_ttbr0_to_kernel(); + unsafe { + super::percpu::Aarch64PerCpu::set_saved_process_cr3(0); + super::percpu::Aarch64PerCpu::set_next_cr3(0); + } +} + +/// Return whether any online CPU still retains `root_phys` in a TTBR0 shadow. +/// +/// TTBR0 values may carry an ASID, so compare only the physical root bits using +/// the same mask as the exception fault lookup paths. +pub fn is_ttbr0_root_live(root_phys: u64) -> bool { + let root_phys = root_phys & TTBR0_ROOT_MASK; + if root_phys == 0 { + return false; + } + + (0..super::constants::MAX_CPUS).any(|cpu_id| { + if !super::smp::is_cpu_online(cpu_id) { + return false; + } + + crate::per_cpu_aarch64::ttbr0_shadow_snapshot(cpu_id) + .map(|(saved_process_ttbr0, next_ttbr0)| { + saved_process_ttbr0 & TTBR0_ROOT_MASK == root_phys + || next_ttbr0 & TTBR0_ROOT_MASK == root_phys + }) + .unwrap_or(false) + }) +} diff --git a/kernel/src/memory/kernel_stack.rs b/kernel/src/memory/kernel_stack.rs index 09450c09..99fa885c 100644 --- a/kernel/src/memory/kernel_stack.rs +++ b/kernel/src/memory/kernel_stack.rs @@ -236,9 +236,9 @@ mod aarch64 { /// - Kernel stacks: 0x5420_0000 to 0x561F_FFFF (32 MB) const ARM64_KERNEL_STACK_PHYS_BASE: u64 = 0x5420_0000; const ARM64_KERNEL_STACK_PHYS_END: u64 = 0x5620_0000; - const ARM64_KERNEL_STACK_BASE: u64 = + pub(crate) const ARM64_KERNEL_STACK_BASE: u64 = crate::arch_impl::aarch64::constants::HHDM_BASE + ARM64_KERNEL_STACK_PHYS_BASE; - const ARM64_KERNEL_STACK_END: u64 = + pub(crate) const ARM64_KERNEL_STACK_END: u64 = crate::arch_impl::aarch64::constants::HHDM_BASE + ARM64_KERNEL_STACK_PHYS_END; /// Stack size for ARM64 (64KB per stack) @@ -248,10 +248,10 @@ mod aarch64 { const ARM64_GUARD_PAGE_SIZE: u64 = 4 * 1024; /// Total slot size (stack + guard) - const ARM64_STACK_SLOT_SIZE: u64 = ARM64_KERNEL_STACK_SIZE + ARM64_GUARD_PAGE_SIZE; + pub(crate) const ARM64_STACK_SLOT_SIZE: u64 = ARM64_KERNEL_STACK_SIZE + ARM64_GUARD_PAGE_SIZE; /// Bitmap to track allocated ARM64 stacks. - const ARM64_MAX_KERNEL_STACKS: usize = + pub(crate) const ARM64_MAX_KERNEL_STACKS: usize = ((ARM64_KERNEL_STACK_END - ARM64_KERNEL_STACK_BASE) / ARM64_STACK_SLOT_SIZE) as usize; const ARM64_BITMAP_SIZE: usize = (ARM64_MAX_KERNEL_STACKS + 63) / 64; static ARM64_STACK_BITMAP: Mutex<[u64; ARM64_BITMAP_SIZE]> = Mutex::new([0; ARM64_BITMAP_SIZE]); @@ -274,6 +274,28 @@ mod aarch64 { } } + /// True when an online CPU still names or has a resume SP inside this slot. + /// + /// A userspace return normally stores the slot top in `user_rsp_scratch`, + /// while a suspended EL1 continuation may store an interior SP. Treat both + /// forms as live so neither reclamation nor allocation can race the final + /// architectural handoff off the old stack. + pub(crate) fn is_kernel_stack_slot_live(stack_top: u64) -> bool { + let stack_bottom = stack_top.saturating_sub(ARM64_KERNEL_STACK_SIZE); + (0..crate::arch_impl::aarch64::constants::MAX_CPUS).any(|cpu_id| { + if !crate::arch_impl::aarch64::smp::is_cpu_online(cpu_id) { + return false; + } + let Some((live_top, live_resume_sp)) = + crate::per_cpu_aarch64::live_stack_snapshot(cpu_id) + else { + return false; + }; + live_top == stack_top + || (live_resume_sp >= stack_bottom && live_resume_sp <= stack_top) + }) + } + /// Allocate a kernel stack for ARM64 /// /// Uses a bitmap over a reserved high-half direct map region so fork-heavy @@ -312,6 +334,14 @@ mod aarch64 { let stack_bottom = VirtAddr::new(slot_base + ARM64_GUARD_PAGE_SIZE); let stack_top = VirtAddr::new(slot_base + ARM64_STACK_SLOT_SIZE); + debug_assert!( + !is_kernel_stack_slot_live(stack_top.as_u64()), + "ARM64 kernel-stack allocator selected live slot {} ({:#x}-{:#x})", + index, + stack_bottom.as_u64(), + stack_top.as_u64() + ); + // ROOT FIX (launcher-spawn EC=0x0/EC=0xe crash, // docs/planning/aarch64-launcher-spawn-crash/ROOT_CAUSE.md): scrub the // ENTIRE slot on every allocation, not just on first use. A bitmap- @@ -406,6 +436,12 @@ pub use aarch64::{ is_in_reused_kstack_region as is_in_reused_kstack_region_aarch64, Aarch64KernelStack, }; +#[cfg(target_arch = "aarch64")] +pub(crate) use aarch64::{ + is_kernel_stack_slot_live, ARM64_KERNEL_STACK_BASE, ARM64_KERNEL_STACK_END, + ARM64_MAX_KERNEL_STACKS, ARM64_STACK_SLOT_SIZE, +}; + /// ARM64: Use the aarch64-specific allocator #[cfg(target_arch = "aarch64")] pub fn allocate_kernel_stack() -> Result { diff --git a/kernel/src/per_cpu_aarch64.rs b/kernel/src/per_cpu_aarch64.rs index b54b3014..1e41b3f6 100644 --- a/kernel/src/per_cpu_aarch64.rs +++ b/kernel/src/per_cpu_aarch64.rs @@ -55,8 +55,18 @@ pub struct PerCpuData { /// Scratch register save area for ERET paths (offset 96) /// Used by assembly to save one register across SP switches during ERET. pub eret_scratch: u64, - /// Padding to match x86_64 layout - _pad3: [u8; 88], + /// Last frame ELR selected by the Rust dispatcher (offset 104). + pub dispatch_elr: u64, + /// Last frame SPSR selected by the Rust dispatcher (offset 112). + pub dispatch_spsr: u64, + /// ELR captured by an assembly ERET invariant redirect (offset 120). + pub eret_guard_elr: u64, + /// SPSR captured by an assembly ERET invariant redirect (offset 128). + pub eret_guard_spsr: u64, + /// Guard source tag, written last to publish the record (offset 136). + pub eret_guard_source: u64, + /// Padding to match the fixed 192-byte per-CPU layout. + _pad3: [u8; 48], } const _: () = assert!( @@ -85,11 +95,29 @@ impl PerCpuData { exception_cleanup_context: 0, _pad3a: [0; 7], eret_scratch: 0, - _pad3: [0; 88], + dispatch_elr: 0, + dispatch_spsr: 0, + eret_guard_elr: 0, + eret_guard_spsr: 0, + eret_guard_source: 0, + _pad3: [0; 48], } } } +const _: () = assert!( + core::mem::offset_of!(PerCpuData, eret_guard_elr) + == crate::arch_impl::aarch64::constants::PERCPU_ERET_GUARD_ELR_OFFSET +); +const _: () = assert!( + core::mem::offset_of!(PerCpuData, eret_guard_spsr) + == crate::arch_impl::aarch64::constants::PERCPU_ERET_GUARD_SPSR_OFFSET +); +const _: () = assert!( + core::mem::offset_of!(PerCpuData, eret_guard_source) + == crate::arch_impl::aarch64::constants::PERCPU_ERET_GUARD_SOURCE_OFFSET +); + /// Per-CPU data for all CPUs (up to MAX_CPUS). /// Each CPU's TPIDR_EL1 points to its own entry in this array. static mut ALL_CPU_DATA: [PerCpuData; crate::arch_impl::aarch64::constants::MAX_CPUS] = [ @@ -107,6 +135,70 @@ static mut ALL_CPU_DATA: [PerCpuData; crate::arch_impl::aarch64::constants::MAX_ static PER_CPU_INITIALIZED: AtomicBool = AtomicBool::new(false); static EARLY_SOFTIRQ_PENDING: AtomicU32 = AtomicU32::new(0); +/// Read the last assembly ERET-guard redirect record for `cpu_id`. +/// The source tag is written last by assembly and acts as the validity word. +pub fn eret_guard_record(cpu_id: usize) -> Option<(u64, u64, u64)> { + if cpu_id >= crate::arch_impl::aarch64::constants::MAX_CPUS { + return None; + } + + let cpu_data = unsafe { &raw const ALL_CPU_DATA[cpu_id] }; + let source = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!((*cpu_data).eret_guard_source)) + }; + if source == 0 { + return None; + } + core::sync::atomic::fence(Ordering::Acquire); + let elr = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!((*cpu_data).eret_guard_elr)) + }; + let spsr = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!((*cpu_data).eret_guard_spsr)) + }; + Some((source, elr, spsr)) +} + +/// Snapshot the two per-CPU pointers that can keep a kernel-stack slot live. +/// +/// These fields are also read and written by exception-return assembly, so use +/// volatile loads rather than borrowing the shared per-CPU object. Callers use +/// this only as a conservative reclamation/allocator exclusion check. +pub fn live_stack_snapshot(cpu_id: usize) -> Option<(u64, u64)> { + if cpu_id >= crate::arch_impl::aarch64::constants::MAX_CPUS { + return None; + } + + let cpu_data = unsafe { &raw const ALL_CPU_DATA[cpu_id] }; + let kernel_stack_top = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!((*cpu_data).kernel_stack_top)) + }; + let user_rsp_scratch = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!((*cpu_data).user_sp_scratch)) + }; + Some((kernel_stack_top, user_rsp_scratch)) +} + +/// Snapshot the per-CPU TTBR0 shadows that can retain a userspace root. +/// +/// Exception-return assembly also reads and writes these fields, so use +/// volatile loads rather than borrowing the shared per-CPU object. Callers +/// combine this conservative snapshot with a scheduling-epoch grace period +/// before returning frames reachable from a retired root to the allocator. +pub fn ttbr0_shadow_snapshot(cpu_id: usize) -> Option<(u64, u64)> { + if cpu_id >= crate::arch_impl::aarch64::constants::MAX_CPUS { + return None; + } + + let cpu_data = unsafe { &raw const ALL_CPU_DATA[cpu_id] }; + let saved_process_ttbr0 = unsafe { + core::ptr::read_volatile(core::ptr::addr_of!((*cpu_data).saved_process_ttbr0)) + }; + let next_ttbr0 = + unsafe { core::ptr::read_volatile(core::ptr::addr_of!((*cpu_data).next_ttbr0)) }; + Some((saved_process_ttbr0, next_ttbr0)) +} + /// Check if per-CPU data has been initialized pub fn is_initialized() -> bool { PER_CPU_INITIALIZED.load(Ordering::Acquire) diff --git a/kernel/src/process/manager.rs b/kernel/src/process/manager.rs index 21cd16e1..09a3ef71 100644 --- a/kernel/src/process/manager.rs +++ b/kernel/src/process/manager.rs @@ -893,6 +893,7 @@ impl ProcessManager { has_started: false, blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -971,6 +972,7 @@ impl ProcessManager { has_started: false, blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -1054,6 +1056,7 @@ impl ProcessManager { has_started: false, blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -1839,6 +1842,7 @@ impl ProcessManager { // dispatch, so the invariant holds regardless of future changes to // either constructor. child_thread.saved_by_inline_schedule = false; + child_thread.inline_schedule_spsr = 0; child_thread.inline_schedule_caller_lr = 0; child_thread.inline_schedule_saved_sp = 0; @@ -2326,6 +2330,7 @@ impl ProcessManager { has_started: true, blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, diff --git a/kernel/src/process/process.rs b/kernel/src/process/process.rs index 3098dfdf..59b31419 100644 --- a/kernel/src/process/process.rs +++ b/kernel/src/process/process.rs @@ -495,29 +495,9 @@ impl Process { /// CRITICAL: No logging — may run under PM lock. #[cfg(not(target_arch = "x86_64"))] pub(crate) fn cleanup_cow_frames(&mut self) { - use crate::memory::arch_stub::{PageTableFlags, PhysFrame}; - use crate::memory::frame_allocator::deallocate_frame; - use crate::memory::frame_metadata::frame_decref; - - // Get the page table for this process - let page_table = match self.page_table.as_ref() { - Some(pt) => pt, - None => return, - }; - - // Walk all user pages and decrement refcounts - let _ = page_table.walk_mapped_pages(|_virt_addr, phys_addr, flags| { - // Only process user-accessible pages - if !flags.contains(PageTableFlags::USER_ACCESSIBLE) { - return; - } - - let frame = PhysFrame::containing_address(phys_addr); - - if frame_decref(frame) { - deallocate_frame(frame); - } - }); + if let Some(page_table) = self.page_table.as_ref() { + cleanup_cow_page_table(page_table); + } } /// Drain and clean up any pending old page tables from previous exec() calls. @@ -588,3 +568,25 @@ impl Process { &mut self.vmas } } + +/// Release the user-frame references reachable from an AArch64 process root. +/// +/// Deferred exit reclamation owns the page table after removing it from the +/// process table, so this operation accepts the page table directly. +#[cfg(target_arch = "aarch64")] +pub(crate) fn cleanup_cow_page_table(page_table: &ProcessPageTable) { + use crate::memory::arch_stub::{PageTableFlags, PhysFrame}; + use crate::memory::frame_allocator::deallocate_frame; + use crate::memory::frame_metadata::frame_decref; + + let _ = page_table.walk_mapped_pages(|_virt_addr, phys_addr, flags| { + if !flags.contains(PageTableFlags::USER_ACCESSIBLE) { + return; + } + + let frame = PhysFrame::containing_address(phys_addr); + if frame_decref(frame) { + deallocate_frame(frame); + } + }); +} diff --git a/kernel/src/syscall/clone.rs b/kernel/src/syscall/clone.rs index 5321da7e..2c75a4c0 100644 --- a/kernel/src/syscall/clone.rs +++ b/kernel/src/syscall/clone.rs @@ -177,6 +177,7 @@ pub fn sys_clone( has_started: false, // Will be set up via first_userspace_entry blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, diff --git a/kernel/src/task/process_task.rs b/kernel/src/task/process_task.rs index 184ceb4a..5c6a52fe 100644 --- a/kernel/src/task/process_task.rs +++ b/kernel/src/task/process_task.rs @@ -60,6 +60,80 @@ static DEFERRED_FAULT_EXIT_BUFFERS: [DeferredFaultExitBuffer; 8] = static DEFERRED_FAULT_EXIT_BUFFERS: [DeferredFaultExitBuffer; 1] = [const { DeferredFaultExitBuffer::new() }]; +#[cfg(target_arch = "aarch64")] +struct PendingProcessReclaim { + page_table: Option>, + old_page_tables: alloc::vec::Vec< + alloc::boxed::Box, + >, + after_epoch: [u64; crate::arch_impl::aarch64::constants::MAX_CPUS], +} + +#[cfg(target_arch = "aarch64")] +impl PendingProcessReclaim { + fn root_is_live(&self) -> bool { + self.page_table + .iter() + .chain(self.old_page_tables.iter()) + .any(|page_table| { + crate::arch_impl::aarch64::is_ttbr0_root_live( + page_table.level_4_frame().start_address().as_u64(), + ) + }) + } + + fn reclaim(mut self) { + if let Some(page_table) = self.page_table.as_ref() { + crate::process::process::cleanup_cow_page_table(page_table); + } + for old_page_table in self.old_page_tables.drain(..) { + old_page_table.cleanup_for_exec(); + } + drop(self.page_table.take()); + } +} + +#[cfg(target_arch = "aarch64")] +static PENDING_PROCESS_RECLAIMS: spin::Mutex> = + spin::Mutex::new(alloc::vec::Vec::new()); + +fn release_process_resources(process: &mut crate::process::Process) { + process.cleanup_cow_frames(); + process.drain_old_page_tables(); + drop(process.page_table.take()); + drop(process.stack.take()); + process.pending_old_page_tables.clear(); +} + +#[cfg(target_arch = "aarch64")] +fn defer_live_process_resources( + process: &mut crate::process::Process, +) -> Option { + let root_is_live = process + .page_table + .iter() + .chain(process.pending_old_page_tables.iter()) + .any(|page_table| { + crate::arch_impl::aarch64::is_ttbr0_root_live( + page_table.level_4_frame().start_address().as_u64(), + ) + }); + if !root_is_live { + return None; + } + + Some(PendingProcessReclaim { + page_table: process.page_table.take(), + old_page_tables: core::mem::take(&mut process.pending_old_page_tables), + after_epoch: scheduler::retirement_grace_target(), + }) +} + +#[cfg(target_arch = "aarch64")] +fn enqueue_process_reclaim(reclaim: PendingProcessReclaim) { + crate::arch_without_interrupts(|| PENDING_PROCESS_RECLAIMS.lock().push(reclaim)); +} + /// Close extracted file descriptor entries outside the PM lock. /// /// This performs the same cleanup as Process::close_all_fds() but operates on @@ -146,14 +220,15 @@ impl ProcessScheduler { // Mark terminated and extract FDs without closing them process.terminate_minimal(exit_code); let fd_entries = process.take_fd_entries(); - // CoW cleanup is fast (no logging, no locks besides frame allocator) - process.cleanup_cow_frames(); - process.drain_old_page_tables(); - - // Free heavy resources immediately (CoW refcounts already decremented) - process.page_table.take(); - process.stack.take(); - process.pending_old_page_tables.clear(); + #[cfg(target_arch = "aarch64")] + if let Some(reclaim) = defer_live_process_resources(process) { + enqueue_process_reclaim(reclaim); + drop(process.stack.take()); + } else { + release_process_resources(process); + } + #[cfg(not(target_arch = "aarch64"))] + release_process_resources(process); #[cfg(feature = "btrt")] crate::test_framework::btrt::on_process_exit(pid.as_u64(), exit_code); @@ -262,6 +337,26 @@ pub fn drain_deferred_fault_sigsegv_exits() { } } +/// Reclaim process frames whose cross-CPU TTBR0 retention has quiesced. +#[cfg(target_arch = "aarch64")] +pub fn reclaim_deferred_process_resources() { + loop { + let reclaim = crate::arch_without_interrupts(|| { + let mut pending = PENDING_PROCESS_RECLAIMS.lock(); + let ready = pending.iter().position(|reclaim| { + scheduler::retirement_grace_elapsed(&reclaim.after_epoch) + && !reclaim.root_is_live() + }); + ready.map(|index| pending.swap_remove(index)) + }); + + match reclaim { + Some(reclaim) => reclaim.reclaim(), + None => break, + } + } +} + /// Extension trait for Thread to support process operations #[allow(dead_code)] pub trait ProcessThread { diff --git a/kernel/src/task/scheduler.rs b/kernel/src/task/scheduler.rs index 9cda8fea..6af07ac4 100644 --- a/kernel/src/task/scheduler.rs +++ b/kernel/src/task/scheduler.rs @@ -528,6 +528,56 @@ const MAX_CPUS: usize = 8; #[cfg(not(target_arch = "aarch64"))] const MAX_CPUS: usize = 1; +/// Scheduler-entry epochs per online AArch64 CPU. +/// +/// A retiring resource records a target two greater than every online CPU's +/// current value. The first bump may be recorded by a handoff that is already +/// in flight on the retiring stack. Requiring a second bump proves that CPU +/// entered the scheduler through a later exception, which can only happen +/// after the in-flight exception return and its old-stack restore completed. +#[cfg(target_arch = "aarch64")] +static SCHEDULING_EPOCHS: [AtomicU64; MAX_CPUS] = + [const { AtomicU64::new(0) }; MAX_CPUS]; + +#[cfg(target_arch = "aarch64")] +#[derive(Clone, Copy)] +struct RetirementGrace { + thread_id: u64, + after_epoch: [u64; MAX_CPUS], +} + +#[cfg(target_arch = "aarch64")] +pub(crate) fn retirement_grace_target() -> [u64; MAX_CPUS] { + let mut target = [0; MAX_CPUS]; + for cpu_id in 0..MAX_CPUS { + if crate::arch_impl::aarch64::smp::is_cpu_online(cpu_id) { + target[cpu_id] = SCHEDULING_EPOCHS[cpu_id] + .load(Ordering::Acquire) + .saturating_add(2); + } + } + target +} + +#[cfg(target_arch = "aarch64")] +pub(crate) fn retirement_grace_elapsed(target: &[u64; MAX_CPUS]) -> bool { + (0..MAX_CPUS).all(|cpu_id| { + target[cpu_id] == 0 + || SCHEDULING_EPOCHS[cpu_id].load(Ordering::Acquire) >= target[cpu_id] + }) +} + +/// Record a scheduler entry for the current CPU. +/// +/// A single entry does not prove the handoff active at that entry has finished; +/// reclamation targets require a second, subsequent entry on every online CPU. +#[cfg(target_arch = "aarch64")] +pub fn note_scheduling_epoch(cpu_id: usize) { + if cpu_id < MAX_CPUS { + SCHEDULING_EPOCHS[cpu_id].fetch_add(1, Ordering::Release); + } +} + /// DIAGNOSTIC: Circular buffer tracking last N cpu_state changes per CPU. /// Each entry: (setter_id, old_thread, new_thread) /// Setter IDs: @@ -769,6 +819,10 @@ pub struct Scheduler { /// Stale entries (threads already woken by ISR or terminated) are harmless — /// wake_expired_timers validates each entry before acting on it. timer_heap: BinaryHeap>, + + /// Per-thread all-CPU grace targets for AArch64 stack reclamation. + #[cfg(target_arch = "aarch64")] + retirement_grace: alloc::vec::Vec, } impl Scheduler { @@ -811,6 +865,8 @@ impl Scheduler { per_cpu_queues, cpu_state, timer_heap: BinaryHeap::new(), + #[cfg(target_arch = "aarch64")] + retirement_grace: alloc::vec::Vec::new(), }; scheduler @@ -887,7 +943,7 @@ impl Scheduler { let _ = (thread_id, thread_name, is_user); } - /// Drop terminated threads that are no longer referenced by any CPU state. + /// Drop terminated threads only after their stack is architecturally dead. /// /// ARM64 userspace kernel stacks are owned by scheduler threads because the /// scheduler clone can outlive the process-table copy until it has fully @@ -902,18 +958,59 @@ impl Scheduler { }); } + let terminated_ids: alloc::vec::Vec = self + .threads + .iter() + .filter(|thread| thread.state == ThreadState::Terminated) + .map(|thread| thread.id()) + .collect(); + self.retirement_grace + .retain(|grace| terminated_ids.contains(&grace.thread_id)); + for thread_id in terminated_ids.iter().copied() { + if !self + .retirement_grace + .iter() + .any(|grace| grace.thread_id == thread_id) + { + self.retirement_grace.push(RetirementGrace { + thread_id, + after_epoch: retirement_grace_target(), + }); + } + } + + let idle_ids: alloc::vec::Vec = self + .cpu_state + .iter() + .map(|state| state.idle_thread) + .collect(); + let graces = &self.retirement_grace; + let mut reclaimed_ids = alloc::vec::Vec::new(); self.threads.retain(|thread| { - if thread.state != ThreadState::Terminated { + if thread.state != ThreadState::Terminated || idle_ids.contains(&thread.id()) { + return true; + } + + let stack_is_live = thread + .kernel_stack_top + .map(|top| { + crate::memory::kernel_stack::is_kernel_stack_slot_live(top.as_u64()) + }) + .unwrap_or(false); + let grace_elapsed = graces + .iter() + .find(|grace| grace.thread_id == thread.id()) + .map(|grace| retirement_grace_elapsed(&grace.after_epoch)) + .unwrap_or(false); + if stack_is_live || !grace_elapsed { return true; } - let thread_id = thread.id(); - (0..MAX_CPUS).any(|cpu| { - self.cpu_state[cpu].current_thread == Some(thread_id) - || self.cpu_state[cpu].previous_thread == Some(thread_id) - || self.cpu_state[cpu].idle_thread == thread_id - }) + reclaimed_ids.push(thread.id()); + false }); + self.retirement_grace + .retain(|grace| !reclaimed_ids.contains(&grace.thread_id)); } /// Add a thread as the current running thread without scheduling. @@ -2480,6 +2577,20 @@ impl Scheduler { }) } + /// Check for a live userspace thread other than the one completing exit. + #[cfg(target_arch = "aarch64")] + pub fn has_userspace_threads_other_than(&self, exiting_thread_id: u64) -> bool { + self.threads.iter().any(|thread| { + thread.id() != exiting_thread_id + && !self + .cpu_state + .iter() + .any(|state| state.idle_thread == thread.id()) + && thread.privilege == super::thread::ThreadPrivilege::User + && thread.state != ThreadState::Terminated + }) + } + /// Remove a thread from all per-CPU queues (used when blocking) pub fn remove_from_ready_queue(&mut self, thread_id: u64) { for q in self.per_cpu_queues.iter_mut() { diff --git a/kernel/src/task/thread.rs b/kernel/src/task/thread.rs index 3e688e2d..f5181fd0 100644 --- a/kernel/src/task/thread.rs +++ b/kernel/src/task/thread.rs @@ -458,6 +458,11 @@ pub struct Thread { /// Matches Linux's cpu_switch_to approach: kernel-to-kernel switches use ret. pub saved_by_inline_schedule: bool, + /// Kernel PSTATE captured by the ret-based inline schedule path. + /// `context.spsr_el1` remains paired with `context.elr_el1`; inline resume + /// metadata must not turn a saved user PC into an apparent EL1 return. + pub inline_schedule_spsr: u64, + /// Diagnostic: caller LR saved in the suspended schedule_from_kernel() frame. /// Used to detect whether the inline-saved kernel frame is already corrupt /// by the time a later exception save overwrites this thread's context. @@ -515,6 +520,7 @@ impl Clone for Thread { has_started: self.has_started, blocked_in_syscall: self.blocked_in_syscall, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: self.inline_schedule_caller_lr, inline_schedule_saved_sp: self.inline_schedule_saved_sp, saved_userspace_context: self.saved_userspace_context.clone(), @@ -581,6 +587,7 @@ impl Thread { has_started: false, // New thread hasn't run yet blocked_in_syscall: false, // New thread is not blocked in syscall saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -642,6 +649,7 @@ impl Thread { has_started: false, blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -690,6 +698,7 @@ impl Thread { has_started: false, // New thread hasn't run yet blocked_in_syscall: false, // New thread is not blocked in syscall saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -737,6 +746,7 @@ impl Thread { has_started: false, blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -797,6 +807,7 @@ impl Thread { has_started: false, // New thread hasn't run yet blocked_in_syscall: false, // New thread is not blocked in syscall saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -852,6 +863,7 @@ impl Thread { has_started: false, blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -932,6 +944,7 @@ impl Thread { has_started: false, // New thread hasn't run yet blocked_in_syscall: false, // New thread is not blocked in syscall saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None, @@ -975,6 +988,7 @@ impl Thread { has_started: false, blocked_in_syscall: false, saved_by_inline_schedule: false, + inline_schedule_spsr: 0, inline_schedule_caller_lr: 0, inline_schedule_saved_sp: 0, saved_userspace_context: None,