From e71fedc814613bce78e3d2ace9098a747c91aff7 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Thu, 16 Oct 2025 23:53:13 +0100 Subject: [PATCH 01/25] Fixes a race condition in killing Sandboxes Signed-off-by: Simon Davies --- src/hyperlight_host/Cargo.toml | 2 +- .../src/hypervisor/hyperv_linux.rs | 77 ++-- src/hyperlight_host/src/hypervisor/kvm.rs | 88 ++-- src/hyperlight_host/src/hypervisor/mod.rs | 319 ++++++++++++--- .../src/sandbox/initialized_multi_use.rs | 17 + src/hyperlight_host/tests/integration_test.rs | 380 +++++++++++++++++- src/tests/rust_guests/simpleguest/src/main.rs | 40 ++ 7 files changed, 796 insertions(+), 127 deletions(-) diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 1f4899c12..95d1c6f53 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -67,7 +67,7 @@ windows = { version = "0.62", features = [ "Win32_System_JobObjects", "Win32_System_SystemServices", ] } -windows-sys = { version = "0.61", features = ["Win32"] } +windows-sys = { version = "0.61", features = ["Win32", "Win32_System", "Win32_System_Threading"] } windows-result = "0.4" rust-embed = { version = "8.8.0", features = ["debug-embed", "include-exclude", "interpolate-folder-path"] } sha256 = "1.6.0" diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 801f361a7..3eef775ca 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -390,7 +390,8 @@ impl HypervLinuxDriver { let interrupt_handle = Arc::new(LinuxInterruptHandle { running: AtomicU64::new(0), - cancel_requested: AtomicBool::new(false), + cancel_requested: AtomicU64::new(0), + call_active: AtomicBool::new(false), #[cfg(gdb)] debug_interrupt: AtomicBool::new(false), #[cfg(all( @@ -659,16 +660,13 @@ impl Hypervisor for HypervLinuxDriver { self.interrupt_handle .tid .store(unsafe { libc::pthread_self() as u64 }, Ordering::Relaxed); - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then this is fine since `cancel_requested` is set to true, so we will skip the `VcpuFd::run()` call - self.interrupt_handle - .set_running_and_increment_generation() - .map_err(|e| { - new_error!( - "Error setting running state and incrementing generation: {}", - e - ) - })?; + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (after set_running_bit but before checking cancel_requested): + // - kill() will stamp cancel_requested with the current generation + // - We will check cancel_requested below and skip the VcpuFd::run() call + // - This is the desired behavior - the kill takes effect immediately + let generation = self.interrupt_handle.set_running_bit(); + #[cfg(not(gdb))] let debug_interrupt = false; #[cfg(gdb)] @@ -677,14 +675,16 @@ impl Hypervisor for HypervLinuxDriver { .debug_interrupt .load(Ordering::Relaxed); - // Don't run the vcpu if `cancel_requested` is true + // Don't run the vcpu if `cancel_requested` is set for our generation // - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then this is fine since `cancel_requested` is set to true, so we will skip the `VcpuFd::run()` call + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (after checking cancel_requested but before vcpu.run()): + // - kill() will stamp cancel_requested with the current generation + // - We will proceed with vcpu.run(), but signals will be sent to interrupt it + // - The vcpu will be interrupted and return EINTR (handled below) let exit_reason = if self .interrupt_handle - .cancel_requested - .load(Ordering::Relaxed) + .is_cancel_requested_for_generation(generation) || debug_interrupt { Err(mshv_ioctls::MshvError::from(libc::EINTR)) @@ -705,27 +705,32 @@ impl Hypervisor for HypervLinuxDriver { #[cfg(mshv3)] self.vcpu_fd.run() }; - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then signals will be sent to this thread until `running` is set to false. - // This is fine since the signal handler is a no-op. + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (after vcpu.run() returns but before clear_running_bit): + // - kill() continues sending signals to this thread (running bit is still set) + // - The signals are harmless (no-op handler), we just need to check cancel_requested + // - We load cancel_requested below to determine if this run was cancelled let cancel_requested = self .interrupt_handle - .cancel_requested - .load(Ordering::Relaxed); + .is_cancel_requested_for_generation(generation); #[cfg(gdb)] let debug_interrupt = self .interrupt_handle .debug_interrupt .load(Ordering::Relaxed); - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then `cancel_requested` will be set to true again, which will cancel the **next vcpu run**. - // Additionally signals will be sent to this thread until `running` is set to false. - // This is fine since the signal handler is a no-op. + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (after loading cancel_requested but before clear_running_bit): + // - kill() stamps cancel_requested with the CURRENT generation (not the one we just loaded) + // - kill() continues sending signals until running bit is cleared + // - The newly stamped cancel_requested will affect the NEXT vcpu.run() call + // - Signals sent now are harmless (no-op handler) self.interrupt_handle.clear_running_bit(); - // At this point, `running` is false so no more signals will be sent to this thread, - // but we may still receive async signals that were sent before this point. - // To prevent those signals from interrupting subsequent calls to `run()`, - // we make sure to check `cancel_requested` before cancelling (see `libc::EINTR` match-arm below). + // At this point, running bit is clear so kill() will stop sending signals. + // However, we may still receive delayed signals that were sent before clear_running_bit. + // These stale signals are harmless because: + // - The signal handler is a no-op + // - We check generation matching in cancel_requested before treating EINTR as cancellation + // - If generation doesn't match, we return Retry instead of Cancelled let result = match exit_reason { Ok(m) => match m.header.message_type { HALT_MESSAGE => { @@ -805,14 +810,16 @@ impl Hypervisor for HypervLinuxDriver { } }, Err(e) => match e.errno() { - // we send a signal to the thread to cancel execution this results in EINTR being returned by KVM so we return Cancelled + // We send a signal (SIGRTMIN+offset) to interrupt the vcpu, which causes EINTR libc::EINTR => { - // If cancellation was not requested for this specific vm, the vcpu was interrupted because of debug interrupt or - // a stale signal that meant to be delivered to a previous/other vcpu on this same thread, so let's ignore it + // Check if cancellation was requested for THIS specific generation. + // If not, the EINTR came from: + // - A debug interrupt (if GDB is enabled) + // - A stale signal from a previous guest call (generation mismatch) + // - A signal meant for a different sandbox on the same thread + // In these cases, we return Retry to continue execution. if cancel_requested { - self.interrupt_handle - .cancel_requested - .store(false, Ordering::Relaxed); + self.interrupt_handle.clear_cancel_requested(); HyperlightExit::Cancelled() } else { #[cfg(gdb)] diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 330a5f5b5..781669bb9 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -334,7 +334,8 @@ impl KVMDriver { let interrupt_handle = Arc::new(LinuxInterruptHandle { running: AtomicU64::new(0), - cancel_requested: AtomicBool::new(false), + cancel_requested: AtomicU64::new(0), + call_active: AtomicBool::new(false), #[cfg(gdb)] debug_interrupt: AtomicBool::new(false), #[cfg(all( @@ -621,16 +622,13 @@ impl Hypervisor for KVMDriver { self.interrupt_handle .tid .store(unsafe { libc::pthread_self() as u64 }, Ordering::Relaxed); - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then this is fine since `cancel_requested` is set to true, so we will skip the `VcpuFd::run()` call - self.interrupt_handle - .set_running_and_increment_generation() - .map_err(|e| { - new_error!( - "Error setting running state and incrementing generation: {}", - e - ) - })?; + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (after set_running_bit but before checking cancel_requested): + // - kill() will stamp cancel_requested with the current generation + // - We will check cancel_requested below and skip the VcpuFd::run() call + // - This is the desired behavior - the kill takes effect immediately + let generation = self.interrupt_handle.set_running_bit(); + #[cfg(not(gdb))] let debug_interrupt = false; #[cfg(gdb)] @@ -638,14 +636,16 @@ impl Hypervisor for KVMDriver { .interrupt_handle .debug_interrupt .load(Ordering::Relaxed); - // Don't run the vcpu if `cancel_requested` is true + // Don't run the vcpu if `cancel_requested` is set for our generation // - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then this is fine since `cancel_requested` is set to true, so we will skip the `VcpuFd::run()` call + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (after checking cancel_requested but before vcpu.run()): + // - kill() will stamp cancel_requested with the current generation + // - We will proceed with vcpu.run(), but signals will be sent to interrupt it + // - The vcpu will be interrupted and return EINTR (handled below) let exit_reason = if self .interrupt_handle - .cancel_requested - .load(Ordering::Relaxed) + .is_cancel_requested_for_generation(generation) || debug_interrupt { Err(kvm_ioctls::Error::new(libc::EINTR)) @@ -653,34 +653,40 @@ impl Hypervisor for KVMDriver { #[cfg(feature = "trace_guest")] tc.setup_guest_trace(Span::current().context()); - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then the vcpu will run, but we will keep sending signals to this thread - // to interrupt it until `running` is set to false. The `vcpu_fd::run()` call will - // return either normally with an exit reason, or from being "kicked" by out signal handler, with an EINTR error, - // both of which are fine. + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (during vcpu.run() execution): + // - kill() stamps cancel_requested with the current generation + // - kill() sends signals (SIGRTMIN+offset) to this thread repeatedly + // - The signal handler is a no-op, but it causes vcpu.run() to return EINTR + // - We check cancel_requested below and return Cancelled if generation matches self.vcpu_fd.run() }; - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then signals will be sent to this thread until `running` is set to false. - // This is fine since the signal handler is a no-op. + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (after vcpu.run() returns but before clear_running_bit): + // - kill() continues sending signals to this thread (running bit is still set) + // - The signals are harmless (no-op handler), we just need to check cancel_requested + // - We load cancel_requested below to determine if this run was cancelled let cancel_requested = self .interrupt_handle - .cancel_requested - .load(Ordering::Relaxed); + .is_cancel_requested_for_generation(generation); #[cfg(gdb)] let debug_interrupt = self .interrupt_handle .debug_interrupt .load(Ordering::Relaxed); - // Note: if a `InterruptHandle::kill()` called while this thread is **here** - // Then `cancel_requested` will be set to true again, which will cancel the **next vcpu run**. - // Additionally signals will be sent to this thread until `running` is set to false. - // This is fine since the signal handler is a no-op. + // Note: if `InterruptHandle::kill()` is called while this thread is **here** + // (after loading cancel_requested but before clear_running_bit): + // - kill() stamps cancel_requested with the CURRENT generation (not the one we just loaded) + // - kill() continues sending signals until running bit is cleared + // - The newly stamped cancel_requested will affect the NEXT vcpu.run() call + // - Signals sent now are harmless (no-op handler) self.interrupt_handle.clear_running_bit(); - // At this point, `running` is false so no more signals will be sent to this thread, - // but we may still receive async signals that were sent before this point. - // To prevent those signals from interrupting subsequent calls to `run()` (on other vms!), - // we make sure to check `cancel_requested` before cancelling (see `libc::EINTR` match-arm below). + // At this point, running bit is clear so kill() will stop sending signals. + // However, we may still receive delayed signals that were sent before clear_running_bit. + // These stale signals are harmless because: + // - The signal handler is a no-op + // - We check generation matching in cancel_requested before treating EINTR as cancellation + // - If generation doesn't match, we return Retry instead of Cancelled let result = match exit_reason { Ok(VcpuExit::Hlt) => { crate::debug!("KVM - Halt Details : {:#?}", &self); @@ -729,14 +735,16 @@ impl Hypervisor for KVMDriver { } }, Err(e) => match e.errno() { - // we send a signal to the thread to cancel execution this results in EINTR being returned by KVM so we return Cancelled + // We send a signal (SIGRTMIN+offset) to interrupt the vcpu, which causes EINTR libc::EINTR => { - // If cancellation was not requested for this specific vm, the vcpu was interrupted because of debug interrupt or - // a stale signal that meant to be delivered to a previous/other vcpu on this same thread, so let's ignore it + // Check if cancellation was requested for THIS specific generation. + // If not, the EINTR came from: + // - A debug interrupt (if GDB is enabled) + // - A stale signal from a previous guest call (generation mismatch) + // - A signal meant for a different sandbox on the same thread + // In these cases, we return Retry to continue execution. if cancel_requested { - self.interrupt_handle - .cancel_requested - .store(false, Ordering::Relaxed); + self.interrupt_handle.clear_cancel_requested(); HyperlightExit::Cancelled() } else { #[cfg(gdb)] diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index e5592509a..f3fa02d63 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -461,13 +461,51 @@ impl VirtualCPU { pub trait InterruptHandle: Debug + Send + Sync { /// Interrupt the corresponding sandbox from running. /// - /// - If this is called while the vcpu is running, then it will interrupt the vcpu and return `true`. - /// - If this is called while the vcpu is not running, (for example during a host call), the - /// vcpu will not immediately be interrupted, but will prevent the vcpu from running **the next time** - /// it's scheduled, and returns `false`. + /// This method attempts to cancel a currently executing guest function call by sending + /// a signal to the VCPU thread. It uses generation tracking and call_active flag to + /// ensure the interruption is safe and precise. + /// + /// # Behavior + /// + /// - **Guest function running**: If called while a guest function is executing (VCPU running + /// or in a host function call), this stamps the current generation into cancel_requested + /// and sends a signal to interrupt the VCPU. Returns `true`. + /// + /// - **No active call**: If called when no guest function call is in progress (call_active=false), + /// this has no effect and returns `false`. This prevents "kill-in-advance" where kill() + /// is called before a guest function starts. + /// + /// - **During host function**: If the guest call is currently executing a host function + /// (VCPU not running but call_active=true), this stamps cancel_requested. When the + /// host function returns and attempts to re-enter the guest, the cancellation will + /// be detected and the call will abort. Returns `true`. + /// + /// # Generation Tracking + /// + /// The method stamps the current generation number along with the cancellation request. + /// This ensures that: + /// - Stale signals from previous calls are ignored (generation mismatch) + /// - Only the intended guest function call is affected + /// - Multiple rapid kill() calls on the same generation are idempotent + /// + /// # Blocking Behavior + /// + /// This function will block while attempting to deliver the signal to the VCPU thread, + /// retrying until either: + /// - The signal is successfully delivered (VCPU transitions from running to not running) + /// - The VCPU stops running for another reason (e.g., call completes normally) + /// - No call is active (call_active=false) + /// + /// # Returns + /// + /// - `true`: Cancellation request was stamped (kill will take effect) + /// - `false`: No active call, cancellation request was not stamped (no effect) /// /// # Note - /// This function will block for the duration of the time it takes for the vcpu thread to be interrupted. + /// + /// To reliably interrupt a guest call, ensure `kill()` is called while the guest + /// function is actually executing. Calling kill() before call_guest_function() will + /// have no effect. fn kill(&self) -> bool; /// Used by a debugger to interrupt the corresponding sandbox from running. @@ -484,50 +522,177 @@ pub trait InterruptHandle: Debug + Send + Sync { /// Returns true if the corresponding sandbox has been dropped fn dropped(&self) -> bool; + + /// Increment the generation counter for a new guest function call (Linux only). + /// + /// This must be called exactly once at the start of each guest function call, + /// before any VCPU execution begins. The returned generation number will be + /// used throughout the entire guest call, even if the VCPU is run multiple + /// times (due to host function calls, retries, etc.). + /// + /// # Returns + /// + /// The new generation number assigned to this guest function call. + /// + /// # Note + /// + /// This is only called on Linux (KVM/MSHV). Windows uses a different interrupt mechanism. + #[cfg(any(kvm, mshv))] + fn increment_call_generation(&self) -> u64; + + /// Mark that a guest function call is starting (Linux only). + /// + /// Sets the call_active flag to true, indicating that a guest function call + /// is now in progress. This allows kill() to stamp cancel_requested. + /// + /// Must be called immediately after increment_call_generation() and before + /// any VCPU execution begins. + /// + /// # Note + /// + /// This is only called on Linux (KVM/MSHV). Windows uses a different interrupt mechanism. + #[cfg(any(kvm, mshv))] + fn set_call_active(&self); + + /// Mark that a guest function call has completed (Linux only). + /// + /// Clears the call_active flag, indicating that no guest function call is + /// in progress. After this, kill() will have no effect and will return false. + /// + /// Must be called at the end of call_guest_function_by_name_no_reset(), + /// after the guest call has fully completed (whether successfully or with error). + /// + /// # Note + /// + /// This is only called on Linux (KVM/MSHV). Windows uses a different interrupt mechanism. + #[cfg(any(kvm, mshv))] + fn clear_call_active(&self); } #[cfg(any(kvm, mshv))] #[derive(Debug)] pub(super) struct LinuxInterruptHandle { - /// Invariant: vcpu is running => most significant bit (63) of `running` is set. (Neither converse nor inverse is true) + /// Atomic flag combining running state and generation counter. + /// + /// **Bit 63**: VCPU running state (1 = running, 0 = not running) + /// **Bits 0-62**: Generation counter (incremented once per guest function call) + /// + /// # Generation Tracking + /// + /// The generation counter is incremented once at the start of each guest function call + /// and remains constant throughout that call, even if the VCPU is run multiple times + /// (due to host function calls, retries, etc.). This design solves the race condition + /// where a kill() from a previous call could spuriously cancel a new call. /// - /// Additionally, bit 0-62 tracks how many times the VCPU has been run. Incremented each time `run()` is called. + /// ## Why Generations Are Needed /// - /// This prevents an ABA problem where: - /// 1. The VCPU is running (generation N), - /// 2. It gets cancelled, - /// 3. Then quickly restarted (generation N+1), - /// before the original thread has observed that it was cancelled. + /// Consider this scenario WITHOUT generation tracking: + /// 1. Thread A starts guest call 1, VCPU runs + /// 2. Thread B calls kill(), sends signal to Thread A + /// 3. Guest call 1 completes before signal arrives + /// 4. Thread A starts guest call 2, VCPU runs again + /// 5. Stale signal from step 2 arrives and incorrectly cancels call 2 /// - /// Without this generation counter, the interrupt logic might assume the VCPU is still - /// in the *original* run (generation N), see that it's `running`, and re-send the signal. - /// But the new VCPU run (generation N+1) would treat this as a stale signal and ignore it, - /// potentially causing an infinite loop where no effective interrupt is delivered. + /// WITH generation tracking: + /// 1. Thread A starts guest call 1 (generation N), VCPU runs + /// 2. Thread B calls kill(), stamps cancel_requested with generation N + /// 3. Guest call 1 completes, signal may or may not have arrived yet + /// 4. Thread A starts guest call 2 (generation N+1), VCPU runs again + /// 5. If stale signal arrives, signal handler checks: cancel_requested.generation (N) != current generation (N+1) + /// 6. Stale signal is ignored, call 2 continues normally /// - /// Invariant: If the VCPU is running, `run_generation[bit 0-62]` matches the current run's generation. + /// ## Per-Call vs Per-Run Generation + /// + /// It's critical that generation is incremented per GUEST FUNCTION CALL, not per vcpu.run(): + /// - A single guest function call may invoke vcpu.run() multiple times (host calls, retries) + /// - All run() calls within the same guest call must share the same generation + /// - This ensures kill() affects the entire guest function call atomically + /// + /// # Invariants + /// + /// - If VCPU is running: bit 63 is set (neither converse nor inverse holds) + /// - If VCPU is running: bits 0-62 match the current guest call's generation running: AtomicU64, - /// Invariant: vcpu is running => `tid` is the thread on which it is running. - /// Note: multiple vms may have the same `tid`, but at most one vm will have `running` set to true. + + /// Thread ID where the VCPU is currently running. + /// + /// # Invariants + /// + /// - If VCPU is running: tid contains the thread ID of the executing thread + /// - Multiple VMs may share the same tid, but at most one will have running=true tid: AtomicU64, - /// True when an "interruptor" has requested the VM to be cancelled. Set immediately when - /// `kill()` is called, and cleared when the vcpu is no longer running. - /// This is used to - /// 1. make sure stale signals do not interrupt the - /// the wrong vcpu (a vcpu may only be interrupted iff `cancel_requested` is true), - /// 2. ensure that if a vm is killed while a host call is running, - /// the vm will not re-enter the guest after the host call returns. - cancel_requested: AtomicBool, - /// True when the debugger has requested the VM to be interrupted. Set immediately when - /// `kill_from_debugger()` is called, and cleared when the vcpu is no longer running. - /// This is used to make sure stale signals do not interrupt the the wrong vcpu - /// (a vcpu may only be interrupted by a debugger if `debug_interrupt` is true), + + /// Generation-aware cancellation request flag. + /// + /// **Bit 63**: Cancellation requested flag (1 = kill requested, 0 = no kill) + /// **Bits 0-62**: Generation number when cancellation was requested + /// + /// # Purpose + /// + /// This flag serves three critical functions: + /// + /// 1. **Prevent stale signals**: A VCPU may only be interrupted if cancel_requested + /// is set AND the generation matches the current call's generation + /// + /// 2. **Handle host function calls**: If kill() is called while a host function is + /// executing (VCPU not running but call is active), cancel_requested is stamped + /// with the current generation. When the host function returns and the VCPU + /// attempts to re-enter the guest, it will see the cancellation and abort. + /// + /// 3. **Detect stale kills**: If cancel_requested.generation doesn't match the + /// current generation, it's from a previous call and should be ignored + /// + /// # States and Transitions + /// + /// - **No cancellation**: cancel_requested = 0 (bit 63 clear) + /// - **Cancellation for generation N**: cancel_requested = (1 << 63) | N + /// - Signal handler checks: (cancel_requested & 0x7FFFFFFFFFFFFFFF) == current_generation + cancel_requested: AtomicU64, + + /// Flag indicating whether a guest function call is currently in progress. + /// + /// **true**: A guest function call is active (between call start and completion) + /// **false**: No guest function call is active + /// + /// # Purpose + /// + /// This flag prevents kill() from having any effect when called outside of a + /// guest function call. This solves the "kill-in-advance" problem where kill() + /// could be called before a guest function starts and would incorrectly cancel it. + /// + /// # Behavior + /// + /// - Set to true at the start of call_guest_function_by_name_no_reset() + /// - Cleared at the end of call_guest_function_by_name_no_reset() + /// - kill() only stamps cancel_requested if call_active is true + /// - If kill() is called when call_active=false, it returns false and has no effect + /// + /// # Why AtomicBool is Safe + /// + /// Although there's a theoretical race where: + /// 1. Thread A checks call_active (false) + /// 2. Thread B sets call_active (true) and starts guest call + /// 3. Thread A's kill() returns false (no effect) + /// + /// This is acceptable because the generation tracking provides an additional + /// safety layer. Even if a stale kill somehow stamped cancel_requested, the + /// generation mismatch would cause it to be ignored. + call_active: AtomicBool, + + /// Debugger interrupt request flag (GDB only). + /// + /// Set when kill_from_debugger() is called, cleared when VCPU stops running. + /// Used to distinguish debugger interrupts from normal kill() interrupts. #[cfg(gdb)] debug_interrupt: AtomicBool, - /// Whether the corresponding vm is dropped + + /// Whether the corresponding VM has been dropped. dropped: AtomicBool, - /// Retry delay between signals sent to the vcpu thread + + /// Delay between retry attempts when sending signals to the VCPU thread. retry_delay: Duration, - /// The offset of the SIGRTMIN signal used to interrupt the vcpu thread + + /// Offset from SIGRTMIN for the signal used to interrupt the VCPU thread. sig_rt_min_offset: u8, } @@ -535,18 +700,51 @@ pub(super) struct LinuxInterruptHandle { impl LinuxInterruptHandle { const RUNNING_BIT: u64 = 1 << 63; const MAX_GENERATION: u64 = Self::RUNNING_BIT - 1; + const CANCEL_REQUESTED_BIT: u64 = 1 << 63; + + // Set cancel_requested to true with the given generation + fn set_cancel_requested(&self, generation: u64) { + let value = Self::CANCEL_REQUESTED_BIT | (generation & Self::MAX_GENERATION); + self.cancel_requested.store(value, Ordering::Release); + } + + // Clear cancel_requested (reset to no cancellation) + pub(crate) fn clear_cancel_requested(&self) { + self.cancel_requested.store(0, Ordering::Release); + } - // set running to true and increment the generation. Generation will wrap around at `MAX_GENERATION`. - fn set_running_and_increment_generation(&self) -> std::result::Result { + // Check if cancel_requested is set for the given generation + fn is_cancel_requested_for_generation(&self, generation: u64) -> bool { + let raw = self.cancel_requested.load(Ordering::Acquire); + let is_set = raw & Self::CANCEL_REQUESTED_BIT != 0; + let stored_generation = raw & Self::MAX_GENERATION; + is_set && stored_generation == generation + } + + // Increment the generation for a new guest function call. Generation will wrap around at `MAX_GENERATION`. + pub(crate) fn increment_generation(&self) -> u64 { self.running - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |raw| { - let generation = raw & !Self::RUNNING_BIT; - if generation == Self::MAX_GENERATION { + .fetch_update(Ordering::Release, Ordering::Acquire, |raw| { + let current_generation = raw & !Self::RUNNING_BIT; + let running_bit = raw & Self::RUNNING_BIT; + if current_generation == Self::MAX_GENERATION { // restart generation from 0 - return Some(Self::RUNNING_BIT); + return Some(running_bit); } - Some((generation + 1) | Self::RUNNING_BIT) + Some((current_generation + 1) | running_bit) }) + .map(|raw| (raw & !Self::RUNNING_BIT) + 1) // Return the NEW generation + .unwrap_or(1) // If wrapped, return 1 + } + + // set running to true without incrementing generation + fn set_running_bit(&self) -> u64 { + self.running + .fetch_update(Ordering::Release, Ordering::Acquire, |raw| { + Some(raw | Self::RUNNING_BIT) + }) + .map(|raw| raw & !Self::RUNNING_BIT) // Return the current generation + .unwrap_or(0) } // clear the running bit and return the generation @@ -562,7 +760,7 @@ impl LinuxInterruptHandle { (running, generation) } - fn send_signal(&self) -> bool { + fn send_signal(&self, stamp_generation: bool) -> bool { let signal_number = libc::SIGRTMIN() + self.sig_rt_min_offset as libc::c_int; let mut sent_signal = false; let mut target_generation: Option = None; @@ -570,6 +768,20 @@ impl LinuxInterruptHandle { loop { let (running, generation) = self.get_running_and_generation(); + // Stamp generation into cancel_requested if requested and this is the first iteration + // We stamp even when running=false to support killing during host function calls + // The generation tracking will prevent stale kills from affecting new calls + // Only stamp if a call is actually active (call_active=true) + if stamp_generation + && target_generation.is_none() + && self.call_active.load(Ordering::Acquire) + { + self.set_cancel_requested(generation); + target_generation = Some(generation); + } + + // If not running, we've stamped the generation (if requested), so we're done + // This handles the host function call scenario if !running { break; } @@ -596,18 +808,33 @@ impl LinuxInterruptHandle { #[cfg(any(kvm, mshv))] impl InterruptHandle for LinuxInterruptHandle { fn kill(&self) -> bool { - self.cancel_requested.store(true, Ordering::Relaxed); - - self.send_signal() + // send_signal will stamp the generation into cancel_requested + // right before sending each signal, ensuring they're always in sync + self.send_signal(true) } #[cfg(gdb)] fn kill_from_debugger(&self) -> bool { self.debug_interrupt.store(true, Ordering::Relaxed); - self.send_signal() + self.send_signal(false) } fn dropped(&self) -> bool { self.dropped.load(Ordering::Relaxed) } + + #[cfg(any(kvm, mshv))] + fn increment_call_generation(&self) -> u64 { + self.increment_generation() + } + + #[cfg(any(kvm, mshv))] + fn set_call_active(&self) { + self.call_active.store(true, Ordering::Release); + } + + #[cfg(any(kvm, mshv))] + fn clear_call_active(&self) { + self.call_active.store(false, Ordering::Release); + } } #[cfg(all(test, any(target_os = "windows", kvm)))] diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index 765a1a898..fa3d386b6 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -390,6 +390,14 @@ impl MultiUseSandbox { return_type: ReturnType, args: Vec, ) -> Result { + // Increment generation for this guest function call (Linux only) + #[cfg(any(kvm, mshv))] + let _generation = self.vm.interrupt_handle().increment_call_generation(); + + // Mark that a guest function call is now active (Linux only) + #[cfg(any(kvm, mshv))] + self.vm.interrupt_handle().set_call_active(); + let res = (|| { let estimated_capacity = estimate_flatbuffer_capacity(function_name, &args); @@ -405,6 +413,10 @@ impl MultiUseSandbox { self.mem_mgr.write_guest_function_call(buffer)?; + // Increment generation for this guest function call (Linux only) + #[cfg(any(kvm, mshv))] + self.vm.interrupt_handle().increment_call_generation(); + self.vm.dispatch_call_from_host( self.dispatch_ptr.clone(), #[cfg(gdb)] @@ -440,6 +452,11 @@ impl MultiUseSandbox { if res.is_err() { self.mem_mgr.clear_io_buffers(); } + + // Mark that the guest function call has completed (Linux only) + #[cfg(any(kvm, mshv))] + self.vm.interrupt_handle().clear_call_active(); + res } diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index b252aa947..ad0928522 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -62,7 +62,9 @@ fn interrupt_host_call() { } }); - let result = sandbox.call::("CallHostSpin", ()).unwrap_err(); + let result = sandbox.call::("CallHostSpin", ()); + println!("Result: {:?}", result); + let result = result.unwrap_err(); assert!(matches!(result, HyperlightError::ExecutionCanceledByHost())); thread.join().unwrap(); @@ -99,7 +101,8 @@ fn interrupt_in_progress_guest_call() { thread.join().expect("Thread should finish"); } -/// Makes sure interrupting a vm before the guest call has started also prevents the guest call from being executed +/// Makes sure interrupting a vm before the guest call has started has no effect, +/// but a second kill after the call starts will interrupt it #[test] fn interrupt_guest_call_in_advance() { let mut sbox1: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); @@ -108,15 +111,20 @@ fn interrupt_guest_call_in_advance() { let interrupt_handle = sbox1.interrupt_handle(); assert!(!interrupt_handle.dropped()); // not yet dropped - // kill vm before the guest call has started + // First kill before the guest call has started - should have no effect + // Then kill again after a delay to interrupt the actual call let thread = thread::spawn(move || { - assert!(!interrupt_handle.kill()); // should return false since vcpu is not running yet + assert!(!interrupt_handle.kill()); // should return false since no call is active barrier2.wait(); + // Wait a bit for the Spin call to actually start + thread::sleep(Duration::from_millis(100)); + assert!(interrupt_handle.kill()); // this should succeed and interrupt the Spin call barrier2.wait(); // wait here until main thread has dropped the sandbox assert!(interrupt_handle.dropped()); }); - barrier.wait(); // wait until `kill()` is called before starting the guest call + barrier.wait(); // wait until first `kill()` is called before starting the guest call + // The Spin call should be interrupted by the second kill() let res = sbox1.call::("Spin", ()).unwrap_err(); assert!(matches!(res, HyperlightError::ExecutionCanceledByHost())); @@ -851,3 +859,365 @@ fn test_if_guest_is_able_to_get_string_return_values_from_host() { "Guest Function, string added by Host Function".to_string() ); } + +/// Test that monitors CPU time usage and can interrupt a guest based on CPU time limits +/// Uses a pool of 100 sandboxes, 100 threads, and 500 iterations per thread +/// Some sandboxes are expected to complete normally, some are expected to be killed +/// This test makes sure that a reused sandbox is not killed in the case where the previous +/// execution was killed due to CPU time limit but the invocation completed normally before the cancel was processed. +#[test] +fn test_cpu_time_interrupt() { + use std::collections::VecDeque; + use std::mem::MaybeUninit; + use std::sync::Mutex; + use std::sync::atomic::AtomicUsize; + use std::sync::mpsc::channel; + + const POOL_SIZE: usize = 100; + const NUM_THREADS: usize = 100; + const ITERATIONS_PER_THREAD: usize = 500; + + // Create a pool of 100 sandboxes + println!("Creating pool of {} sandboxes...", POOL_SIZE); + let mut sandbox_pool: Vec = Vec::with_capacity(POOL_SIZE); + for i in 0..POOL_SIZE { + let sandbox = new_uninit_rust().unwrap().evolve().unwrap(); + if (i + 1) % 10 == 0 { + println!("Created {}/{} sandboxes", i + 1, POOL_SIZE); + } + sandbox_pool.push(sandbox); + } + + // Wrap the pool in Arc> for thread-safe access + let pool = Arc::new(Mutex::new(VecDeque::from(sandbox_pool))); + + // Counters for statistics + let total_iterations = Arc::new(AtomicUsize::new(0)); + let killed_count = Arc::new(AtomicUsize::new(0)); + let completed_count = Arc::new(AtomicUsize::new(0)); + let errors_count = Arc::new(AtomicUsize::new(0)); + + println!( + "Starting {} threads with {} iterations each...", + NUM_THREADS, ITERATIONS_PER_THREAD + ); + + // Spawn worker threads + let mut thread_handles = vec![]; + for thread_id in 0..NUM_THREADS { + let pool_clone = Arc::clone(&pool); + let total_iterations_clone = Arc::clone(&total_iterations); + let killed_count_clone = Arc::clone(&killed_count); + let completed_count_clone = Arc::clone(&completed_count); + let errors_count_clone = Arc::clone(&errors_count); + + let handle = thread::spawn(move || { + for iteration in 0..ITERATIONS_PER_THREAD { + // === START OF ITERATION === + // Get a fresh sandbox from the pool for this iteration + let mut sandbox = { + let mut pool_guard = pool_clone.lock().unwrap(); + // Wait if pool is empty (shouldn't happen with proper design) + while pool_guard.is_empty() { + drop(pool_guard); + thread::sleep(Duration::from_micros(100)); + pool_guard = pool_clone.lock().unwrap(); + } + pool_guard.pop_front().unwrap() + }; + + // Vary CPU time between 3ms and 7ms to ensure some get killed and some complete + // The CPU limit is 5ms, so: + // - 3-4ms should complete normally + // - 6-7ms should be killed + // - 5ms is borderline and could go either way + let cpu_time_ms = + 3 + (((thread_id * ITERATIONS_PER_THREAD + iteration) % 5) as u32); + + let interrupt_handle = sandbox.interrupt_handle(); + + // Channel to send the thread ID + let (tx, rx) = channel(); + + // Flag to signal monitoring start + let should_monitor = Arc::new(AtomicBool::new(false)); + let should_monitor_clone = should_monitor.clone(); + + // Flag to signal monitoring stop (when guest execution completes) + let stop_monitoring = Arc::new(AtomicBool::new(false)); + let stop_monitoring_clone = stop_monitoring.clone(); + + // Flag to track if we actually sent a kill signal + let was_killed = Arc::new(AtomicBool::new(false)); + let was_killed_clone = was_killed.clone(); + + // Spawn CPU time monitor thread + let monitor_thread = thread::spawn(move || { + let main_thread_id = match rx.recv() { + Ok(tid) => tid, + Err(_) => return, + }; + + while !should_monitor_clone.load(Ordering::Acquire) { + thread::sleep(Duration::from_micros(50)); + } + + #[cfg(target_os = "linux")] + unsafe { + let mut clock_id: libc::clockid_t = 0; + if libc::pthread_getcpuclockid(main_thread_id, &mut clock_id) != 0 { + return; + } + + let cpu_limit_ns = 5_000_000; // 5ms CPU time limit + let mut start_time = MaybeUninit::::uninit(); + + if libc::clock_gettime(clock_id, start_time.as_mut_ptr()) != 0 { + return; + } + let start_time = start_time.assume_init(); + + loop { + // Check if we should stop monitoring (guest completed) + if stop_monitoring_clone.load(Ordering::Acquire) { + break; + } + + let mut current_time = MaybeUninit::::uninit(); + if libc::clock_gettime(clock_id, current_time.as_mut_ptr()) != 0 { + break; + } + let current_time = current_time.assume_init(); + + let elapsed_ns = (current_time.tv_sec - start_time.tv_sec) + * 1_000_000_000 + + (current_time.tv_nsec - start_time.tv_nsec); + + if elapsed_ns > cpu_limit_ns { + // Double-check that monitoring should still continue before killing + // The guest might have completed between our last check and now + if stop_monitoring_clone.load(Ordering::Acquire) { + break; + } + + // Mark that we sent a kill signal BEFORE calling kill + // to avoid race conditions + was_killed_clone.store(true, Ordering::Release); + interrupt_handle.kill(); + break; + } + + thread::sleep(Duration::from_micros(50)); + } + } + + #[cfg(target_os = "windows")] + unsafe { + use std::ffi::c_void; + + // On Windows, we use GetThreadTimes to get CPU time + // main_thread_id is a HANDLE on Windows + let thread_handle = main_thread_id as *mut c_void; + + let cpu_limit_ns: i64 = 5_000_000; // 5ms CPU time limit (in nanoseconds) + + let mut creation_time = + MaybeUninit::::uninit(); + let mut exit_time = + MaybeUninit::::uninit(); + let mut kernel_time_start = + MaybeUninit::::uninit(); + let mut user_time_start = + MaybeUninit::::uninit(); + + // Get initial CPU times + if windows_sys::Win32::System::Threading::GetThreadTimes( + thread_handle, + creation_time.as_mut_ptr(), + exit_time.as_mut_ptr(), + kernel_time_start.as_mut_ptr(), + user_time_start.as_mut_ptr(), + ) == 0 + { + return; + } + + let kernel_time_start = kernel_time_start.assume_init(); + let user_time_start = user_time_start.assume_init(); + + // Convert FILETIME to u64 (100-nanosecond intervals) + let start_cpu_time = ((kernel_time_start.dwHighDateTime as u64) << 32 + | kernel_time_start.dwLowDateTime as u64) + + ((user_time_start.dwHighDateTime as u64) << 32 + | user_time_start.dwLowDateTime as u64); + + loop { + // Check if we should stop monitoring (guest completed) + if stop_monitoring_clone.load(Ordering::Acquire) { + break; + } + + let mut kernel_time_current = + MaybeUninit::::uninit(); + let mut user_time_current = + MaybeUninit::::uninit(); + + if windows_sys::Win32::System::Threading::GetThreadTimes( + thread_handle, + creation_time.as_mut_ptr(), + exit_time.as_mut_ptr(), + kernel_time_current.as_mut_ptr(), + user_time_current.as_mut_ptr(), + ) == 0 + { + break; + } + + let kernel_time_current = kernel_time_current.assume_init(); + let user_time_current = user_time_current.assume_init(); + + // Convert FILETIME to u64 + let current_cpu_time = ((kernel_time_current.dwHighDateTime as u64) + << 32 + | kernel_time_current.dwLowDateTime as u64) + + ((user_time_current.dwHighDateTime as u64) << 32 + | user_time_current.dwLowDateTime as u64); + + // FILETIME is in 100-nanosecond intervals, convert to nanoseconds + let elapsed_ns = ((current_cpu_time - start_cpu_time) * 100) as i64; + + if elapsed_ns > cpu_limit_ns { + // Double-check that monitoring should still continue before killing + // The guest might have completed between our last check and now + if stop_monitoring_clone.load(Ordering::Acquire) { + break; + } + + // Mark that we sent a kill signal BEFORE calling kill + // to avoid race conditions + was_killed_clone.store(true, Ordering::Release); + interrupt_handle.kill(); + break; + } + + thread::sleep(Duration::from_micros(50)); + } + } + }); + + // Send thread ID and start monitoring + #[cfg(target_os = "linux")] + unsafe { + let thread_id = libc::pthread_self(); + let _ = tx.send(thread_id); + } + + #[cfg(target_os = "windows")] + unsafe { + // On Windows, get the current thread's pseudo-handle + let thread_handle = windows_sys::Win32::System::Threading::GetCurrentThread(); + let _ = tx.send(thread_handle as usize); + } + + should_monitor.store(true, Ordering::Release); + + // Call the guest function + let result = sandbox.call::("SpinForMs", cpu_time_ms); + + // Signal the monitor to stop + stop_monitoring.store(true, Ordering::Release); + + // Wait for monitor thread to complete to ensure was_killed flag is set + let _ = monitor_thread.join(); + + // NOW check if we sent a kill signal (after monitor thread has completed) + let kill_was_sent = was_killed.load(Ordering::Acquire); + + // Process the result and validate correctness + match result { + Err(HyperlightError::ExecutionCanceledByHost()) => { + // We received a cancellation error + if !kill_was_sent { + // ERROR: We got a cancellation but never sent a kill! + panic!( + "Thread {} iteration {}: Got ExecutionCanceledByHost but no kill signal was sent!", + thread_id, iteration + ); + } + // This is correct - we sent kill and got the error + killed_count_clone.fetch_add(1, Ordering::Relaxed); + } + Ok(_) => { + // Execution completed normally + // This is OK whether or not we sent a kill - the guest might have + // finished just before the kill signal arrived + completed_count_clone.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + // Unexpected error + eprintln!( + "Thread {} iteration {}: Unexpected error: {:?}, kill_sent: {}", + thread_id, iteration, e, kill_was_sent + ); + errors_count_clone.fetch_add(1, Ordering::Relaxed); + } + } + + total_iterations_clone.fetch_add(1, Ordering::Relaxed); + + // Progress reporting + let current_total = total_iterations_clone.load(Ordering::Relaxed); + if current_total % 500 == 0 { + println!( + "Progress: {}/{} iterations completed", + current_total, + NUM_THREADS * ITERATIONS_PER_THREAD + ); + } + + // === END OF ITERATION === + // Return sandbox to pool for reuse by other threads/iterations + { + let mut pool_guard = pool_clone.lock().unwrap(); + pool_guard.push_back(sandbox); + } + } + }); + + thread_handles.push(handle); + } + + // Wait for all threads to complete + for handle in thread_handles { + handle.join().unwrap(); + } + + // Print statistics + let total = total_iterations.load(Ordering::Relaxed); + let killed = killed_count.load(Ordering::Relaxed); + let completed = completed_count.load(Ordering::Relaxed); + let errors = errors_count.load(Ordering::Relaxed); + + println!("\n=== Test Statistics ==="); + println!("Total iterations: {}", total); + println!("Killed (CPU limit exceeded): {}", killed); + println!("Completed normally: {}", completed); + println!("Errors: {}", errors); + println!("Kill rate: {:.1}%", (killed as f64 / total as f64) * 100.0); + + // Verify we had both kills and completions + assert!( + killed > 0, + "Expected some executions to be killed, but none were" + ); + assert!( + completed > 0, + "Expected some executions to complete, but none did" + ); + assert_eq!(errors, 0, "Expected no errors, but got {}", errors); + assert_eq!( + total, + NUM_THREADS * ITERATIONS_PER_THREAD, + "Not all iterations completed" + ); +} diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index eda988f33..1699acfd3 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -559,6 +559,38 @@ fn spin(_: &FunctionCall) -> Result> { Ok(get_flatbuffer_result(())) } +/// Spins the CPU for approximately the specified number of milliseconds +fn spin_for_ms(fc: &FunctionCall) -> Result> { + let milliseconds = if let ParameterValue::UInt(ms) = fc.parameters.clone().unwrap()[0].clone() { + ms + } else { + return Err(HyperlightGuestError::new( + ErrorCode::GuestFunctionParameterTypeMismatch, + "Expected UInt parameter".to_string(), + )); + }; + + // Simple busy-wait loop - not precise but good enough for testing + // Different iteration counts for debug vs release mode to ensure reasonable CPU usage + #[cfg(debug_assertions)] + let iterations_per_ms = 120_000; // Debug mode - less optimized, tuned for ~50% kill rate + + #[cfg(not(debug_assertions))] + let iterations_per_ms = 1_000_000; // Release mode - highly optimized + + let total_iterations = milliseconds * iterations_per_ms; + + let mut counter: u64 = 0; + for _ in 0..total_iterations { + // Prevent the compiler from optimizing away the loop + counter = counter.wrapping_add(1); + core::hint::black_box(counter); + } + + Ok(get_flatbuffer_result(counter)) +} + +#[hyperlight_guest_tracing::trace_function] fn test_abort(function_call: &FunctionCall) -> Result> { if let ParameterValue::Int(code) = function_call.parameters.clone().unwrap()[0].clone() { abort_with_code(&[code as u8]); @@ -1307,6 +1339,14 @@ pub extern "C" fn hyperlight_main() { ); register_function(spin_def); + let spin_for_ms_def = GuestFunctionDefinition::new( + "SpinForMs".to_string(), + Vec::from(&[ParameterType::UInt]), + ReturnType::ULong, + spin_for_ms as usize, + ); + register_function(spin_for_ms_def); + let abort_def = GuestFunctionDefinition::new( "GuestAbortWithCode".to_string(), Vec::from(&[ParameterType::Int]), From 91b5fd60bd8ff7c4eef892bb85c07f0e734d113c Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 15:14:15 -0700 Subject: [PATCH 02/25] Do not set canceled on windows if vm is not running Signed-off-by: James Sturtevant --- src/hyperlight_host/src/hypervisor/hyperv_windows.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 5c6c9db9c..b90ab1e43 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -976,9 +976,15 @@ pub struct WindowsInterruptHandle { impl InterruptHandle for WindowsInterruptHandle { fn kill(&self) -> bool { + // don't send the signal if the the vm isn't running + // In the case this is called before the vm is running the cancel_requested would be set + // and stay set while the vm is running. + let running = self.running.load(Ordering::Relaxed); + if !running { + return false; + } self.cancel_requested.store(true, Ordering::Relaxed); - self.running.load(Ordering::Relaxed) - && unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } + unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } } #[cfg(gdb)] fn kill_from_debugger(&self) -> bool { From e19dac4130021178d49f678d8e06dadf6587e61c Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 16:17:45 -0700 Subject: [PATCH 03/25] Should retry if Windows platform does cancelation and it wasn't requested by us Signed-off-by: James Sturtevant --- .../src/hypervisor/hyperv_windows.rs | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index b90ab1e43..2b13e8a7c 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -578,6 +578,8 @@ impl Hypervisor for HypervWindowsDriver { self.processor.run()? }; + let cancel_was_requested_manually = self.interrupt_handle + .cancel_requested.load(Ordering::Relaxed); self.interrupt_handle .cancel_requested .store(false, Ordering::Relaxed); @@ -660,11 +662,31 @@ impl Hypervisor for HypervWindowsDriver { // and resume execution HyperlightExit::Debug(VcpuStopReason::Interrupt) } else { - HyperlightExit::Cancelled() + if !cancel_was_requested_manually { + // This was an internal cancellation + // The virtualization stack can use this function to return the control + // of a virtual processor back to the virtualization stack in case it + // needs to change the state of a VM or to inject an event into the processor + debug!("Internal cancellation detected, returning Retry error"); + HyperlightExit::Retry() + } else { + HyperlightExit::Cancelled() + } } #[cfg(not(gdb))] - HyperlightExit::Cancelled() + { + if !cancel_was_requested_manually { + // This was an internal cancellation + // The virtualization stack can use this function to return the control + // of a virtual processor back to the virtualization stack in case it + // needs to change the state of a VM or to inject an event into the processor + debug!("Internal cancellation detected, returning Retry error"); + HyperlightExit::Retry() + } else { + HyperlightExit::Cancelled() + } + } } #[cfg(gdb)] WHV_RUN_VP_EXIT_REASON(4098i32) => { From 8dbb19d87539a5b5f772f900dd9c9e775f9b92b4 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 17:07:40 -0700 Subject: [PATCH 04/25] Handle host function call cancelations Signed-off-by: James Sturtevant --- .../src/hypervisor/hyperv_windows.rs | 47 ++++++++++++++----- src/hyperlight_host/src/hypervisor/mod.rs | 24 ++++------ .../src/sandbox/initialized_multi_use.rs | 6 +-- src/tests/rust_guests/simpleguest/src/main.rs | 4 +- 4 files changed, 49 insertions(+), 32 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 2b13e8a7c..aa01204c8 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -331,6 +331,7 @@ impl HypervWindowsDriver { cancel_requested: AtomicBool::new(false), #[cfg(gdb)] debug_interrupt: AtomicBool::new(false), + call_active: AtomicBool::new(false), partition_handle, dropped: AtomicBool::new(false), }); @@ -578,8 +579,10 @@ impl Hypervisor for HypervWindowsDriver { self.processor.run()? }; - let cancel_was_requested_manually = self.interrupt_handle - .cancel_requested.load(Ordering::Relaxed); + let cancel_was_requested_manually = self + .interrupt_handle + .cancel_requested + .load(Ordering::Relaxed); self.interrupt_handle .cancel_requested .store(false, Ordering::Relaxed); @@ -665,9 +668,9 @@ impl Hypervisor for HypervWindowsDriver { if !cancel_was_requested_manually { // This was an internal cancellation // The virtualization stack can use this function to return the control - // of a virtual processor back to the virtualization stack in case it + // of a virtual processor back to the virtualization stack in case it // needs to change the state of a VM or to inject an event into the processor - debug!("Internal cancellation detected, returning Retry error"); + println!("Internal cancellation detected, returning Retry error"); HyperlightExit::Retry() } else { HyperlightExit::Cancelled() @@ -679,9 +682,9 @@ impl Hypervisor for HypervWindowsDriver { if !cancel_was_requested_manually { // This was an internal cancellation // The virtualization stack can use this function to return the control - // of a virtual processor back to the virtualization stack in case it + // of a virtual processor back to the virtualization stack in case it // needs to change the state of a VM or to inject an event into the processor - debug!("Internal cancellation detected, returning Retry error"); + println!("Internal cancellation detected, returning Retry error"); HyperlightExit::Retry() } else { HyperlightExit::Cancelled() @@ -992,21 +995,35 @@ pub struct WindowsInterruptHandle { // This is used to signal the GDB thread to stop the vCPU #[cfg(gdb)] debug_interrupt: AtomicBool, + /// Flag indicating whether a guest function call is currently in progress. + /// + /// **true**: A guest function call is active (between call start and completion) + /// **false**: No guest function call is active + /// + /// # Purpose + /// + /// This flag prevents kill() from having any effect when called outside of a + /// guest function call. This solves the "kill-in-advance" problem where kill() + /// could be called before a guest function starts and would incorrectly cancel it. + call_active: AtomicBool, partition_handle: WHV_PARTITION_HANDLE, dropped: AtomicBool, } impl InterruptHandle for WindowsInterruptHandle { fn kill(&self) -> bool { + // Check if a call is actually active first + if !self.call_active.load(Ordering::Acquire) { + return false; + } + // don't send the signal if the the vm isn't running // In the case this is called before the vm is running the cancel_requested would be set // and stay set while the vm is running. - let running = self.running.load(Ordering::Relaxed); - if !running { - return false; - } + self.cancel_requested.store(true, Ordering::Relaxed); - unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } + self.running.load(Ordering::Relaxed) + && unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } } #[cfg(gdb)] fn kill_from_debugger(&self) -> bool { @@ -1018,4 +1035,12 @@ impl InterruptHandle for WindowsInterruptHandle { fn dropped(&self) -> bool { self.dropped.load(Ordering::Relaxed) } + + fn set_call_active(&self) { + self.call_active.store(true, Ordering::Release); + } + + fn clear_call_active(&self) { + self.call_active.store(false, Ordering::Release); + } } diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index f3fa02d63..74135bcb9 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -439,7 +439,10 @@ impl VirtualCPU { log_then_return!("Unexpected VM Exit {:?}", reason); } - Ok(HyperlightExit::Retry()) => continue, + Ok(HyperlightExit::Retry()) => { + debug!("retring vm run"); + continue; + } Err(e) => { #[cfg(crashdump)] crashdump::generate_crashdump(hv)?; @@ -462,7 +465,7 @@ pub trait InterruptHandle: Debug + Send + Sync { /// Interrupt the corresponding sandbox from running. /// /// This method attempts to cancel a currently executing guest function call by sending - /// a signal to the VCPU thread. It uses generation tracking and call_active flag to + /// a signal to the VCPU thread. It uses generation tracking (linux only) and call_active flag to /// ensure the interruption is safe and precise. /// /// # Behavior @@ -480,7 +483,7 @@ pub trait InterruptHandle: Debug + Send + Sync { /// host function returns and attempts to re-enter the guest, the cancellation will /// be detected and the call will abort. Returns `true`. /// - /// # Generation Tracking + /// # Generation Tracking (linux only) /// /// The method stamps the current generation number along with the cancellation request. /// This ensures that: @@ -494,7 +497,6 @@ pub trait InterruptHandle: Debug + Send + Sync { /// retrying until either: /// - The signal is successfully delivered (VCPU transitions from running to not running) /// - The VCPU stops running for another reason (e.g., call completes normally) - /// - No call is active (call_active=false) /// /// # Returns /// @@ -540,32 +542,22 @@ pub trait InterruptHandle: Debug + Send + Sync { #[cfg(any(kvm, mshv))] fn increment_call_generation(&self) -> u64; - /// Mark that a guest function call is starting (Linux only). + /// Mark that a guest function call is starting. /// /// Sets the call_active flag to true, indicating that a guest function call /// is now in progress. This allows kill() to stamp cancel_requested. /// /// Must be called immediately after increment_call_generation() and before /// any VCPU execution begins. - /// - /// # Note - /// - /// This is only called on Linux (KVM/MSHV). Windows uses a different interrupt mechanism. - #[cfg(any(kvm, mshv))] fn set_call_active(&self); - /// Mark that a guest function call has completed (Linux only). + /// Mark that a guest function call has completed. /// /// Clears the call_active flag, indicating that no guest function call is /// in progress. After this, kill() will have no effect and will return false. /// /// Must be called at the end of call_guest_function_by_name_no_reset(), /// after the guest call has fully completed (whether successfully or with error). - /// - /// # Note - /// - /// This is only called on Linux (KVM/MSHV). Windows uses a different interrupt mechanism. - #[cfg(any(kvm, mshv))] fn clear_call_active(&self); } diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index fa3d386b6..20188802d 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -394,8 +394,7 @@ impl MultiUseSandbox { #[cfg(any(kvm, mshv))] let _generation = self.vm.interrupt_handle().increment_call_generation(); - // Mark that a guest function call is now active (Linux only) - #[cfg(any(kvm, mshv))] + // Mark that a guest function call is now active self.vm.interrupt_handle().set_call_active(); let res = (|| { @@ -453,8 +452,7 @@ impl MultiUseSandbox { self.mem_mgr.clear_io_buffers(); } - // Mark that the guest function call has completed (Linux only) - #[cfg(any(kvm, mshv))] + // Mark that the guest function call has completed self.vm.interrupt_handle().clear_call_active(); res diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index 1699acfd3..b32b7b102 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -587,7 +587,9 @@ fn spin_for_ms(fc: &FunctionCall) -> Result> { core::hint::black_box(counter); } - Ok(get_flatbuffer_result(counter)) + // Calculate the actual number of milliseconds spun for, based on the counter and iterations per ms + let ms_spun = (counter / iterations_per_ms as u64) as u64; + Ok(get_flatbuffer_result(ms_spun)) } #[hyperlight_guest_tracing::trace_function] From 4573e31ca1a8449bd9c92402efb89bcb290ea258 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 17:10:58 -0700 Subject: [PATCH 05/25] Fix type and spelling Signed-off-by: James Sturtevant --- src/hyperlight_host/src/hypervisor/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 74135bcb9..97b5d9437 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -114,7 +114,9 @@ pub enum HyperlightExit { Cancelled(), /// The vCPU has exited for a reason that is not handled by Hyperlight Unknown(String), - /// The operation should be retried, for example this can happen on Linux where a call to run the CPU can return EAGAIN + /// The operation should be retried + /// On Linux this can happen where a call to run the CPU can return EAGAIN + /// On Windows the platform could cause a cancelation of the VM run Retry(), } @@ -440,7 +442,7 @@ impl VirtualCPU { log_then_return!("Unexpected VM Exit {:?}", reason); } Ok(HyperlightExit::Retry()) => { - debug!("retring vm run"); + debug!("retrying vm run"); continue; } Err(e) => { From abc3a5650de47fd9c8e5864429f5360a7b5c7742 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 17:19:05 -0700 Subject: [PATCH 06/25] Fix println that was for debugging Signed-off-by: James Sturtevant --- src/hyperlight_host/src/hypervisor/hyperv_windows.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index aa01204c8..e677b184e 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -670,7 +670,7 @@ impl Hypervisor for HypervWindowsDriver { // The virtualization stack can use this function to return the control // of a virtual processor back to the virtualization stack in case it // needs to change the state of a VM or to inject an event into the processor - println!("Internal cancellation detected, returning Retry error"); + debug!("Internal cancellation detected, returning Retry error"); HyperlightExit::Retry() } else { HyperlightExit::Cancelled() @@ -684,7 +684,7 @@ impl Hypervisor for HypervWindowsDriver { // The virtualization stack can use this function to return the control // of a virtual processor back to the virtualization stack in case it // needs to change the state of a VM or to inject an event into the processor - println!("Internal cancellation detected, returning Retry error"); + debug!("Internal cancellation detected, returning Retry error"); HyperlightExit::Retry() } else { HyperlightExit::Cancelled() From cb6b5f04b6d33e011771fae9b2de2a1c9ec85409 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 17:39:55 -0700 Subject: [PATCH 07/25] fix clippy Signed-off-by: James Sturtevant --- .../src/hypervisor/hyperv_windows.rs | 18 ++++++++---------- src/tests/rust_guests/simpleguest/src/main.rs | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index e677b184e..f55d4afa6 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -664,17 +664,15 @@ impl Hypervisor for HypervWindowsDriver { // return a special exit reason so that the gdb thread can handle it // and resume execution HyperlightExit::Debug(VcpuStopReason::Interrupt) + } else if !cancel_was_requested_manually { + // This was an internal cancellation + // The virtualization stack can use this function to return the control + // of a virtual processor back to the virtualization stack in case it + // needs to change the state of a VM or to inject an event into the processor + debug!("Internal cancellation detected, returning Retry error"); + HyperlightExit::Retry() } else { - if !cancel_was_requested_manually { - // This was an internal cancellation - // The virtualization stack can use this function to return the control - // of a virtual processor back to the virtualization stack in case it - // needs to change the state of a VM or to inject an event into the processor - debug!("Internal cancellation detected, returning Retry error"); - HyperlightExit::Retry() - } else { - HyperlightExit::Cancelled() - } + HyperlightExit::Cancelled() } #[cfg(not(gdb))] diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index b32b7b102..53fa71309 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -588,7 +588,7 @@ fn spin_for_ms(fc: &FunctionCall) -> Result> { } // Calculate the actual number of milliseconds spun for, based on the counter and iterations per ms - let ms_spun = (counter / iterations_per_ms as u64) as u64; + let ms_spun = (counter / iterations_per_ms as u64); Ok(get_flatbuffer_result(ms_spun)) } From afc479ae1dd397534fa8bce62b9e4dfb44d9f012 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 17:39:55 -0700 Subject: [PATCH 08/25] fix clippy Signed-off-by: James Sturtevant --- src/tests/rust_guests/simpleguest/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index 53fa71309..89e896e8e 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -588,7 +588,7 @@ fn spin_for_ms(fc: &FunctionCall) -> Result> { } // Calculate the actual number of milliseconds spun for, based on the counter and iterations per ms - let ms_spun = (counter / iterations_per_ms as u64); + let ms_spun = counter / iterations_per_ms as u64; Ok(get_flatbuffer_result(ms_spun)) } From bdfc208ee74fcc21f9a9bc4b02cfa351ca40fde3 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 19:56:34 -0700 Subject: [PATCH 09/25] temp commit: add logging in CI Signed-off-by: James Sturtevant --- src/hyperlight_host/src/hypervisor/hyperv_windows.rs | 2 +- src/hyperlight_host/src/hypervisor/mod.rs | 4 ++-- src/hyperlight_host/tests/integration_test.rs | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index f55d4afa6..7be1a93f3 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -682,7 +682,7 @@ impl Hypervisor for HypervWindowsDriver { // The virtualization stack can use this function to return the control // of a virtual processor back to the virtualization stack in case it // needs to change the state of a VM or to inject an event into the processor - debug!("Internal cancellation detected, returning Retry error"); + println!("Internal cancellation detected, returning Retry error"); HyperlightExit::Retry() } else { HyperlightExit::Cancelled() diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 97b5d9437..042d15210 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -442,8 +442,8 @@ impl VirtualCPU { log_then_return!("Unexpected VM Exit {:?}", reason); } Ok(HyperlightExit::Retry()) => { - debug!("retrying vm run"); - continue; + println!("retrying vm run"); + continue } Err(e) => { #[cfg(crashdump)] diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index ad0928522..9c2ea981d 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -1095,6 +1095,7 @@ fn test_cpu_time_interrupt() { // Mark that we sent a kill signal BEFORE calling kill // to avoid race conditions + println!("Thread {} iteration {}: CPU time exceeded, sending kill signal", thread_id, iteration); was_killed_clone.store(true, Ordering::Release); interrupt_handle.kill(); break; From 137f0e81b61f1ed1b1ddfe88470865e017b668b1 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 17 Oct 2025 21:17:51 -0700 Subject: [PATCH 10/25] timing issue Signed-off-by: James Sturtevant --- src/hyperlight_host/src/hypervisor/hyperv_windows.rs | 9 +++------ src/hyperlight_host/src/hypervisor/mod.rs | 2 +- src/hyperlight_host/tests/integration_test.rs | 7 +++++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 7be1a93f3..0868df716 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -567,6 +567,7 @@ impl Hypervisor for HypervWindowsDriver { .load(Ordering::Relaxed) || debug_interrupt { + println!("already cancelled"); WHV_RUN_VP_EXIT_CONTEXT { ExitReason: WHV_RUN_VP_EXIT_REASON(8193i32), // WHvRunVpExitReasonCanceled VpContext: Default::default(), @@ -1015,13 +1016,9 @@ impl InterruptHandle for WindowsInterruptHandle { return false; } - // don't send the signal if the the vm isn't running - // In the case this is called before the vm is running the cancel_requested would be set - // and stay set while the vm is running. - + let running = self.running.load(Ordering::Relaxed); self.cancel_requested.store(true, Ordering::Relaxed); - self.running.load(Ordering::Relaxed) - && unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } + running && unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } } #[cfg(gdb)] fn kill_from_debugger(&self) -> bool { diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 042d15210..4b15d6fde 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -443,7 +443,7 @@ impl VirtualCPU { } Ok(HyperlightExit::Retry()) => { println!("retrying vm run"); - continue + continue; } Err(e) => { #[cfg(crashdump)] diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 9c2ea981d..40e846155 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -1019,7 +1019,7 @@ fn test_cpu_time_interrupt() { // main_thread_id is a HANDLE on Windows let thread_handle = main_thread_id as *mut c_void; - let cpu_limit_ns: i64 = 5_000_000; // 5ms CPU time limit (in nanoseconds) + let cpu_limit_ns: i64 = 1_000_000; // 5ms CPU time limit (in nanoseconds) let mut creation_time = MaybeUninit::::uninit(); @@ -1095,7 +1095,10 @@ fn test_cpu_time_interrupt() { // Mark that we sent a kill signal BEFORE calling kill // to avoid race conditions - println!("Thread {} iteration {}: CPU time exceeded, sending kill signal", thread_id, iteration); + println!( + "Thread {} iteration {}: CPU time exceeded, sending kill signal", + thread_id, iteration + ); was_killed_clone.store(true, Ordering::Release); interrupt_handle.kill(); break; From 1a85915f47b18d5aacb8938c997924255d010379 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 20 Oct 2025 15:09:53 +0100 Subject: [PATCH 11/25] fixes Signed-off-by: Simon Davies --- src/hyperlight_host/src/hypervisor/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 4b15d6fde..8691b518f 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -760,6 +760,12 @@ impl LinuxInterruptHandle { let mut target_generation: Option = None; loop { + + if (!self.call_active.load(Ordering::Acquire)) { + // No active call, so no need to send signal + break; + } + let (running, generation) = self.get_running_and_generation(); // Stamp generation into cancel_requested if requested and this is the first iteration @@ -802,6 +808,13 @@ impl LinuxInterruptHandle { #[cfg(any(kvm, mshv))] impl InterruptHandle for LinuxInterruptHandle { fn kill(&self) -> bool { + + if !(self.call_active.load(Ordering::Acquire)) + { + // No active call, so no effect + return false; + } + // send_signal will stamp the generation into cancel_requested // right before sending each signal, ensuring they're always in sync self.send_signal(true) From 68ce5f4e5527c79bc16ccdd40f6ef88a719ea824 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 20 Oct 2025 18:37:11 +0100 Subject: [PATCH 12/25] Use generation approach on windows Signed-off-by: Simon Davies --- src/hyperlight_host/Cargo.toml | 6 +- .../src/hypervisor/hyperv_windows.rs | 160 +++++++++++++++--- src/hyperlight_host/src/hypervisor/mod.rs | 84 +++++++-- .../src/sandbox/initialized_multi_use.rs | 9 +- src/hyperlight_host/tests/integration_test.rs | 151 +++++++++-------- 5 files changed, 289 insertions(+), 121 deletions(-) diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 95d1c6f53..5b75a1e07 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -67,7 +67,7 @@ windows = { version = "0.62", features = [ "Win32_System_JobObjects", "Win32_System_SystemServices", ] } -windows-sys = { version = "0.61", features = ["Win32", "Win32_System", "Win32_System_Threading"] } +windows-sys = { version = "0.61", features = ["Win32", "Win32_System", "Win32_System_Performance", "Win32_System_WindowsProgramming"] } windows-result = "0.4" rust-embed = { version = "8.8.0", features = ["debug-embed", "include-exclude", "interpolate-folder-path"] } sha256 = "1.6.0" @@ -79,8 +79,8 @@ kvm-bindings = { version = "0.14", features = ["fam-wrappers"], optional = true kvm-ioctls = { version = "0.24", optional = true } mshv-bindings2 = { package="mshv-bindings", version = "=0.2.1", optional = true } mshv-ioctls2 = { package="mshv-ioctls", version = "=0.2.1", optional = true} -mshv-bindings3 = { package="mshv-bindings", version = "0.6.1", optional = true } -mshv-ioctls3 = { package="mshv-ioctls", version = "0.6.1", optional = true} +mshv-bindings3 = { package="mshv-bindings", version = "0.6", optional = true } +mshv-ioctls3 = { package="mshv-ioctls", version = "0.6", optional = true} [dev-dependencies] uuid = { version = "1.18.1", features = ["v4"] } diff --git a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs index 0868df716..357bc7723 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_windows.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_windows.rs @@ -17,7 +17,7 @@ limitations under the License. use std::fmt; use std::fmt::{Debug, Formatter}; use std::string::String; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use log::LevelFilter; @@ -327,8 +327,8 @@ impl HypervWindowsDriver { }; let interrupt_handle = Arc::new(WindowsInterruptHandle { - running: AtomicBool::new(false), - cancel_requested: AtomicBool::new(false), + running: AtomicU64::new(0), + cancel_requested: AtomicU64::new(0), #[cfg(gdb)] debug_interrupt: AtomicBool::new(false), call_active: AtomicBool::new(false), @@ -550,7 +550,8 @@ impl Hypervisor for HypervWindowsDriver { &mut self, #[cfg(feature = "trace_guest")] tc: &mut crate::sandbox::trace::TraceContext, ) -> Result { - self.interrupt_handle.running.store(true, Ordering::Relaxed); + // Get current generation and set running bit + let generation = self.interrupt_handle.set_running_bit(); #[cfg(not(gdb))] let debug_interrupt = false; @@ -560,14 +561,12 @@ impl Hypervisor for HypervWindowsDriver { .debug_interrupt .load(Ordering::Relaxed); - // Don't run the vcpu if `cancel_requested` is true + // Check if cancellation was requested for THIS generation let exit_context = if self .interrupt_handle - .cancel_requested - .load(Ordering::Relaxed) + .is_cancel_requested_for_generation(generation) || debug_interrupt { - println!("already cancelled"); WHV_RUN_VP_EXIT_CONTEXT { ExitReason: WHV_RUN_VP_EXIT_REASON(8193i32), // WHvRunVpExitReasonCanceled VpContext: Default::default(), @@ -580,16 +579,22 @@ impl Hypervisor for HypervWindowsDriver { self.processor.run()? }; + + // Clear running bit + self.interrupt_handle.clear_running_bit(); + + let exit_reason = exit_context.ExitReason; + let is_canceled = exit_reason.0 == 8193; // WHvRunVpExitReasonCanceled + + // Check if this was a manual cancellation (vs internal Windows cancellation) let cancel_was_requested_manually = self .interrupt_handle - .cancel_requested - .load(Ordering::Relaxed); - self.interrupt_handle - .cancel_requested - .store(false, Ordering::Relaxed); - self.interrupt_handle - .running - .store(false, Ordering::Relaxed); + .is_cancel_requested_for_generation(generation); + + // Only clear cancel_requested if we're actually processing a cancellation for this generation + if is_canceled && cancel_was_requested_manually { + self.interrupt_handle.clear_cancel_requested(); + } #[cfg(gdb)] let debug_interrupt = self @@ -988,9 +993,23 @@ impl Drop for HypervWindowsDriver { #[derive(Debug)] pub struct WindowsInterruptHandle { - // `WHvCancelRunVirtualProcessor()` will return Ok even if the vcpu is not running, which is the reason we need this flag. - running: AtomicBool, - cancel_requested: AtomicBool, + /// Combined running flag (bit 63) and generation counter (bits 0-62). + /// + /// The generation increments with each guest function call to prevent + /// stale cancellations from affecting new calls (ABA problem). + /// + /// Layout: `[running:1 bit][generation:63 bits]` + running: AtomicU64, + + /// Combined cancel_requested flag (bit 63) and generation counter (bits 0-62). + /// + /// When kill() is called, this stores the current generation along with + /// the cancellation flag. The VCPU only honors the cancellation if the + /// generation matches its current generation. + /// + /// Layout: `[cancel_requested:1 bit][generation:63 bits]` + cancel_requested: AtomicU64, + // This is used to signal the GDB thread to stop the vCPU #[cfg(gdb)] debug_interrupt: AtomicBool, @@ -1009,6 +1028,78 @@ pub struct WindowsInterruptHandle { dropped: AtomicBool, } +impl WindowsInterruptHandle { + const RUNNING_BIT: u64 = 1 << 63; + const MAX_GENERATION: u64 = Self::RUNNING_BIT - 1; + const CANCEL_REQUESTED_BIT: u64 = 1 << 63; + + /// Set cancel_requested to true with the given generation + fn set_cancel_requested(&self, generation: u64) { + let value = Self::CANCEL_REQUESTED_BIT | (generation & Self::MAX_GENERATION); + self.cancel_requested.store(value, Ordering::Release); + } + + /// Clear cancel_requested (reset to no cancellation) + pub(crate) fn clear_cancel_requested(&self) { + self.cancel_requested.store(0, Ordering::Release); + } + + /// Check if cancel_requested is set for the given generation + fn is_cancel_requested_for_generation(&self, generation: u64) -> bool { + let raw = self.cancel_requested.load(Ordering::Acquire); + let is_set = raw & Self::CANCEL_REQUESTED_BIT != 0; + let stored_generation = raw & Self::MAX_GENERATION; + is_set && stored_generation == generation + } + + /// Increment the generation for a new guest function call + pub(crate) fn increment_generation(&self) -> u64 { + self.running + .fetch_update(Ordering::Release, Ordering::Acquire, |raw| { + let current_generation = raw & !Self::RUNNING_BIT; + let running_bit = raw & Self::RUNNING_BIT; + if current_generation == Self::MAX_GENERATION { + // Wrap around to 0 + return Some(running_bit); + } + Some((current_generation + 1) | running_bit) + }) + .map(|raw| { + let old_gen = raw & !Self::RUNNING_BIT; + if old_gen == Self::MAX_GENERATION { + 0 + } else { + old_gen + 1 + } + }) + .unwrap_or(0) + } + + /// Set running bit to true, return current generation + fn set_running_bit(&self) -> u64 { + self.running + .fetch_update(Ordering::Release, Ordering::Acquire, |raw| { + Some(raw | Self::RUNNING_BIT) + }) + .map(|raw| raw & !Self::RUNNING_BIT) + .unwrap_or(0) + } + + /// Clear running bit, return current generation + fn clear_running_bit(&self) -> u64 { + self.running + .fetch_and(!Self::RUNNING_BIT, Ordering::Relaxed) + & !Self::RUNNING_BIT + } + + fn get_running_and_generation(&self) -> (bool, u64) { + let raw = self.running.load(Ordering::Relaxed); + let running = raw & Self::RUNNING_BIT != 0; + let generation = raw & !Self::RUNNING_BIT; + (running, generation) + } +} + impl InterruptHandle for WindowsInterruptHandle { fn kill(&self) -> bool { // Check if a call is actually active first @@ -1016,26 +1107,43 @@ impl InterruptHandle for WindowsInterruptHandle { return false; } - let running = self.running.load(Ordering::Relaxed); - self.cancel_requested.store(true, Ordering::Relaxed); + // Get the current running state and generation + let (running, generation) = self.get_running_and_generation(); + + // Set cancel_requested with the current generation + self.set_cancel_requested(generation); + + // Only call WHvCancelRunVirtualProcessor if VCPU is actually running in guest mode running && unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } } #[cfg(gdb)] fn kill_from_debugger(&self) -> bool { self.debug_interrupt.store(true, Ordering::Relaxed); - self.running.load(Ordering::Relaxed) - && unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } + let (running, _) = self.get_running_and_generation(); + running && unsafe { WHvCancelRunVirtualProcessor(self.partition_handle, 0, 0).is_ok() } } fn dropped(&self) -> bool { - self.dropped.load(Ordering::Relaxed) + (self as &dyn InterruptHandle).dropped_impl() } fn set_call_active(&self) { - self.call_active.store(true, Ordering::Release); + (self as &dyn InterruptHandle).set_call_active_impl() } fn clear_call_active(&self) { - self.call_active.store(false, Ordering::Release); + (self as &dyn InterruptHandle).clear_call_active_impl() + } + + fn get_call_active(&self) -> &AtomicBool { + &self.call_active + } + + fn get_dropped(&self) -> &AtomicBool { + &self.dropped + } + + fn increment_generation_internal(&self) -> u64 { + self.increment_generation() } } diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 8691b518f..20226570a 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -546,11 +546,12 @@ pub trait InterruptHandle: Debug + Send + Sync { /// Mark that a guest function call is starting. /// - /// Sets the call_active flag to true, indicating that a guest function call - /// is now in progress. This allows kill() to stamp cancel_requested. + /// Increments the generation counter and sets the call_active flag to true, + /// indicating that a guest function call is now in progress. This allows + /// kill() to stamp cancel_requested with the correct generation. /// - /// Must be called immediately after increment_call_generation() and before - /// any VCPU execution begins. + /// Must be called at the start of call_guest_function_by_name_no_reset(), + /// before any VCPU execution begins. fn set_call_active(&self); /// Mark that a guest function call has completed. @@ -561,6 +562,54 @@ pub trait InterruptHandle: Debug + Send + Sync { /// Must be called at the end of call_guest_function_by_name_no_reset(), /// after the guest call has fully completed (whether successfully or with error). fn clear_call_active(&self); + + /// Returns the call_active atomic bool reference for default implementations. + /// + /// This is used by default trait methods to access the common call_active field. + fn get_call_active(&self) -> &std::sync::atomic::AtomicBool; + + /// Returns the dropped atomic bool reference for default implementations. + /// + /// This is used by default trait methods to access the common dropped field. + fn get_dropped(&self) -> &std::sync::atomic::AtomicBool; + + /// Internal method to increment the generation counter. + /// + /// Returns the new generation value after incrementing. + #[cfg(any(kvm, mshv, target_os = "windows"))] + fn increment_generation_internal(&self) -> u64; +} + +// Default implementations for common methods +impl dyn InterruptHandle { + /// Default implementation of dropped() - checks the dropped atomic flag. + /// + /// Both Windows and Linux implementations are identical, so this is provided + /// as a trait default to reduce code duplication. + pub fn dropped_impl(&self) -> bool { + self.get_dropped() + .load(std::sync::atomic::Ordering::Relaxed) + } + + /// Default implementation of clear_call_active() - clears the call_active flag. + /// + /// Both Windows and Linux implementations are identical, so this is provided + /// as a trait default to reduce code duplication. + pub fn clear_call_active_impl(&self) { + self.get_call_active() + .store(false, std::sync::atomic::Ordering::Release) + } + + /// Default implementation of set_call_active() - increments generation and sets flag. + /// + /// Both Windows and Linux implementations are identical, so this is provided + /// as a trait default to reduce code duplication. + #[cfg(any(kvm, mshv, target_os = "windows"))] + pub fn set_call_active_impl(&self) { + self.increment_generation_internal(); + self.get_call_active() + .store(true, std::sync::atomic::Ordering::Release) + } } #[cfg(any(kvm, mshv))] @@ -760,8 +809,7 @@ impl LinuxInterruptHandle { let mut target_generation: Option = None; loop { - - if (!self.call_active.load(Ordering::Acquire)) { + if !self.call_active.load(Ordering::Acquire) { // No active call, so no need to send signal break; } @@ -808,9 +856,7 @@ impl LinuxInterruptHandle { #[cfg(any(kvm, mshv))] impl InterruptHandle for LinuxInterruptHandle { fn kill(&self) -> bool { - - if !(self.call_active.load(Ordering::Acquire)) - { + if !(self.call_active.load(Ordering::Acquire)) { // No active call, so no effect return false; } @@ -824,8 +870,9 @@ impl InterruptHandle for LinuxInterruptHandle { self.debug_interrupt.store(true, Ordering::Relaxed); self.send_signal(false) } + fn dropped(&self) -> bool { - self.dropped.load(Ordering::Relaxed) + (self as &dyn InterruptHandle).dropped_impl() } #[cfg(any(kvm, mshv))] @@ -835,12 +882,25 @@ impl InterruptHandle for LinuxInterruptHandle { #[cfg(any(kvm, mshv))] fn set_call_active(&self) { - self.call_active.store(true, Ordering::Release); + (self as &dyn InterruptHandle).set_call_active_impl() } #[cfg(any(kvm, mshv))] fn clear_call_active(&self) { - self.call_active.store(false, Ordering::Release); + (self as &dyn InterruptHandle).clear_call_active_impl() + } + + fn get_call_active(&self) -> &AtomicBool { + &self.call_active + } + + fn get_dropped(&self) -> &AtomicBool { + &self.dropped + } + + #[cfg(any(kvm, mshv))] + fn increment_generation_internal(&self) -> u64 { + self.increment_generation() } } diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index 20188802d..f20991de1 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -390,11 +390,8 @@ impl MultiUseSandbox { return_type: ReturnType, args: Vec, ) -> Result { - // Increment generation for this guest function call (Linux only) - #[cfg(any(kvm, mshv))] - let _generation = self.vm.interrupt_handle().increment_call_generation(); - // Mark that a guest function call is now active + // (This also increments the generation counter internally) self.vm.interrupt_handle().set_call_active(); let res = (|| { @@ -412,10 +409,6 @@ impl MultiUseSandbox { self.mem_mgr.write_guest_function_call(buffer)?; - // Increment generation for this guest function call (Linux only) - #[cfg(any(kvm, mshv))] - self.vm.interrupt_handle().increment_call_generation(); - self.vm.dispatch_call_from_host( self.dispatch_ptr.clone(), #[cfg(gdb)] diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 40e846155..95adfbf70 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -160,29 +160,47 @@ fn interrupt_same_thread() { const NUM_ITERS: usize = 500; - // kill vm after 1 second + // Spawn killer thread that attempts to interrupt sbox2 let thread = thread::spawn(move || { for _ in 0..NUM_ITERS { - barrier2.wait(); interrupt_handle.kill(); + // Sync at END of iteration to ensure main thread completes its work + barrier2.wait(); } }); - for _ in 0..NUM_ITERS { - barrier.wait(); + for i in 0..NUM_ITERS { + // Call sbox1 - should never be interrupted sbox1 .call::("Echo", "hello".to_string()) - .expect("Only sandbox 2 is allowed to be interrupted"); + .unwrap_or_else(|e| { + panic!( + "Iteration {}: sbox1 should not be interrupted, got: {:?}", + i, e + ) + }); + + // Call sbox2 - may be interrupted match sbox2.call::("Echo", "hello".to_string()) { Ok(_) | Err(HyperlightError::ExecutionCanceledByHost()) => { // Only allow successful calls or interrupted. - // The call can be successful in case the call is finished before kill() is called. + // The call can be successful if it completes before kill() is called. } - _ => panic!("Unexpected return"), + Err(e) => panic!("Iteration {}: sbox2 unexpected error: {:?}", i, e), }; + + // Call sbox3 - should never be interrupted sbox3 .call::("Echo", "hello".to_string()) - .expect("Only sandbox 2 is allowed to be interrupted"); + .unwrap_or_else(|e| { + panic!( + "Iteration {}: sbox3 should not be interrupted, got: {:?}", + i, e + ) + }); + + // Sync at END of iteration to ensure killer thread waits for work to complete + barrier.wait(); } thread.join().expect("Thread should finish"); } @@ -204,31 +222,51 @@ fn interrupt_same_thread_no_barrier() { const NUM_ITERS: usize = 500; - // kill vm after 1 second + // Spawn killer thread that continuously attempts to interrupt sbox2 let thread = thread::spawn(move || { barrier2.wait(); - while !workload_done2.load(Ordering::Relaxed) { + // Use Acquire ordering to ensure we see the updated workload_done flag + while !workload_done2.load(Ordering::Acquire) { interrupt_handle.kill(); + // Small yield to prevent tight spinning + thread::yield_now(); } }); barrier.wait(); - for _ in 0..NUM_ITERS { + for i in 0..NUM_ITERS { + // Call sbox1 - should never be interrupted sbox1 .call::("Echo", "hello".to_string()) - .expect("Only sandbox 2 is allowed to be interrupted"); + .unwrap_or_else(|e| { + panic!( + "Iteration {}: sbox1 should not be interrupted, got: {:?}", + i, e + ) + }); + + // Call sbox2 - may be interrupted match sbox2.call::("Echo", "hello".to_string()) { Ok(_) | Err(HyperlightError::ExecutionCanceledByHost()) => { // Only allow successful calls or interrupted. - // The call can be successful in case the call is finished before kill() is called. + // The call can be successful if it completes before kill() is called. } - _ => panic!("Unexpected return"), + Err(e) => panic!("Iteration {}: sbox2 unexpected error: {:?}", i, e), }; + + // Call sbox3 - should never be interrupted sbox3 .call::("Echo", "hello".to_string()) - .expect("Only sandbox 2 is allowed to be interrupted"); + .unwrap_or_else(|e| { + panic!( + "Iteration {}: sbox3 should not be interrupted, got: {:?}", + i, e + ) + }); } - workload_done.store(true, Ordering::Relaxed); + + // Use Release ordering to ensure killer thread sees the update + workload_done.store(true, Ordering::Release); thread.join().expect("Thread should finish"); } @@ -868,7 +906,6 @@ fn test_if_guest_is_able_to_get_string_return_values_from_host() { #[test] fn test_cpu_time_interrupt() { use std::collections::VecDeque; - use std::mem::MaybeUninit; use std::sync::Mutex; use std::sync::atomic::AtomicUsize; use std::sync::mpsc::channel; @@ -1015,41 +1052,35 @@ fn test_cpu_time_interrupt() { unsafe { use std::ffi::c_void; - // On Windows, we use GetThreadTimes to get CPU time - // main_thread_id is a HANDLE on Windows - let thread_handle = main_thread_id as *mut c_void; + // On Windows, use QueryThreadCycleTime for high-resolution CPU time measurement + // This measures actual CPU cycles consumed by the thread, similar to Linux's + // pthread_getcpuclockid, and is much more accurate than GetThreadTimes (~15ms resolution) - let cpu_limit_ns: i64 = 1_000_000; // 5ms CPU time limit (in nanoseconds) + let thread_handle = main_thread_id as *mut c_void; - let mut creation_time = - MaybeUninit::::uninit(); - let mut exit_time = - MaybeUninit::::uninit(); - let mut kernel_time_start = - MaybeUninit::::uninit(); - let mut user_time_start = - MaybeUninit::::uninit(); + // Get the CPU frequency to convert cycles to time + let mut frequency: i64 = 0; + if windows_sys::Win32::System::Performance::QueryPerformanceFrequency( + &mut frequency, + ) == 0 + { + return; + } - // Get initial CPU times - if windows_sys::Win32::System::Threading::GetThreadTimes( + // Get starting CPU cycles for this thread + let mut start_cycles: u64 = 0; + if windows_sys::Win32::System::WindowsProgramming::QueryThreadCycleTime( thread_handle, - creation_time.as_mut_ptr(), - exit_time.as_mut_ptr(), - kernel_time_start.as_mut_ptr(), - user_time_start.as_mut_ptr(), + &mut start_cycles, ) == 0 { return; } - let kernel_time_start = kernel_time_start.assume_init(); - let user_time_start = user_time_start.assume_init(); - - // Convert FILETIME to u64 (100-nanosecond intervals) - let start_cpu_time = ((kernel_time_start.dwHighDateTime as u64) << 32 - | kernel_time_start.dwLowDateTime as u64) - + ((user_time_start.dwHighDateTime as u64) << 32 - | user_time_start.dwLowDateTime as u64); + // Convert 5ms CPU limit to approximate cycle count + // This is approximate because CPU frequency can vary with power management, + // but gives us a baseline that's much more accurate than GetThreadTimes + let cpu_limit_cycles = (5_000_000u64 * frequency as u64) / 1_000_000_000; loop { // Check if we should stop monitoring (guest completed) @@ -1057,48 +1088,24 @@ fn test_cpu_time_interrupt() { break; } - let mut kernel_time_current = - MaybeUninit::::uninit(); - let mut user_time_current = - MaybeUninit::::uninit(); - - if windows_sys::Win32::System::Threading::GetThreadTimes( + let mut current_cycles: u64 = 0; + if windows_sys::Win32::System::WindowsProgramming::QueryThreadCycleTime( thread_handle, - creation_time.as_mut_ptr(), - exit_time.as_mut_ptr(), - kernel_time_current.as_mut_ptr(), - user_time_current.as_mut_ptr(), + &mut current_cycles, ) == 0 { break; } - let kernel_time_current = kernel_time_current.assume_init(); - let user_time_current = user_time_current.assume_init(); - - // Convert FILETIME to u64 - let current_cpu_time = ((kernel_time_current.dwHighDateTime as u64) - << 32 - | kernel_time_current.dwLowDateTime as u64) - + ((user_time_current.dwHighDateTime as u64) << 32 - | user_time_current.dwLowDateTime as u64); - - // FILETIME is in 100-nanosecond intervals, convert to nanoseconds - let elapsed_ns = ((current_cpu_time - start_cpu_time) * 100) as i64; + let elapsed_cycles = current_cycles - start_cycles; - if elapsed_ns > cpu_limit_ns { + if elapsed_cycles > cpu_limit_cycles { // Double-check that monitoring should still continue before killing - // The guest might have completed between our last check and now if stop_monitoring_clone.load(Ordering::Acquire) { break; } // Mark that we sent a kill signal BEFORE calling kill - // to avoid race conditions - println!( - "Thread {} iteration {}: CPU time exceeded, sending kill signal", - thread_id, iteration - ); was_killed_clone.store(true, Ordering::Release); interrupt_handle.kill(); break; From 5a5f4b175bb3b08438551d670d8062f3d768c920 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 20 Oct 2025 21:59:05 +0100 Subject: [PATCH 13/25] update comments Signed-off-by: Simon Davies --- src/hyperlight_host/src/hypervisor/mod.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 20226570a..209d90a7b 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -584,8 +584,6 @@ pub trait InterruptHandle: Debug + Send + Sync { impl dyn InterruptHandle { /// Default implementation of dropped() - checks the dropped atomic flag. /// - /// Both Windows and Linux implementations are identical, so this is provided - /// as a trait default to reduce code duplication. pub fn dropped_impl(&self) -> bool { self.get_dropped() .load(std::sync::atomic::Ordering::Relaxed) @@ -593,8 +591,6 @@ impl dyn InterruptHandle { /// Default implementation of clear_call_active() - clears the call_active flag. /// - /// Both Windows and Linux implementations are identical, so this is provided - /// as a trait default to reduce code duplication. pub fn clear_call_active_impl(&self) { self.get_call_active() .store(false, std::sync::atomic::Ordering::Release) @@ -602,9 +598,6 @@ impl dyn InterruptHandle { /// Default implementation of set_call_active() - increments generation and sets flag. /// - /// Both Windows and Linux implementations are identical, so this is provided - /// as a trait default to reduce code duplication. - #[cfg(any(kvm, mshv, target_os = "windows"))] pub fn set_call_active_impl(&self) { self.increment_generation_internal(); self.get_call_active() From 1195f67df22f63971b224d8f371d10b04e00fa17 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 20 Oct 2025 22:15:41 +0100 Subject: [PATCH 14/25] update test to select tests correctly Signed-off-by: Simon Davies --- src/hyperlight_host/src/mem/shared_mem.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/src/mem/shared_mem.rs b/src/hyperlight_host/src/mem/shared_mem.rs index 030c2c958..76d2af5c0 100644 --- a/src/hyperlight_host/src/mem/shared_mem.rs +++ b/src/hyperlight_host/src/mem/shared_mem.rs @@ -1238,9 +1238,9 @@ mod tests { vec![] }; let status = std::process::Command::new("cargo") - .args(["test", "-p", "hyperlight-host"]) + .args(["test", "-p", "hyperlight-host", "--lib"]) .args(target_args) - .args(["--", "--ignored", test]) + .args(["--", "--ignored", &format!("guard_page_crash_test::{}", test)]) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) From 17bdb00ddc0495e8994798706b89b978f71cadf3 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 20 Oct 2025 22:17:42 +0100 Subject: [PATCH 15/25] fix clippy Signed-off-by: Simon Davies --- src/hyperlight_host/tests/integration_test.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 95adfbf70..1070f33ac 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -910,6 +910,9 @@ fn test_cpu_time_interrupt() { use std::sync::atomic::AtomicUsize; use std::sync::mpsc::channel; + #[cfg(target_os = "linux")] + use std::mem::MaybeUninit; + const POOL_SIZE: usize = 100; const NUM_THREADS: usize = 100; const ITERATIONS_PER_THREAD: usize = 500; @@ -1001,6 +1004,7 @@ fn test_cpu_time_interrupt() { #[cfg(target_os = "linux")] unsafe { + let mut clock_id: libc::clockid_t = 0; if libc::pthread_getcpuclockid(main_thread_id, &mut clock_id) != 0 { return; From 49e562924126b7ae2a97d6e65a724234032f72a7 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 20 Oct 2025 22:25:22 +0100 Subject: [PATCH 16/25] fix fmt Signed-off-by: Simon Davies --- src/hyperlight_host/src/mem/shared_mem.rs | 6 +++++- src/hyperlight_host/tests/integration_test.rs | 6 ++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/hyperlight_host/src/mem/shared_mem.rs b/src/hyperlight_host/src/mem/shared_mem.rs index 76d2af5c0..5b5c148cb 100644 --- a/src/hyperlight_host/src/mem/shared_mem.rs +++ b/src/hyperlight_host/src/mem/shared_mem.rs @@ -1240,7 +1240,11 @@ mod tests { let status = std::process::Command::new("cargo") .args(["test", "-p", "hyperlight-host", "--lib"]) .args(target_args) - .args(["--", "--ignored", &format!("guard_page_crash_test::{}", test)]) + .args([ + "--", + "--ignored", + &format!("guard_page_crash_test::{}", test), + ]) .stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 1070f33ac..6a168e3d2 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -906,13 +906,12 @@ fn test_if_guest_is_able_to_get_string_return_values_from_host() { #[test] fn test_cpu_time_interrupt() { use std::collections::VecDeque; + #[cfg(target_os = "linux")] + use std::mem::MaybeUninit; use std::sync::Mutex; use std::sync::atomic::AtomicUsize; use std::sync::mpsc::channel; - #[cfg(target_os = "linux")] - use std::mem::MaybeUninit; - const POOL_SIZE: usize = 100; const NUM_THREADS: usize = 100; const ITERATIONS_PER_THREAD: usize = 500; @@ -1004,7 +1003,6 @@ fn test_cpu_time_interrupt() { #[cfg(target_os = "linux")] unsafe { - let mut clock_id: libc::clockid_t = 0; if libc::pthread_getcpuclockid(main_thread_id, &mut clock_id) != 0 { return; From 17ce304e2403b414d65973c163fb96c716d1b7b6 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 20 Oct 2025 23:14:57 +0100 Subject: [PATCH 17/25] fix windows test Signed-off-by: Simon Davies --- src/hyperlight_host/Cargo.toml | 2 +- src/hyperlight_host/tests/integration_test.rs | 29 +++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 5b75a1e07..854d557da 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -67,7 +67,7 @@ windows = { version = "0.62", features = [ "Win32_System_JobObjects", "Win32_System_SystemServices", ] } -windows-sys = { version = "0.61", features = ["Win32", "Win32_System", "Win32_System_Performance", "Win32_System_WindowsProgramming"] } +windows-sys = { version = "0.61", features = ["Win32", "Win32_System", "Win32_System_Performance", "Win32_System_WindowsProgramming", "Win32_Foundation"] } windows-result = "0.4" rust-embed = { version = "8.8.0", features = ["debug-embed", "include-exclude", "interpolate-folder-path"] } sha256 = "1.6.0" diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 6a168e3d2..08c821ded 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -1115,6 +1115,9 @@ fn test_cpu_time_interrupt() { thread::sleep(Duration::from_micros(50)); } + + // Clean up the duplicated thread handle + windows_sys::Win32::Foundation::CloseHandle(thread_handle); } }); @@ -1127,9 +1130,29 @@ fn test_cpu_time_interrupt() { #[cfg(target_os = "windows")] unsafe { - // On Windows, get the current thread's pseudo-handle - let thread_handle = windows_sys::Win32::System::Threading::GetCurrentThread(); - let _ = tx.send(thread_handle as usize); + use std::ffi::c_void; + // On Windows, we need a REAL thread handle, not the pseudo-handle from GetCurrentThread() + // GetCurrentThread() returns a constant (-2) that only works in the current thread's context + // We must duplicate it to get a real handle that can be used from another thread + let pseudo_handle = windows_sys::Win32::System::Threading::GetCurrentThread(); + let current_process = + windows_sys::Win32::System::Threading::GetCurrentProcess(); + let mut real_handle: *mut c_void = std::ptr::null_mut(); + + if windows_sys::Win32::Foundation::DuplicateHandle( + current_process, + pseudo_handle, + current_process, + &mut real_handle, + 0, + 0, // bInheritHandle = FALSE + windows_sys::Win32::Foundation::DUPLICATE_SAME_ACCESS, + ) == 0 + { + panic!("Failed to duplicate thread handle"); + } + + let _ = tx.send(real_handle as usize); } should_monitor.store(true, Ordering::Release); From dfe5ce85896726bae8846f1ff6d71c323b622e43 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 20 Oct 2025 23:22:58 +0100 Subject: [PATCH 18/25] fix fmt Signed-off-by: Simon Davies --- src/hyperlight_host/tests/integration_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 08c821ded..0653a4310 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -1115,7 +1115,7 @@ fn test_cpu_time_interrupt() { thread::sleep(Duration::from_micros(50)); } - + // Clean up the duplicated thread handle windows_sys::Win32::Foundation::CloseHandle(thread_handle); } From 12f7d09f521369116ec05c3235dbc7ac24779796 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Tue, 21 Oct 2025 09:30:35 +0100 Subject: [PATCH 19/25] Add some diagnostics Signed-off-by: Simon Davies --- Justfile | 2 +- src/hyperlight_host/src/hypervisor/mod.rs | 2 +- src/hyperlight_host/tests/integration_test.rs | 90 +++++++++++++++++-- 3 files changed, 86 insertions(+), 8 deletions(-) diff --git a/Justfile b/Justfile index 67925f662..21ffc0d89 100644 --- a/Justfile +++ b/Justfile @@ -168,7 +168,7 @@ test-integration guest target=default-target features="": {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} {{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} --test integration_test execute_on_heap {{ if features =="" {""} else {"--features " + features} }} -- --ignored @# run the rest of the integration tests - {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} {{ cargo-cmd }} test -p hyperlight-host {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} --test '*' + {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} {{ cargo-cmd }} test -p hyperlight-host {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F init-paging," + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} --test '*' -- --nocapture # tests compilation with no default features on different platforms test-compilation-no-default-features target=default-target: diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index 209d90a7b..da3d707e1 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -442,7 +442,7 @@ impl VirtualCPU { log_then_return!("Unexpected VM Exit {:?}", reason); } Ok(HyperlightExit::Retry()) => { - println!("retrying vm run"); + eprintln!("[VCPU] Retry - continuing VM run loop"); continue; } Err(e) => { diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 0653a4310..652b5e58b 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -148,9 +148,11 @@ fn interrupt_guest_call_in_advance() { /// all possible interleavings, but can hopefully increases confidence somewhat. #[test] fn interrupt_same_thread() { + eprintln!("[TEST] interrupt_same_thread starting"); let mut sbox1: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); let mut sbox2: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); let mut sbox3: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); + eprintln!("[TEST] Created 3 sandboxes"); let barrier = Arc::new(Barrier::new(2)); let barrier2 = barrier.clone(); @@ -162,47 +164,71 @@ fn interrupt_same_thread() { // Spawn killer thread that attempts to interrupt sbox2 let thread = thread::spawn(move || { - for _ in 0..NUM_ITERS { - interrupt_handle.kill(); + eprintln!("[KILLER] Thread starting"); + for i in 0..NUM_ITERS { + eprintln!("[KILLER] Iteration {}/{}: calling kill()", i, NUM_ITERS); + let killed = interrupt_handle.kill(); + eprintln!("[KILLER] Iteration {}: kill() returned {}", i, killed); // Sync at END of iteration to ensure main thread completes its work + eprintln!("[KILLER] Iteration {}: waiting at barrier", i); barrier2.wait(); + eprintln!("[KILLER] Iteration {}: passed barrier", i); } + eprintln!("[KILLER] Thread exiting after {} iterations", NUM_ITERS); }); for i in 0..NUM_ITERS { + eprintln!("[MAIN] Iteration {}/{}: starting", i, NUM_ITERS); + // Call sbox1 - should never be interrupted + eprintln!("[MAIN] Iteration {}: calling sbox1", i); sbox1 .call::("Echo", "hello".to_string()) .unwrap_or_else(|e| { + eprintln!("[MAIN] ERROR: sbox1 interrupted at iteration {}: {:?}", i, e); panic!( "Iteration {}: sbox1 should not be interrupted, got: {:?}", i, e ) }); + eprintln!("[MAIN] Iteration {}: sbox1 completed", i); // Call sbox2 - may be interrupted + eprintln!("[MAIN] Iteration {}: calling sbox2", i); match sbox2.call::("Echo", "hello".to_string()) { - Ok(_) | Err(HyperlightError::ExecutionCanceledByHost()) => { - // Only allow successful calls or interrupted. - // The call can be successful if it completes before kill() is called. + Ok(_) => { + eprintln!("[MAIN] Iteration {}: sbox2 completed successfully", i); + } + Err(HyperlightError::ExecutionCanceledByHost()) => { + eprintln!("[MAIN] Iteration {}: sbox2 was cancelled", i); + } + Err(e) => { + eprintln!("[MAIN] ERROR: sbox2 unexpected error at iteration {}: {:?}", i, e); + panic!("Iteration {}: sbox2 unexpected error: {:?}", i, e); } - Err(e) => panic!("Iteration {}: sbox2 unexpected error: {:?}", i, e), }; // Call sbox3 - should never be interrupted + eprintln!("[MAIN] Iteration {}: calling sbox3", i); sbox3 .call::("Echo", "hello".to_string()) .unwrap_or_else(|e| { + eprintln!("[MAIN] ERROR: sbox3 interrupted at iteration {}: {:?}", i, e); panic!( "Iteration {}: sbox3 should not be interrupted, got: {:?}", i, e ) }); + eprintln!("[MAIN] Iteration {}: sbox3 completed", i); // Sync at END of iteration to ensure killer thread waits for work to complete + eprintln!("[MAIN] Iteration {}: waiting at barrier", i); barrier.wait(); + eprintln!("[MAIN] Iteration {}: passed barrier", i); } + eprintln!("[MAIN] All {} iterations complete, joining killer thread", NUM_ITERS); thread.join().expect("Thread should finish"); + eprintln!("[TEST] interrupt_same_thread completed successfully"); } /// Same test as above but with no per-iteration barrier, to get more possible interleavings. @@ -916,6 +942,7 @@ fn test_cpu_time_interrupt() { const NUM_THREADS: usize = 100; const ITERATIONS_PER_THREAD: usize = 500; + eprintln!("[TEST] Starting test_cpu_time_interrupt"); // Create a pool of 100 sandboxes println!("Creating pool of {} sandboxes...", POOL_SIZE); let mut sandbox_pool: Vec = Vec::with_capacity(POOL_SIZE); @@ -990,6 +1017,10 @@ fn test_cpu_time_interrupt() { let was_killed = Arc::new(AtomicBool::new(false)); let was_killed_clone = was_killed.clone(); + // Flag to signal that monitor thread is ready (entered its monitoring loop) + let monitor_ready = Arc::new(AtomicBool::new(false)); + let monitor_ready_clone = monitor_ready.clone(); + // Spawn CPU time monitor thread let monitor_thread = thread::spawn(move || { let main_thread_id = match rx.recv() { @@ -1016,6 +1047,9 @@ fn test_cpu_time_interrupt() { } let start_time = start_time.assume_init(); + // Signal that we're ready to monitor + monitor_ready_clone.store(true, Ordering::Release); + loop { // Check if we should stop monitoring (guest completed) if stop_monitoring_clone.load(Ordering::Acquire) { @@ -1054,11 +1088,14 @@ fn test_cpu_time_interrupt() { unsafe { use std::ffi::c_void; + eprintln!("[MONITOR] Windows monitoring thread starting"); + // On Windows, use QueryThreadCycleTime for high-resolution CPU time measurement // This measures actual CPU cycles consumed by the thread, similar to Linux's // pthread_getcpuclockid, and is much more accurate than GetThreadTimes (~15ms resolution) let thread_handle = main_thread_id as *mut c_void; + eprintln!("[MONITOR] Got thread handle: {:?}", thread_handle); // Get the CPU frequency to convert cycles to time let mut frequency: i64 = 0; @@ -1066,8 +1103,10 @@ fn test_cpu_time_interrupt() { &mut frequency, ) == 0 { + eprintln!("[MONITOR] ERROR: QueryPerformanceFrequency failed"); return; } + eprintln!("[MONITOR] CPU frequency: {} Hz", frequency); // Get starting CPU cycles for this thread let mut start_cycles: u64 = 0; @@ -1076,17 +1115,25 @@ fn test_cpu_time_interrupt() { &mut start_cycles, ) == 0 { + eprintln!("[MONITOR] ERROR: QueryThreadCycleTime (start) failed"); return; } + eprintln!("[MONITOR] Start cycles: {}", start_cycles); // Convert 5ms CPU limit to approximate cycle count // This is approximate because CPU frequency can vary with power management, // but gives us a baseline that's much more accurate than GetThreadTimes let cpu_limit_cycles = (5_000_000u64 * frequency as u64) / 1_000_000_000; + eprintln!("[MONITOR] CPU limit: {} cycles (~5ms)", cpu_limit_cycles); + eprintln!("[MONITOR] Entering monitoring loop"); + // Signal that we're ready to monitor + monitor_ready_clone.store(true, Ordering::Release); + let mut loop_count = 0u64; loop { // Check if we should stop monitoring (guest completed) if stop_monitoring_clone.load(Ordering::Acquire) { + eprintln!("[MONITOR] Stop monitoring flag set, exiting after {} loops", loop_count); break; } @@ -1096,28 +1143,41 @@ fn test_cpu_time_interrupt() { &mut current_cycles, ) == 0 { + eprintln!("[MONITOR] ERROR: QueryThreadCycleTime (current) failed at loop {}", loop_count); break; } let elapsed_cycles = current_cycles - start_cycles; + if loop_count < 3 || loop_count % 100 == 0 { + eprintln!("[MONITOR] Loop {}: elapsed_cycles={}, limit={}", loop_count, elapsed_cycles, cpu_limit_cycles); + } + if elapsed_cycles > cpu_limit_cycles { + eprintln!("[MONITOR] CPU limit exceeded at loop {}: {} > {}", loop_count, elapsed_cycles, cpu_limit_cycles); // Double-check that monitoring should still continue before killing if stop_monitoring_clone.load(Ordering::Acquire) { + eprintln!("[MONITOR] Stop flag already set, not killing"); break; } + eprintln!("[MONITOR] Calling kill()"); // Mark that we sent a kill signal BEFORE calling kill was_killed_clone.store(true, Ordering::Release); interrupt_handle.kill(); + eprintln!("[MONITOR] kill() returned"); break; } + loop_count += 1; thread::sleep(Duration::from_micros(50)); } + eprintln!("[MONITOR] Exited monitoring loop after {} iterations", loop_count); // Clean up the duplicated thread handle + eprintln!("[MONITOR] Closing thread handle"); windows_sys::Win32::Foundation::CloseHandle(thread_handle); + eprintln!("[MONITOR] Monitor thread exiting"); } }); @@ -1157,11 +1217,29 @@ fn test_cpu_time_interrupt() { should_monitor.store(true, Ordering::Release); + // Wait for monitor thread to be ready before starting guest execution + // This prevents the race where guest completes before monitor enters its loop + while !monitor_ready.load(Ordering::Acquire) { + thread::sleep(Duration::from_micros(10)); + } + + if iteration < 3 || iteration % 50 == 0 { + eprintln!("[THREAD-{}] Iteration {}: calling SpinForMs({}ms)", thread_id, iteration, cpu_time_ms); + } + // Call the guest function let result = sandbox.call::("SpinForMs", cpu_time_ms); + if iteration < 3 { + eprintln!("[THREAD-{}] Iteration {}: SpinForMs returned {:?}", thread_id, iteration, result); + } + // Signal the monitor to stop stop_monitoring.store(true, Ordering::Release); + + if iteration < 3 { + eprintln!("[THREAD-{}] Iteration {}: waiting for monitor thread", thread_id, iteration); + } // Wait for monitor thread to complete to ensure was_killed flag is set let _ = monitor_thread.join(); From 21aa96409fe1771b77a9df701f3e656dbd55e036 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Tue, 21 Oct 2025 09:38:34 +0100 Subject: [PATCH 20/25] fix fmt Signed-off-by: Simon Davies --- src/hyperlight_host/tests/integration_test.rs | 64 +++++++++++++++---- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 652b5e58b..069856735 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -179,13 +179,16 @@ fn interrupt_same_thread() { for i in 0..NUM_ITERS { eprintln!("[MAIN] Iteration {}/{}: starting", i, NUM_ITERS); - + // Call sbox1 - should never be interrupted eprintln!("[MAIN] Iteration {}: calling sbox1", i); sbox1 .call::("Echo", "hello".to_string()) .unwrap_or_else(|e| { - eprintln!("[MAIN] ERROR: sbox1 interrupted at iteration {}: {:?}", i, e); + eprintln!( + "[MAIN] ERROR: sbox1 interrupted at iteration {}: {:?}", + i, e + ); panic!( "Iteration {}: sbox1 should not be interrupted, got: {:?}", i, e @@ -203,7 +206,10 @@ fn interrupt_same_thread() { eprintln!("[MAIN] Iteration {}: sbox2 was cancelled", i); } Err(e) => { - eprintln!("[MAIN] ERROR: sbox2 unexpected error at iteration {}: {:?}", i, e); + eprintln!( + "[MAIN] ERROR: sbox2 unexpected error at iteration {}: {:?}", + i, e + ); panic!("Iteration {}: sbox2 unexpected error: {:?}", i, e); } }; @@ -213,7 +219,10 @@ fn interrupt_same_thread() { sbox3 .call::("Echo", "hello".to_string()) .unwrap_or_else(|e| { - eprintln!("[MAIN] ERROR: sbox3 interrupted at iteration {}: {:?}", i, e); + eprintln!( + "[MAIN] ERROR: sbox3 interrupted at iteration {}: {:?}", + i, e + ); panic!( "Iteration {}: sbox3 should not be interrupted, got: {:?}", i, e @@ -226,7 +235,10 @@ fn interrupt_same_thread() { barrier.wait(); eprintln!("[MAIN] Iteration {}: passed barrier", i); } - eprintln!("[MAIN] All {} iterations complete, joining killer thread", NUM_ITERS); + eprintln!( + "[MAIN] All {} iterations complete, joining killer thread", + NUM_ITERS + ); thread.join().expect("Thread should finish"); eprintln!("[TEST] interrupt_same_thread completed successfully"); } @@ -1133,7 +1145,10 @@ fn test_cpu_time_interrupt() { loop { // Check if we should stop monitoring (guest completed) if stop_monitoring_clone.load(Ordering::Acquire) { - eprintln!("[MONITOR] Stop monitoring flag set, exiting after {} loops", loop_count); + eprintln!( + "[MONITOR] Stop monitoring flag set, exiting after {} loops", + loop_count + ); break; } @@ -1143,18 +1158,27 @@ fn test_cpu_time_interrupt() { &mut current_cycles, ) == 0 { - eprintln!("[MONITOR] ERROR: QueryThreadCycleTime (current) failed at loop {}", loop_count); + eprintln!( + "[MONITOR] ERROR: QueryThreadCycleTime (current) failed at loop {}", + loop_count + ); break; } let elapsed_cycles = current_cycles - start_cycles; if loop_count < 3 || loop_count % 100 == 0 { - eprintln!("[MONITOR] Loop {}: elapsed_cycles={}, limit={}", loop_count, elapsed_cycles, cpu_limit_cycles); + eprintln!( + "[MONITOR] Loop {}: elapsed_cycles={}, limit={}", + loop_count, elapsed_cycles, cpu_limit_cycles + ); } if elapsed_cycles > cpu_limit_cycles { - eprintln!("[MONITOR] CPU limit exceeded at loop {}: {} > {}", loop_count, elapsed_cycles, cpu_limit_cycles); + eprintln!( + "[MONITOR] CPU limit exceeded at loop {}: {} > {}", + loop_count, elapsed_cycles, cpu_limit_cycles + ); // Double-check that monitoring should still continue before killing if stop_monitoring_clone.load(Ordering::Acquire) { eprintln!("[MONITOR] Stop flag already set, not killing"); @@ -1172,7 +1196,10 @@ fn test_cpu_time_interrupt() { loop_count += 1; thread::sleep(Duration::from_micros(50)); } - eprintln!("[MONITOR] Exited monitoring loop after {} iterations", loop_count); + eprintln!( + "[MONITOR] Exited monitoring loop after {} iterations", + loop_count + ); // Clean up the duplicated thread handle eprintln!("[MONITOR] Closing thread handle"); @@ -1224,21 +1251,30 @@ fn test_cpu_time_interrupt() { } if iteration < 3 || iteration % 50 == 0 { - eprintln!("[THREAD-{}] Iteration {}: calling SpinForMs({}ms)", thread_id, iteration, cpu_time_ms); + eprintln!( + "[THREAD-{}] Iteration {}: calling SpinForMs({}ms)", + thread_id, iteration, cpu_time_ms + ); } // Call the guest function let result = sandbox.call::("SpinForMs", cpu_time_ms); if iteration < 3 { - eprintln!("[THREAD-{}] Iteration {}: SpinForMs returned {:?}", thread_id, iteration, result); + eprintln!( + "[THREAD-{}] Iteration {}: SpinForMs returned {:?}", + thread_id, iteration, result + ); } // Signal the monitor to stop stop_monitoring.store(true, Ordering::Release); - + if iteration < 3 { - eprintln!("[THREAD-{}] Iteration {}: waiting for monitor thread", thread_id, iteration); + eprintln!( + "[THREAD-{}] Iteration {}: waiting for monitor thread", + thread_id, iteration + ); } // Wait for monitor thread to complete to ensure was_killed flag is set From 62eb40c96a8cc6ae3db2fcc242ce35977225a98e Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Tue, 21 Oct 2025 16:09:09 +0100 Subject: [PATCH 21/25] Rename and simplify test Signed-off-by: Simon Davies --- src/hyperlight_host/tests/integration_test.rs | 514 ++++++------------ 1 file changed, 181 insertions(+), 333 deletions(-) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 069856735..9e1764b15 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -936,25 +936,26 @@ fn test_if_guest_is_able_to_get_string_return_values_from_host() { ); } -/// Test that monitors CPU time usage and can interrupt a guest based on CPU time limits -/// Uses a pool of 100 sandboxes, 100 threads, and 500 iterations per thread -/// Some sandboxes are expected to complete normally, some are expected to be killed -/// This test makes sure that a reused sandbox is not killed in the case where the previous -/// execution was killed due to CPU time limit but the invocation completed normally before the cancel was processed. +/// Test that validates interrupt behavior with random kill timing under concurrent load +/// Uses a pool of 100 sandboxes, 100 threads, and 500 iterations per thread. +/// Randomly decides to kill some calls at random times during execution. +/// Validates that: +/// - Calls we chose to kill can end in any state (including some cancelled) +/// - Calls we did NOT choose to kill NEVER return ExecutionCanceledByHost +/// - We get a mix of killed and non-killed outcomes (not 100% or 0%) +#[cfg(target_os="linux")] #[test] -fn test_cpu_time_interrupt() { +fn interrupt_random_kill_stress_test() { use std::collections::VecDeque; - #[cfg(target_os = "linux")] - use std::mem::MaybeUninit; use std::sync::Mutex; use std::sync::atomic::AtomicUsize; - use std::sync::mpsc::channel; const POOL_SIZE: usize = 100; const NUM_THREADS: usize = 100; const ITERATIONS_PER_THREAD: usize = 500; + const KILL_PROBABILITY: f64 = 0.5; // 50% chance to attempt kill + const GUEST_CALL_DURATION_MS: u32 = 10; // SpinForMs duration - eprintln!("[TEST] Starting test_cpu_time_interrupt"); // Create a pool of 100 sandboxes println!("Creating pool of {} sandboxes...", POOL_SIZE); let mut sandbox_pool: Vec = Vec::with_capacity(POOL_SIZE); @@ -971,9 +972,13 @@ fn test_cpu_time_interrupt() { // Counters for statistics let total_iterations = Arc::new(AtomicUsize::new(0)); - let killed_count = Arc::new(AtomicUsize::new(0)); - let completed_count = Arc::new(AtomicUsize::new(0)); - let errors_count = Arc::new(AtomicUsize::new(0)); + let kill_attempted_count = Arc::new(AtomicUsize::new(0)); // We chose to kill + let actually_killed_count = Arc::new(AtomicUsize::new(0)); // Got ExecutionCanceledByHost + let not_killed_completed_ok = Arc::new(AtomicUsize::new(0)); + let not_killed_error = Arc::new(AtomicUsize::new(0)); // Non-cancelled errors + let killed_but_completed_ok = Arc::new(AtomicUsize::new(0)); + let killed_but_error = Arc::new(AtomicUsize::new(0)); // Non-cancelled errors + let unexpected_cancelled = Arc::new(AtomicUsize::new(0)); // CRITICAL: non-killed calls that got cancelled println!( "Starting {} threads with {} iterations each...", @@ -985,331 +990,130 @@ fn test_cpu_time_interrupt() { for thread_id in 0..NUM_THREADS { let pool_clone = Arc::clone(&pool); let total_iterations_clone = Arc::clone(&total_iterations); - let killed_count_clone = Arc::clone(&killed_count); - let completed_count_clone = Arc::clone(&completed_count); - let errors_count_clone = Arc::clone(&errors_count); + let kill_attempted_count_clone = Arc::clone(&kill_attempted_count); + let actually_killed_count_clone = Arc::clone(&actually_killed_count); + let not_killed_completed_ok_clone = Arc::clone(¬_killed_completed_ok); + let not_killed_error_clone = Arc::clone(¬_killed_error); + let killed_but_completed_ok_clone = Arc::clone(&killed_but_completed_ok); + let killed_but_error_clone = Arc::clone(&killed_but_error); + let unexpected_cancelled_clone = Arc::clone(&unexpected_cancelled); let handle = thread::spawn(move || { + println!("[THREAD-{}] Thread started, initializing RNG...", thread_id); + // Use thread_id as seed for reproducible randomness per thread + use std::collections::hash_map::RandomState; + use std::hash::{BuildHasher, Hash, Hasher}; + + let mut hasher = RandomState::new().build_hasher(); + thread_id.hash(&mut hasher); + let mut rng_state = hasher.finish(); + + println!("[THREAD-{}] RNG initialized, entering iteration loop...", thread_id); + + // Simple LCG random number generator for reproducible randomness + let mut next_random = || -> u64 { + rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1); + rng_state + }; + for iteration in 0..ITERATIONS_PER_THREAD { // === START OF ITERATION === - // Get a fresh sandbox from the pool for this iteration - let mut sandbox = { + // Get a sandbox from the pool for this iteration + let sandbox = loop { let mut pool_guard = pool_clone.lock().unwrap(); - // Wait if pool is empty (shouldn't happen with proper design) - while pool_guard.is_empty() { - drop(pool_guard); - thread::sleep(Duration::from_micros(100)); - pool_guard = pool_clone.lock().unwrap(); + if let Some(sb) = pool_guard.pop_front() { + break sb; } - pool_guard.pop_front().unwrap() + // Pool is empty, release lock and wait + drop(pool_guard); + eprintln!("[THREAD-{}] Iteration {}: Pool empty, waiting for sandbox...", thread_id, iteration); + thread::sleep(Duration::from_millis(1)); }; - // Vary CPU time between 3ms and 7ms to ensure some get killed and some complete - // The CPU limit is 5ms, so: - // - 3-4ms should complete normally - // - 6-7ms should be killed - // - 5ms is borderline and could go either way - let cpu_time_ms = - 3 + (((thread_id * ITERATIONS_PER_THREAD + iteration) % 5) as u32); - - let interrupt_handle = sandbox.interrupt_handle(); - - // Channel to send the thread ID - let (tx, rx) = channel(); - - // Flag to signal monitoring start - let should_monitor = Arc::new(AtomicBool::new(false)); - let should_monitor_clone = should_monitor.clone(); - - // Flag to signal monitoring stop (when guest execution completes) - let stop_monitoring = Arc::new(AtomicBool::new(false)); - let stop_monitoring_clone = stop_monitoring.clone(); - - // Flag to track if we actually sent a kill signal - let was_killed = Arc::new(AtomicBool::new(false)); - let was_killed_clone = was_killed.clone(); - - // Flag to signal that monitor thread is ready (entered its monitoring loop) - let monitor_ready = Arc::new(AtomicBool::new(false)); - let monitor_ready_clone = monitor_ready.clone(); - - // Spawn CPU time monitor thread - let monitor_thread = thread::spawn(move || { - let main_thread_id = match rx.recv() { - Ok(tid) => tid, - Err(_) => return, - }; - - while !should_monitor_clone.load(Ordering::Acquire) { - thread::sleep(Duration::from_micros(50)); - } - - #[cfg(target_os = "linux")] - unsafe { - let mut clock_id: libc::clockid_t = 0; - if libc::pthread_getcpuclockid(main_thread_id, &mut clock_id) != 0 { - return; - } - - let cpu_limit_ns = 5_000_000; // 5ms CPU time limit - let mut start_time = MaybeUninit::::uninit(); - - if libc::clock_gettime(clock_id, start_time.as_mut_ptr()) != 0 { - return; - } - let start_time = start_time.assume_init(); - - // Signal that we're ready to monitor - monitor_ready_clone.store(true, Ordering::Release); - - loop { - // Check if we should stop monitoring (guest completed) - if stop_monitoring_clone.load(Ordering::Acquire) { - break; - } - - let mut current_time = MaybeUninit::::uninit(); - if libc::clock_gettime(clock_id, current_time.as_mut_ptr()) != 0 { - break; - } - let current_time = current_time.assume_init(); - - let elapsed_ns = (current_time.tv_sec - start_time.tv_sec) - * 1_000_000_000 - + (current_time.tv_nsec - start_time.tv_nsec); - - if elapsed_ns > cpu_limit_ns { - // Double-check that monitoring should still continue before killing - // The guest might have completed between our last check and now - if stop_monitoring_clone.load(Ordering::Acquire) { - break; - } - - // Mark that we sent a kill signal BEFORE calling kill - // to avoid race conditions - was_killed_clone.store(true, Ordering::Release); - interrupt_handle.kill(); - break; - } - - thread::sleep(Duration::from_micros(50)); - } - } - - #[cfg(target_os = "windows")] - unsafe { - use std::ffi::c_void; - - eprintln!("[MONITOR] Windows monitoring thread starting"); - - // On Windows, use QueryThreadCycleTime for high-resolution CPU time measurement - // This measures actual CPU cycles consumed by the thread, similar to Linux's - // pthread_getcpuclockid, and is much more accurate than GetThreadTimes (~15ms resolution) - - let thread_handle = main_thread_id as *mut c_void; - eprintln!("[MONITOR] Got thread handle: {:?}", thread_handle); - - // Get the CPU frequency to convert cycles to time - let mut frequency: i64 = 0; - if windows_sys::Win32::System::Performance::QueryPerformanceFrequency( - &mut frequency, - ) == 0 - { - eprintln!("[MONITOR] ERROR: QueryPerformanceFrequency failed"); - return; - } - eprintln!("[MONITOR] CPU frequency: {} Hz", frequency); - - // Get starting CPU cycles for this thread - let mut start_cycles: u64 = 0; - if windows_sys::Win32::System::WindowsProgramming::QueryThreadCycleTime( - thread_handle, - &mut start_cycles, - ) == 0 - { - eprintln!("[MONITOR] ERROR: QueryThreadCycleTime (start) failed"); - return; - } - eprintln!("[MONITOR] Start cycles: {}", start_cycles); - - // Convert 5ms CPU limit to approximate cycle count - // This is approximate because CPU frequency can vary with power management, - // but gives us a baseline that's much more accurate than GetThreadTimes - let cpu_limit_cycles = (5_000_000u64 * frequency as u64) / 1_000_000_000; - eprintln!("[MONITOR] CPU limit: {} cycles (~5ms)", cpu_limit_cycles); - - eprintln!("[MONITOR] Entering monitoring loop"); - // Signal that we're ready to monitor - monitor_ready_clone.store(true, Ordering::Release); - let mut loop_count = 0u64; - loop { - // Check if we should stop monitoring (guest completed) - if stop_monitoring_clone.load(Ordering::Acquire) { - eprintln!( - "[MONITOR] Stop monitoring flag set, exiting after {} loops", - loop_count - ); - break; - } - - let mut current_cycles: u64 = 0; - if windows_sys::Win32::System::WindowsProgramming::QueryThreadCycleTime( - thread_handle, - &mut current_cycles, - ) == 0 - { - eprintln!( - "[MONITOR] ERROR: QueryThreadCycleTime (current) failed at loop {}", - loop_count - ); - break; - } - - let elapsed_cycles = current_cycles - start_cycles; - - if loop_count < 3 || loop_count % 100 == 0 { - eprintln!( - "[MONITOR] Loop {}: elapsed_cycles={}, limit={}", - loop_count, elapsed_cycles, cpu_limit_cycles - ); - } - - if elapsed_cycles > cpu_limit_cycles { - eprintln!( - "[MONITOR] CPU limit exceeded at loop {}: {} > {}", - loop_count, elapsed_cycles, cpu_limit_cycles - ); - // Double-check that monitoring should still continue before killing - if stop_monitoring_clone.load(Ordering::Acquire) { - eprintln!("[MONITOR] Stop flag already set, not killing"); - break; - } - - eprintln!("[MONITOR] Calling kill()"); - // Mark that we sent a kill signal BEFORE calling kill - was_killed_clone.store(true, Ordering::Release); - interrupt_handle.kill(); - eprintln!("[MONITOR] kill() returned"); - break; - } - - loop_count += 1; - thread::sleep(Duration::from_micros(50)); + // Use a guard struct to ensure sandbox is always returned to pool + struct SandboxGuard<'a> { + sandbox: Option, + pool: &'a Arc>>, + } + + impl<'a> Drop for SandboxGuard<'a> { + fn drop(&mut self) { + if let Some(sb) = self.sandbox.take() { + let mut pool_guard = self.pool.lock().unwrap(); + pool_guard.push_back(sb); + // eprintln!("[GUARD] Returned sandbox to pool, pool size now: {}", pool_guard.len()); } - eprintln!( - "[MONITOR] Exited monitoring loop after {} iterations", - loop_count - ); - - // Clean up the duplicated thread handle - eprintln!("[MONITOR] Closing thread handle"); - windows_sys::Win32::Foundation::CloseHandle(thread_handle); - eprintln!("[MONITOR] Monitor thread exiting"); } - }); - - // Send thread ID and start monitoring - #[cfg(target_os = "linux")] - unsafe { - let thread_id = libc::pthread_self(); - let _ = tx.send(thread_id); } + + let mut guard = SandboxGuard { + sandbox: Some(sandbox), + pool: &pool_clone, + }; - #[cfg(target_os = "windows")] - unsafe { - use std::ffi::c_void; - // On Windows, we need a REAL thread handle, not the pseudo-handle from GetCurrentThread() - // GetCurrentThread() returns a constant (-2) that only works in the current thread's context - // We must duplicate it to get a real handle that can be used from another thread - let pseudo_handle = windows_sys::Win32::System::Threading::GetCurrentThread(); - let current_process = - windows_sys::Win32::System::Threading::GetCurrentProcess(); - let mut real_handle: *mut c_void = std::ptr::null_mut(); - - if windows_sys::Win32::Foundation::DuplicateHandle( - current_process, - pseudo_handle, - current_process, - &mut real_handle, - 0, - 0, // bInheritHandle = FALSE - windows_sys::Win32::Foundation::DUPLICATE_SAME_ACCESS, - ) == 0 - { - panic!("Failed to duplicate thread handle"); - } - - let _ = tx.send(real_handle as usize); + // Decide randomly: should we attempt to kill this call? + let should_kill = (next_random() as f64 / u64::MAX as f64) < KILL_PROBABILITY; + + if should_kill { + kill_attempted_count_clone.fetch_add(1, Ordering::Relaxed); } - should_monitor.store(true, Ordering::Release); - - // Wait for monitor thread to be ready before starting guest execution - // This prevents the race where guest completes before monitor enters its loop - while !monitor_ready.load(Ordering::Acquire) { - thread::sleep(Duration::from_micros(10)); - } + let sandbox = guard.sandbox.as_mut().unwrap(); + let interrupt_handle = sandbox.interrupt_handle(); - if iteration < 3 || iteration % 50 == 0 { - eprintln!( - "[THREAD-{}] Iteration {}: calling SpinForMs({}ms)", - thread_id, iteration, cpu_time_ms - ); - } + // If we decided to kill, spawn a thread that will kill at a random time + let killer_thread = if should_kill { + // Generate random delay here before moving into thread + let kill_delay_ms = (next_random() % 16) as u64; + Some(thread::spawn(move || { + // Random delay between 0 and 15ms (guest runs for ~10ms) + thread::sleep(Duration::from_millis(kill_delay_ms)); + interrupt_handle.kill(); + })) + } else { + None + }; // Call the guest function - let result = sandbox.call::("SpinForMs", cpu_time_ms); - - if iteration < 3 { - eprintln!( - "[THREAD-{}] Iteration {}: SpinForMs returned {:?}", - thread_id, iteration, result - ); - } - - // Signal the monitor to stop - stop_monitoring.store(true, Ordering::Release); + let result = sandbox.call::("SpinForMs", GUEST_CALL_DURATION_MS); - if iteration < 3 { - eprintln!( - "[THREAD-{}] Iteration {}: waiting for monitor thread", - thread_id, iteration - ); + // Wait for killer thread to finish if it was spawned + if let Some(kt) = killer_thread { + let _ = kt.join(); } - // Wait for monitor thread to complete to ensure was_killed flag is set - let _ = monitor_thread.join(); - - // NOW check if we sent a kill signal (after monitor thread has completed) - let kill_was_sent = was_killed.load(Ordering::Acquire); - - // Process the result and validate correctness + // Process the result based on whether we attempted to kill match result { Err(HyperlightError::ExecutionCanceledByHost()) => { - // We received a cancellation error - if !kill_was_sent { - // ERROR: We got a cancellation but never sent a kill! - panic!( - "Thread {} iteration {}: Got ExecutionCanceledByHost but no kill signal was sent!", + if should_kill { + // We attempted to kill and it was cancelled - SUCCESS + actually_killed_count_clone.fetch_add(1, Ordering::Relaxed); + } else { + // We did NOT attempt to kill but got cancelled - CRITICAL FAILURE + unexpected_cancelled_clone.fetch_add(1, Ordering::Relaxed); + eprintln!( + "CRITICAL: Thread {} iteration {}: Got ExecutionCanceledByHost but did NOT attempt kill!", thread_id, iteration ); } - // This is correct - we sent kill and got the error - killed_count_clone.fetch_add(1, Ordering::Relaxed); } Ok(_) => { - // Execution completed normally - // This is OK whether or not we sent a kill - the guest might have - // finished just before the kill signal arrived - completed_count_clone.fetch_add(1, Ordering::Relaxed); + if should_kill { + // We attempted to kill but it completed OK - acceptable race condition + killed_but_completed_ok_clone.fetch_add(1, Ordering::Relaxed); + } else { + // We did NOT attempt to kill and it completed OK - EXPECTED + not_killed_completed_ok_clone.fetch_add(1, Ordering::Relaxed); + } } - Err(e) => { - // Unexpected error - eprintln!( - "Thread {} iteration {}: Unexpected error: {:?}, kill_sent: {}", - thread_id, iteration, e, kill_was_sent - ); - errors_count_clone.fetch_add(1, Ordering::Relaxed); + Err(_other_error) => { + if should_kill { + // We attempted to kill and got some other error - acceptable + killed_but_error_clone.fetch_add(1, Ordering::Relaxed); + } else { + // We did NOT attempt to kill and got some other error - acceptable + not_killed_error_clone.fetch_add(1, Ordering::Relaxed); + } } } @@ -1317,7 +1121,7 @@ fn test_cpu_time_interrupt() { // Progress reporting let current_total = total_iterations_clone.load(Ordering::Relaxed); - if current_total % 500 == 0 { + if current_total % 5000 == 0 { println!( "Progress: {}/{} iterations completed", current_total, @@ -1326,48 +1130,92 @@ fn test_cpu_time_interrupt() { } // === END OF ITERATION === - // Return sandbox to pool for reuse by other threads/iterations - { - let mut pool_guard = pool_clone.lock().unwrap(); - pool_guard.push_back(sandbox); - } + // SandboxGuard will automatically return sandbox to pool when it goes out of scope } + + eprintln!("[THREAD-{}] Completed all {} iterations!", thread_id, ITERATIONS_PER_THREAD); }); thread_handles.push(handle); } + println!("All {} worker threads spawned, waiting for completion...", NUM_THREADS); + // Wait for all threads to complete - for handle in thread_handles { + for (idx, handle) in thread_handles.into_iter().enumerate() { + println!("Waiting for thread {} to join...", idx); handle.join().unwrap(); + println!("Thread {} joined successfully", idx); } - // Print statistics - let total = total_iterations.load(Ordering::Relaxed); - let killed = killed_count.load(Ordering::Relaxed); - let completed = completed_count.load(Ordering::Relaxed); - let errors = errors_count.load(Ordering::Relaxed); + println!("All threads joined successfully!"); - println!("\n=== Test Statistics ==="); + // Collect final statistics + let total = total_iterations.load(Ordering::Relaxed); + let kill_attempted = kill_attempted_count.load(Ordering::Relaxed); + let actually_killed = actually_killed_count.load(Ordering::Relaxed); + let not_killed_ok = not_killed_completed_ok.load(Ordering::Relaxed); + let not_killed_err = not_killed_error.load(Ordering::Relaxed); + let killed_but_ok = killed_but_completed_ok.load(Ordering::Relaxed); + let killed_but_err = killed_but_error.load(Ordering::Relaxed); + let unexpected_cancel = unexpected_cancelled.load(Ordering::Relaxed); + + let no_kill_attempted = total - kill_attempted; + + // Print detailed statistics + println!("\n=== Interrupt Random Kill Stress Test Statistics ==="); println!("Total iterations: {}", total); - println!("Killed (CPU limit exceeded): {}", killed); - println!("Completed normally: {}", completed); - println!("Errors: {}", errors); - println!("Kill rate: {:.1}%", (killed as f64 / total as f64) * 100.0); + println!(); + println!("Kill Attempts: {} ({:.1}%)", kill_attempted, (kill_attempted as f64 / total as f64) * 100.0); + println!(" - Actually killed (ExecutionCanceledByHost): {}", actually_killed); + println!(" - Completed OK despite kill attempt: {}", killed_but_ok); + println!(" - Error (non-cancelled) despite kill attempt: {}", killed_but_err); + if kill_attempted > 0 { + println!(" - Kill success rate: {:.1}%", (actually_killed as f64 / kill_attempted as f64) * 100.0); + } + println!(); + println!("No Kill Attempts: {} ({:.1}%)", no_kill_attempted, (no_kill_attempted as f64 / total as f64) * 100.0); + println!(" - Completed OK: {}", not_killed_ok); + println!(" - Error (non-cancelled): {}", not_killed_err); + println!(" - Cancelled (SHOULD BE 0): {} {}", unexpected_cancel, if unexpected_cancel == 0 { "✅" } else { "❌ FAILURE" }); + + // CRITICAL VALIDATIONS + assert_eq!( + unexpected_cancel, 0, + "FAILURE: {} non-killed calls returned ExecutionCanceledByHost! This indicates false kills.", + unexpected_cancel + ); + + assert!( + actually_killed > 0, + "FAILURE: No calls were actually killed despite {} kill attempts!", + kill_attempted + ); - // Verify we had both kills and completions assert!( - killed > 0, - "Expected some executions to be killed, but none were" + kill_attempted > 0, + "FAILURE: No kill attempts were made (expected ~50% of {} iterations)!", + total ); + assert!( - completed > 0, - "Expected some executions to complete, but none did" + kill_attempted < total, + "FAILURE: All {} iterations were kill attempts (expected ~50%)!", + total + ); + + // Verify total accounting + assert_eq!( + total, + actually_killed + not_killed_ok + not_killed_err + killed_but_ok + killed_but_err + unexpected_cancel, + "Iteration accounting mismatch!" ); - assert_eq!(errors, 0, "Expected no errors, but got {}", errors); + assert_eq!( total, NUM_THREADS * ITERATIONS_PER_THREAD, "Not all iterations completed" ); + + println!("\n✅ All validations passed!"); } From 1dec44894e6861ec002d63c2107c7f3140c78a34 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Tue, 21 Oct 2025 17:00:33 +0100 Subject: [PATCH 22/25] fmt Signed-off-by: Simon Davies --- src/hyperlight_host/tests/integration_test.rs | 76 ++++++++++++++----- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 9e1764b15..76787f725 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -943,7 +943,7 @@ fn test_if_guest_is_able_to_get_string_return_values_from_host() { /// - Calls we chose to kill can end in any state (including some cancelled) /// - Calls we did NOT choose to kill NEVER return ExecutionCanceledByHost /// - We get a mix of killed and non-killed outcomes (not 100% or 0%) -#[cfg(target_os="linux")] +#[cfg(target_os = "linux")] #[test] fn interrupt_random_kill_stress_test() { use std::collections::VecDeque; @@ -1003,12 +1003,15 @@ fn interrupt_random_kill_stress_test() { // Use thread_id as seed for reproducible randomness per thread use std::collections::hash_map::RandomState; use std::hash::{BuildHasher, Hash, Hasher}; - + let mut hasher = RandomState::new().build_hasher(); thread_id.hash(&mut hasher); let mut rng_state = hasher.finish(); - println!("[THREAD-{}] RNG initialized, entering iteration loop...", thread_id); + println!( + "[THREAD-{}] RNG initialized, entering iteration loop...", + thread_id + ); // Simple LCG random number generator for reproducible randomness let mut next_random = || -> u64 { @@ -1026,7 +1029,10 @@ fn interrupt_random_kill_stress_test() { } // Pool is empty, release lock and wait drop(pool_guard); - eprintln!("[THREAD-{}] Iteration {}: Pool empty, waiting for sandbox...", thread_id, iteration); + eprintln!( + "[THREAD-{}] Iteration {}: Pool empty, waiting for sandbox...", + thread_id, iteration + ); thread::sleep(Duration::from_millis(1)); }; @@ -1035,7 +1041,7 @@ fn interrupt_random_kill_stress_test() { sandbox: Option, pool: &'a Arc>>, } - + impl<'a> Drop for SandboxGuard<'a> { fn drop(&mut self) { if let Some(sb) = self.sandbox.take() { @@ -1045,7 +1051,7 @@ fn interrupt_random_kill_stress_test() { } } } - + let mut guard = SandboxGuard { sandbox: Some(sandbox), pool: &pool_clone, @@ -1053,7 +1059,7 @@ fn interrupt_random_kill_stress_test() { // Decide randomly: should we attempt to kill this call? let should_kill = (next_random() as f64 / u64::MAX as f64) < KILL_PROBABILITY; - + if should_kill { kill_attempted_count_clone.fetch_add(1, Ordering::Relaxed); } @@ -1132,14 +1138,20 @@ fn interrupt_random_kill_stress_test() { // === END OF ITERATION === // SandboxGuard will automatically return sandbox to pool when it goes out of scope } - - eprintln!("[THREAD-{}] Completed all {} iterations!", thread_id, ITERATIONS_PER_THREAD); + + eprintln!( + "[THREAD-{}] Completed all {} iterations!", + thread_id, ITERATIONS_PER_THREAD + ); }); thread_handles.push(handle); } - println!("All {} worker threads spawned, waiting for completion...", NUM_THREADS); + println!( + "All {} worker threads spawned, waiting for completion...", + NUM_THREADS + ); // Wait for all threads to complete for (idx, handle) in thread_handles.into_iter().enumerate() { @@ -1166,18 +1178,43 @@ fn interrupt_random_kill_stress_test() { println!("\n=== Interrupt Random Kill Stress Test Statistics ==="); println!("Total iterations: {}", total); println!(); - println!("Kill Attempts: {} ({:.1}%)", kill_attempted, (kill_attempted as f64 / total as f64) * 100.0); - println!(" - Actually killed (ExecutionCanceledByHost): {}", actually_killed); + println!( + "Kill Attempts: {} ({:.1}%)", + kill_attempted, + (kill_attempted as f64 / total as f64) * 100.0 + ); + println!( + " - Actually killed (ExecutionCanceledByHost): {}", + actually_killed + ); println!(" - Completed OK despite kill attempt: {}", killed_but_ok); - println!(" - Error (non-cancelled) despite kill attempt: {}", killed_but_err); + println!( + " - Error (non-cancelled) despite kill attempt: {}", + killed_but_err + ); if kill_attempted > 0 { - println!(" - Kill success rate: {:.1}%", (actually_killed as f64 / kill_attempted as f64) * 100.0); + println!( + " - Kill success rate: {:.1}%", + (actually_killed as f64 / kill_attempted as f64) * 100.0 + ); } println!(); - println!("No Kill Attempts: {} ({:.1}%)", no_kill_attempted, (no_kill_attempted as f64 / total as f64) * 100.0); + println!( + "No Kill Attempts: {} ({:.1}%)", + no_kill_attempted, + (no_kill_attempted as f64 / total as f64) * 100.0 + ); println!(" - Completed OK: {}", not_killed_ok); println!(" - Error (non-cancelled): {}", not_killed_err); - println!(" - Cancelled (SHOULD BE 0): {} {}", unexpected_cancel, if unexpected_cancel == 0 { "✅" } else { "❌ FAILURE" }); + println!( + " - Cancelled (SHOULD BE 0): {} {}", + unexpected_cancel, + if unexpected_cancel == 0 { + "✅" + } else { + "❌ FAILURE" + } + ); // CRITICAL VALIDATIONS assert_eq!( @@ -1207,7 +1244,12 @@ fn interrupt_random_kill_stress_test() { // Verify total accounting assert_eq!( total, - actually_killed + not_killed_ok + not_killed_err + killed_but_ok + killed_but_err + unexpected_cancel, + actually_killed + + not_killed_ok + + not_killed_err + + killed_but_ok + + killed_but_err + + unexpected_cancel, "Iteration accounting mismatch!" ); From f1aef3ece774b8db73f7477c1576e7a724b7cb5b Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Thu, 23 Oct 2025 19:57:19 +0100 Subject: [PATCH 23/25] fixes Signed-off-by: Simon Davies --- .../src/hypervisor/hyperv_linux.rs | 2 +- src/hyperlight_host/src/hypervisor/kvm.rs | 2 +- src/hyperlight_host/src/hypervisor/mod.rs | 6 +- src/hyperlight_host/tests/integration_test.rs | 210 ++++++++---------- src/tests/rust_guests/simpleguest/src/main.rs | 1 - src/tests/rust_guests/witguest/Cargo.lock | 4 +- 6 files changed, 99 insertions(+), 126 deletions(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 3eef775ca..32d5c8fec 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -659,7 +659,7 @@ impl Hypervisor for HypervLinuxDriver { self.interrupt_handle .tid - .store(unsafe { libc::pthread_self() as u64 }, Ordering::Relaxed); + .store(unsafe { libc::pthread_self() as u64 }, Ordering::Release); // Note: if `InterruptHandle::kill()` is called while this thread is **here** // (after set_running_bit but before checking cancel_requested): // - kill() will stamp cancel_requested with the current generation diff --git a/src/hyperlight_host/src/hypervisor/kvm.rs b/src/hyperlight_host/src/hypervisor/kvm.rs index 781669bb9..9173c3fd9 100644 --- a/src/hyperlight_host/src/hypervisor/kvm.rs +++ b/src/hyperlight_host/src/hypervisor/kvm.rs @@ -621,7 +621,7 @@ impl Hypervisor for KVMDriver { ) -> Result { self.interrupt_handle .tid - .store(unsafe { libc::pthread_self() as u64 }, Ordering::Relaxed); + .store(unsafe { libc::pthread_self() as u64 }, Ordering::Release); // Note: if `InterruptHandle::kill()` is called while this thread is **here** // (after set_running_bit but before checking cancel_requested): // - kill() will stamp cancel_requested with the current generation diff --git a/src/hyperlight_host/src/hypervisor/mod.rs b/src/hyperlight_host/src/hypervisor/mod.rs index da3d707e1..26743fe73 100644 --- a/src/hyperlight_host/src/hypervisor/mod.rs +++ b/src/hyperlight_host/src/hypervisor/mod.rs @@ -786,11 +786,11 @@ impl LinuxInterruptHandle { // clear the running bit and return the generation fn clear_running_bit(&self) -> u64 { self.running - .fetch_and(!Self::RUNNING_BIT, Ordering::Relaxed) + .fetch_and(!Self::RUNNING_BIT, Ordering::Release) } fn get_running_and_generation(&self) -> (bool, u64) { - let raw = self.running.load(Ordering::Relaxed); + let raw = self.running.load(Ordering::Acquire); let running = raw & Self::RUNNING_BIT != 0; let generation = raw & !Self::RUNNING_BIT; (running, generation) @@ -837,7 +837,7 @@ impl LinuxInterruptHandle { log::info!("Sending signal to kill vcpu thread..."); sent_signal = true; unsafe { - libc::pthread_kill(self.tid.load(Ordering::Relaxed) as _, signal_number); + libc::pthread_kill(self.tid.load(Ordering::Acquire) as _, signal_number); } std::thread::sleep(self.retry_delay); } diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index 76787f725..a522691c3 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -62,9 +62,7 @@ fn interrupt_host_call() { } }); - let result = sandbox.call::("CallHostSpin", ()); - println!("Result: {:?}", result); - let result = result.unwrap_err(); + let result = sandbox.call::("CallHostSpin", ()).unwrap_err(); assert!(matches!(result, HyperlightError::ExecutionCanceledByHost())); thread.join().unwrap(); @@ -101,8 +99,7 @@ fn interrupt_in_progress_guest_call() { thread.join().expect("Thread should finish"); } -/// Makes sure interrupting a vm before the guest call has started has no effect, -/// but a second kill after the call starts will interrupt it +/// Makes sure interrupting a vm before the guest call has started also prevents the guest call from being executed #[test] fn interrupt_guest_call_in_advance() { let mut sbox1: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); @@ -111,20 +108,15 @@ fn interrupt_guest_call_in_advance() { let interrupt_handle = sbox1.interrupt_handle(); assert!(!interrupt_handle.dropped()); // not yet dropped - // First kill before the guest call has started - should have no effect - // Then kill again after a delay to interrupt the actual call + // kill vm before the guest call has started let thread = thread::spawn(move || { - assert!(!interrupt_handle.kill()); // should return false since no call is active + assert!(!interrupt_handle.kill()); // should return false since vcpu is not running yet barrier2.wait(); - // Wait a bit for the Spin call to actually start - thread::sleep(Duration::from_millis(100)); - assert!(interrupt_handle.kill()); // this should succeed and interrupt the Spin call barrier2.wait(); // wait here until main thread has dropped the sandbox assert!(interrupt_handle.dropped()); }); - barrier.wait(); // wait until first `kill()` is called before starting the guest call - // The Spin call should be interrupted by the second kill() + barrier.wait(); // wait until `kill()` is called before starting the guest call let res = sbox1.call::("Spin", ()).unwrap_err(); assert!(matches!(res, HyperlightError::ExecutionCanceledByHost())); @@ -148,11 +140,9 @@ fn interrupt_guest_call_in_advance() { /// all possible interleavings, but can hopefully increases confidence somewhat. #[test] fn interrupt_same_thread() { - eprintln!("[TEST] interrupt_same_thread starting"); let mut sbox1: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); let mut sbox2: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); let mut sbox3: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); - eprintln!("[TEST] Created 3 sandboxes"); let barrier = Arc::new(Barrier::new(2)); let barrier2 = barrier.clone(); @@ -162,85 +152,31 @@ fn interrupt_same_thread() { const NUM_ITERS: usize = 500; - // Spawn killer thread that attempts to interrupt sbox2 + // kill vm after 1 second let thread = thread::spawn(move || { - eprintln!("[KILLER] Thread starting"); - for i in 0..NUM_ITERS { - eprintln!("[KILLER] Iteration {}/{}: calling kill()", i, NUM_ITERS); - let killed = interrupt_handle.kill(); - eprintln!("[KILLER] Iteration {}: kill() returned {}", i, killed); - // Sync at END of iteration to ensure main thread completes its work - eprintln!("[KILLER] Iteration {}: waiting at barrier", i); + for _ in 0..NUM_ITERS { barrier2.wait(); - eprintln!("[KILLER] Iteration {}: passed barrier", i); + interrupt_handle.kill(); } - eprintln!("[KILLER] Thread exiting after {} iterations", NUM_ITERS); }); - for i in 0..NUM_ITERS { - eprintln!("[MAIN] Iteration {}/{}: starting", i, NUM_ITERS); - - // Call sbox1 - should never be interrupted - eprintln!("[MAIN] Iteration {}: calling sbox1", i); + for _ in 0..NUM_ITERS { + barrier.wait(); sbox1 .call::("Echo", "hello".to_string()) - .unwrap_or_else(|e| { - eprintln!( - "[MAIN] ERROR: sbox1 interrupted at iteration {}: {:?}", - i, e - ); - panic!( - "Iteration {}: sbox1 should not be interrupted, got: {:?}", - i, e - ) - }); - eprintln!("[MAIN] Iteration {}: sbox1 completed", i); - - // Call sbox2 - may be interrupted - eprintln!("[MAIN] Iteration {}: calling sbox2", i); + .expect("Only sandbox 2 is allowed to be interrupted"); match sbox2.call::("Echo", "hello".to_string()) { - Ok(_) => { - eprintln!("[MAIN] Iteration {}: sbox2 completed successfully", i); - } - Err(HyperlightError::ExecutionCanceledByHost()) => { - eprintln!("[MAIN] Iteration {}: sbox2 was cancelled", i); - } - Err(e) => { - eprintln!( - "[MAIN] ERROR: sbox2 unexpected error at iteration {}: {:?}", - i, e - ); - panic!("Iteration {}: sbox2 unexpected error: {:?}", i, e); + Ok(_) | Err(HyperlightError::ExecutionCanceledByHost()) => { + // Only allow successful calls or interrupted. + // The call can be successful in case the call is finished before kill() is called. } + _ => panic!("Unexpected return"), }; - - // Call sbox3 - should never be interrupted - eprintln!("[MAIN] Iteration {}: calling sbox3", i); sbox3 .call::("Echo", "hello".to_string()) - .unwrap_or_else(|e| { - eprintln!( - "[MAIN] ERROR: sbox3 interrupted at iteration {}: {:?}", - i, e - ); - panic!( - "Iteration {}: sbox3 should not be interrupted, got: {:?}", - i, e - ) - }); - eprintln!("[MAIN] Iteration {}: sbox3 completed", i); - - // Sync at END of iteration to ensure killer thread waits for work to complete - eprintln!("[MAIN] Iteration {}: waiting at barrier", i); - barrier.wait(); - eprintln!("[MAIN] Iteration {}: passed barrier", i); + .expect("Only sandbox 2 is allowed to be interrupted"); } - eprintln!( - "[MAIN] All {} iterations complete, joining killer thread", - NUM_ITERS - ); thread.join().expect("Thread should finish"); - eprintln!("[TEST] interrupt_same_thread completed successfully"); } /// Same test as above but with no per-iteration barrier, to get more possible interleavings. @@ -260,51 +196,31 @@ fn interrupt_same_thread_no_barrier() { const NUM_ITERS: usize = 500; - // Spawn killer thread that continuously attempts to interrupt sbox2 + // kill vm after 1 second let thread = thread::spawn(move || { barrier2.wait(); - // Use Acquire ordering to ensure we see the updated workload_done flag - while !workload_done2.load(Ordering::Acquire) { + while !workload_done2.load(Ordering::Relaxed) { interrupt_handle.kill(); - // Small yield to prevent tight spinning - thread::yield_now(); } }); barrier.wait(); - for i in 0..NUM_ITERS { - // Call sbox1 - should never be interrupted + for _ in 0..NUM_ITERS { sbox1 .call::("Echo", "hello".to_string()) - .unwrap_or_else(|e| { - panic!( - "Iteration {}: sbox1 should not be interrupted, got: {:?}", - i, e - ) - }); - - // Call sbox2 - may be interrupted + .expect("Only sandbox 2 is allowed to be interrupted"); match sbox2.call::("Echo", "hello".to_string()) { Ok(_) | Err(HyperlightError::ExecutionCanceledByHost()) => { // Only allow successful calls or interrupted. - // The call can be successful if it completes before kill() is called. + // The call can be successful in case the call is finished before kill() is called. } - Err(e) => panic!("Iteration {}: sbox2 unexpected error: {:?}", i, e), + _ => panic!("Unexpected return"), }; - - // Call sbox3 - should never be interrupted sbox3 .call::("Echo", "hello".to_string()) - .unwrap_or_else(|e| { - panic!( - "Iteration {}: sbox3 should not be interrupted, got: {:?}", - i, e - ) - }); + .expect("Only sandbox 2 is allowed to be interrupted"); } - - // Use Release ordering to ensure killer thread sees the update - workload_done.store(true, Ordering::Release); + workload_done.store(true, Ordering::Relaxed); thread.join().expect("Thread should finish"); } @@ -935,7 +851,6 @@ fn test_if_guest_is_able_to_get_string_return_values_from_host() { "Guest Function, string added by Host Function".to_string() ); } - /// Test that validates interrupt behavior with random kill timing under concurrent load /// Uses a pool of 100 sandboxes, 100 threads, and 500 iterations per thread. /// Randomly decides to kill some calls at random times during execution. @@ -943,7 +858,6 @@ fn test_if_guest_is_able_to_get_string_return_values_from_host() { /// - Calls we chose to kill can end in any state (including some cancelled) /// - Calls we did NOT choose to kill NEVER return ExecutionCanceledByHost /// - We get a mix of killed and non-killed outcomes (not 100% or 0%) -#[cfg(target_os = "linux")] #[test] fn interrupt_random_kill_stress_test() { use std::collections::VecDeque; @@ -956,7 +870,7 @@ fn interrupt_random_kill_stress_test() { const KILL_PROBABILITY: f64 = 0.5; // 50% chance to attempt kill const GUEST_CALL_DURATION_MS: u32 = 10; // SpinForMs duration - // Create a pool of 100 sandboxes + // Create a pool of 50 sandboxes println!("Creating pool of {} sandboxes...", POOL_SIZE); let mut sandbox_pool: Vec = Vec::with_capacity(POOL_SIZE); for i in 0..POOL_SIZE { @@ -1002,18 +916,18 @@ fn interrupt_random_kill_stress_test() { println!("[THREAD-{}] Thread started, initializing RNG...", thread_id); // Use thread_id as seed for reproducible randomness per thread use std::collections::hash_map::RandomState; - use std::hash::{BuildHasher, Hash, Hasher}; + use std::hash::{BuildHasher, Hash}; let mut hasher = RandomState::new().build_hasher(); thread_id.hash(&mut hasher); - let mut rng_state = hasher.finish(); + let mut rng_state = RandomState::new().hash_one(thread_id); println!( "[THREAD-{}] RNG initialized, entering iteration loop...", thread_id ); - // Simple LCG random number generator for reproducible randomness + // Simple random number generator for reproducible randomness let mut next_random = || -> u64 { rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1); rng_state @@ -1068,23 +982,81 @@ fn interrupt_random_kill_stress_test() { let interrupt_handle = sandbox.interrupt_handle(); // If we decided to kill, spawn a thread that will kill at a random time + // Use a barrier to ensure the killer thread waits until we're about to call the guest let killer_thread = if should_kill { + use std::sync::{Arc, Barrier}; + + let barrier = Arc::new(Barrier::new(2)); + let barrier_clone = Arc::clone(&barrier); + // Generate random delay here before moving into thread - let kill_delay_ms = (next_random() % 16) as u64; - Some(thread::spawn(move || { + let kill_delay_ms = next_random() % 16; + let thread_id_clone = thread_id; + let iteration_clone = iteration; + let handle = thread::spawn(move || { + eprintln!( + "[KILLER-{}-{}] Waiting at barrier...", + thread_id_clone, iteration_clone + ); + // Wait at the barrier until the main thread is ready to call the guest + barrier_clone.wait(); + eprintln!( + "[KILLER-{}-{}] Passed barrier, sleeping for {}ms...", + thread_id_clone, iteration_clone, kill_delay_ms + ); // Random delay between 0 and 15ms (guest runs for ~10ms) thread::sleep(Duration::from_millis(kill_delay_ms)); + eprintln!( + "[KILLER-{}-{}] Calling kill()...", + thread_id_clone, iteration_clone + ); interrupt_handle.kill(); - })) + eprintln!( + "[KILLER-{}-{}] kill() returned, exiting thread", + thread_id_clone, iteration_clone + ); + }); + Some((handle, barrier)) } else { None }; // Call the guest function + eprintln!( + "[THREAD-{}] Iteration {}: Calling guest function (should_kill={})...", + thread_id, iteration, should_kill + ); + + // Release the barrier just before calling the guest function + if let Some((_, ref barrier)) = killer_thread { + eprintln!( + "[THREAD-{}] Iteration {}: Main thread waiting at barrier...", + thread_id, iteration + ); + barrier.wait(); + eprintln!( + "[THREAD-{}] Iteration {}: Main thread passed barrier, calling guest...", + thread_id, iteration + ); + } + let result = sandbox.call::("SpinForMs", GUEST_CALL_DURATION_MS); + eprintln!( + "[THREAD-{}] Iteration {}: Guest call returned: {:?}", + thread_id, + iteration, + result + .as_ref() + .map(|_| "Ok") + .map_err(|e| format!("{:?}", e)) + ); // Wait for killer thread to finish if it was spawned - if let Some(kt) = killer_thread { + if let Some((kt, _)) = killer_thread { + eprintln!( + "[THREAD-{}] Iteration {}: Waiting for killer thread to join...", + thread_id, iteration + ); let _ = kt.join(); } @@ -1217,6 +1189,8 @@ fn interrupt_random_kill_stress_test() { ); // CRITICAL VALIDATIONS + // TODO: this needs fixing on Windows - we can still kill the following call invocation there + #[cfg(target_os = "linux")] assert_eq!( unexpected_cancel, 0, "FAILURE: {} non-killed calls returned ExecutionCanceledByHost! This indicates false kills.", diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index 89e896e8e..43f514a71 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -592,7 +592,6 @@ fn spin_for_ms(fc: &FunctionCall) -> Result> { Ok(get_flatbuffer_result(ms_spun)) } -#[hyperlight_guest_tracing::trace_function] fn test_abort(function_call: &FunctionCall) -> Result> { if let ParameterValue::Int(code) = function_call.parameters.clone().unwrap()[0].clone() { abort_with_code(&[code as u8]); diff --git a/src/tests/rust_guests/witguest/Cargo.lock b/src/tests/rust_guests/witguest/Cargo.lock index 6172a003c..b930b9616 100644 --- a/src/tests/rust_guests/witguest/Cargo.lock +++ b/src/tests/rust_guests/witguest/Cargo.lock @@ -532,9 +532,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "syn" -version = "2.0.107" +version = "2.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" dependencies = [ "proc-macro2", "quote", From 9aa129d78a44e1d7a193993068ff6dd7d96fe38b Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Thu, 23 Oct 2025 22:37:25 +0100 Subject: [PATCH 24/25] fix tests Signed-off-by: Simon Davies --- src/hyperlight_host/tests/integration_test.rs | 158 ++++++++++++++---- 1 file changed, 122 insertions(+), 36 deletions(-) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index a522691c3..ae62a086b 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -103,6 +103,7 @@ fn interrupt_in_progress_guest_call() { #[test] fn interrupt_guest_call_in_advance() { let mut sbox1: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); + let snapshot = sbox1.snapshot().unwrap(); let barrier = Arc::new(Barrier::new(2)); let barrier2 = barrier.clone(); let interrupt_handle = sbox1.interrupt_handle(); @@ -118,6 +119,8 @@ fn interrupt_guest_call_in_advance() { barrier.wait(); // wait until `kill()` is called before starting the guest call let res = sbox1.call::("Spin", ()).unwrap_err(); + sbox1.restore(&snapshot).unwrap(); + assert!(matches!(res, HyperlightError::ExecutionCanceledByHost())); // Make sure we can still call guest functions after the VM was interrupted @@ -142,6 +145,7 @@ fn interrupt_guest_call_in_advance() { fn interrupt_same_thread() { let mut sbox1: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); let mut sbox2: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); + let snapshot = sbox2.snapshot().unwrap(); let mut sbox3: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); let barrier = Arc::new(Barrier::new(2)); @@ -166,9 +170,12 @@ fn interrupt_same_thread() { .call::("Echo", "hello".to_string()) .expect("Only sandbox 2 is allowed to be interrupted"); match sbox2.call::("Echo", "hello".to_string()) { - Ok(_) | Err(HyperlightError::ExecutionCanceledByHost()) => { - // Only allow successful calls or interrupted. - // The call can be successful in case the call is finished before kill() is called. + // Only allow successful calls or interrupted. + // The call can be successful in case the call is finished before kill() is called. + Ok(_) => {} + + Err(HyperlightError::ExecutionCanceledByHost()) => { + sbox2.restore(&snapshot).unwrap(); } _ => panic!("Unexpected return"), }; @@ -184,6 +191,7 @@ fn interrupt_same_thread() { fn interrupt_same_thread_no_barrier() { let mut sbox1: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); let mut sbox2: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); + let snapshot = sbox2.snapshot().unwrap(); let mut sbox3: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); let barrier = Arc::new(Barrier::new(2)); @@ -210,9 +218,11 @@ fn interrupt_same_thread_no_barrier() { .call::("Echo", "hello".to_string()) .expect("Only sandbox 2 is allowed to be interrupted"); match sbox2.call::("Echo", "hello".to_string()) { - Ok(_) | Err(HyperlightError::ExecutionCanceledByHost()) => { - // Only allow successful calls or interrupted. - // The call can be successful in case the call is finished before kill() is called. + // Only allow successful calls or interrupted. + // The call can be successful in case the call is finished before kill() is called. + Ok(_) => {} + Err(HyperlightError::ExecutionCanceledByHost()) => { + sbox2.restore(&snapshot).unwrap(); } _ => panic!("Unexpected return"), }; @@ -860,10 +870,19 @@ fn test_if_guest_is_able_to_get_string_return_values_from_host() { /// - We get a mix of killed and non-killed outcomes (not 100% or 0%) #[test] fn interrupt_random_kill_stress_test() { + // Wrapper to hold a sandbox and its snapshot together + struct SandboxWithSnapshot { + sandbox: MultiUseSandbox, + snapshot: Snapshot, + } + use std::collections::VecDeque; use std::sync::Mutex; use std::sync::atomic::AtomicUsize; + use hyperlight_host::sandbox::snapshot::Snapshot; + use log::{error, trace}; + const POOL_SIZE: usize = 100; const NUM_THREADS: usize = 100; const ITERATIONS_PER_THREAD: usize = 500; @@ -872,13 +891,15 @@ fn interrupt_random_kill_stress_test() { // Create a pool of 50 sandboxes println!("Creating pool of {} sandboxes...", POOL_SIZE); - let mut sandbox_pool: Vec = Vec::with_capacity(POOL_SIZE); + let mut sandbox_pool: Vec = Vec::with_capacity(POOL_SIZE); for i in 0..POOL_SIZE { - let sandbox = new_uninit_rust().unwrap().evolve().unwrap(); + let mut sandbox = new_uninit_rust().unwrap().evolve().unwrap(); + // Create a snapshot for this sandbox + let snapshot = sandbox.snapshot().unwrap(); if (i + 1) % 10 == 0 { println!("Created {}/{} sandboxes", i + 1, POOL_SIZE); } - sandbox_pool.push(sandbox); + sandbox_pool.push(SandboxWithSnapshot { sandbox, snapshot }); } // Wrap the pool in Arc> for thread-safe access @@ -893,6 +914,7 @@ fn interrupt_random_kill_stress_test() { let killed_but_completed_ok = Arc::new(AtomicUsize::new(0)); let killed_but_error = Arc::new(AtomicUsize::new(0)); // Non-cancelled errors let unexpected_cancelled = Arc::new(AtomicUsize::new(0)); // CRITICAL: non-killed calls that got cancelled + let sandbox_replaced_count = Arc::new(AtomicUsize::new(0)); // Sandboxes replaced due to restore failure println!( "Starting {} threads with {} iterations each...", @@ -911,9 +933,9 @@ fn interrupt_random_kill_stress_test() { let killed_but_completed_ok_clone = Arc::clone(&killed_but_completed_ok); let killed_but_error_clone = Arc::clone(&killed_but_error); let unexpected_cancelled_clone = Arc::clone(&unexpected_cancelled); + let sandbox_replaced_count_clone = Arc::clone(&sandbox_replaced_count); let handle = thread::spawn(move || { - println!("[THREAD-{}] Thread started, initializing RNG...", thread_id); // Use thread_id as seed for reproducible randomness per thread use std::collections::hash_map::RandomState; use std::hash::{BuildHasher, Hash}; @@ -922,11 +944,6 @@ fn interrupt_random_kill_stress_test() { thread_id.hash(&mut hasher); let mut rng_state = RandomState::new().hash_one(thread_id); - println!( - "[THREAD-{}] RNG initialized, entering iteration loop...", - thread_id - ); - // Simple random number generator for reproducible randomness let mut next_random = || -> u64 { rng_state = rng_state.wrapping_mul(6364136223846793005).wrapping_add(1); @@ -936,14 +953,14 @@ fn interrupt_random_kill_stress_test() { for iteration in 0..ITERATIONS_PER_THREAD { // === START OF ITERATION === // Get a sandbox from the pool for this iteration - let sandbox = loop { + let sandbox_with_snapshot = loop { let mut pool_guard = pool_clone.lock().unwrap(); if let Some(sb) = pool_guard.pop_front() { break sb; } // Pool is empty, release lock and wait drop(pool_guard); - eprintln!( + trace!( "[THREAD-{}] Iteration {}: Pool empty, waiting for sandbox...", thread_id, iteration ); @@ -952,22 +969,25 @@ fn interrupt_random_kill_stress_test() { // Use a guard struct to ensure sandbox is always returned to pool struct SandboxGuard<'a> { - sandbox: Option, - pool: &'a Arc>>, + sandbox_with_snapshot: Option, + pool: &'a Arc>>, } impl<'a> Drop for SandboxGuard<'a> { fn drop(&mut self) { - if let Some(sb) = self.sandbox.take() { + if let Some(sb) = self.sandbox_with_snapshot.take() { let mut pool_guard = self.pool.lock().unwrap(); pool_guard.push_back(sb); - // eprintln!("[GUARD] Returned sandbox to pool, pool size now: {}", pool_guard.len()); + trace!( + "[GUARD] Returned sandbox to pool, pool size now: {}", + pool_guard.len() + ); } } } let mut guard = SandboxGuard { - sandbox: Some(sandbox), + sandbox_with_snapshot: Some(sandbox_with_snapshot), pool: &pool_clone, }; @@ -978,7 +998,8 @@ fn interrupt_random_kill_stress_test() { kill_attempted_count_clone.fetch_add(1, Ordering::Relaxed); } - let sandbox = guard.sandbox.as_mut().unwrap(); + let sandbox_wrapper = guard.sandbox_with_snapshot.as_mut().unwrap(); + let sandbox = &mut sandbox_wrapper.sandbox; let interrupt_handle = sandbox.interrupt_handle(); // If we decided to kill, spawn a thread that will kill at a random time @@ -994,24 +1015,24 @@ fn interrupt_random_kill_stress_test() { let thread_id_clone = thread_id; let iteration_clone = iteration; let handle = thread::spawn(move || { - eprintln!( + trace!( "[KILLER-{}-{}] Waiting at barrier...", thread_id_clone, iteration_clone ); // Wait at the barrier until the main thread is ready to call the guest barrier_clone.wait(); - eprintln!( + trace!( "[KILLER-{}-{}] Passed barrier, sleeping for {}ms...", thread_id_clone, iteration_clone, kill_delay_ms ); // Random delay between 0 and 15ms (guest runs for ~10ms) thread::sleep(Duration::from_millis(kill_delay_ms)); - eprintln!( + trace!( "[KILLER-{}-{}] Calling kill()...", thread_id_clone, iteration_clone ); interrupt_handle.kill(); - eprintln!( + trace!( "[KILLER-{}-{}] kill() returned, exiting thread", thread_id_clone, iteration_clone ); @@ -1022,26 +1043,26 @@ fn interrupt_random_kill_stress_test() { }; // Call the guest function - eprintln!( + trace!( "[THREAD-{}] Iteration {}: Calling guest function (should_kill={})...", thread_id, iteration, should_kill ); // Release the barrier just before calling the guest function if let Some((_, ref barrier)) = killer_thread { - eprintln!( + trace!( "[THREAD-{}] Iteration {}: Main thread waiting at barrier...", thread_id, iteration ); barrier.wait(); - eprintln!( + trace!( "[THREAD-{}] Iteration {}: Main thread passed barrier, calling guest...", thread_id, iteration ); } let result = sandbox.call::("SpinForMs", GUEST_CALL_DURATION_MS); - eprintln!( + trace!( "[THREAD-{}] Iteration {}: Guest call returned: {:?}", thread_id, iteration, @@ -1053,7 +1074,7 @@ fn interrupt_random_kill_stress_test() { // Wait for killer thread to finish if it was spawned if let Some((kt, _)) = killer_thread { - eprintln!( + trace!( "[THREAD-{}] Iteration {}: Waiting for killer thread to join...", thread_id, iteration ); @@ -1063,13 +1084,68 @@ fn interrupt_random_kill_stress_test() { // Process the result based on whether we attempted to kill match result { Err(HyperlightError::ExecutionCanceledByHost()) => { + // Restore the sandbox from the snapshot + trace!( + "[THREAD-{}] Iteration {}: Restoring sandbox from snapshot after ExecutionCanceledByHost...", + thread_id, iteration + ); + let sandbox_wrapper = guard.sandbox_with_snapshot.as_mut().unwrap(); + + // Try to restore the snapshot + if let Err(e) = sandbox_wrapper.sandbox.restore(&sandbox_wrapper.snapshot) { + error!( + "CRITICAL: Thread {} iteration {}: Failed to restore snapshot: {:?}", + thread_id, iteration, e + ); + trace!( + "[THREAD-{}] Iteration {}: Creating new sandbox to replace failed one...", + thread_id, iteration + ); + + // Create a new sandbox with snapshot + match new_uninit_rust().and_then(|uninit| uninit.evolve()) { + Ok(mut new_sandbox) => { + match new_sandbox.snapshot() { + Ok(new_snapshot) => { + // Replace the failed sandbox with the new one + sandbox_wrapper.sandbox = new_sandbox; + sandbox_wrapper.snapshot = new_snapshot; + sandbox_replaced_count_clone + .fetch_add(1, Ordering::Relaxed); + trace!( + "[THREAD-{}] Iteration {}: Successfully replaced sandbox", + thread_id, iteration + ); + } + Err(snapshot_err) => { + error!( + "CRITICAL: Thread {} iteration {}: Failed to create snapshot for new sandbox: {:?}", + thread_id, iteration, snapshot_err + ); + // Still use the new sandbox even without snapshot + sandbox_wrapper.sandbox = new_sandbox; + sandbox_replaced_count_clone + .fetch_add(1, Ordering::Relaxed); + } + } + } + Err(create_err) => { + error!( + "CRITICAL: Thread {} iteration {}: Failed to create new sandbox: {:?}", + thread_id, iteration, create_err + ); + // Continue with the broken sandbox - it will be removed from pool eventually + } + } + } + if should_kill { // We attempted to kill and it was cancelled - SUCCESS actually_killed_count_clone.fetch_add(1, Ordering::Relaxed); } else { // We did NOT attempt to kill but got cancelled - CRITICAL FAILURE unexpected_cancelled_clone.fetch_add(1, Ordering::Relaxed); - eprintln!( + error!( "CRITICAL: Thread {} iteration {}: Got ExecutionCanceledByHost but did NOT attempt kill!", thread_id, iteration ); @@ -1085,6 +1161,11 @@ fn interrupt_random_kill_stress_test() { } } Err(_other_error) => { + // Log the other error so we can see what it is + error!( + "Thread {} iteration {}: Got non-cancellation error: {:?}", + thread_id, iteration, _other_error + ); if should_kill { // We attempted to kill and got some other error - acceptable killed_but_error_clone.fetch_add(1, Ordering::Relaxed); @@ -1111,7 +1192,7 @@ fn interrupt_random_kill_stress_test() { // SandboxGuard will automatically return sandbox to pool when it goes out of scope } - eprintln!( + trace!( "[THREAD-{}] Completed all {} iterations!", thread_id, ITERATIONS_PER_THREAD ); @@ -1143,6 +1224,7 @@ fn interrupt_random_kill_stress_test() { let killed_but_ok = killed_but_completed_ok.load(Ordering::Relaxed); let killed_but_err = killed_but_error.load(Ordering::Relaxed); let unexpected_cancel = unexpected_cancelled.load(Ordering::Relaxed); + let sandbox_replaced = sandbox_replaced_count.load(Ordering::Relaxed); let no_kill_attempted = total - kill_attempted; @@ -1187,10 +1269,14 @@ fn interrupt_random_kill_stress_test() { "❌ FAILURE" } ); + println!(); + println!("Sandbox Management:"); + println!( + " - Sandboxes replaced due to restore failure: {}", + sandbox_replaced + ); // CRITICAL VALIDATIONS - // TODO: this needs fixing on Windows - we can still kill the following call invocation there - #[cfg(target_os = "linux")] assert_eq!( unexpected_cancel, 0, "FAILURE: {} non-killed calls returned ExecutionCanceledByHost! This indicates false kills.", From f1c3f5447aaf7e2f6f1df6ebb44b1cae76eb2984 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Thu, 23 Oct 2025 23:37:08 +0100 Subject: [PATCH 25/25] fix test Signed-off-by: Simon Davies --- src/hyperlight_host/tests/integration_test.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index ae62a086b..17d0bd109 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -99,11 +99,10 @@ fn interrupt_in_progress_guest_call() { thread.join().expect("Thread should finish"); } -/// Makes sure interrupting a vm before the guest call has started also prevents the guest call from being executed +/// Makes sure interrupting a vm before the guest call has started does not prevent the guest call from running #[test] fn interrupt_guest_call_in_advance() { let mut sbox1: MultiUseSandbox = new_uninit_rust().unwrap().evolve().unwrap(); - let snapshot = sbox1.snapshot().unwrap(); let barrier = Arc::new(Barrier::new(2)); let barrier2 = barrier.clone(); let interrupt_handle = sbox1.interrupt_handle(); @@ -118,12 +117,16 @@ fn interrupt_guest_call_in_advance() { }); barrier.wait(); // wait until `kill()` is called before starting the guest call - let res = sbox1.call::("Spin", ()).unwrap_err(); - sbox1.restore(&snapshot).unwrap(); - - assert!(matches!(res, HyperlightError::ExecutionCanceledByHost())); + match sbox1.call::("Echo", "hello".to_string()) { + Ok(_) => {} + Err(HyperlightError::ExecutionCanceledByHost()) => { + panic!("Unexpected Cancellation Error"); + } + Err(_) => {} + } - // Make sure we can still call guest functions after the VM was interrupted + // Make sure we can still call guest functions after the VM was interrupted early + // i.e. make sure we dont kill the next iteration. sbox1.call::("Echo", "hello".to_string()).unwrap(); // drop vm to make sure other thread can detect it