From 620193b5c139fdcd234fd660dc5708e9cffa4ad7 Mon Sep 17 00:00:00 2001 From: Chris Denton Date: Sun, 3 Mar 2024 22:06:00 +0000 Subject: [PATCH] Windows: Implement mutex using futex Well, the Windows equivalent: `WaitOnAddress`, `WakeByAddressSingle` and `WakeByAddressAll`. --- library/std/src/sys/locks/condvar/mod.rs | 7 ++- .../locks/condvar/{windows.rs => windows7.rs} | 0 library/std/src/sys/locks/mutex/futex.rs | 54 +++++++++++------- library/std/src/sys/locks/mutex/mod.rs | 7 ++- .../locks/mutex/{windows.rs => windows7.rs} | 0 library/std/src/sys/locks/rwlock/mod.rs | 7 ++- .../locks/rwlock/{windows.rs => windows7.rs} | 0 library/std/src/sys/pal/windows/c.rs | 4 ++ library/std/src/sys/pal/windows/futex.rs | 57 +++++++++++++++++++ library/std/src/sys/pal/windows/mod.rs | 2 + .../miri/src/shims/windows/foreign_items.rs | 6 ++ src/tools/miri/src/shims/windows/sync.rs | 15 +++++ 12 files changed, 129 insertions(+), 30 deletions(-) rename library/std/src/sys/locks/condvar/{windows.rs => windows7.rs} (100%) rename library/std/src/sys/locks/mutex/{windows.rs => windows7.rs} (100%) rename library/std/src/sys/locks/rwlock/{windows.rs => windows7.rs} (100%) create mode 100644 library/std/src/sys/pal/windows/futex.rs diff --git a/library/std/src/sys/locks/condvar/mod.rs b/library/std/src/sys/locks/condvar/mod.rs index 126a42a2a4c00..6849cacf88e76 100644 --- a/library/std/src/sys/locks/condvar/mod.rs +++ b/library/std/src/sys/locks/condvar/mod.rs @@ -1,5 +1,6 @@ cfg_if::cfg_if! { if #[cfg(any( + all(target_os = "windows", not(target_vendor="win7")), target_os = "linux", target_os = "android", target_os = "freebsd", @@ -14,9 +15,9 @@ cfg_if::cfg_if! { } else if #[cfg(target_family = "unix")] { mod pthread; pub use pthread::Condvar; - } else if #[cfg(target_os = "windows")] { - mod windows; - pub use windows::Condvar; + } else if #[cfg(all(target_os = "windows", target_vendor = "win7"))] { + mod windows7; + pub use windows7::Condvar; } else if #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] { mod sgx; pub use sgx::Condvar; diff --git a/library/std/src/sys/locks/condvar/windows.rs b/library/std/src/sys/locks/condvar/windows7.rs similarity index 100% rename from library/std/src/sys/locks/condvar/windows.rs rename to library/std/src/sys/locks/condvar/windows7.rs diff --git a/library/std/src/sys/locks/mutex/futex.rs b/library/std/src/sys/locks/mutex/futex.rs index c01229586c302..7427cae94d68a 100644 --- a/library/std/src/sys/locks/mutex/futex.rs +++ b/library/std/src/sys/locks/mutex/futex.rs @@ -1,30 +1,42 @@ use crate::sync::atomic::{ - AtomicU32, + self, Ordering::{Acquire, Relaxed, Release}, }; use crate::sys::futex::{futex_wait, futex_wake}; +cfg_if::cfg_if! { +if #[cfg(windows)] { + // On Windows we can have a smol futex + type Atomic = atomic::AtomicU8; + type State = u8; +} else { + type Atomic = atomic::AtomicU32; + type State = u32; +} +} + pub struct Mutex { - /// 0: unlocked - /// 1: locked, no other threads waiting - /// 2: locked, and other threads waiting (contended) - futex: AtomicU32, + futex: Atomic, } +const UNLOCKED: State = 0; +const LOCKED: State = 1; // locked, no other threads waiting +const CONTENDED: State = 2; // locked, and other threads waiting (contended) + impl Mutex { #[inline] pub const fn new() -> Self { - Self { futex: AtomicU32::new(0) } + Self { futex: Atomic::new(UNLOCKED) } } #[inline] pub fn try_lock(&self) -> bool { - self.futex.compare_exchange(0, 1, Acquire, Relaxed).is_ok() + self.futex.compare_exchange(UNLOCKED, LOCKED, Acquire, Relaxed).is_ok() } #[inline] pub fn lock(&self) { - if self.futex.compare_exchange(0, 1, Acquire, Relaxed).is_err() { + if self.futex.compare_exchange(UNLOCKED, LOCKED, Acquire, Relaxed).is_err() { self.lock_contended(); } } @@ -36,8 +48,8 @@ impl Mutex { // If it's unlocked now, attempt to take the lock // without marking it as contended. - if state == 0 { - match self.futex.compare_exchange(0, 1, Acquire, Relaxed) { + if state == UNLOCKED { + match self.futex.compare_exchange(UNLOCKED, LOCKED, Acquire, Relaxed) { Ok(_) => return, // Locked! Err(s) => state = s, } @@ -45,31 +57,31 @@ impl Mutex { loop { // Put the lock in contended state. - // We avoid an unnecessary write if it as already set to 2, + // We avoid an unnecessary write if it as already set to CONTENDED, // to be friendlier for the caches. - if state != 2 && self.futex.swap(2, Acquire) == 0 { - // We changed it from 0 to 2, so we just successfully locked it. + if state != CONTENDED && self.futex.swap(CONTENDED, Acquire) == UNLOCKED { + // We changed it from UNLOCKED to CONTENDED, so we just successfully locked it. return; } - // Wait for the futex to change state, assuming it is still 2. - futex_wait(&self.futex, 2, None); + // Wait for the futex to change state, assuming it is still CONTENDED. + futex_wait(&self.futex, CONTENDED, None); // Spin again after waking up. state = self.spin(); } } - fn spin(&self) -> u32 { + fn spin(&self) -> State { let mut spin = 100; loop { // We only use `load` (and not `swap` or `compare_exchange`) // while spinning, to be easier on the caches. let state = self.futex.load(Relaxed); - // We stop spinning when the mutex is unlocked (0), - // but also when it's contended (2). - if state != 1 || spin == 0 { + // We stop spinning when the mutex is UNLOCKED, + // but also when it's CONTENDED. + if state != LOCKED || spin == 0 { return state; } @@ -80,9 +92,9 @@ impl Mutex { #[inline] pub unsafe fn unlock(&self) { - if self.futex.swap(0, Release) == 2 { + if self.futex.swap(UNLOCKED, Release) == CONTENDED { // We only wake up one thread. When that thread locks the mutex, it - // will mark the mutex as contended (2) (see lock_contended above), + // will mark the mutex as CONTENDED (see lock_contended above), // which makes sure that any other waiting threads will also be // woken up eventually. self.wake(); diff --git a/library/std/src/sys/locks/mutex/mod.rs b/library/std/src/sys/locks/mutex/mod.rs index 710cb91fb1473..73d9bd273de17 100644 --- a/library/std/src/sys/locks/mutex/mod.rs +++ b/library/std/src/sys/locks/mutex/mod.rs @@ -1,5 +1,6 @@ cfg_if::cfg_if! { if #[cfg(any( + all(target_os = "windows", not(target_vendor = "win7")), target_os = "linux", target_os = "android", target_os = "freebsd", @@ -19,9 +20,9 @@ cfg_if::cfg_if! { ))] { mod pthread; pub use pthread::{Mutex, raw}; - } else if #[cfg(target_os = "windows")] { - mod windows; - pub use windows::{Mutex, raw}; + } else if #[cfg(all(target_os = "windows", target_vendor = "win7"))] { + mod windows7; + pub use windows7::{Mutex, raw}; } else if #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] { mod sgx; pub use sgx::Mutex; diff --git a/library/std/src/sys/locks/mutex/windows.rs b/library/std/src/sys/locks/mutex/windows7.rs similarity index 100% rename from library/std/src/sys/locks/mutex/windows.rs rename to library/std/src/sys/locks/mutex/windows7.rs diff --git a/library/std/src/sys/locks/rwlock/mod.rs b/library/std/src/sys/locks/rwlock/mod.rs index 0564f1fe6fab9..675931c64bdd3 100644 --- a/library/std/src/sys/locks/rwlock/mod.rs +++ b/library/std/src/sys/locks/rwlock/mod.rs @@ -1,5 +1,6 @@ cfg_if::cfg_if! { if #[cfg(any( + all(target_os = "windows", not(target_vendor = "win7")), target_os = "linux", target_os = "android", target_os = "freebsd", @@ -14,9 +15,9 @@ cfg_if::cfg_if! { } else if #[cfg(target_family = "unix")] { mod queue; pub use queue::RwLock; - } else if #[cfg(target_os = "windows")] { - mod windows; - pub use windows::RwLock; + } else if #[cfg(all(target_os = "windows", target_vendor = "win7"))] { + mod windows7; + pub use windows7::RwLock; } else if #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] { mod sgx; pub use sgx::RwLock; diff --git a/library/std/src/sys/locks/rwlock/windows.rs b/library/std/src/sys/locks/rwlock/windows7.rs similarity index 100% rename from library/std/src/sys/locks/rwlock/windows.rs rename to library/std/src/sys/locks/rwlock/windows7.rs diff --git a/library/std/src/sys/pal/windows/c.rs b/library/std/src/sys/pal/windows/c.rs index afa9240940494..584e17cd19677 100644 --- a/library/std/src/sys/pal/windows/c.rs +++ b/library/std/src/sys/pal/windows/c.rs @@ -36,6 +36,7 @@ pub type LPVOID = *mut c_void; pub type LPWCH = *mut WCHAR; pub type LPWSTR = *mut WCHAR; +#[cfg(target_vendor = "win7")] pub type PSRWLOCK = *mut SRWLOCK; pub type socklen_t = c_int; @@ -50,7 +51,9 @@ pub const INVALID_HANDLE_VALUE: HANDLE = ::core::ptr::without_provenance_mut(-1i pub const EXIT_SUCCESS: u32 = 0; pub const EXIT_FAILURE: u32 = 1; +#[cfg(target_vendor = "win7")] pub const CONDITION_VARIABLE_INIT: CONDITION_VARIABLE = CONDITION_VARIABLE { Ptr: ptr::null_mut() }; +#[cfg(target_vendor = "win7")] pub const SRWLOCK_INIT: SRWLOCK = SRWLOCK { Ptr: ptr::null_mut() }; pub const INIT_ONCE_STATIC_INIT: INIT_ONCE = INIT_ONCE { Ptr: ptr::null_mut() }; @@ -373,6 +376,7 @@ extern "system" { dwmilliseconds: u32, ) -> BOOL; pub fn WakeByAddressSingle(address: *const c_void); + pub fn WakeByAddressAll(address: *const c_void); } #[cfg(target_vendor = "win7")] diff --git a/library/std/src/sys/pal/windows/futex.rs b/library/std/src/sys/pal/windows/futex.rs new file mode 100644 index 0000000000000..2fe3bec0051ac --- /dev/null +++ b/library/std/src/sys/pal/windows/futex.rs @@ -0,0 +1,57 @@ +use super::api; +use crate::sys::c; +use crate::sys::dur2timeout; +use core::ffi::c_void; +use core::mem; +use core::ptr; +use core::time::Duration; + +#[inline(always)] +pub fn wait_on_address(address: &T, compare: U, timeout: Option) -> bool { + assert_eq!(mem::size_of::(), mem::size_of::()); + unsafe { + let addr = ptr::addr_of!(*address).cast::(); + let size = mem::size_of::(); + let compare_addr = ptr::addr_of!(compare).cast::(); + let timeout = timeout.map(dur2timeout).unwrap_or(c::INFINITE); + c::WaitOnAddress(addr, compare_addr, size, timeout) == c::TRUE + } +} + +#[inline(always)] +pub fn wake_by_address_single(address: &T) -> bool { + unsafe { + let addr = ptr::addr_of!(*address).cast::(); + c::WakeByAddressSingle(addr); + false + } +} + +#[inline(always)] +pub fn wake_by_address_all(address: &T) { + unsafe { + let addr = ptr::addr_of!(*address).cast::(); + c::WakeByAddressAll(addr); + } +} + +#[inline(always)] +pub fn futex_wait(futex: &T, expected: U, timeout: Option) -> bool { + // return false only on timeout + if wait_on_address(futex, expected, timeout) { + true + } else { + api::get_last_error().code != c::ERROR_TIMEOUT + } +} + +#[inline(always)] +pub fn futex_wake(futex: &T) -> bool { + wake_by_address_single(futex); + false +} + +#[inline(always)] +pub fn futex_wake_all(futex: &T) { + wake_by_address_all(futex) +} diff --git a/library/std/src/sys/pal/windows/mod.rs b/library/std/src/sys/pal/windows/mod.rs index a53c4034d0685..6a561518fad65 100644 --- a/library/std/src/sys/pal/windows/mod.rs +++ b/library/std/src/sys/pal/windows/mod.rs @@ -17,6 +17,8 @@ pub mod args; pub mod c; pub mod env; pub mod fs; +#[cfg(not(target_vendor = "win7"))] +pub mod futex; pub mod handle; pub mod io; pub mod net; diff --git a/src/tools/miri/src/shims/windows/foreign_items.rs b/src/tools/miri/src/shims/windows/foreign_items.rs index 734737a86dd58..f56ea06dbe35e 100644 --- a/src/tools/miri/src/shims/windows/foreign_items.rs +++ b/src/tools/miri/src/shims/windows/foreign_items.rs @@ -366,6 +366,12 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { this.WakeByAddressSingle(ptr_op)?; } + "WakeByAddressAll" => { + let [ptr_op] = + this.check_shim(abi, Abi::System { unwind: false }, link_name, args)?; + + this.WakeByAddressAll(ptr_op)?; + } // Dynamic symbol loading "GetProcAddress" => { diff --git a/src/tools/miri/src/shims/windows/sync.rs b/src/tools/miri/src/shims/windows/sync.rs index 2b9801fea68e6..1ce385aaabad4 100644 --- a/src/tools/miri/src/shims/windows/sync.rs +++ b/src/tools/miri/src/shims/windows/sync.rs @@ -384,6 +384,21 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { Ok(()) } + fn WakeByAddressAll(&mut self, ptr_op: &OpTy<'tcx, Provenance>) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let ptr = this.read_pointer(ptr_op)?; + + // See the Linux futex implementation for why this fence exists. + this.atomic_fence(AtomicFenceOrd::SeqCst)?; + + while let Some(thread) = this.futex_wake(ptr.addr().bytes(), u32::MAX) { + this.unblock_thread(thread); + this.unregister_timeout_callback_if_exists(thread); + } + + Ok(()) + } fn SleepConditionVariableSRW( &mut self,