diff --git a/.github/scripts/ci-common.sh b/.github/scripts/ci-common.sh index 35e450bdbd..287d1c980b 100644 --- a/.github/scripts/ci-common.sh +++ b/.github/scripts/ci-common.sh @@ -67,7 +67,12 @@ init_non_exclusive_features() { IFS='='; feature=($line); unset IFS; if [[ ! -z "$feature" ]]; then # Trim whitespaces - features[i]=$(echo $feature) + feature_name=$(echo $feature) + # jemalloc does not support Windows + if [[ $os == "windows" && $feature_name == "malloc_jemalloc" ]]; then + continue + fi + features[i]=$feature_name let "i++" fi fi @@ -118,6 +123,11 @@ init_exclusive_features() { if [[ ! -z "$feature" ]]; then # Trim whitespaces features[i]=$(echo $feature) + # jemalloc does not support Windows + if [[ $os == "windows" && $feature_name == "malloc_jemalloc" ]]; then + continue + fi + features[i]=$feature_name let "i++" fi fi diff --git a/.github/scripts/ci-setup-x86_64-pc-windows-msvc.sh b/.github/scripts/ci-setup-x86_64-pc-windows-msvc.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/minimal-tests-core.yml b/.github/workflows/minimal-tests-core.yml index 2a113ca850..69edf67362 100644 --- a/.github/workflows/minimal-tests-core.yml +++ b/.github/workflows/minimal-tests-core.yml @@ -37,11 +37,16 @@ jobs: - { os: ubuntu-22.04, triple: x86_64-unknown-linux-gnu } - { os: ubuntu-22.04, triple: i686-unknown-linux-gnu } - { os: macos-15, triple: x86_64-apple-darwin } + - { os: windows-latest, triple: x86_64-pc-windows-msvc } rust: ${{ fromJson(needs.setup-test-matrix.outputs.rust )}} name: minimal-tests-core/${{ matrix.target.triple }}/${{ matrix.rust }} runs-on: ${{ matrix.target.os }} + defaults: + run: + shell: bash + env: # This determines the default target which cargo-build, cargo-test, etc. use. CARGO_BUILD_TARGET: "${{ matrix.target.triple }}" @@ -88,11 +93,16 @@ jobs: - { os: ubuntu-22.04, triple: x86_64-unknown-linux-gnu } - { os: ubuntu-22.04, triple: i686-unknown-linux-gnu } - { os: macos-15, triple: x86_64-apple-darwin } + - { os: windows-latest, triple: x86_64-pc-windows-msvc } rust: ${{ fromJson(needs.setup-test-matrix.outputs.rust )}} name: style-check/${{ matrix.target.triple }}/${{ matrix.rust }} runs-on: ${{ matrix.target.os }} + defaults: + run: + shell: bash + env: # This determines the default target which cargo-build, cargo-test, etc. use. CARGO_BUILD_TARGET: "${{ matrix.target.triple }}" diff --git a/Cargo.toml b/Cargo.toml index 5ca7a0358e..e244adb175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,15 @@ strum = "0.27.1" strum_macros = "0.27.1" sysinfo = "0.33.1" +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = [ + "Win32_Foundation", + "Win32_System_Memory", + "Win32_System_SystemInformation", + "Win32_System_Threading", + "Win32_System_Diagnostics_Debug", +] } + [dev-dependencies] paste = "1.0.8" rand = "0.9.0" diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 4f0564b9e5..6505fd2e60 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -13,7 +13,6 @@ use crate::util::object_forwarding; use crate::util::{copy::*, object_enum}; use crate::util::{Address, ObjectReference}; use crate::vm::*; -use libc::{mprotect, PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -293,8 +292,8 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - unsafe { - mprotect(start.to_mut_ptr(), extent, PROT_NONE); + if let Err(e) = crate::util::memory::mprotect(start, extent) { + panic!("Failed to protect memory: {:?}", e); } trace!("Protect {:x} {:x}", start, start + extent); } @@ -308,12 +307,12 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - unsafe { - mprotect( - start.to_mut_ptr(), - extent, - PROT_READ | PROT_WRITE | PROT_EXEC, - ); + if let Err(e) = crate::util::memory::munprotect( + start, + extent, + crate::util::memory::MmapProtection::ReadWriteExec, + ) { + panic!("Failed to unprotect memory: {:?}", e); } trace!("Unprotect {:x} {:x}", start, start + extent); } diff --git a/src/scheduler/affinity.rs b/src/scheduler/affinity.rs index 331ec330b8..819489c128 100644 --- a/src/scheduler/affinity.rs +++ b/src/scheduler/affinity.rs @@ -20,7 +20,17 @@ pub fn get_total_num_cpus() -> u16 { } } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "windows")] +/// Return the total number of cores allocated to the program. +pub fn get_total_num_cpus() -> u16 { + unsafe { + windows_sys::Win32::System::Threading::GetActiveProcessorCount( + windows_sys::Win32::System::Threading::ALL_PROCESSOR_GROUPS, + ) as u16 + } +} + +#[cfg(not(any(target_os = "linux", target_os = "windows")))] /// Return the total number of cores allocated to the program. pub fn get_total_num_cpus() -> u16 { unimplemented!() @@ -59,7 +69,18 @@ fn bind_current_thread_to_core(cpu: CoreId) { } } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "windows")] +/// Bind the current thread to the specified core. +fn bind_current_thread_to_core(cpu: CoreId) { + unsafe { + windows_sys::Win32::System::Threading::SetThreadAffinityMask( + windows_sys::Win32::System::Threading::GetCurrentThread(), + 1 << cpu, + ); + } +} + +#[cfg(not(any(target_os = "linux", target_os = "windows")))] /// Bind the current thread to the specified core. fn bind_current_thread_to_core(_cpu: CoreId) { unimplemented!() @@ -79,7 +100,22 @@ fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { } } -#[cfg(not(any(target_os = "linux", target_os = "android")))] +#[cfg(target_os = "windows")] +/// Bind the current thread to the specified core. +fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { + let mut mask = 0; + for cpu in cpuset { + mask |= 1 << cpu; + } + unsafe { + windows_sys::Win32::System::Threading::SetThreadAffinityMask( + windows_sys::Win32::System::Threading::GetCurrentThread(), + mask, + ); + } +} + +#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "windows")))] /// Bind the current thread to the specified core. fn bind_current_thread_to_cpuset(_cpuset: &[CoreId]) { unimplemented!() diff --git a/src/util/malloc/library.rs b/src/util/malloc/library.rs index 2177a06144..e716502a04 100644 --- a/src/util/malloc/library.rs +++ b/src/util/malloc/library.rs @@ -2,10 +2,18 @@ #[cfg(feature = "malloc_jemalloc")] pub use self::jemalloc::*; -#[cfg(not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc",)))] +#[cfg(all( + not(target_os = "windows"), + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] pub use self::libc_malloc::*; #[cfg(feature = "malloc_mimalloc")] pub use self::mimalloc::*; +#[cfg(all( + target_os = "windows", + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] +pub use self::win_malloc::*; /// When we count page usage of library malloc, we assume they allocate in pages. For some malloc implementations, /// they may use a larger page (e.g. mimalloc's 64K page). For libraries that we are not sure, we assume they use @@ -43,7 +51,10 @@ mod mimalloc { } /// If no malloc lib is specified, use the libc implementation -#[cfg(not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc",)))] +#[cfg(all( + not(target_os = "windows"), + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] mod libc_malloc { // Normal 4K page pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE; @@ -61,3 +72,42 @@ mod libc_malloc { #[cfg(target_os = "macos")] pub use self::malloc_size as malloc_usable_size; } + +/// Windows malloc implementation using HeapAlloc +#[cfg(all( + target_os = "windows", + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] +mod win_malloc { + // Normal 4K page + pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE; + + use std::ffi::c_void; + use windows_sys::Win32::System::Memory::*; + + pub unsafe fn malloc(size: usize) -> *mut c_void { + HeapAlloc(GetProcessHeap(), 0, size) + } + + pub unsafe fn free(ptr: *mut c_void) { + if !ptr.is_null() { + HeapFree(GetProcessHeap(), 0, ptr); + } + } + + pub unsafe fn calloc(nmemb: usize, size: usize) -> *mut c_void { + let total = nmemb * size; + HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, total) + } + + pub unsafe fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void { + if ptr.is_null() { + return malloc(size); + } + HeapReAlloc(GetProcessHeap(), 0, ptr, size) + } + + pub unsafe fn malloc_usable_size(ptr: *const c_void) -> usize { + HeapSize(GetProcessHeap(), 0, ptr) + } +} diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 5d27cf79cb..19400a485b 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -4,6 +4,12 @@ use crate::util::Address; use crate::vm::VMBinding; /// Allocate with alignment. This also guarantees the memory is zero initialized. +/// This uses posix_memalign, which is not available on Windows. +/// This would somehow affect `MallocMarkSweep` performance on Windows. +#[cfg(all( + not(target_os = "windows"), + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] pub fn align_alloc(size: usize, align: usize) -> Address { let mut ptr = std::ptr::null_mut::(); let ptr_ptr = std::ptr::addr_of_mut!(ptr); @@ -71,28 +77,41 @@ pub fn alloc(size: usize, align: usize, offset: usize) -> (Addres let mut is_offset_malloc = false; // malloc returns 16 bytes aligned address. // So if the alignment is smaller than 16 bytes, we do not need to align. - if align <= 16 && offset == 0 { - let raw = unsafe { calloc(1, size) }; - address = Address::from_mut_ptr(raw); - debug_assert!(address.is_aligned_to(align)); - } else if align > 16 && offset == 0 { - address = align_alloc(size, align); - debug_assert!( - address.is_aligned_to(align), - "Address: {:x} is not aligned to the given alignment: {}", - address, - align - ); - } else { - address = align_offset_alloc::(size, align, offset); - is_offset_malloc = true; - debug_assert!( - (address + offset).is_aligned_to(align), - "Address: {:x} is not aligned to the given alignment: {} at offset: {}", - address, - align, - offset - ); + + match (align, offset) { + (a, 0) if a <= 16 => { + let raw = unsafe { calloc(1, size) }; + address = Address::from_mut_ptr(raw); + debug_assert!(address.is_aligned_to(align)); + } + #[cfg(all( + not(target_os = "windows"), + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) + ))] + // On non-Windows platforms with posix_memalign, we can use align_alloc for alignments > 16 + // However, on Windows, there is no equivalent function. + // The memory alloc by `align_alloc` may not be freed correctly by `free`. + // So we use offset allocation for all alignments > 16 on Windows. + (a, 0) if a > 16 => { + address = align_alloc(size, align); + debug_assert!( + address.is_aligned_to(align), + "Address: {:x} is not aligned to the given alignment: {}", + address, + align + ); + } + _ => { + address = align_offset_alloc::(size, align, offset); + is_offset_malloc = true; + debug_assert!( + (address + offset).is_aligned_to(align), + "Address: {:x} is not aligned to the given alignment: {} at offset: {}", + address, + align, + offset + ); + } } (address, is_offset_malloc) } diff --git a/src/util/memory.rs b/src/util/memory.rs index 9a9d4d16dc..2e267ae17b 100644 --- a/src/util/memory.rs +++ b/src/util/memory.rs @@ -3,6 +3,7 @@ use crate::util::opaque_pointer::*; use crate::util::Address; use crate::vm::{Collection, VMBinding}; use bytemuck::NoUninit; +#[cfg(not(target_os = "windows"))] use libc::{PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; use std::io::{Error, Result}; use sysinfo::MemoryRefreshKind; @@ -14,6 +15,10 @@ const MMAP_FLAGS: libc::c_int = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_F #[cfg(target_os = "macos")] // MAP_FIXED is used instead of MAP_FIXED_NOREPLACE (which is not available on macOS). We are at the risk of overwriting pre-existing mappings. const MMAP_FLAGS: libc::c_int = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED; +#[cfg(target_os = "windows")] +const MMAP_FLAGS: libc::c_int = 0; // Not used on Windows +#[cfg(target_os = "windows")] +const MAP_NORESERVE: libc::c_int = 0x4000; // Custom flag for Windows emulation /// Strategy for performing mmap #[derive(Debug, Copy, Clone)] @@ -64,7 +69,8 @@ pub enum MmapProtection { } impl MmapProtection { - /// Turn the protection enum into the native flags + /// Turn the protection enum into the native flags on non-Windows platforms + #[cfg(not(target_os = "windows"))] pub fn into_native_flags(self) -> libc::c_int { match self { Self::ReadWrite => PROT_READ | PROT_WRITE, @@ -72,6 +78,17 @@ impl MmapProtection { Self::NoAccess => PROT_NONE, } } + + /// Turn the protection enum into the native flags on Windows platforms + #[cfg(target_os = "windows")] + pub fn into_native_flags(self) -> u32 { + use windows_sys::Win32::System::Memory::*; + match self { + Self::ReadWrite => PAGE_READWRITE, + Self::ReadWriteExec => PAGE_EXECUTE_READWRITE, + Self::NoAccess => PAGE_NOACCESS, + } + } } /// Support for huge pages @@ -157,10 +174,20 @@ impl std::fmt::Display for MmapAnnotation<'_> { /// Check the result from an mmap function in this module. /// Return true if the mmap has failed due to an existing conflicting mapping. pub(crate) fn result_is_mapped(result: Result<()>) -> bool { + #[cfg(not(target_os = "windows"))] match result { Ok(_) => false, Err(err) => err.raw_os_error().unwrap() == libc::EEXIST, } + #[cfg(target_os = "windows")] + match result { + Ok(_) => false, + Err(err) => { + // ERROR_INVALID_ADDRESS may be returned if the address is already mapped or invalid + err.raw_os_error().unwrap() + == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 + } + } } /// Set a range of memory to 0. @@ -191,10 +218,14 @@ pub unsafe fn dzmmap( strategy: MmapStrategy, anno: &MmapAnnotation, ) -> Result<()> { + #[cfg(not(target_os = "windows"))] let flags = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED; + #[cfg(target_os = "windows")] + let flags = 0; // Not used let ret = mmap_fixed(start, size, flags, strategy, anno); // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - #[cfg(not(target_os = "linux"))] + // On Windows, MEM_COMMIT guarantees zero-initialized pages. + #[cfg(not(any(target_os = "linux", target_os = "windows")))] if ret.is_ok() { zero(start, size) } @@ -210,10 +241,15 @@ pub fn dzmmap_noreplace( strategy: MmapStrategy, anno: &MmapAnnotation, ) -> Result<()> { + #[cfg(not(target_os = "windows"))] let flags = MMAP_FLAGS; + #[cfg(target_os = "windows")] + let flags = 0; // Not used + let ret = mmap_fixed(start, size, flags, strategy, anno); // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - #[cfg(not(target_os = "linux"))] + // On Windows, MEM_COMMIT guarantees zero-initialized pages. + #[cfg(not(any(target_os = "linux", target_os = "windows")))] if ret.is_ok() { zero(start, size) } @@ -231,7 +267,10 @@ pub fn mmap_noreserve( anno: &MmapAnnotation, ) -> Result<()> { strategy.prot = MmapProtection::NoAccess; + #[cfg(not(target_os = "windows"))] let flags = MMAP_FLAGS | libc::MAP_NORESERVE; + #[cfg(target_os = "windows")] + let flags = MAP_NORESERVE; mmap_fixed(start, size, flags, strategy, anno) } @@ -242,64 +281,187 @@ fn mmap_fixed( strategy: MmapStrategy, _anno: &MmapAnnotation, ) -> Result<()> { - let ptr = start.to_mut_ptr(); - let prot = strategy.prot.into_native_flags(); - wrap_libc_call( - &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, - ptr, - )?; - - #[cfg(all( - any(target_os = "linux", target_os = "android"), - not(feature = "no_mmap_annotation") - ))] + #[cfg(not(target_os = "windows"))] { - // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that - // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not - // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` - // for other reasons, too. That includes `start` being an invalid address, and the - // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But - // since this prctl is used for debugging, we log the error instead of panicking. - let anno_str = _anno.to_string(); - let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); - let result = wrap_libc_call( - &|| unsafe { - libc::prctl( - libc::PR_SET_VMA, - libc::PR_SET_VMA_ANON_NAME, - start.to_ptr::(), - size, - anno_cstr.as_ptr(), - ) - }, - 0, - ); - if let Err(e) = result { - debug!("Error while calling prctl: {e}"); + let ptr = start.to_mut_ptr(); + let prot = strategy.prot.into_native_flags(); + wrap_libc_call( + &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, + ptr, + )?; + + #[cfg(all( + any(target_os = "linux", target_os = "android"), + not(feature = "no_mmap_annotation") + ))] + { + // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that + // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not + // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` + // for other reasons, too. That includes `start` being an invalid address, and the + // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But + // since this prctl is used for debugging, we log the error instead of panicking. + let anno_str = _anno.to_string(); + let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); + let result = wrap_libc_call( + &|| unsafe { + libc::prctl( + libc::PR_SET_VMA, + libc::PR_SET_VMA_ANON_NAME, + start.to_ptr::(), + size, + anno_cstr.as_ptr(), + ) + }, + 0, + ); + if let Err(e) = result { + debug!("Error while calling prctl: {e}"); + } + } + + match strategy.huge_page { + HugePageSupport::No => Ok(()), + HugePageSupport::TransparentHugePages => { + #[cfg(target_os = "linux")] + { + wrap_libc_call( + &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, + 0, + ) + } + // Setting the transparent hugepage option to true will not pass + // the validation on non-Linux OSes + #[cfg(not(target_os = "linux"))] + unreachable!() + } } } - match strategy.huge_page { - HugePageSupport::No => Ok(()), - HugePageSupport::TransparentHugePages => { - #[cfg(target_os = "linux")] - { - wrap_libc_call( - &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, - 0, - ) + #[cfg(target_os = "windows")] + { + use std::io; + use windows_sys::Win32::System::Memory::{ + VirtualAlloc, VirtualQuery, MEMORY_BASIC_INFORMATION, MEM_COMMIT, MEM_FREE, MEM_RESERVE, + }; + + let ptr: *mut u8 = start.to_mut_ptr(); + let prot = strategy.prot.into_native_flags(); + + // Has to COMMIT inmediately if: + // - not MAP_NORESERVE + // - and protection is not NoAccess + let commit = + (flags & MAP_NORESERVE) == 0 && !matches!(strategy.prot, MmapProtection::NoAccess); + + // Scan the region [ptr, ptr + size) to understand its current state + unsafe { + let mut addr = ptr; + let end = ptr.add(size); + + let mut saw_free = false; + let mut saw_reserved = false; + let mut saw_committed = false; + + while addr < end { + let mut mbi: MEMORY_BASIC_INFORMATION = std::mem::zeroed(); + let q = VirtualQuery( + addr as *const _, + &mut mbi, + std::mem::size_of::(), + ); + if q == 0 { + return Err(io::Error::last_os_error()); + } + + let region_base = mbi.BaseAddress as *mut u8; + let region_size = mbi.RegionSize; + let region_end = region_base.add(region_size); + + // Calculate the intersection of [addr, end) and [region_base, region_end) + let _sub_begin = if addr > region_base { + addr + } else { + region_base + }; + let _sub_end = if end < region_end { end } else { region_end }; + + match mbi.State { + MEM_FREE => saw_free = true, + MEM_RESERVE => saw_reserved = true, + MEM_COMMIT => saw_committed = true, + _ => { + return Err(io::Error::other("Unexpected memory state in mmap_fixed")); + } + } + + // Jump to the next region (VirtualQuery always returns "continuous regions with the same attributes") + addr = region_end; + } + + // 1. All FREE: make a new mapping in the region + // 2. All RESERVE/COMMIT: treat as an existing mapping, can just COMMIT or succeed directly + // 3. MIX of FREE + others: not allowed (semantically similar to MAP_FIXED_NOREPLACE) + if saw_free && (saw_reserved || saw_committed) { + return Err(io::Error::from_raw_os_error( + windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32, + )); + } + + if saw_free && !saw_reserved && !saw_committed { + // All FREE: make a new mapping in the region + let mut allocation_type = MEM_RESERVE; + if commit { + allocation_type |= MEM_COMMIT; + } + + let res = VirtualAlloc(ptr as *mut _, size, allocation_type, prot); + if res.is_null() { + return Err(io::Error::last_os_error()); + } + + Ok(()) + } else { + // This behavior is similar to mmap with MAP_FIXED on Linux. + // If the region is already mapped, we just ensure the required commitment. + // If commit is not needed, we just return Ok. + if commit { + let res = VirtualAlloc(ptr as *mut _, size, MEM_COMMIT, prot); + if res.is_null() { + return Err(io::Error::last_os_error()); + } + } + Ok(()) } - // Setting the transparent hugepage option to true will not pass - // the validation on non-Linux OSes - #[cfg(not(target_os = "linux"))] - unreachable!() } } } /// Unmap the given memory (in page granularity). This wraps the unsafe libc munmap call. pub fn munmap(start: Address, size: usize) -> Result<()> { - wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0) + #[cfg(not(target_os = "windows"))] + return wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0); + + #[cfg(target_os = "windows")] + { + use windows_sys::Win32::System::Memory::*; + // Using MEM_DECOMMIT will decommit the memory but leave the address space reserved. + // This is the safest way to emulate munmap on Windows, as MEM_RELEASE would free + // the entire allocation, which could be larger than the requested size. + let res = unsafe { VirtualFree(start.to_mut_ptr(), size, MEM_DECOMMIT) }; + if res == 0 { + // If decommit fails, we try to release the memory. This might happen if the memory was + // only reserved. + let res_release = unsafe { VirtualFree(start.to_mut_ptr(), 0, MEM_RELEASE) }; + if res_release == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } else { + Ok(()) + } + } } /// Properly handle errors from a mmap Result, including invoking the binding code in the case of @@ -329,18 +491,37 @@ pub fn handle_mmap_error( // further check the error if let Some(os_errno) = error.raw_os_error() { // If it is OOM, we invoke out_of_memory() through the VM interface. + #[cfg(not(target_os = "windows"))] if os_errno == libc::ENOMEM { // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. trace!("Signal MmapOutOfMemory!"); VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); unreachable!() } + #[cfg(target_os = "windows")] + if os_errno == windows_sys::Win32::Foundation::ERROR_NOT_ENOUGH_MEMORY as i32 { + // ERROR_NOT_ENOUGH_MEMORY + trace!("Signal MmapOutOfMemory!"); + VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); + unreachable!() + } } } ErrorKind::AlreadyExists => { panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?"); } - _ => {} + _ => { + #[cfg(target_os = "windows")] + if let Some(os_errno) = error.raw_os_error() { + // If it is invalid address, we provide a more specific panic message. + if os_errno == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 { + // ERROR_INVALID_ADDRESS + trace!("Signal MmapOutOfMemory!"); + VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); + unreachable!() + } + } + } } panic!("Unexpected mmap failure: {:?}", error) } @@ -380,19 +561,49 @@ pub(crate) fn panic_if_unmapped(_start: Address, _size: usize, _anno: &MmapAnnot /// Unprotect the given memory (in page granularity) to allow access (PROT_READ/WRITE/EXEC). pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - let prot = prot.into_native_flags(); - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, - 0, - ) + #[cfg(not(target_os = "windows"))] + { + let prot = prot.into_native_flags(); + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, + 0, + ) + } + #[cfg(target_os = "windows")] + { + use windows_sys::Win32::System::Memory::*; + let prot = prot.into_native_flags(); + let mut old_protect = 0; + let res = unsafe { VirtualProtect(start.to_mut_ptr(), size, prot, &mut old_protect) }; + if res == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } } /// Protect the given memory (in page granularity) to forbid any access (PROT_NONE). pub fn mprotect(start: Address, size: usize) -> Result<()> { - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, PROT_NONE) }, - 0, - ) + #[cfg(not(target_os = "windows"))] + { + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, PROT_NONE) }, + 0, + ) + } + #[cfg(target_os = "windows")] + { + use windows_sys::Win32::System::Memory::*; + let mut old_protect = 0; + let res = + unsafe { VirtualProtect(start.to_mut_ptr(), size, PAGE_NOACCESS, &mut old_protect) }; + if res == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } } fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { diff --git a/src/util/raw_memory_freelist.rs b/src/util/raw_memory_freelist.rs index 23f6844ed7..f02e6ad971 100644 --- a/src/util/raw_memory_freelist.rs +++ b/src/util/raw_memory_freelist.rs @@ -222,9 +222,7 @@ impl Drop for RawMemoryFreeList { fn drop(&mut self) { let len = self.high_water - self.base; if len != 0 { - unsafe { - ::libc::munmap(self.base.as_usize() as _, len); - } + let _ = crate::util::memory::munmap(self.base, len); } } } diff --git a/src/util/rust_util/mod.rs b/src/util/rust_util/mod.rs index f31f68b5ec..10f57c560b 100644 --- a/src/util/rust_util/mod.rs +++ b/src/util/rust_util/mod.rs @@ -108,15 +108,22 @@ unsafe impl Sync for InitializeOnce {} /// Create a formatted string that makes the best effort idenfying the current process and thread. pub fn debug_process_thread_id() -> String { - let pid = unsafe { libc::getpid() }; #[cfg(target_os = "linux")] { + let pid = unsafe { libc::getpid() }; // `gettid()` is Linux-specific. let tid = unsafe { libc::gettid() }; format!("PID: {}, TID: {}", pid, tid) } - #[cfg(not(target_os = "linux"))] + #[cfg(target_os = "windows")] { + let pid = unsafe { windows_sys::Win32::System::Threading::GetCurrentProcessId() }; + let tid = unsafe { windows_sys::Win32::System::Threading::GetCurrentThreadId() }; + format!("PID: {}, TID: {}", pid, tid) + } + #[cfg(not(any(target_os = "linux", target_os = "windows")))] + { + let pid = unsafe { libc::getpid() }; // TODO: When we support other platforms, use platform-specific methods to get thread // identifiers. format!("PID: {}", pid) diff --git a/src/util/test_util/mod.rs b/src/util/test_util/mod.rs index 4540cee3b8..be71905b97 100644 --- a/src/util/test_util/mod.rs +++ b/src/util/test_util/mod.rs @@ -53,6 +53,9 @@ const TEST_ADDRESS: Address = #[cfg(target_os = "macos")] const TEST_ADDRESS: Address = crate::util::conversions::chunk_align_down(unsafe { Address::from_usize(0x2_0000_0000) }); +#[cfg(target_os = "windows")] +const TEST_ADDRESS: Address = + crate::util::conversions::chunk_align_down(unsafe { Address::from_usize(0x5_0000_0000) }); // util::heap::layout::mmapper::csm pub(crate) const CHUNK_STATE_MMAPPER_TEST_REGION: MmapTestRegion = diff --git a/src/vm/tests/mock_tests/mock_test_malloc_ms.rs b/src/vm/tests/mock_tests/mock_test_malloc_ms.rs index 1e386bc0d1..e9c9d28b1e 100644 --- a/src/vm/tests/mock_tests/mock_test_malloc_ms.rs +++ b/src/vm/tests/mock_tests/mock_test_malloc_ms.rs @@ -18,7 +18,21 @@ fn test_malloc() { assert!((address4 + 4_isize).is_aligned_to(64)); assert!(!bool1); - assert!(!bool2); + + // Since Windows HeapAlloc only guarantees 16-byte alignment, the allocation with 32-byte alignment + // without offset will be treated as an offset allocation. + if cfg!(all( + not(target_os = "windows"), + not(any( + feature = "malloc_jemalloc", + feature = "malloc_mimalloc" + )) + )) { + assert!(!bool2); + } else { + assert!(bool2); + } + assert!(bool3); assert!(bool4); @@ -30,8 +44,13 @@ fn test_malloc() { unsafe { malloc_ms_util::free(address1.to_mut_ptr()); } - unsafe { - malloc_ms_util::free(address2.to_mut_ptr()); + + if !bool2 { + unsafe { + malloc_ms_util::free(address2.to_mut_ptr()); + } + } else { + malloc_ms_util::offset_free(address2); } malloc_ms_util::offset_free(address3); malloc_ms_util::offset_free(address4);