diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs index 69a65e6b0f024..47775f9684008 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs @@ -15,7 +15,7 @@ pub(crate) fn target() -> Target { data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: Arch::AArch64, options: TargetOptions { - features: "+v8a".into(), + features: "+v8a,+outline-atomics".into(), max_atomic_width: Some(128), stack_probes: StackProbeType::Inline, supported_sanitizers: SanitizerSet::ADDRESS diff --git a/library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs b/library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs similarity index 97% rename from library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs rename to library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs index 01d7fb4732918..df0cf76502223 100644 --- a/library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs +++ b/library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs @@ -196,7 +196,7 @@ macro_rules! compare_and_swap { "cbnz w17, 0b", "1:", "ret", - have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS, + have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } } } @@ -228,7 +228,7 @@ macro_rules! compare_and_swap_i128 { "cbnz w15, 0b", "1:", "ret", - have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS, + have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } } } @@ -256,7 +256,7 @@ macro_rules! swap { concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"), "cbnz w17, 0b", "ret", - have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS, + have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } } } @@ -286,7 +286,7 @@ macro_rules! fetch_op { concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"), "cbnz w15, 0b", "ret", - have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS, + have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } } } diff --git a/library/compiler-builtins/compiler-builtins/src/lib.rs b/library/compiler-builtins/compiler-builtins/src/lib.rs index a9dd6e531c5d3..c993209699be4 100644 --- a/library/compiler-builtins/compiler-builtins/src/lib.rs +++ b/library/compiler-builtins/compiler-builtins/src/lib.rs @@ -55,8 +55,8 @@ pub mod arm; #[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))] pub mod aarch64; -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] -pub mod aarch64_linux; +#[cfg(all(target_arch = "aarch64", target_feature = "outline-atomics"))] +pub mod aarch64_outline_atomics; #[cfg(all( kernel_user_helpers, diff --git a/library/std/src/sys/configure_builtins.rs b/library/std/src/sys/configure_builtins.rs index 9d776b778dcbe..f569ec0f51298 100644 --- a/library/std/src/sys/configure_builtins.rs +++ b/library/std/src/sys/configure_builtins.rs @@ -1,13 +1,49 @@ -/// Hook into .init_array to enable LSE atomic operations at startup, if -/// supported. -#[cfg(all(target_arch = "aarch64", target_os = "linux", not(feature = "compiler-builtins-c")))] +/// Enable LSE atomic operations at startup, if supported. +/// +/// Linker sections are based on what [`ctor`] does, with priorities to run slightly before user +/// code: +/// +/// - Apple uses the section `__mod_init_func`, `mod_init_funcs` is needed to set +/// `S_MOD_INIT_FUNC_POINTERS`. There doesn't seem to be a way to indicate priorities. +/// - Windows uses `.CRT$XCT`, which is run before user constructors (these should use `.CRT$XCU`). +/// - ELF uses `.init_array` with a priority of 90, which runs before our `ARGV_INIT_ARRAY` +/// initializer (priority 99). Both are within the 0-100 implementation-reserved range, per docs +/// for the [`prio-ctor-dtor`] warning, and this matches compiler-rt's `CONSTRUCTOR_PRIORITY`. +/// +/// To save startup time, the initializer is only run if outline atomic routines from +/// compiler-builtins may be used. If LSE is known to be available then the calls are never +/// emitted, and if we build the C intrinsics then it has its own initializer using the symbol +/// `__aarch64_have_lse_atomics`. +/// +/// Initialization is done in a global constructor to so we get the same behavior regardless of +/// whether Rust's `init` is used, or if we are in a `dylib` or `no_main` situation (as opposed +/// to doing it as part of pre-main startup). This also matches C implementations. +/// +/// Ideally `core` would have something similar, but detecting the CPU features requires the +/// auxiliary vector from the OS. We do the initialization in `std` rather than as part of +/// `compiler-builtins` because a builtins->std dependency isn't possible, and inlining parts of +/// `std-detect` would be much messier. +/// +/// [`ctor`]: https://github.com/mmastrac/rust-ctor/blob/63382b833ddcbfb8b064f4e86bfa1ed4026ff356/shared/src/macros/mod.rs#L522-L534 +/// [`prio-ctor-dtor`]: https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html +#[cfg(all( + target_arch = "aarch64", + target_feature = "outline-atomics", + not(target_feature = "lse"), + not(feature = "compiler-builtins-c"), +))] #[used] -#[unsafe(link_section = ".init_array.90")] +#[cfg_attr(target_vendor = "apple", unsafe(link_section = "__DATA,__mod_init_func,mod_init_funcs"))] +#[cfg_attr(target_os = "windows", unsafe(link_section = ".CRT$XCT"))] +#[cfg_attr( + not(any(target_vendor = "apple", target_os = "windows")), + unsafe(link_section = ".init_array.90") +)] static RUST_LSE_INIT: extern "C" fn() = { extern "C" fn init_lse() { use crate::arch; - // This is provided by compiler-builtins::aarch64_linux. + // This is provided by compiler-builtins::aarch64_outline_atomics. unsafe extern "C" { fn __rust_enable_lse(); }