Skip to content

Commit

Permalink
[SOL] remove stubs for atomics
Browse files Browse the repository at this point in the history 
(Pseudo) atomics are now implemented in SBF
anza-xyz/llvm-project#23
  • Loading branch information
@alessandrod @dmakarov
alessandrod authored and dmakarov committed Mar 4, 2022
1 parent 4a441cb commit c0f2ef0
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 192 deletions.
175 changes: 32 additions & 143 deletions library/core/src/sync/atomic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2342,10 +2342,7 @@ fn strongest_failure_ordering(order: Ordering) -> Ordering {
}

#[inline]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_store`.
unsafe {
match order {
Expand All @@ -2356,18 +2353,10 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
AcqRel => panic!("there is no such thing as an acquire/release store"),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_store`.
unsafe {
*dst = val;
}
}

#[inline]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_load`.
unsafe {
match order {
Expand All @@ -2378,19 +2367,11 @@ unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
AcqRel => panic!("there is no such thing as an acquire/release load"),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_load`.
unsafe {
*dst
}
}

#[inline]
#[cfg(target_has_atomic = "8")]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_swap`.
unsafe {
match order {
Expand All @@ -2401,22 +2382,12 @@ unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
SeqCst => intrinsics::atomic_xchg(dst, val),
}
}
#[cfg(any(target_arch = "bpf" ,target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_swap`.
unsafe {
let old = *dst;
*dst = val;
old
}
}

/// Returns the previous value (like __sync_fetch_and_add).
#[inline]
#[cfg(target_has_atomic = "8")]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_add<T: Copy + crate::ops::Add<Output = T>>(dst: *mut T, val: T, order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_add`.
unsafe {
match order {
Expand All @@ -2427,22 +2398,12 @@ unsafe fn atomic_add<T: Copy + crate::ops::Add<Output = T>>(dst: *mut T, val: T,
SeqCst => intrinsics::atomic_xadd(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_add`.
unsafe {
let old = *dst;
*dst = old + val;
old
}
}

/// Returns the previous value (like __sync_fetch_and_sub).
#[inline]
#[cfg(target_has_atomic = "8")]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_sub<T: Copy + crate::ops::Sub<Output = T>>(dst: *mut T, val: T, order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_sub`.
unsafe {
match order {
Expand All @@ -2453,58 +2414,35 @@ unsafe fn atomic_sub<T: Copy + crate::ops::Sub<Output = T>>(dst: *mut T, val: T,
SeqCst => intrinsics::atomic_xsub(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_sub`.
unsafe {
let old = *dst;
*dst = old - val;
old
}
}

#[inline]
#[cfg(target_has_atomic = "8")]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
unsafe fn atomic_compare_exchange<T: Copy + crate::cmp::PartialEq>(
dst: *mut T,
old: T,
new: T,
success: Ordering,
failure: Ordering,
) -> Result<T, T> {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
{
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
let (val, ok) = unsafe {
match (success, failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
};
if ok { Ok(val) } else { Err(val) }
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
unsafe {
let current = *dst;
if current == old {
*dst = new;
Ok(current)
} else {
Err(current)
let (val, ok) = unsafe {
match (success, failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
}
};
if ok { Ok(val) } else { Err(val) }
}

#[inline]
Expand All @@ -2516,38 +2454,24 @@ unsafe fn atomic_compare_exchange_weak<T: Copy + crate::cmp::PartialEq>(
_success: Ordering,
_failure: Ordering,
) -> Result<T, T> {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
{
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`.
let (val, ok) = unsafe {
match (_success, _failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchgweak_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
};
if ok { Ok(val) } else { Err(val) }
}
#[cfg(any(target_arch = "sbf", target_arch = "bpf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`.
unsafe {
let current = *dst;
if current == old {
*dst = new;
Ok(current)
} else {
Err(current)
let (val, ok) = unsafe {
match (_success, _failure) {
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acq(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchgweak_rel(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel(dst, old, new),
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acq_failrelaxed(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_failrelaxed(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_failrelaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_failacq(dst, old, new),
(_, AcqRel) => panic!("there is no such thing as an acquire/release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
_ => panic!("a failure ordering can't be stronger than a success ordering"),
}
}
};
if ok { Ok(val) } else { Err(val) }
}

#[inline]
Expand Down Expand Up @@ -2614,7 +2538,6 @@ unsafe fn atomic_xor<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
#[inline]
#[cfg(target_has_atomic = "8")]
unsafe fn atomic_max<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_max`
unsafe {
match _order {
Expand All @@ -2625,18 +2548,12 @@ unsafe fn atomic_max<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ord
SeqCst => intrinsics::atomic_max(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_max`
unsafe {
crate::cmp::max(*dst, val)
}
}

/// returns the min value (signed comparison)
#[inline]
#[cfg(target_has_atomic = "8")]
unsafe fn atomic_min<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_min`
unsafe {
match _order {
Expand All @@ -2647,18 +2564,12 @@ unsafe fn atomic_min<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ord
SeqCst => intrinsics::atomic_min(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_min`
unsafe {
crate::cmp::min(*dst, val)
}
}

/// returns the max value (unsigned comparison)
#[inline]
#[cfg(target_has_atomic = "8")]
unsafe fn atomic_umax<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_umax`
unsafe {
match _order {
Expand All @@ -2669,18 +2580,12 @@ unsafe fn atomic_umax<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Or
SeqCst => intrinsics::atomic_umax(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_umax`
unsafe {
crate::cmp::max(*dst, val)
}
}

/// returns the min value (unsigned comparison)
#[inline]
#[cfg(target_has_atomic = "8")]
unsafe fn atomic_umin<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Ordering) -> T {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: the caller must uphold the safety contract for `atomic_umin`
unsafe {
match _order {
Expand All @@ -2691,11 +2596,6 @@ unsafe fn atomic_umin<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Or
SeqCst => intrinsics::atomic_umin(dst, val),
}
}
#[cfg(any(target_arch = "bpf", target_arch = "sbf"))]
// SAFETY: the caller must uphold the safety contract for `atomic_umin`
unsafe {
crate::cmp::min(*dst, val)
}
}

/// An atomic fence.
Expand Down Expand Up @@ -2775,16 +2675,8 @@ unsafe fn atomic_umin<T: Copy + crate::cmp::Ord>(dst: *mut T, val: T, _order: Or
#[inline]
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_diagnostic_item = "fence"]
#[cfg_attr(any(target_arch = "bpf", target_arch = "sbf", target_arch = "wasm32"), allow(unused_variables))]
pub fn fence(order: Ordering) {
#[cfg(not(any(target_arch = "wasm32", target_arch = "bpf", target_arch = "sbf")))]
// SAFETY: using an atomic fence is safe.
// On wasm32 and SBF it looks like fences aren't implemented in LLVM yet in that
// they will cause LLVM to abort. The wasm instruction set doesn't have
// fences right now. There's discussion online about the best way for tools
// to conventionally implement fences at
// https://github.com/WebAssembly/tool-conventions/issues/59. We should
// follow that discussion and implement a solution when one comes about!
unsafe {
match order {
Acquire => intrinsics::atomic_fence_acq(),
Expand Down Expand Up @@ -2865,10 +2757,7 @@ pub fn fence(order: Ordering) {
#[inline]
#[stable(feature = "compiler_fences", since = "1.21.0")]
#[rustc_diagnostic_item = "compiler_fence"]
#[cfg_attr(target_arch = "bpf", allow(unused_variables))]
#[cfg_attr(target_arch = "sbf", allow(unused_variables))]
pub fn compiler_fence(order: Ordering) {
#[cfg(all(not(target_arch = "bpf"), not(target_arch = "sbf")))]
// SAFETY: using an atomic fence is safe.
unsafe {
match order {
Expand Down
Loading

0 comments on commit c0f2ef0

Please sign in to comment.