Skip to content

Commit

Permalink
AtomicCell: Use atomic-maybe-uninit
Browse files Browse the repository at this point in the history
  • Loading branch information
@taiki-e
taiki-e committed Dec 2, 2023
1 parent a57e655 commit 1718eec
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 67 deletions.
1 change: 1 addition & 0 deletions crossbeam-utils/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ default = ["std"]
std = []

[dependencies]
atomic-maybe-uninit = "0.3"
cfg-if = "1"

# Enable the use of loom for concurrency testing.
Expand Down
3 changes: 3 additions & 0 deletions crossbeam-utils/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,7 @@ fn main() {
if sanitize.contains("thread") {
println!("cargo:rustc-cfg=crossbeam_sanitize_thread");
}
if !sanitize.is_empty() {
println!("cargo:rustc-cfg=crossbeam_atomic_cell_force_fallback");
}
}
186 changes: 134 additions & 52 deletions crossbeam-utils/src/atomic/atomic_cell.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ impl<T> AtomicCell<T> {
/// # Examples
///
/// ```
/// # // Always use fallback for now on environments that do not support inline assembly.
/// # if cfg!(any(miri, crossbeam_loom, crossbeam_atomic_cell_force_fallback)) { return; }
/// use crossbeam_utils::atomic::AtomicCell;
///
/// // This type is internally represented as `AtomicUsize` so we can just use atomic
Expand Down Expand Up @@ -305,21 +307,37 @@ macro_rules! atomic {
loop {
atomic!(@check, $t, AtomicUnit, $a, $atomic_op);

atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op);
atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op);
atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op);
#[cfg(target_has_atomic = "64")]
atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op);
// TODO: AtomicU128 is unstable
// atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op);
// Always use fallback for now on environments that do not support inline assembly.
#[cfg(not(any(
miri,
crossbeam_loom,
crossbeam_atomic_cell_force_fallback,
)))]
{
atomic_maybe_uninit::cfg_has_atomic_8! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u8>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_16! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u16>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_32! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u32>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_64! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u64>, $a, $atomic_op);
}
atomic_maybe_uninit::cfg_has_atomic_128! {
atomic!(@check, $t, atomic_maybe_uninit::AtomicMaybeUninit<u128>, $a, $atomic_op);
}
}

break $fallback_op;
}
};
}

macro_rules! impl_arithmetic {
($t:ty, fallback, $example:tt) => {
($t:ty, fetch_update, $example:tt) => {
impl AtomicCell<$t> {
/// Increments the current value by `val` and returns the previous value.
///
Expand All @@ -337,11 +355,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_add(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_add(val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old.wrapping_add(val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_add(val);
old
}
}
}

/// Decrements the current value by `val` and returns the previous value.
Expand All @@ -360,11 +386,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_sub(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_sub(val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old.wrapping_sub(val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = value.wrapping_sub(val);
old
}
}
}

/// Applies bitwise "and" to the current value and returns the previous value.
Expand All @@ -381,11 +415,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_and(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value &= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old & val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value &= val;
old
}
}
}

/// Applies bitwise "nand" to the current value and returns the previous value.
Expand All @@ -402,11 +444,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_nand(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = !(old & val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(!(old & val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = !(old & val);
old
}
}
}

/// Applies bitwise "or" to the current value and returns the previous value.
Expand All @@ -423,11 +473,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_or(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value |= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old | val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value |= val;
old
}
}
}

/// Applies bitwise "xor" to the current value and returns the previous value.
Expand All @@ -444,11 +502,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_xor(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value ^= val;
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(old ^ val)).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value ^= val;
old
}
}
}

/// Compares and sets the maximum of the current value and `val`,
Expand All @@ -466,11 +532,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_max(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::max(old, val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(cmp::max(old, val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::max(old, val);
old
}
}
}

/// Compares and sets the minimum of the current value and `val`,
Expand All @@ -488,11 +562,19 @@ macro_rules! impl_arithmetic {
/// ```
#[inline]
pub fn fetch_min(&self, val: $t) -> $t {
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::min(old, val);
old
atomic! {
$t, _a,
{
self.fetch_update(|old| Some(cmp::min(old, val))).unwrap()
},
{
let _guard = lock(self.as_ptr() as usize).write();
let value = unsafe { &mut *(self.as_ptr()) };
let old = *value;
*value = cmp::min(old, val);
old
}
}
}
}
};
Expand Down Expand Up @@ -760,15 +842,15 @@ impl_arithmetic!(u64, AtomicU64, "let a = AtomicCell::new(7u64);");
#[cfg(target_has_atomic = "64")]
impl_arithmetic!(i64, AtomicI64, "let a = AtomicCell::new(7i64);");
#[cfg(not(target_has_atomic = "64"))]
impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);");
impl_arithmetic!(u64, fetch_update, "let a = AtomicCell::new(7u64);");
#[cfg(not(target_has_atomic = "64"))]
impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);");
impl_arithmetic!(i64, fetch_update, "let a = AtomicCell::new(7i64);");

// TODO: AtomicU128 is unstable
// impl_arithmetic!(u128, AtomicU128, "let a = AtomicCell::new(7u128);");
// impl_arithmetic!(i128, AtomicI128, "let a = AtomicCell::new(7i128);");
impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);");
impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);");
impl_arithmetic!(u128, fetch_update, "let a = AtomicCell::new(7u128);");
impl_arithmetic!(i128, fetch_update, "let a = AtomicCell::new(7i128);");

impl_arithmetic!(usize, AtomicUsize, "let a = AtomicCell::new(7usize);");
impl_arithmetic!(isize, AtomicIsize, "let a = AtomicCell::new(7isize);");
Expand Down
16 changes: 7 additions & 9 deletions crossbeam-utils/src/atomic/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@
//! * [`AtomicConsume`], for reading from primitive atomic types with "consume" ordering.

#[cfg(target_has_atomic = "ptr")]
// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic
// types have a different in-memory representation than the underlying type.
// TODO: The latest loom supports fences, so fallback using seqlock may be available.
#[cfg(not(crossbeam_loom))]
atomic_maybe_uninit::cfg_has_atomic_cas! {
cfg_if::cfg_if! {
// Use "wide" sequence lock if the pointer width <= 32 for preventing its counter against wrap
// around.
Expand All @@ -23,15 +27,9 @@ cfg_if::cfg_if! {
}
}

#[cfg(target_has_atomic = "ptr")]
// We cannot provide AtomicCell under cfg(crossbeam_loom) because loom's atomic
// types have a different in-memory representation than the underlying type.
// TODO: The latest loom supports fences, so fallback using seqlock may be available.
#[cfg(not(crossbeam_loom))]
mod atomic_cell;
mod consume;

#[cfg(target_has_atomic = "ptr")]
#[cfg(not(crossbeam_loom))]
pub use self::atomic_cell::AtomicCell;
}

mod consume;
pub use self::consume::AtomicConsume;
23 changes: 17 additions & 6 deletions crossbeam-utils/tests/atomic_cell.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@ use crossbeam_utils::atomic::AtomicCell;

#[test]
fn is_lock_free() {
// Always use fallback for now on environments that do not support inline assembly.
if cfg!(any(
miri,
crossbeam_loom,
crossbeam_atomic_cell_force_fallback,
)) {
return;
}

struct UsizeWrap(usize);
struct U8Wrap(bool);
struct I16Wrap(i16);
Expand Down Expand Up @@ -44,8 +53,9 @@ fn is_lock_free() {
cfg!(target_has_atomic = "64")
);

// AtomicU128 is unstable
assert!(!AtomicCell::<u128>::is_lock_free());
// TODO
// // AtomicU128 is unstable
// assert!(!AtomicCell::<u128>::is_lock_free());
}

#[test]
Expand Down Expand Up @@ -318,10 +328,11 @@ fn issue_748() {
}

assert_eq!(mem::size_of::<Test>(), 8);
assert_eq!(
AtomicCell::<Test>::is_lock_free(),
cfg!(target_has_atomic = "64")
);
// TODO
// assert_eq!(
// AtomicCell::<Test>::is_lock_free(),
// cfg!(target_has_atomic = "64")
// );
let x = AtomicCell::new(Test::FieldLess);
assert_eq!(x.load(), Test::FieldLess);
}
Expand Down

0 comments on commit 1718eec

Please sign in to comment.