// Necessary for implementing atomic methods for `AtomicUnit` #![allow(clippy::unit_arg)] use crate::primitive::sync::atomic::{self, Ordering}; use crate::CachePadded; use core::cell::UnsafeCell; use core::cmp; use core::fmt; use core::mem::{self, ManuallyDrop, MaybeUninit}; use core::panic::{RefUnwindSafe, UnwindSafe}; use core::ptr; use super::seq_lock::SeqLock; /// A thread-safe mutable memory location. /// /// This type is equivalent to [`Cell`], except it can also be shared among multiple threads. /// /// Operations on `AtomicCell`s use atomic instructions whenever possible, and synchronize using /// global locks otherwise. You can call [`AtomicCell::::is_lock_free()`] to check whether /// atomic instructions or locks will be used. /// /// Atomic loads use the [`Acquire`] ordering and atomic stores use the [`Release`] ordering. /// /// [`Cell`]: std::cell::Cell /// [`AtomicCell::::is_lock_free()`]: AtomicCell::is_lock_free /// [`Acquire`]: std::sync::atomic::Ordering::Acquire /// [`Release`]: std::sync::atomic::Ordering::Release #[repr(transparent)] pub struct AtomicCell { /// The inner value. /// /// If this value can be transmuted into a primitive atomic type, it will be treated as such. /// Otherwise, all potentially concurrent operations on this data will be protected by a global /// lock. /// /// Using MaybeUninit to prevent code outside the cell from observing partially initialized state: /// /// /// Note: /// - we'll never store uninitialized `T` due to our API only using initialized `T`. /// - this `MaybeUninit` does *not* fix . value: UnsafeCell>, } unsafe impl Send for AtomicCell {} unsafe impl Sync for AtomicCell {} impl UnwindSafe for AtomicCell {} impl RefUnwindSafe for AtomicCell {} impl AtomicCell { /// Creates a new atomic cell initialized with `val`. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(7); /// ``` pub const fn new(val: T) -> AtomicCell { AtomicCell { value: UnsafeCell::new(MaybeUninit::new(val)), } } /// Consumes the atomic and returns the contained value. /// /// This is safe because passing `self` by value guarantees that no other threads are /// concurrently accessing the atomic data. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(7); /// let v = a.into_inner(); /// /// assert_eq!(v, 7); /// ``` pub fn into_inner(self) -> T { let this = ManuallyDrop::new(self); // SAFETY: // - passing `self` by value guarantees that no other threads are concurrently // accessing the atomic data // - the raw pointer passed in is valid because we got it from an owned value. // - `ManuallyDrop` prevents double dropping `T` unsafe { this.as_ptr().read() } } /// Returns `true` if operations on values of this type are lock-free. /// /// If the compiler or the platform doesn't support the necessary atomic instructions, /// `AtomicCell` will use global locks for every potentially concurrent atomic operation. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// // This type is internally represented as `AtomicUsize` so we can just use atomic /// // operations provided by it. /// assert_eq!(AtomicCell::::is_lock_free(), true); /// /// // A wrapper struct around `isize`. /// struct Foo { /// bar: isize, /// } /// // `AtomicCell` will be internally represented as `AtomicIsize`. /// assert_eq!(AtomicCell::::is_lock_free(), true); /// /// // Operations on zero-sized types are always lock-free. /// assert_eq!(AtomicCell::<()>::is_lock_free(), true); /// /// // Very large types cannot be represented as any of the standard atomic types, so atomic /// // operations on them will have to use global locks for synchronization. /// assert_eq!(AtomicCell::<[u8; 1000]>::is_lock_free(), false); /// ``` pub const fn is_lock_free() -> bool { atomic_is_lock_free::() } /// Stores `val` into the atomic cell. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(7); /// /// assert_eq!(a.load(), 7); /// a.store(8); /// assert_eq!(a.load(), 8); /// ``` pub fn store(&self, val: T) { if mem::needs_drop::() { drop(self.swap(val)); } else { unsafe { atomic_store(self.as_ptr(), val); } } } /// Stores `val` into the atomic cell and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(7); /// /// assert_eq!(a.load(), 7); /// assert_eq!(a.swap(8), 7); /// assert_eq!(a.load(), 8); /// ``` pub fn swap(&self, val: T) -> T { unsafe { atomic_swap(self.as_ptr(), val) } } /// Returns a raw pointer to the underlying data in this atomic cell. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(5); /// /// let ptr = a.as_ptr(); /// ``` #[inline] pub fn as_ptr(&self) -> *mut T { self.value.get().cast::() } } impl AtomicCell { /// Takes the value of the atomic cell, leaving `Default::default()` in its place. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(5); /// let five = a.take(); /// /// assert_eq!(five, 5); /// assert_eq!(a.into_inner(), 0); /// ``` pub fn take(&self) -> T { self.swap(Default::default()) } } impl AtomicCell { /// Loads a value from the atomic cell. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(7); /// /// assert_eq!(a.load(), 7); /// ``` pub fn load(&self) -> T { unsafe { atomic_load(self.as_ptr()) } } } impl AtomicCell { /// If the current value equals `current`, stores `new` into the atomic cell. /// /// The return value is always the previous value. If it is equal to `current`, then the value /// was updated. /// /// # Examples /// /// ``` /// # #![allow(deprecated)] /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(1); /// /// assert_eq!(a.compare_and_swap(2, 3), 1); /// assert_eq!(a.load(), 1); /// /// assert_eq!(a.compare_and_swap(1, 2), 1); /// assert_eq!(a.load(), 2); /// ``` // TODO: remove in the next major version. #[deprecated(note = "Use `compare_exchange` instead")] pub fn compare_and_swap(&self, current: T, new: T) -> T { match self.compare_exchange(current, new) { Ok(v) => v, Err(v) => v, } } /// If the current value equals `current`, stores `new` into the atomic cell. /// /// The return value is a result indicating whether the new value was written and containing /// the previous value. On success this value is guaranteed to be equal to `current`. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(1); /// /// assert_eq!(a.compare_exchange(2, 3), Err(1)); /// assert_eq!(a.load(), 1); /// /// assert_eq!(a.compare_exchange(1, 2), Ok(1)); /// assert_eq!(a.load(), 2); /// ``` pub fn compare_exchange(&self, current: T, new: T) -> Result { unsafe { atomic_compare_exchange_weak(self.as_ptr(), current, new) } } /// Fetches the value, and applies a function to it that returns an optional /// new value. Returns a `Result` of `Ok(previous_value)` if the function returned `Some(_)`, else /// `Err(previous_value)`. /// /// Note: This may call the function multiple times if the value has been changed from other threads in /// the meantime, as long as the function returns `Some(_)`, but the function will have been applied /// only once to the stored value. /// /// # Examples /// /// ```rust /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(7); /// assert_eq!(a.fetch_update(|_| None), Err(7)); /// assert_eq!(a.fetch_update(|a| Some(a + 1)), Ok(7)); /// assert_eq!(a.fetch_update(|a| Some(a + 1)), Ok(8)); /// assert_eq!(a.load(), 9); /// ``` #[inline] pub fn fetch_update(&self, mut f: F) -> Result where F: FnMut(T) -> Option, { let mut prev = self.load(); while let Some(next) = f(prev) { match self.compare_exchange(prev, next) { x @ Ok(_) => return x, Err(next_prev) => prev = next_prev, } } Err(prev) } } // `MaybeUninit` prevents `T` from being dropped, so we need to implement `Drop` // for `AtomicCell` to avoid leaks of non-`Copy` types. impl Drop for AtomicCell { fn drop(&mut self) { if mem::needs_drop::() { // SAFETY: // - the mutable reference guarantees that no other threads are concurrently accessing the atomic data // - the raw pointer passed in is valid because we got it from a reference // - `MaybeUninit` prevents double dropping `T` unsafe { self.as_ptr().drop_in_place(); } } } } macro_rules! atomic { // If values of type `$t` can be transmuted into values of the primitive atomic type `$atomic`, // declares variable `$a` of type `$atomic` and executes `$atomic_op`, breaking out of the loop. (@check, $t:ty, $atomic:ty, $a:ident, $atomic_op:expr) => { if can_transmute::<$t, $atomic>() { let $a: &$atomic; break $atomic_op; } }; // If values of type `$t` can be transmuted into values of a primitive atomic type, declares // variable `$a` of that type and executes `$atomic_op`. Otherwise, just executes // `$fallback_op`. ($t:ty, $a:ident, $atomic_op:expr, $fallback_op:expr) => { loop { atomic!(@check, $t, AtomicUnit, $a, $atomic_op); atomic!(@check, $t, atomic::AtomicU8, $a, $atomic_op); atomic!(@check, $t, atomic::AtomicU16, $a, $atomic_op); atomic!(@check, $t, atomic::AtomicU32, $a, $atomic_op); #[cfg(target_has_atomic = "64")] atomic!(@check, $t, atomic::AtomicU64, $a, $atomic_op); // TODO: AtomicU128 is unstable // atomic!(@check, $t, atomic::AtomicU128, $a, $atomic_op); break $fallback_op; } }; } macro_rules! impl_arithmetic { ($t:ty, fallback, $example:tt) => { impl AtomicCell<$t> { /// Increments the current value by `val` and returns the previous value. /// /// The addition wraps on overflow. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_add(3), 7); /// assert_eq!(a.load(), 10); /// ``` #[inline] pub fn fetch_add(&self, val: $t) -> $t { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = value.wrapping_add(val); old } /// Decrements the current value by `val` and returns the previous value. /// /// The subtraction wraps on overflow. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_sub(3), 7); /// assert_eq!(a.load(), 4); /// ``` #[inline] pub fn fetch_sub(&self, val: $t) -> $t { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = value.wrapping_sub(val); old } /// Applies bitwise "and" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_and(3), 7); /// assert_eq!(a.load(), 3); /// ``` #[inline] pub fn fetch_and(&self, val: $t) -> $t { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value &= val; old } /// Applies bitwise "nand" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_nand(3), 7); /// assert_eq!(a.load(), !(7 & 3)); /// ``` #[inline] pub fn fetch_nand(&self, val: $t) -> $t { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = !(old & val); old } /// Applies bitwise "or" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_or(16), 7); /// assert_eq!(a.load(), 23); /// ``` #[inline] pub fn fetch_or(&self, val: $t) -> $t { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value |= val; old } /// Applies bitwise "xor" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_xor(2), 7); /// assert_eq!(a.load(), 5); /// ``` #[inline] pub fn fetch_xor(&self, val: $t) -> $t { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value ^= val; old } /// Compares and sets the maximum of the current value and `val`, /// and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_max(2), 7); /// assert_eq!(a.load(), 7); /// ``` #[inline] pub fn fetch_max(&self, val: $t) -> $t { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = cmp::max(old, val); old } /// Compares and sets the minimum of the current value and `val`, /// and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_min(2), 7); /// assert_eq!(a.load(), 2); /// ``` #[inline] pub fn fetch_min(&self, val: $t) -> $t { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = cmp::min(old, val); old } } }; ($t:ty, $atomic:ident, $example:tt) => { impl AtomicCell<$t> { /// Increments the current value by `val` and returns the previous value. /// /// The addition wraps on overflow. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_add(3), 7); /// assert_eq!(a.load(), 10); /// ``` #[inline] pub fn fetch_add(&self, val: $t) -> $t { atomic! { $t, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; a.fetch_add(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = value.wrapping_add(val); old } } } /// Decrements the current value by `val` and returns the previous value. /// /// The subtraction wraps on overflow. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_sub(3), 7); /// assert_eq!(a.load(), 4); /// ``` #[inline] pub fn fetch_sub(&self, val: $t) -> $t { atomic! { $t, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; a.fetch_sub(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = value.wrapping_sub(val); old } } } /// Applies bitwise "and" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_and(3), 7); /// assert_eq!(a.load(), 3); /// ``` #[inline] pub fn fetch_and(&self, val: $t) -> $t { atomic! { $t, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; a.fetch_and(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value &= val; old } } } /// Applies bitwise "nand" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_nand(3), 7); /// assert_eq!(a.load(), !(7 & 3)); /// ``` #[inline] pub fn fetch_nand(&self, val: $t) -> $t { atomic! { $t, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; a.fetch_nand(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = !(old & val); old } } } /// Applies bitwise "or" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_or(16), 7); /// assert_eq!(a.load(), 23); /// ``` #[inline] pub fn fetch_or(&self, val: $t) -> $t { atomic! { $t, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; a.fetch_or(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value |= val; old } } } /// Applies bitwise "xor" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_xor(2), 7); /// assert_eq!(a.load(), 5); /// ``` #[inline] pub fn fetch_xor(&self, val: $t) -> $t { atomic! { $t, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; a.fetch_xor(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value ^= val; old } } } /// Compares and sets the maximum of the current value and `val`, /// and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_max(9), 7); /// assert_eq!(a.load(), 9); /// ``` #[inline] pub fn fetch_max(&self, val: $t) -> $t { atomic! { $t, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; a.fetch_max(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = cmp::max(old, val); old } } } /// Compares and sets the minimum of the current value and `val`, /// and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// #[doc = $example] /// /// assert_eq!(a.fetch_min(2), 7); /// assert_eq!(a.load(), 2); /// ``` #[inline] pub fn fetch_min(&self, val: $t) -> $t { atomic! { $t, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::$atomic) }; a.fetch_min(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = cmp::min(old, val); old } } } } }; } impl_arithmetic!(u8, AtomicU8, "let a = AtomicCell::new(7u8);"); impl_arithmetic!(i8, AtomicI8, "let a = AtomicCell::new(7i8);"); impl_arithmetic!(u16, AtomicU16, "let a = AtomicCell::new(7u16);"); impl_arithmetic!(i16, AtomicI16, "let a = AtomicCell::new(7i16);"); impl_arithmetic!(u32, AtomicU32, "let a = AtomicCell::new(7u32);"); impl_arithmetic!(i32, AtomicI32, "let a = AtomicCell::new(7i32);"); #[cfg(target_has_atomic = "64")] impl_arithmetic!(u64, AtomicU64, "let a = AtomicCell::new(7u64);"); #[cfg(target_has_atomic = "64")] impl_arithmetic!(i64, AtomicI64, "let a = AtomicCell::new(7i64);"); #[cfg(not(target_has_atomic = "64"))] impl_arithmetic!(u64, fallback, "let a = AtomicCell::new(7u64);"); #[cfg(not(target_has_atomic = "64"))] impl_arithmetic!(i64, fallback, "let a = AtomicCell::new(7i64);"); // TODO: AtomicU128 is unstable // impl_arithmetic!(u128, AtomicU128, "let a = AtomicCell::new(7u128);"); // impl_arithmetic!(i128, AtomicI128, "let a = AtomicCell::new(7i128);"); impl_arithmetic!(u128, fallback, "let a = AtomicCell::new(7u128);"); impl_arithmetic!(i128, fallback, "let a = AtomicCell::new(7i128);"); impl_arithmetic!(usize, AtomicUsize, "let a = AtomicCell::new(7usize);"); impl_arithmetic!(isize, AtomicIsize, "let a = AtomicCell::new(7isize);"); impl AtomicCell { /// Applies logical "and" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(true); /// /// assert_eq!(a.fetch_and(true), true); /// assert_eq!(a.load(), true); /// /// assert_eq!(a.fetch_and(false), true); /// assert_eq!(a.load(), false); /// ``` #[inline] pub fn fetch_and(&self, val: bool) -> bool { atomic! { bool, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; a.fetch_and(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value &= val; old } } } /// Applies logical "nand" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(true); /// /// assert_eq!(a.fetch_nand(false), true); /// assert_eq!(a.load(), true); /// /// assert_eq!(a.fetch_nand(true), true); /// assert_eq!(a.load(), false); /// /// assert_eq!(a.fetch_nand(false), false); /// assert_eq!(a.load(), true); /// ``` #[inline] pub fn fetch_nand(&self, val: bool) -> bool { atomic! { bool, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; a.fetch_nand(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value = !(old & val); old } } } /// Applies logical "or" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(false); /// /// assert_eq!(a.fetch_or(false), false); /// assert_eq!(a.load(), false); /// /// assert_eq!(a.fetch_or(true), false); /// assert_eq!(a.load(), true); /// ``` #[inline] pub fn fetch_or(&self, val: bool) -> bool { atomic! { bool, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; a.fetch_or(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value |= val; old } } } /// Applies logical "xor" to the current value and returns the previous value. /// /// # Examples /// /// ``` /// use crossbeam_utils::atomic::AtomicCell; /// /// let a = AtomicCell::new(true); /// /// assert_eq!(a.fetch_xor(false), true); /// assert_eq!(a.load(), true); /// /// assert_eq!(a.fetch_xor(true), true); /// assert_eq!(a.load(), false); /// ``` #[inline] pub fn fetch_xor(&self, val: bool) -> bool { atomic! { bool, _a, { let a = unsafe { &*(self.as_ptr() as *const atomic::AtomicBool) }; a.fetch_xor(val, Ordering::AcqRel) }, { let _guard = lock(self.as_ptr() as usize).write(); let value = unsafe { &mut *(self.as_ptr()) }; let old = *value; *value ^= val; old } } } } impl Default for AtomicCell { fn default() -> AtomicCell { AtomicCell::new(T::default()) } } impl From for AtomicCell { #[inline] fn from(val: T) -> AtomicCell { AtomicCell::new(val) } } impl fmt::Debug for AtomicCell { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("AtomicCell") .field("value", &self.load()) .finish() } } /// Returns `true` if values of type `A` can be transmuted into values of type `B`. const fn can_transmute() -> bool { // Sizes must be equal, but alignment of `A` must be greater or equal than that of `B`. (mem::size_of::() == mem::size_of::()) & (mem::align_of::() >= mem::align_of::()) } /// Returns a reference to the global lock associated with the `AtomicCell` at address `addr`. /// /// This function is used to protect atomic data which doesn't fit into any of the primitive atomic /// types in `std::sync::atomic`. Operations on such atomics must therefore use a global lock. /// /// However, there is not only one global lock but an array of many locks, and one of them is /// picked based on the given address. Having many locks reduces contention and improves /// scalability. #[inline] #[must_use] fn lock(addr: usize) -> &'static SeqLock { // The number of locks is a prime number because we want to make sure `addr % LEN` gets // dispersed across all locks. // // Note that addresses are always aligned to some power of 2, depending on type `T` in // `AtomicCell`. If `LEN` was an even number, then `addr % LEN` would be an even number, // too, which means only half of the locks would get utilized! // // It is also possible for addresses to accidentally get aligned to a number that is not a // power of 2. Consider this example: // // ``` // #[repr(C)] // struct Foo { // a: AtomicCell, // b: u8, // c: u8, // } // ``` // // Now, if we have a slice of type `&[Foo]`, it is possible that field `a` in all items gets // stored at addresses that are multiples of 3. It'd be too bad if `LEN` was divisible by 3. // In order to protect from such cases, we simply choose a large prime number for `LEN`. const LEN: usize = 67; #[allow(clippy::declare_interior_mutable_const)] const L: CachePadded = CachePadded::new(SeqLock::new()); static LOCKS: [CachePadded; LEN] = [L; LEN]; // If the modulus is a constant number, the compiler will use crazy math to transform this into // a sequence of cheap arithmetic operations rather than using the slow modulo instruction. &LOCKS[addr % LEN] } /// An atomic `()`. /// /// All operations are noops. struct AtomicUnit; impl AtomicUnit { #[inline] fn load(&self, _order: Ordering) {} #[inline] fn store(&self, _val: (), _order: Ordering) {} #[inline] fn swap(&self, _val: (), _order: Ordering) {} #[inline] fn compare_exchange_weak( &self, _current: (), _new: (), _success: Ordering, _failure: Ordering, ) -> Result<(), ()> { Ok(()) } } /// Returns `true` if operations on `AtomicCell` are lock-free. const fn atomic_is_lock_free() -> bool { atomic! { T, _a, true, false } } /// Atomically reads data from `src`. /// /// This operation uses the `Acquire` ordering. If possible, an atomic instructions is used, and a /// global lock otherwise. unsafe fn atomic_load(src: *mut T) -> T where T: Copy, { atomic! { T, a, { a = &*(src as *const _ as *const _); mem::transmute_copy(&a.load(Ordering::Acquire)) }, { let lock = lock(src as usize); // Try doing an optimistic read first. if let Some(stamp) = lock.optimistic_read() { // We need a volatile read here because other threads might concurrently modify the // value. In theory, data races are *always* UB, even if we use volatile reads and // discard the data when a data race is detected. The proper solution would be to // do atomic reads and atomic writes, but we can't atomically read and write all // kinds of data since `AtomicU8` is not available on stable Rust yet. // Load as `MaybeUninit` because we may load a value that is not valid as `T`. let val = ptr::read_volatile(src.cast::>()); if lock.validate_read(stamp) { return val.assume_init(); } } // Grab a regular write lock so that writers don't starve this load. let guard = lock.write(); let val = ptr::read(src); // The value hasn't been changed. Drop the guard without incrementing the stamp. guard.abort(); val } } } /// Atomically writes `val` to `dst`. /// /// This operation uses the `Release` ordering. If possible, an atomic instructions is used, and a /// global lock otherwise. unsafe fn atomic_store(dst: *mut T, val: T) { atomic! { T, a, { a = &*(dst as *const _ as *const _); a.store(mem::transmute_copy(&val), Ordering::Release); mem::forget(val); }, { let _guard = lock(dst as usize).write(); ptr::write(dst, val); } } } /// Atomically swaps data at `dst` with `val`. /// /// This operation uses the `AcqRel` ordering. If possible, an atomic instructions is used, and a /// global lock otherwise. unsafe fn atomic_swap(dst: *mut T, val: T) -> T { atomic! { T, a, { a = &*(dst as *const _ as *const _); let res = mem::transmute_copy(&a.swap(mem::transmute_copy(&val), Ordering::AcqRel)); mem::forget(val); res }, { let _guard = lock(dst as usize).write(); ptr::replace(dst, val) } } } /// Atomically compares data at `dst` to `current` and, if equal byte-for-byte, exchanges data at /// `dst` with `new`. /// /// Returns the old value on success, or the current value at `dst` on failure. /// /// This operation uses the `AcqRel` ordering. If possible, an atomic instructions is used, and a /// global lock otherwise. #[allow(clippy::let_unit_value)] unsafe fn atomic_compare_exchange_weak(dst: *mut T, mut current: T, new: T) -> Result where T: Copy + Eq, { atomic! { T, a, { a = &*(dst as *const _ as *const _); let mut current_raw = mem::transmute_copy(¤t); let new_raw = mem::transmute_copy(&new); loop { match a.compare_exchange_weak( current_raw, new_raw, Ordering::AcqRel, Ordering::Acquire, ) { Ok(_) => break Ok(current), Err(previous_raw) => { let previous = mem::transmute_copy(&previous_raw); if !T::eq(&previous, ¤t) { break Err(previous); } // The compare-exchange operation has failed and didn't store `new`. The // failure is either spurious, or `previous` was semantically equal to // `current` but not byte-equal. Let's retry with `previous` as the new // `current`. current = previous; current_raw = previous_raw; } } } }, { let guard = lock(dst as usize).write(); if T::eq(&*dst, ¤t) { Ok(ptr::replace(dst, new)) } else { let val = ptr::read(dst); // The value hasn't been changed. Drop the guard without incrementing the stamp. guard.abort(); Err(val) } } } }