diff options
Diffstat (limited to 'vendor/portable-atomic/src/imp/interrupt/mod.rs')
| -rw-r--r-- | vendor/portable-atomic/src/imp/interrupt/mod.rs | 903 |
1 files changed, 0 insertions, 903 deletions
diff --git a/vendor/portable-atomic/src/imp/interrupt/mod.rs b/vendor/portable-atomic/src/imp/interrupt/mod.rs deleted file mode 100644 index e0ed0f6..0000000 --- a/vendor/portable-atomic/src/imp/interrupt/mod.rs +++ /dev/null @@ -1,903 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Critical section based fallback implementations -// -// This module supports two different critical section implementations: -// - Built-in "disable all interrupts". -// - Call into the `critical-section` crate (which allows the user to plug any implementation). -// -// The `critical-section`-based fallback is enabled when the user asks for it with the `critical-section` -// Cargo feature. -// -// The "disable interrupts" fallback is not sound on multi-core systems. -// Also, this uses privileged instructions to disable interrupts, so it usually -// doesn't work on unprivileged mode. Using this fallback in an environment where privileged -// instructions are not available is also usually considered **unsound**, -// although the details are system-dependent. -// -// Therefore, this implementation will only be enabled in one of the following cases: -// -// - When the user explicitly declares that the system is single-core and that -// privileged instructions are available using an unsafe cfg. -// - When we can safely assume that the system is single-core and that -// privileged instructions are available on the system. -// -// AVR, which is single core[^avr1] and LLVM also generates code that disables -// interrupts [^avr2] in atomic ops by default, is considered the latter. -// MSP430 as well. -// -// See also README.md of this directory. -// -// [^avr1]: https://github.com/llvm/llvm-project/blob/llvmorg-17.0.0-rc2/llvm/lib/Target/AVR/AVRExpandPseudoInsts.cpp#L1074 -// [^avr2]: https://github.com/llvm/llvm-project/blob/llvmorg-17.0.0-rc2/llvm/test/CodeGen/AVR/atomics/load16.ll#L5 - -// On some platforms, atomic load/store can be implemented in a more efficient -// way than disabling interrupts. On MSP430, some RMWs that do not return the -// previous value can also be optimized. -// -// Note: On single-core systems, it is okay to use critical session-based -// CAS together with atomic load/store. The load/store will not be -// called while interrupts are disabled, and since the load/store is -// atomic, it is not affected by interrupts even if interrupts are enabled. -#[cfg(not(any(target_arch = "avr", feature = "critical-section")))] -use arch::atomic; - -#[cfg(not(feature = "critical-section"))] -#[cfg_attr( - all( - target_arch = "arm", - any(target_feature = "mclass", portable_atomic_target_feature = "mclass"), - ), - path = "armv6m.rs" -)] -#[cfg_attr( - all( - target_arch = "arm", - not(any(target_feature = "mclass", portable_atomic_target_feature = "mclass")), - ), - path = "armv4t.rs" -)] -#[cfg_attr(target_arch = "avr", path = "avr.rs")] -#[cfg_attr(target_arch = "msp430", path = "msp430.rs")] -#[cfg_attr(any(target_arch = "riscv32", target_arch = "riscv64"), path = "riscv.rs")] -#[cfg_attr(target_arch = "xtensa", path = "xtensa.rs")] -mod arch; - -use core::{cell::UnsafeCell, sync::atomic::Ordering}; - -// Critical section implementations might use locks internally. -#[cfg(feature = "critical-section")] -const IS_ALWAYS_LOCK_FREE: bool = false; - -// Consider atomic operations based on disabling interrupts on single-core -// systems are lock-free. (We consider the pre-v6 ARM Linux's atomic operations -// provided in a similar way by the Linux kernel to be lock-free.) -#[cfg(not(feature = "critical-section"))] -const IS_ALWAYS_LOCK_FREE: bool = true; - -#[cfg(feature = "critical-section")] -#[inline] -fn with<F, R>(f: F) -> R -where - F: FnOnce() -> R, -{ - critical_section::with(|_| f()) -} - -#[cfg(not(feature = "critical-section"))] -#[inline] -fn with<F, R>(f: F) -> R -where - F: FnOnce() -> R, -{ - // Get current interrupt state and disable interrupts - let state = arch::disable(); - - let r = f(); - - // Restore interrupt state - // SAFETY: the state was retrieved by the previous `disable`. - unsafe { arch::restore(state) } - - r -} - -#[cfg_attr(target_pointer_width = "16", repr(C, align(2)))] -#[cfg_attr(target_pointer_width = "32", repr(C, align(4)))] -#[cfg_attr(target_pointer_width = "64", repr(C, align(8)))] -#[cfg_attr(target_pointer_width = "128", repr(C, align(16)))] -pub(crate) struct AtomicPtr<T> { - p: UnsafeCell<*mut T>, -} - -// SAFETY: any data races are prevented by disabling interrupts or -// atomic intrinsics (see module-level comments). -unsafe impl<T> Send for AtomicPtr<T> {} -// SAFETY: any data races are prevented by disabling interrupts or -// atomic intrinsics (see module-level comments). -unsafe impl<T> Sync for AtomicPtr<T> {} - -impl<T> AtomicPtr<T> { - #[inline] - pub(crate) const fn new(p: *mut T) -> Self { - Self { p: UnsafeCell::new(p) } - } - - #[inline] - pub(crate) fn is_lock_free() -> bool { - Self::is_always_lock_free() - } - #[inline] - pub(crate) const fn is_always_lock_free() -> bool { - IS_ALWAYS_LOCK_FREE - } - - #[inline] - pub(crate) fn get_mut(&mut self) -> &mut *mut T { - // SAFETY: the mutable reference guarantees unique ownership. - // (UnsafeCell::get_mut requires Rust 1.50) - unsafe { &mut *self.p.get() } - } - - #[inline] - pub(crate) fn into_inner(self) -> *mut T { - self.p.into_inner() - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn load(&self, order: Ordering) -> *mut T { - crate::utils::assert_load_ordering(order); - #[cfg(not(any(target_arch = "avr", feature = "critical-section")))] - { - self.as_native().load(order) - } - #[cfg(any(target_arch = "avr", feature = "critical-section"))] - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { self.p.get().read() }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn store(&self, ptr: *mut T, order: Ordering) { - crate::utils::assert_store_ordering(order); - #[cfg(not(any(target_arch = "avr", feature = "critical-section")))] - { - self.as_native().store(ptr, order); - } - #[cfg(any(target_arch = "avr", feature = "critical-section"))] - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { self.p.get().write(ptr) }); - } - - #[inline] - pub(crate) fn swap(&self, ptr: *mut T, order: Ordering) -> *mut T { - let _ = order; - #[cfg(portable_atomic_force_amo)] - { - self.as_native().swap(ptr, order) - } - #[cfg(not(portable_atomic_force_amo))] - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.p.get().read(); - self.p.get().write(ptr); - prev - }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn compare_exchange( - &self, - current: *mut T, - new: *mut T, - success: Ordering, - failure: Ordering, - ) -> Result<*mut T, *mut T> { - crate::utils::assert_compare_exchange_ordering(success, failure); - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.p.get().read(); - if prev == current { - self.p.get().write(new); - Ok(prev) - } else { - Err(prev) - } - }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn compare_exchange_weak( - &self, - current: *mut T, - new: *mut T, - success: Ordering, - failure: Ordering, - ) -> Result<*mut T, *mut T> { - self.compare_exchange(current, new, success, failure) - } - - #[inline] - pub(crate) const fn as_ptr(&self) -> *mut *mut T { - self.p.get() - } - - #[cfg(not(any(target_arch = "avr", feature = "critical-section")))] - #[inline] - fn as_native(&self) -> &atomic::AtomicPtr<T> { - // SAFETY: AtomicPtr and atomic::AtomicPtr have the same layout and - // guarantee atomicity in a compatible way. (see module-level comments) - unsafe { &*(self as *const Self as *const atomic::AtomicPtr<T>) } - } -} - -macro_rules! atomic_int { - (base, $atomic_type:ident, $int_type:ident, $align:literal) => { - #[repr(C, align($align))] - pub(crate) struct $atomic_type { - v: UnsafeCell<$int_type>, - } - - // Send is implicitly implemented. - // SAFETY: any data races are prevented by disabling interrupts or - // atomic intrinsics (see module-level comments). - unsafe impl Sync for $atomic_type {} - - impl $atomic_type { - #[inline] - pub(crate) const fn new(v: $int_type) -> Self { - Self { v: UnsafeCell::new(v) } - } - - #[inline] - pub(crate) fn is_lock_free() -> bool { - Self::is_always_lock_free() - } - #[inline] - pub(crate) const fn is_always_lock_free() -> bool { - IS_ALWAYS_LOCK_FREE - } - - #[inline] - pub(crate) fn get_mut(&mut self) -> &mut $int_type { - // SAFETY: the mutable reference guarantees unique ownership. - // (UnsafeCell::get_mut requires Rust 1.50) - unsafe { &mut *self.v.get() } - } - - #[inline] - pub(crate) fn into_inner(self) -> $int_type { - self.v.into_inner() - } - - #[inline] - pub(crate) const fn as_ptr(&self) -> *mut $int_type { - self.v.get() - } - } - }; - (load_store_atomic $([$kind:ident])?, $atomic_type:ident, $int_type:ident, $align:literal) => { - atomic_int!(base, $atomic_type, $int_type, $align); - #[cfg(not(portable_atomic_force_amo))] - atomic_int!(cas[emulate], $atomic_type, $int_type); - #[cfg(portable_atomic_force_amo)] - atomic_int!(cas $([$kind])?, $atomic_type, $int_type); - impl $atomic_type { - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn load(&self, order: Ordering) -> $int_type { - crate::utils::assert_load_ordering(order); - #[cfg(not(any(target_arch = "avr", feature = "critical-section")))] - { - self.as_native().load(order) - } - #[cfg(any(target_arch = "avr", feature = "critical-section"))] - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { self.v.get().read() }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn store(&self, val: $int_type, order: Ordering) { - crate::utils::assert_store_ordering(order); - #[cfg(not(any(target_arch = "avr", feature = "critical-section")))] - { - self.as_native().store(val, order); - } - #[cfg(any(target_arch = "avr", feature = "critical-section"))] - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { self.v.get().write(val) }); - } - - #[cfg(not(any(target_arch = "avr", feature = "critical-section")))] - #[inline] - fn as_native(&self) -> &atomic::$atomic_type { - // SAFETY: $atomic_type and atomic::$atomic_type have the same layout and - // guarantee atomicity in a compatible way. (see module-level comments) - unsafe { &*(self as *const Self as *const atomic::$atomic_type) } - } - } - - #[cfg(not(all(target_arch = "msp430", not(feature = "critical-section"))))] - impl_default_no_fetch_ops!($atomic_type, $int_type); - impl_default_bit_opts!($atomic_type, $int_type); - #[cfg(not(all(target_arch = "msp430", not(feature = "critical-section"))))] - impl $atomic_type { - #[inline] - pub(crate) fn not(&self, order: Ordering) { - self.fetch_not(order); - } - } - #[cfg(all(target_arch = "msp430", not(feature = "critical-section")))] - impl $atomic_type { - #[inline] - pub(crate) fn add(&self, val: $int_type, order: Ordering) { - self.as_native().add(val, order); - } - #[inline] - pub(crate) fn sub(&self, val: $int_type, order: Ordering) { - self.as_native().sub(val, order); - } - #[inline] - pub(crate) fn and(&self, val: $int_type, order: Ordering) { - self.as_native().and(val, order); - } - #[inline] - pub(crate) fn or(&self, val: $int_type, order: Ordering) { - self.as_native().or(val, order); - } - #[inline] - pub(crate) fn xor(&self, val: $int_type, order: Ordering) { - self.as_native().xor(val, order); - } - #[inline] - pub(crate) fn not(&self, order: Ordering) { - self.as_native().not(order); - } - } - }; - (load_store_critical_session, $atomic_type:ident, $int_type:ident, $align:literal) => { - atomic_int!(base, $atomic_type, $int_type, $align); - atomic_int!(cas[emulate], $atomic_type, $int_type); - impl_default_no_fetch_ops!($atomic_type, $int_type); - impl_default_bit_opts!($atomic_type, $int_type); - impl $atomic_type { - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn load(&self, order: Ordering) -> $int_type { - crate::utils::assert_load_ordering(order); - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { self.v.get().read() }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn store(&self, val: $int_type, order: Ordering) { - crate::utils::assert_store_ordering(order); - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { self.v.get().write(val) }); - } - - #[inline] - pub(crate) fn not(&self, order: Ordering) { - self.fetch_not(order); - } - } - }; - (cas[emulate], $atomic_type:ident, $int_type:ident) => { - impl $atomic_type { - #[inline] - pub(crate) fn swap(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(val); - prev - }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn compare_exchange( - &self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering, - ) -> Result<$int_type, $int_type> { - crate::utils::assert_compare_exchange_ordering(success, failure); - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - if prev == current { - self.v.get().write(new); - Ok(prev) - } else { - Err(prev) - } - }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn compare_exchange_weak( - &self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering, - ) -> Result<$int_type, $int_type> { - self.compare_exchange(current, new, success, failure) - } - - #[inline] - pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev.wrapping_add(val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev.wrapping_sub(val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_and(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev & val); - prev - }) - } - - #[inline] - pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(!(prev & val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_or(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev | val); - prev - }) - } - - #[inline] - pub(crate) fn fetch_xor(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev ^ val); - prev - }) - } - - #[inline] - pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(core::cmp::max(prev, val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(core::cmp::min(prev, val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_not(&self, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(!prev); - prev - }) - } - - #[inline] - pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev.wrapping_neg()); - prev - }) - } - #[inline] - pub(crate) fn neg(&self, order: Ordering) { - self.fetch_neg(order); - } - } - }; - // cfg(portable_atomic_force_amo) 32-bit(RV32)/{32,64}-bit(RV64) RMW - (cas, $atomic_type:ident, $int_type:ident) => { - impl $atomic_type { - #[inline] - pub(crate) fn swap(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().swap(val, order) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn compare_exchange( - &self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering, - ) -> Result<$int_type, $int_type> { - crate::utils::assert_compare_exchange_ordering(success, failure); - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - if prev == current { - self.v.get().write(new); - Ok(prev) - } else { - Err(prev) - } - }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn compare_exchange_weak( - &self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering, - ) -> Result<$int_type, $int_type> { - self.compare_exchange(current, new, success, failure) - } - - #[inline] - pub(crate) fn fetch_add(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_add(val, order) - } - #[inline] - pub(crate) fn fetch_sub(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_sub(val, order) - } - #[inline] - pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_and(val, order) - } - - #[inline] - pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(!(prev & val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_or(val, order) - } - #[inline] - pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_xor(val, order) - } - #[inline] - pub(crate) fn fetch_max(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_max(val, order) - } - #[inline] - pub(crate) fn fetch_min(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_min(val, order) - } - #[inline] - pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type { - self.as_native().fetch_not(order) - } - - #[inline] - pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev.wrapping_neg()); - prev - }) - } - #[inline] - pub(crate) fn neg(&self, order: Ordering) { - self.fetch_neg(order); - } - } - }; - // cfg(portable_atomic_force_amo) {8,16}-bit RMW - (cas[sub_word], $atomic_type:ident, $int_type:ident) => { - impl $atomic_type { - #[inline] - pub(crate) fn swap(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(val); - prev - }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn compare_exchange( - &self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering, - ) -> Result<$int_type, $int_type> { - crate::utils::assert_compare_exchange_ordering(success, failure); - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - if prev == current { - self.v.get().write(new); - Ok(prev) - } else { - Err(prev) - } - }) - } - - #[inline] - #[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)] - pub(crate) fn compare_exchange_weak( - &self, - current: $int_type, - new: $int_type, - success: Ordering, - failure: Ordering, - ) -> Result<$int_type, $int_type> { - self.compare_exchange(current, new, success, failure) - } - - #[inline] - pub(crate) fn fetch_add(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev.wrapping_add(val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_sub(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev.wrapping_sub(val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_and(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_and(val, order) - } - - #[inline] - pub(crate) fn fetch_nand(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(!(prev & val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_or(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_or(val, order) - } - #[inline] - pub(crate) fn fetch_xor(&self, val: $int_type, order: Ordering) -> $int_type { - self.as_native().fetch_xor(val, order) - } - - #[inline] - pub(crate) fn fetch_max(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(core::cmp::max(prev, val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_min(&self, val: $int_type, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(core::cmp::min(prev, val)); - prev - }) - } - - #[inline] - pub(crate) fn fetch_not(&self, order: Ordering) -> $int_type { - self.as_native().fetch_not(order) - } - - #[inline] - pub(crate) fn fetch_neg(&self, _order: Ordering) -> $int_type { - // SAFETY: any data races are prevented by disabling interrupts (see - // module-level comments) and the raw pointer is valid because we got it - // from a reference. - with(|| unsafe { - let prev = self.v.get().read(); - self.v.get().write(prev.wrapping_neg()); - prev - }) - } - #[inline] - pub(crate) fn neg(&self, order: Ordering) { - self.fetch_neg(order); - } - } - }; -} - -#[cfg(target_pointer_width = "16")] -atomic_int!(load_store_atomic, AtomicIsize, isize, 2); -#[cfg(target_pointer_width = "16")] -atomic_int!(load_store_atomic, AtomicUsize, usize, 2); -#[cfg(target_pointer_width = "32")] -atomic_int!(load_store_atomic, AtomicIsize, isize, 4); -#[cfg(target_pointer_width = "32")] -atomic_int!(load_store_atomic, AtomicUsize, usize, 4); -#[cfg(target_pointer_width = "64")] -atomic_int!(load_store_atomic, AtomicIsize, isize, 8); -#[cfg(target_pointer_width = "64")] -atomic_int!(load_store_atomic, AtomicUsize, usize, 8); -#[cfg(target_pointer_width = "128")] -atomic_int!(load_store_atomic, AtomicIsize, isize, 16); -#[cfg(target_pointer_width = "128")] -atomic_int!(load_store_atomic, AtomicUsize, usize, 16); - -atomic_int!(load_store_atomic[sub_word], AtomicI8, i8, 1); -atomic_int!(load_store_atomic[sub_word], AtomicU8, u8, 1); -atomic_int!(load_store_atomic[sub_word], AtomicI16, i16, 2); -atomic_int!(load_store_atomic[sub_word], AtomicU16, u16, 2); - -#[cfg(not(target_pointer_width = "16"))] -atomic_int!(load_store_atomic, AtomicI32, i32, 4); -#[cfg(not(target_pointer_width = "16"))] -atomic_int!(load_store_atomic, AtomicU32, u32, 4); -#[cfg(target_pointer_width = "16")] -#[cfg(any(test, feature = "fallback"))] -atomic_int!(load_store_critical_session, AtomicI32, i32, 4); -#[cfg(target_pointer_width = "16")] -#[cfg(any(test, feature = "fallback"))] -atomic_int!(load_store_critical_session, AtomicU32, u32, 4); - -#[cfg(not(any(target_pointer_width = "16", target_pointer_width = "32")))] -atomic_int!(load_store_atomic, AtomicI64, i64, 8); -#[cfg(not(any(target_pointer_width = "16", target_pointer_width = "32")))] -atomic_int!(load_store_atomic, AtomicU64, u64, 8); -#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))] -#[cfg(any(test, feature = "fallback"))] -atomic_int!(load_store_critical_session, AtomicI64, i64, 8); -#[cfg(any(target_pointer_width = "16", target_pointer_width = "32"))] -#[cfg(any(test, feature = "fallback"))] -atomic_int!(load_store_critical_session, AtomicU64, u64, 8); - -#[cfg(any(test, feature = "fallback"))] -atomic_int!(load_store_critical_session, AtomicI128, i128, 16); -#[cfg(any(test, feature = "fallback"))] -atomic_int!(load_store_critical_session, AtomicU128, u128, 16); - -#[cfg(test)] -mod tests { - use super::*; - - test_atomic_ptr_single_thread!(); - test_atomic_int_single_thread!(i8); - test_atomic_int_single_thread!(u8); - test_atomic_int_single_thread!(i16); - test_atomic_int_single_thread!(u16); - test_atomic_int_single_thread!(i32); - test_atomic_int_single_thread!(u32); - test_atomic_int_single_thread!(i64); - test_atomic_int_single_thread!(u64); - test_atomic_int_single_thread!(i128); - test_atomic_int_single_thread!(u128); - test_atomic_int_single_thread!(isize); - test_atomic_int_single_thread!(usize); -} |