aboutsummaryrefslogtreecommitdiff
path: root/vendor/portable-atomic/src/utils.rs
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/portable-atomic/src/utils.rs')
-rw-r--r--vendor/portable-atomic/src/utils.rs437
1 files changed, 0 insertions, 437 deletions
diff --git a/vendor/portable-atomic/src/utils.rs b/vendor/portable-atomic/src/utils.rs
deleted file mode 100644
index f73e794..0000000
--- a/vendor/portable-atomic/src/utils.rs
+++ /dev/null
@@ -1,437 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-#![cfg_attr(not(all(test, feature = "float")), allow(dead_code, unused_macros))]
-
-#[macro_use]
-#[path = "gen/utils.rs"]
-mod gen;
-
-use core::sync::atomic::Ordering;
-
-macro_rules! static_assert {
- ($cond:expr $(,)?) => {{
- let [] = [(); true as usize - $crate::utils::_assert_is_bool($cond) as usize];
- }};
-}
-pub(crate) const fn _assert_is_bool(v: bool) -> bool {
- v
-}
-
-macro_rules! static_assert_layout {
- ($atomic_type:ty, $value_type:ty) => {
- static_assert!(
- core::mem::align_of::<$atomic_type>() == core::mem::size_of::<$atomic_type>()
- );
- static_assert!(core::mem::size_of::<$atomic_type>() == core::mem::size_of::<$value_type>());
- };
-}
-
-// #[doc = concat!(...)] requires Rust 1.54
-macro_rules! doc_comment {
- ($doc:expr, $($tt:tt)*) => {
- #[doc = $doc]
- $($tt)*
- };
-}
-
-// Adapted from https://github.com/BurntSushi/memchr/blob/2.4.1/src/memchr/x86/mod.rs#L9-L71.
-/// # Safety
-///
-/// - the caller must uphold the safety contract for the function returned by $detect_body.
-/// - the memory pointed by the function pointer returned by $detect_body must be visible from any threads.
-///
-/// The second requirement is always met if the function pointer is to the function definition.
-/// (Currently, all uses of this macro in our code are in this case.)
-#[allow(unused_macros)]
-#[cfg(not(portable_atomic_no_outline_atomics))]
-#[cfg(any(
- target_arch = "aarch64",
- target_arch = "arm",
- target_arch = "powerpc64",
- all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
-))]
-macro_rules! ifunc {
- (unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)? { $($detect_body:tt)* }) => {{
- type FnTy = unsafe fn($($arg_ty),*) $(-> $ret_ty)?;
- static FUNC: core::sync::atomic::AtomicPtr<()>
- = core::sync::atomic::AtomicPtr::new(detect as *mut ());
- #[cold]
- unsafe fn detect($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
- let func: FnTy = { $($detect_body)* };
- FUNC.store(func as *mut (), core::sync::atomic::Ordering::Relaxed);
- // SAFETY: the caller must uphold the safety contract for the function returned by $detect_body.
- unsafe { func($($arg_pat),*) }
- }
- // SAFETY: `FnTy` is a function pointer, which is always safe to transmute with a `*mut ()`.
- // (To force the caller to use unsafe block for this macro, do not use
- // unsafe block here.)
- let func = {
- core::mem::transmute::<*mut (), FnTy>(FUNC.load(core::sync::atomic::Ordering::Relaxed))
- };
- // SAFETY: the caller must uphold the safety contract for the function returned by $detect_body.
- // (To force the caller to use unsafe block for this macro, do not use
- // unsafe block here.)
- func($($arg_pat),*)
- }};
-}
-
-#[allow(unused_macros)]
-#[cfg(not(portable_atomic_no_outline_atomics))]
-#[cfg(any(
- target_arch = "aarch64",
- target_arch = "arm",
- target_arch = "powerpc64",
- all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
-))]
-macro_rules! fn_alias {
- (
- $(#[$($fn_attr:tt)*])*
- $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
- $(#[$($alias_attr:tt)*])*
- $new:ident = $from:ident($($last_args:tt)*);
- $($rest:tt)*
- ) => {
- $(#[$($fn_attr)*])*
- $(#[$($alias_attr)*])*
- $vis unsafe fn $new($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
- // SAFETY: the caller must uphold the safety contract.
- unsafe { $from($($arg_pat,)* $($last_args)*) }
- }
- fn_alias! {
- $(#[$($fn_attr)*])*
- $vis unsafe fn($($arg_pat: $arg_ty),*) $(-> $ret_ty)?;
- $($rest)*
- }
- };
- (
- $(#[$($attr:tt)*])*
- $vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
- ) => {}
-}
-
-/// Make the given function const if the given condition is true.
-macro_rules! const_fn {
- (
- const_if: #[cfg($($cfg:tt)+)];
- $(#[$($attr:tt)*])*
- $vis:vis const fn $($rest:tt)*
- ) => {
- #[cfg($($cfg)+)]
- $(#[$($attr)*])*
- $vis const fn $($rest)*
- #[cfg(not($($cfg)+))]
- $(#[$($attr)*])*
- $vis fn $($rest)*
- };
-}
-
-/// Implements `core::fmt::Debug` and `serde::{Serialize, Deserialize}` (when serde
-/// feature is enabled) for atomic bool, integer, or float.
-macro_rules! impl_debug_and_serde {
- ($atomic_type:ident) => {
- impl fmt::Debug for $atomic_type {
- #[allow(clippy::missing_inline_in_public_items)] // fmt is not hot path
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- // std atomic types use Relaxed in Debug::fmt: https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/sync/atomic.rs#L2024
- fmt::Debug::fmt(&self.load(Ordering::Relaxed), f)
- }
- }
- #[cfg(feature = "serde")]
- #[cfg_attr(portable_atomic_doc_cfg, doc(cfg(feature = "serde")))]
- impl serde::ser::Serialize for $atomic_type {
- #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl
- fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
- where
- S: serde::ser::Serializer,
- {
- // https://github.com/serde-rs/serde/blob/v1.0.152/serde/src/ser/impls.rs#L958-L959
- self.load(Ordering::Relaxed).serialize(serializer)
- }
- }
- #[cfg(feature = "serde")]
- #[cfg_attr(portable_atomic_doc_cfg, doc(cfg(feature = "serde")))]
- impl<'de> serde::de::Deserialize<'de> for $atomic_type {
- #[allow(clippy::missing_inline_in_public_items)] // serde doesn't use inline on std atomic's Serialize/Deserialize impl
- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where
- D: serde::de::Deserializer<'de>,
- {
- serde::de::Deserialize::deserialize(deserializer).map(Self::new)
- }
- }
- };
-}
-
-// We do not provide `nand` because it cannot be optimized on neither x86 nor MSP430.
-// https://godbolt.org/z/7TzjKqYvE
-macro_rules! impl_default_no_fetch_ops {
- ($atomic_type:ident, bool) => {
- impl $atomic_type {
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn and(&self, val: bool, order: Ordering) {
- self.fetch_and(val, order);
- }
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn or(&self, val: bool, order: Ordering) {
- self.fetch_or(val, order);
- }
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn xor(&self, val: bool, order: Ordering) {
- self.fetch_xor(val, order);
- }
- }
- };
- ($atomic_type:ident, $int_type:ident) => {
- impl $atomic_type {
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn add(&self, val: $int_type, order: Ordering) {
- self.fetch_add(val, order);
- }
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn sub(&self, val: $int_type, order: Ordering) {
- self.fetch_sub(val, order);
- }
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn and(&self, val: $int_type, order: Ordering) {
- self.fetch_and(val, order);
- }
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn or(&self, val: $int_type, order: Ordering) {
- self.fetch_or(val, order);
- }
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn xor(&self, val: $int_type, order: Ordering) {
- self.fetch_xor(val, order);
- }
- }
- };
-}
-macro_rules! impl_default_bit_opts {
- ($atomic_type:ident, $int_type:ident) => {
- impl $atomic_type {
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn bit_set(&self, bit: u32, order: Ordering) -> bool {
- let mask = <$int_type>::wrapping_shl(1, bit);
- self.fetch_or(mask, order) & mask != 0
- }
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn bit_clear(&self, bit: u32, order: Ordering) -> bool {
- let mask = <$int_type>::wrapping_shl(1, bit);
- self.fetch_and(!mask, order) & mask != 0
- }
- #[inline]
- #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
- pub(crate) fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {
- let mask = <$int_type>::wrapping_shl(1, bit);
- self.fetch_xor(mask, order) & mask != 0
- }
- }
- };
-}
-
-// This just outputs the input as is, but can be used like an item-level block by using it with cfg.
-macro_rules! items {
- ($($tt:tt)*) => {
- $($tt)*
- };
-}
-
-// https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/sync/atomic.rs#L3155
-#[inline]
-#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
-pub(crate) fn assert_load_ordering(order: Ordering) {
- match order {
- Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
- Ordering::Release => panic!("there is no such thing as a release load"),
- Ordering::AcqRel => panic!("there is no such thing as an acquire-release load"),
- _ => unreachable!("{:?}", order),
- }
-}
-
-// https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/sync/atomic.rs#L3140
-#[inline]
-#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
-pub(crate) fn assert_store_ordering(order: Ordering) {
- match order {
- Ordering::Release | Ordering::Relaxed | Ordering::SeqCst => {}
- Ordering::Acquire => panic!("there is no such thing as an acquire store"),
- Ordering::AcqRel => panic!("there is no such thing as an acquire-release store"),
- _ => unreachable!("{:?}", order),
- }
-}
-
-// https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/sync/atomic.rs#L3221
-#[inline]
-#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
-pub(crate) fn assert_compare_exchange_ordering(success: Ordering, failure: Ordering) {
- match success {
- Ordering::AcqRel
- | Ordering::Acquire
- | Ordering::Relaxed
- | Ordering::Release
- | Ordering::SeqCst => {}
- _ => unreachable!("{:?}, {:?}", success, failure),
- }
- match failure {
- Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
- Ordering::Release => panic!("there is no such thing as a release failure ordering"),
- Ordering::AcqRel => panic!("there is no such thing as an acquire-release failure ordering"),
- _ => unreachable!("{:?}, {:?}", success, failure),
- }
-}
-
-// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0418r2.html
-// https://github.com/rust-lang/rust/pull/98383
-#[allow(dead_code)]
-#[inline]
-pub(crate) fn upgrade_success_ordering(success: Ordering, failure: Ordering) -> Ordering {
- match (success, failure) {
- (Ordering::Relaxed, Ordering::Acquire) => Ordering::Acquire,
- (Ordering::Release, Ordering::Acquire) => Ordering::AcqRel,
- (_, Ordering::SeqCst) => Ordering::SeqCst,
- _ => success,
- }
-}
-
-/// Zero-extends the given 32-bit pointer to `MaybeUninit<u64>`.
-/// This is used for 64-bit architecture's 32-bit ABI (e.g., AArch64 ILP32 ABI).
-/// See ptr_reg! macro in src/gen/utils.rs for details.
-#[cfg(not(portable_atomic_no_asm_maybe_uninit))]
-#[cfg(target_pointer_width = "32")]
-#[allow(dead_code)]
-#[inline]
-pub(crate) fn zero_extend64_ptr(v: *mut ()) -> core::mem::MaybeUninit<u64> {
- #[repr(C)]
- struct ZeroExtended {
- #[cfg(target_endian = "big")]
- pad: *mut (),
- v: *mut (),
- #[cfg(target_endian = "little")]
- pad: *mut (),
- }
- // SAFETY: we can safely transmute any 64-bit value to MaybeUninit<u64>.
- unsafe { core::mem::transmute(ZeroExtended { v, pad: core::ptr::null_mut() }) }
-}
-
-#[allow(dead_code)]
-#[cfg(any(
- target_arch = "aarch64",
- target_arch = "powerpc64",
- target_arch = "s390x",
- target_arch = "x86_64",
-))]
-/// A 128-bit value represented as a pair of 64-bit values.
-///
-/// This type is `#[repr(C)]`, both fields have the same in-memory representation
-/// and are plain old data types, so access to the fields is always safe.
-#[derive(Clone, Copy)]
-#[repr(C)]
-pub(crate) union U128 {
- pub(crate) whole: u128,
- pub(crate) pair: Pair<u64>,
-}
-#[allow(dead_code)]
-#[cfg(target_arch = "arm")]
-/// A 64-bit value represented as a pair of 32-bit values.
-///
-/// This type is `#[repr(C)]`, both fields have the same in-memory representation
-/// and are plain old data types, so access to the fields is always safe.
-#[derive(Clone, Copy)]
-#[repr(C)]
-pub(crate) union U64 {
- pub(crate) whole: u64,
- pub(crate) pair: Pair<u32>,
-}
-#[allow(dead_code)]
-#[derive(Clone, Copy)]
-#[repr(C)]
-pub(crate) struct Pair<T: Copy> {
- // little endian order
- #[cfg(any(target_endian = "little", target_arch = "aarch64", target_arch = "arm"))]
- pub(crate) lo: T,
- pub(crate) hi: T,
- // big endian order
- #[cfg(not(any(target_endian = "little", target_arch = "aarch64", target_arch = "arm")))]
- pub(crate) lo: T,
-}
-
-#[allow(dead_code)]
-type MinWord = u32;
-#[cfg(target_arch = "riscv32")]
-type RegSize = u32;
-#[cfg(target_arch = "riscv64")]
-type RegSize = u64;
-// Adapted from https://github.com/taiki-e/atomic-maybe-uninit/blob/v0.3.0/src/utils.rs#L210.
-// Helper for implementing sub-word atomic operations using word-sized LL/SC loop or CAS loop.
-//
-// Refs: https://github.com/llvm/llvm-project/blob/llvmorg-17.0.0-rc2/llvm/lib/CodeGen/AtomicExpandPass.cpp#L699
-// (aligned_ptr, shift, mask)
-#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
-#[allow(dead_code)]
-#[inline]
-pub(crate) fn create_sub_word_mask_values<T>(ptr: *mut T) -> (*mut MinWord, RegSize, RegSize) {
- use core::mem;
- const SHIFT_MASK: bool = !cfg!(any(
- target_arch = "riscv32",
- target_arch = "riscv64",
- target_arch = "loongarch64",
- target_arch = "s390x",
- ));
- let ptr_mask = mem::size_of::<MinWord>() - 1;
- let aligned_ptr = strict::with_addr(ptr, ptr as usize & !ptr_mask) as *mut MinWord;
- let ptr_lsb = if SHIFT_MASK {
- ptr as usize & ptr_mask
- } else {
- // We use 32-bit wrapping shift instructions in asm on these platforms.
- ptr as usize
- };
- let shift = if cfg!(any(target_endian = "little", target_arch = "s390x")) {
- ptr_lsb.wrapping_mul(8)
- } else {
- (ptr_lsb ^ (mem::size_of::<MinWord>() - mem::size_of::<T>())).wrapping_mul(8)
- };
- let mut mask: RegSize = (1 << (mem::size_of::<T>() * 8)) - 1; // !(0 as T) as RegSize
- if SHIFT_MASK {
- mask <<= shift;
- }
- (aligned_ptr, shift as RegSize, mask)
-}
-
-/// Emulate strict provenance.
-///
-/// Once strict_provenance is stable, migrate to the standard library's APIs.
-#[cfg(any(miri, target_arch = "riscv32", target_arch = "riscv64"))]
-#[allow(dead_code)]
-pub(crate) mod strict {
- /// Replace the address portion of this pointer with a new address.
- #[inline]
- #[must_use]
- pub(crate) fn with_addr<T>(ptr: *mut T, addr: usize) -> *mut T {
- // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic.
- //
- // In the mean-time, this operation is defined to be "as if" it was
- // a wrapping_add, so we can emulate it as such. This should properly
- // restore pointer provenance even under today's compiler.
- let offset = addr.wrapping_sub(ptr as usize);
-
- // This is the canonical desugaring of this operation.
- (ptr as *mut u8).wrapping_add(offset) as *mut T
- }
-
- /// Run an operation of some kind on a pointer.
- #[inline]
- #[must_use]
- pub(crate) fn map_addr<T>(ptr: *mut T, f: impl FnOnce(usize) -> usize) -> *mut T {
- with_addr(ptr, f(ptr as usize))
- }
-}