From 1b6a04ca5504955c571d1c97504fb45ea0befee4 Mon Sep 17 00:00:00 2001 From: Valentin Popov Date: Mon, 8 Jan 2024 01:21:28 +0400 Subject: Initial vendor packages Signed-off-by: Valentin Popov --- vendor/portable-atomic/src/imp/mod.rs | 449 ++++++++++++++++++++++++++++++++++ 1 file changed, 449 insertions(+) create mode 100644 vendor/portable-atomic/src/imp/mod.rs (limited to 'vendor/portable-atomic/src/imp/mod.rs') diff --git a/vendor/portable-atomic/src/imp/mod.rs b/vendor/portable-atomic/src/imp/mod.rs new file mode 100644 index 0000000..cea71eb --- /dev/null +++ b/vendor/portable-atomic/src/imp/mod.rs @@ -0,0 +1,449 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// ----------------------------------------------------------------------------- +// Lock-free implementations + +#[cfg(not(any( + all( + portable_atomic_no_atomic_load_store, + not(all(target_arch = "bpf", not(feature = "critical-section"))), + ), + portable_atomic_unsafe_assume_single_core, + target_arch = "avr", + target_arch = "msp430", +)))] +#[cfg_attr( + portable_atomic_no_cfg_target_has_atomic, + cfg(not(all( + any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"), + portable_atomic_no_atomic_cas, + ))) +)] +#[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(not(all( + any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"), + not(target_has_atomic = "ptr"), + ))) +)] +mod core_atomic; + +// aarch64 128-bit atomics +#[cfg(all( + target_arch = "aarch64", + any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), +))] +// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly. +#[cfg_attr( + all(any(miri, portable_atomic_sanitize_thread), portable_atomic_new_atomic_intrinsics), + path = "atomic128/intrinsics.rs" +)] +#[cfg_attr( + not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_new_atomic_intrinsics)), + path = "atomic128/aarch64.rs" +)] +mod aarch64; + +// x86_64 128-bit atomics +#[cfg(all( + target_arch = "x86_64", + any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), + any( + target_feature = "cmpxchg16b", + portable_atomic_target_feature = "cmpxchg16b", + all( + feature = "fallback", + not(portable_atomic_no_cmpxchg16b_target_feature), + not(portable_atomic_no_outline_atomics), + not(any(target_env = "sgx", miri)), + ), + ), +))] +// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly. +#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "atomic128/intrinsics.rs")] +#[cfg_attr(not(any(miri, portable_atomic_sanitize_thread)), path = "atomic128/x86_64.rs")] +mod x86_64; + +// powerpc64 128-bit atomics +#[cfg(all( + target_arch = "powerpc64", + portable_atomic_unstable_asm_experimental_arch, + any( + target_feature = "quadword-atomics", + portable_atomic_target_feature = "quadword-atomics", + all( + feature = "fallback", + not(portable_atomic_no_outline_atomics), + any(test, portable_atomic_outline_atomics), // TODO(powerpc64): currently disabled by default + any( + all( + target_os = "linux", + any( + target_env = "gnu", + all( + any(target_env = "musl", target_env = "ohos"), + not(target_feature = "crt-static"), + ), + portable_atomic_outline_atomics, + ), + ), + target_os = "android", + target_os = "freebsd", + ), + not(any(miri, portable_atomic_sanitize_thread)), + ), + ), +))] +// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly. +#[cfg_attr( + all(any(miri, portable_atomic_sanitize_thread), portable_atomic_llvm_15), + path = "atomic128/intrinsics.rs" +)] +#[cfg_attr( + not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_llvm_15)), + path = "atomic128/powerpc64.rs" +)] +mod powerpc64; + +// s390x 128-bit atomics +#[cfg(all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch))] +// Use intrinsics.rs on Miri and Sanitizer that do not support inline assembly. +#[cfg_attr(any(miri, portable_atomic_sanitize_thread), path = "atomic128/intrinsics.rs")] +#[cfg_attr(not(any(miri, portable_atomic_sanitize_thread)), path = "atomic128/s390x.rs")] +mod s390x; + +// pre-v6 ARM Linux 64-bit atomics +#[cfg(feature = "fallback")] +// Miri and Sanitizer do not support inline assembly. +#[cfg(all( + target_arch = "arm", + not(any(miri, portable_atomic_sanitize_thread)), + not(portable_atomic_no_asm), + any(target_os = "linux", target_os = "android"), + not(any(target_feature = "v6", portable_atomic_target_feature = "v6")), + not(portable_atomic_no_outline_atomics), +))] +#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))] +#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))] +mod arm_linux; + +// MSP430 atomics +#[cfg(target_arch = "msp430")] +pub(crate) mod msp430; + +// atomic load/store for RISC-V without A-extension +#[cfg(any(test, not(feature = "critical-section")))] +#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))] +#[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(any(test, not(target_has_atomic = "ptr"))) +)] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +mod riscv; + +// x86-specific optimizations +// Miri and Sanitizer do not support inline assembly. +#[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + not(any(miri, portable_atomic_sanitize_thread)), + not(portable_atomic_no_asm), +))] +mod x86; + +// ----------------------------------------------------------------------------- +// Lock-based fallback implementations + +#[cfg(feature = "fallback")] +#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] +#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))] +#[cfg(any( + test, + not(any( + all( + target_arch = "aarch64", + any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), + ), + all( + target_arch = "x86_64", + any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), + any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"), + ), + all( + target_arch = "powerpc64", + portable_atomic_unstable_asm_experimental_arch, + any( + target_feature = "quadword-atomics", + portable_atomic_target_feature = "quadword-atomics", + ), + ), + all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch), + )) +))] +mod fallback; + +// ----------------------------------------------------------------------------- +// Critical section based fallback implementations + +// On AVR, we always use critical section based fallback implementation. +// AVR can be safely assumed to be single-core, so this is sound. +// https://github.com/llvm/llvm-project/blob/llvmorg-17.0.0-rc2/llvm/lib/Target/AVR/AVRExpandPseudoInsts.cpp#L1074 +// MSP430 as well. +#[cfg(any( + all(test, target_os = "none"), + portable_atomic_unsafe_assume_single_core, + feature = "critical-section", + target_arch = "avr", + target_arch = "msp430", +))] +#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))] +#[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(any(test, not(target_has_atomic = "ptr"))) +)] +#[cfg(any( + target_arch = "arm", + target_arch = "avr", + target_arch = "msp430", + target_arch = "riscv32", + target_arch = "riscv64", + target_arch = "xtensa", + feature = "critical-section", +))] +mod interrupt; + +// ----------------------------------------------------------------------------- +// Atomic float implementations + +#[cfg(feature = "float")] +pub(crate) mod float; + +// ----------------------------------------------------------------------------- + +#[cfg(not(any( + portable_atomic_no_atomic_load_store, + portable_atomic_unsafe_assume_single_core, + target_arch = "avr", + target_arch = "msp430", +)))] +#[cfg_attr( + portable_atomic_no_cfg_target_has_atomic, + cfg(not(all( + any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"), + portable_atomic_no_atomic_cas, + ))) +)] +#[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(not(all( + any(target_arch = "riscv32", target_arch = "riscv64", feature = "critical-section"), + not(target_has_atomic = "ptr"), + ))) +)] +items! { + pub(crate) use self::core_atomic::{ + AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8, + AtomicUsize, + }; + #[cfg_attr( + portable_atomic_no_cfg_target_has_atomic, + cfg(any( + not(portable_atomic_no_atomic_64), + not(any(target_pointer_width = "16", target_pointer_width = "32")), + )) + )] + #[cfg_attr( + not(portable_atomic_no_cfg_target_has_atomic), + cfg(any( + target_has_atomic = "64", + not(any(target_pointer_width = "16", target_pointer_width = "32")), + )) + )] + pub(crate) use self::core_atomic::{AtomicI64, AtomicU64}; +} +// bpf +#[cfg(all( + target_arch = "bpf", + portable_atomic_no_atomic_load_store, + not(feature = "critical-section"), +))] +pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize}; + +// RISC-V without A-extension & !(assume single core | critical section) +#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))] +#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))] +#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +items! { + pub(crate) use self::riscv::{ + AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8, + AtomicUsize, + }; + #[cfg(target_arch = "riscv64")] + pub(crate) use self::riscv::{AtomicI64, AtomicU64}; +} + +// no core atomic CAS & (assume single core | critical section) => critical section based fallback +#[cfg(any( + portable_atomic_unsafe_assume_single_core, + feature = "critical-section", + target_arch = "avr", + target_arch = "msp430", +))] +#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))] +#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))] +items! { + pub(crate) use self::interrupt::{ + AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize, + }; + #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))] + pub(crate) use self::interrupt::{AtomicI32, AtomicU32}; + #[cfg(any( + not(any(target_pointer_width = "16", target_pointer_width = "32")), + feature = "fallback", + ))] + pub(crate) use self::interrupt::{AtomicI64, AtomicU64}; + #[cfg(feature = "fallback")] + pub(crate) use self::interrupt::{AtomicI128, AtomicU128}; +} + +// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback +#[cfg(feature = "fallback")] +#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))] +#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))] +items! { + #[cfg(not(all( + target_arch = "arm", + not(any(miri, portable_atomic_sanitize_thread)), + not(portable_atomic_no_asm), + any(target_os = "linux", target_os = "android"), + not(any(target_feature = "v6", portable_atomic_target_feature = "v6")), + not(portable_atomic_no_outline_atomics), + )))] + #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))] + #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))] + pub(crate) use self::fallback::{AtomicI64, AtomicU64}; + #[cfg(not(any( + all( + target_arch = "aarch64", + any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), + ), + all( + target_arch = "x86_64", + any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), + any( + target_feature = "cmpxchg16b", + portable_atomic_target_feature = "cmpxchg16b", + all( + feature = "fallback", + not(portable_atomic_no_cmpxchg16b_target_feature), + not(portable_atomic_no_outline_atomics), + not(any(target_env = "sgx", miri)), + ), + ), + ), + all( + target_arch = "powerpc64", + portable_atomic_unstable_asm_experimental_arch, + any( + target_feature = "quadword-atomics", + portable_atomic_target_feature = "quadword-atomics", + all( + feature = "fallback", + not(portable_atomic_no_outline_atomics), + portable_atomic_outline_atomics, // TODO(powerpc64): currently disabled by default + any( + all( + target_os = "linux", + any( + target_env = "gnu", + all( + any(target_env = "musl", target_env = "ohos"), + not(target_feature = "crt-static"), + ), + portable_atomic_outline_atomics, + ), + ), + target_os = "android", + target_os = "freebsd", + ), + not(any(miri, portable_atomic_sanitize_thread)), + ), + ), + ), + all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch), + )))] + pub(crate) use self::fallback::{AtomicI128, AtomicU128}; +} + +// 64-bit atomics (platform-specific) +// pre-v6 ARM Linux +#[cfg(feature = "fallback")] +#[cfg(all( + target_arch = "arm", + not(any(miri, portable_atomic_sanitize_thread)), + not(portable_atomic_no_asm), + any(target_os = "linux", target_os = "android"), + not(any(target_feature = "v6", portable_atomic_target_feature = "v6")), + not(portable_atomic_no_outline_atomics), +))] +#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))] +#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))] +pub(crate) use self::arm_linux::{AtomicI64, AtomicU64}; + +// 128-bit atomics (platform-specific) +// aarch64 +#[cfg(all( + target_arch = "aarch64", + any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), +))] +pub(crate) use self::aarch64::{AtomicI128, AtomicU128}; +// x86_64 & (cmpxchg16b | outline-atomics) +#[cfg(all( + target_arch = "x86_64", + any(not(portable_atomic_no_asm), portable_atomic_unstable_asm), + any( + target_feature = "cmpxchg16b", + portable_atomic_target_feature = "cmpxchg16b", + all( + feature = "fallback", + not(portable_atomic_no_cmpxchg16b_target_feature), + not(portable_atomic_no_outline_atomics), + not(any(target_env = "sgx", miri)), + ), + ), +))] +pub(crate) use self::x86_64::{AtomicI128, AtomicU128}; +// powerpc64 & (pwr8 | outline-atomics) +#[cfg(all( + target_arch = "powerpc64", + portable_atomic_unstable_asm_experimental_arch, + any( + target_feature = "quadword-atomics", + portable_atomic_target_feature = "quadword-atomics", + all( + feature = "fallback", + not(portable_atomic_no_outline_atomics), + portable_atomic_outline_atomics, // TODO(powerpc64): currently disabled by default + any( + all( + target_os = "linux", + any( + target_env = "gnu", + all( + any(target_env = "musl", target_env = "ohos"), + not(target_feature = "crt-static"), + ), + portable_atomic_outline_atomics, + ), + ), + target_os = "android", + target_os = "freebsd", + ), + not(any(miri, portable_atomic_sanitize_thread)), + ), + ), +))] +pub(crate) use self::powerpc64::{AtomicI128, AtomicU128}; +// s390x +#[cfg(all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch))] +pub(crate) use self::s390x::{AtomicI128, AtomicU128}; -- cgit v1.2.3