diff options
Diffstat (limited to 'vendor/portable-atomic/src/imp/atomic128/x86_64.rs')
| -rw-r--r-- | vendor/portable-atomic/src/imp/atomic128/x86_64.rs | 854 |
1 files changed, 0 insertions, 854 deletions
diff --git a/vendor/portable-atomic/src/imp/atomic128/x86_64.rs b/vendor/portable-atomic/src/imp/atomic128/x86_64.rs deleted file mode 100644 index 3b9d141..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/x86_64.rs +++ /dev/null @@ -1,854 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Atomic{I,U}128 implementation on x86_64 using CMPXCHG16B (DWCAS). -// -// Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use -// this module and use intrinsics.rs instead. -// -// Refs: -// - x86 and amd64 instruction reference https://www.felixcloutier.com/x86 -// - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit -// -// Generated asm: -// - x86_64 (+cmpxchg16b) https://godbolt.org/z/55n54WeKr - -include!("macros.rs"); - -#[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))] -#[path = "../fallback/outline_atomics.rs"] -mod fallback; - -#[cfg(not(portable_atomic_no_outline_atomics))] -#[cfg(not(target_env = "sgx"))] -#[path = "detect/x86_64.rs"] -mod detect; - -#[cfg(not(portable_atomic_no_asm))] -use core::arch::asm; -use core::sync::atomic::Ordering; - -use crate::utils::{Pair, U128}; - -// Asserts that the function is called in the correct context. -macro_rules! debug_assert_cmpxchg16b { - () => { - #[cfg(not(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b", - )))] - { - debug_assert!(detect::detect().has_cmpxchg16b()); - } - }; -} -#[cfg(not(any(portable_atomic_no_outline_atomics, target_env = "sgx")))] -#[cfg(target_feature = "sse")] -macro_rules! debug_assert_vmovdqa_atomic { - () => {{ - debug_assert_cmpxchg16b!(); - debug_assert!(detect::detect().has_vmovdqa_atomic()); - }}; -} - -#[allow(unused_macros)] -#[cfg(target_pointer_width = "32")] -macro_rules! ptr_modifier { - () => { - ":e" - }; -} -#[allow(unused_macros)] -#[cfg(target_pointer_width = "64")] -macro_rules! ptr_modifier { - () => { - "" - }; -} - -#[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - target_feature(enable = "cmpxchg16b") -)] -#[inline] -unsafe fn cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) { - debug_assert!(dst as usize % 16 == 0); - debug_assert_cmpxchg16b!(); - - // SAFETY: the caller must guarantee that `dst` is valid for both writes and - // reads, 16-byte aligned (required by CMPXCHG16B), that there are no - // concurrent non-atomic operations, and that the CPU supports CMPXCHG16B. - // - // If the value at `dst` (destination operand) and rdx:rax are equal, the - // 128-bit value in rcx:rbx is stored in the `dst`, otherwise the value at - // `dst` is loaded to rdx:rax. - // - // The ZF flag is set if the value at `dst` and rdx:rax are equal, - // otherwise it is cleared. Other flags are unaffected. - // - // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b - unsafe { - // cmpxchg16b is always SeqCst. - let r: u8; - let old = U128 { whole: old }; - let new = U128 { whole: new }; - let (prev_lo, prev_hi); - macro_rules! cmpxchg16b { - ($rdi:tt) => { - asm!( - // rbx is reserved by LLVM - "xchg {rbx_tmp}, rbx", - concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"), - "sete r8b", - "mov rbx, {rbx_tmp}", // restore rbx - rbx_tmp = inout(reg) new.pair.lo => _, - in("rcx") new.pair.hi, - inout("rax") old.pair.lo => prev_lo, - inout("rdx") old.pair.hi => prev_hi, - in($rdi) dst, - out("r8b") r, - // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag. - options(nostack), - ) - }; - } - #[cfg(target_pointer_width = "32")] - cmpxchg16b!("edi"); - #[cfg(target_pointer_width = "64")] - cmpxchg16b!("rdi"); - (U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole, r != 0) - } -} - -// VMOVDQA is atomic on Intel and AMD CPUs with AVX. -// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details. -// -// Refs: https://www.felixcloutier.com/x86/movdqa:vmovdqa32:vmovdqa64 -// -// Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled. -// https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html -#[cfg(not(any(portable_atomic_no_outline_atomics, target_env = "sgx")))] -#[cfg(target_feature = "sse")] -#[target_feature(enable = "avx")] -#[inline] -unsafe fn atomic_load_vmovdqa(src: *mut u128) -> u128 { - debug_assert!(src as usize % 16 == 0); - debug_assert_vmovdqa_atomic!(); - - // SAFETY: the caller must uphold the safety contract. - // - // atomic load by vmovdqa is always SeqCst. - unsafe { - let out: core::arch::x86_64::__m128; - asm!( - concat!("vmovdqa {out}, xmmword ptr [{src", ptr_modifier!(), "}]"), - src = in(reg) src, - out = out(xmm_reg) out, - options(nostack, preserves_flags), - ); - core::mem::transmute(out) - } -} -#[cfg(not(any(portable_atomic_no_outline_atomics, target_env = "sgx")))] -#[cfg(target_feature = "sse")] -#[target_feature(enable = "avx")] -#[inline] -unsafe fn atomic_store_vmovdqa(dst: *mut u128, val: u128, order: Ordering) { - debug_assert!(dst as usize % 16 == 0); - debug_assert_vmovdqa_atomic!(); - - // SAFETY: the caller must uphold the safety contract. - unsafe { - let val: core::arch::x86_64::__m128 = core::mem::transmute(val); - match order { - // Relaxed and Release stores are equivalent. - Ordering::Relaxed | Ordering::Release => { - asm!( - concat!("vmovdqa xmmword ptr [{dst", ptr_modifier!(), "}], {val}"), - dst = in(reg) dst, - val = in(xmm_reg) val, - options(nostack, preserves_flags), - ); - } - Ordering::SeqCst => { - asm!( - concat!("vmovdqa xmmword ptr [{dst", ptr_modifier!(), "}], {val}"), - "mfence", - dst = in(reg) dst, - val = in(xmm_reg) val, - options(nostack, preserves_flags), - ); - } - _ => unreachable!("{:?}", order), - } - } -} - -#[cfg(not(all( - any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"), - any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")), -)))] -macro_rules! load_store_detect { - ( - vmovdqa = $vmovdqa:ident - cmpxchg16b = $cmpxchg16b:ident - fallback = $fallback:ident - ) => {{ - let cpuid = detect::detect(); - #[cfg(not(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b", - )))] - { - // Check CMPXCHG16B first to prevent mixing atomic and non-atomic access. - if cpuid.has_cmpxchg16b() { - // We do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled. - #[cfg(target_feature = "sse")] - { - if cpuid.has_vmovdqa_atomic() { - $vmovdqa - } else { - $cmpxchg16b - } - } - #[cfg(not(target_feature = "sse"))] - { - $cmpxchg16b - } - } else { - fallback::$fallback - } - } - #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] - { - if cpuid.has_vmovdqa_atomic() { - $vmovdqa - } else { - $cmpxchg16b - } - } - }}; -} - -#[inline] -unsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 { - // Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled. - // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html - // SGX doesn't support CPUID. - #[cfg(all( - any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"), - any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")), - ))] - // SAFETY: the caller must uphold the safety contract. - // cfg guarantees that CMPXCHG16B is available at compile-time. - unsafe { - // cmpxchg16b is always SeqCst. - atomic_load_cmpxchg16b(src) - } - #[cfg(not(all( - any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"), - any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")), - )))] - // SAFETY: the caller must uphold the safety contract. - unsafe { - ifunc!(unsafe fn(src: *mut u128) -> u128 { - load_store_detect! { - vmovdqa = atomic_load_vmovdqa - cmpxchg16b = atomic_load_cmpxchg16b - // Use SeqCst because cmpxchg16b and atomic load by vmovdqa is always SeqCst. - fallback = atomic_load_seqcst - } - }) - } -} -#[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - target_feature(enable = "cmpxchg16b") -)] -#[inline] -unsafe fn atomic_load_cmpxchg16b(src: *mut u128) -> u128 { - debug_assert!(src as usize % 16 == 0); - debug_assert_cmpxchg16b!(); - - // SAFETY: the caller must guarantee that `src` is valid for both writes and - // reads, 16-byte aligned, and that there are no concurrent non-atomic operations. - // cfg guarantees that the CPU supports CMPXCHG16B. - // - // See cmpxchg16b function for more. - // - // We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows - // omitting the storing of condition flags and avoid use of xchg to handle rbx. - unsafe { - // cmpxchg16b is always SeqCst. - let (out_lo, out_hi); - macro_rules! cmpxchg16b { - ($rdi:tt) => { - asm!( - // rbx is reserved by LLVM - "mov {rbx_tmp}, rbx", - "xor rbx, rbx", // zeroed rbx - concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"), - "mov rbx, {rbx_tmp}", // restore rbx - // set old/new args of cmpxchg16b to 0 (rbx is zeroed after saved to rbx_tmp, to avoid xchg) - rbx_tmp = out(reg) _, - in("rcx") 0_u64, - inout("rax") 0_u64 => out_lo, - inout("rdx") 0_u64 => out_hi, - in($rdi) src, - // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag. - options(nostack), - ) - }; - } - #[cfg(target_pointer_width = "32")] - cmpxchg16b!("edi"); - #[cfg(target_pointer_width = "64")] - cmpxchg16b!("rdi"); - U128 { pair: Pair { lo: out_lo, hi: out_hi } }.whole - } -} - -#[inline] -unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) { - // Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled. - // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html - // SGX doesn't support CPUID. - #[cfg(all( - any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"), - any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")), - ))] - // SAFETY: the caller must uphold the safety contract. - // cfg guarantees that CMPXCHG16B is available at compile-time. - unsafe { - // cmpxchg16b is always SeqCst. - let _ = order; - atomic_store_cmpxchg16b(dst, val); - } - #[cfg(not(all( - any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"), - any(portable_atomic_no_outline_atomics, target_env = "sgx", not(target_feature = "sse")), - )))] - // SAFETY: the caller must uphold the safety contract. - unsafe { - #[cfg(target_feature = "sse")] - fn_alias! { - #[target_feature(enable = "avx")] - unsafe fn(dst: *mut u128, val: u128); - // atomic store by vmovdqa has at least release semantics. - atomic_store_vmovdqa_non_seqcst = atomic_store_vmovdqa(Ordering::Release); - atomic_store_vmovdqa_seqcst = atomic_store_vmovdqa(Ordering::SeqCst); - } - match order { - // Relaxed and Release stores are equivalent in all implementations - // that may be called here (vmovdqa, asm-based cmpxchg16b, and fallback). - // core::arch's cmpxchg16b will never called here. - Ordering::Relaxed | Ordering::Release => { - ifunc!(unsafe fn(dst: *mut u128, val: u128) { - load_store_detect! { - vmovdqa = atomic_store_vmovdqa_non_seqcst - cmpxchg16b = atomic_store_cmpxchg16b - fallback = atomic_store_non_seqcst - } - }); - } - Ordering::SeqCst => { - ifunc!(unsafe fn(dst: *mut u128, val: u128) { - load_store_detect! { - vmovdqa = atomic_store_vmovdqa_seqcst - cmpxchg16b = atomic_store_cmpxchg16b - fallback = atomic_store_seqcst - } - }); - } - _ => unreachable!("{:?}", order), - } - } -} -#[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - target_feature(enable = "cmpxchg16b") -)] -unsafe fn atomic_store_cmpxchg16b(dst: *mut u128, val: u128) { - // SAFETY: the caller must uphold the safety contract. - unsafe { - // cmpxchg16b is always SeqCst. - atomic_swap_cmpxchg16b(dst, val, Ordering::SeqCst); - } -} - -#[inline] -unsafe fn atomic_compare_exchange( - dst: *mut u128, - old: u128, - new: u128, - _success: Ordering, - _failure: Ordering, -) -> Result<u128, u128> { - #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] - // SAFETY: the caller must guarantee that `dst` is valid for both writes and - // reads, 16-byte aligned, that there are no concurrent non-atomic operations, - // and cfg guarantees that CMPXCHG16B is available at compile-time. - let (prev, ok) = unsafe { cmpxchg16b(dst, old, new) }; - #[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))] - // SAFETY: the caller must guarantee that `dst` is valid for both writes and - // reads, 16-byte aligned, and that there are no different kinds of concurrent accesses. - let (prev, ok) = unsafe { - ifunc!(unsafe fn(dst: *mut u128, old: u128, new: u128) -> (u128, bool) { - if detect::detect().has_cmpxchg16b() { - cmpxchg16b - } else { - // Use SeqCst because cmpxchg16b is always SeqCst. - fallback::atomic_compare_exchange_seqcst - } - }) - }; - if ok { - Ok(prev) - } else { - Err(prev) - } -} - -// cmpxchg16b is always strong. -use atomic_compare_exchange as atomic_compare_exchange_weak; - -#[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] -use atomic_swap_cmpxchg16b as atomic_swap; -#[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - target_feature(enable = "cmpxchg16b") -)] -#[inline] -unsafe fn atomic_swap_cmpxchg16b(dst: *mut u128, val: u128, _order: Ordering) -> u128 { - debug_assert!(dst as usize % 16 == 0); - debug_assert_cmpxchg16b!(); - - // SAFETY: the caller must guarantee that `dst` is valid for both writes and - // reads, 16-byte aligned, and that there are no concurrent non-atomic operations. - // cfg guarantees that the CPU supports CMPXCHG16B. - // - // See cmpxchg16b function for more. - // - // We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows - // omitting the storing/comparing of condition flags and reducing uses of xchg/mov to handle rbx. - // - // Do not use atomic_rmw_cas_3 because it needs extra MOV to implement swap. - unsafe { - // cmpxchg16b is always SeqCst. - let val = U128 { whole: val }; - let (mut prev_lo, mut prev_hi); - macro_rules! cmpxchg16b { - ($rdi:tt) => { - asm!( - // rbx is reserved by LLVM - "xchg {rbx_tmp}, rbx", - // This is not single-copy atomic reads, but this is ok because subsequent - // CAS will check for consistency. - // - // This is based on the code generated for the first load in DW RMWs by LLVM. - // - // Note that the C++20 memory model does not allow mixed-sized atomic access, - // so we must use inline assembly to implement this. - // (i.e., byte-wise atomic based on the standard library's atomic types - // cannot be used here). - concat!("mov rax, qword ptr [", $rdi, "]"), - concat!("mov rdx, qword ptr [", $rdi, " + 8]"), - "2:", - concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"), - "jne 2b", - "mov rbx, {rbx_tmp}", // restore rbx - rbx_tmp = inout(reg) val.pair.lo => _, - in("rcx") val.pair.hi, - out("rax") prev_lo, - out("rdx") prev_hi, - in($rdi) dst, - // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag. - options(nostack), - ) - }; - } - #[cfg(target_pointer_width = "32")] - cmpxchg16b!("edi"); - #[cfg(target_pointer_width = "64")] - cmpxchg16b!("rdi"); - U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole - } -} - -/// Atomic RMW by CAS loop (3 arguments) -/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;` -/// -/// `$op` can use the following registers: -/// - rsi/r8 pair: val argument (read-only for `$op`) -/// - rax/rdx pair: previous value loaded (read-only for `$op`) -/// - rbx/rcx pair: new value that will be stored -// We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows -// omitting the storing/comparing of condition flags and reducing uses of xchg/mov to handle rbx. -macro_rules! atomic_rmw_cas_3 { - ($name:ident as $reexport_name:ident, $($op:tt)*) => { - #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] - use $name as $reexport_name; - #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - target_feature(enable = "cmpxchg16b") - )] - #[inline] - unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 { - debug_assert!(dst as usize % 16 == 0); - debug_assert_cmpxchg16b!(); - // SAFETY: the caller must guarantee that `dst` is valid for both writes and - // reads, 16-byte aligned, and that there are no concurrent non-atomic operations. - // cfg guarantees that the CPU supports CMPXCHG16B. - // - // See cmpxchg16b function for more. - unsafe { - // cmpxchg16b is always SeqCst. - let val = U128 { whole: val }; - let (mut prev_lo, mut prev_hi); - macro_rules! cmpxchg16b { - ($rdi:tt) => { - asm!( - // rbx is reserved by LLVM - "mov {rbx_tmp}, rbx", - // This is not single-copy atomic reads, but this is ok because subsequent - // CAS will check for consistency. - // - // This is based on the code generated for the first load in DW RMWs by LLVM. - // - // Note that the C++20 memory model does not allow mixed-sized atomic access, - // so we must use inline assembly to implement this. - // (i.e., byte-wise atomic based on the standard library's atomic types - // cannot be used here). - concat!("mov rax, qword ptr [", $rdi, "]"), - concat!("mov rdx, qword ptr [", $rdi, " + 8]"), - "2:", - $($op)* - concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"), - "jne 2b", - "mov rbx, {rbx_tmp}", // restore rbx - rbx_tmp = out(reg) _, - out("rcx") _, - out("rax") prev_lo, - out("rdx") prev_hi, - in($rdi) dst, - in("rsi") val.pair.lo, - in("r8") val.pair.hi, - // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag. - options(nostack), - ) - }; - } - #[cfg(target_pointer_width = "32")] - cmpxchg16b!("edi"); - #[cfg(target_pointer_width = "64")] - cmpxchg16b!("rdi"); - U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole - } - } - }; -} -/// Atomic RMW by CAS loop (2 arguments) -/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;` -/// -/// `$op` can use the following registers: -/// - rax/rdx pair: previous value loaded (read-only for `$op`) -/// - rbx/rcx pair: new value that will be stored -// We could use CAS loop by atomic_compare_exchange here, but using an inline assembly allows -// omitting the storing of condition flags and avoid use of xchg to handle rbx. -macro_rules! atomic_rmw_cas_2 { - ($name:ident as $reexport_name:ident, $($op:tt)*) => { - #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] - use $name as $reexport_name; - #[cfg_attr( - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - target_feature(enable = "cmpxchg16b") - )] - #[inline] - unsafe fn $name(dst: *mut u128, _order: Ordering) -> u128 { - debug_assert!(dst as usize % 16 == 0); - debug_assert_cmpxchg16b!(); - // SAFETY: the caller must guarantee that `dst` is valid for both writes and - // reads, 16-byte aligned, and that there are no concurrent non-atomic operations. - // cfg guarantees that the CPU supports CMPXCHG16B. - // - // See cmpxchg16b function for more. - unsafe { - // cmpxchg16b is always SeqCst. - let (mut prev_lo, mut prev_hi); - macro_rules! cmpxchg16b { - ($rdi:tt) => { - asm!( - // rbx is reserved by LLVM - "mov {rbx_tmp}, rbx", - // This is not single-copy atomic reads, but this is ok because subsequent - // CAS will check for consistency. - // - // This is based on the code generated for the first load in DW RMWs by LLVM. - // - // Note that the C++20 memory model does not allow mixed-sized atomic access, - // so we must use inline assembly to implement this. - // (i.e., byte-wise atomic based on the standard library's atomic types - // cannot be used here). - concat!("mov rax, qword ptr [", $rdi, "]"), - concat!("mov rdx, qword ptr [", $rdi, " + 8]"), - "2:", - $($op)* - concat!("lock cmpxchg16b xmmword ptr [", $rdi, "]"), - "jne 2b", - "mov rbx, {rbx_tmp}", // restore rbx - rbx_tmp = out(reg) _, - out("rcx") _, - out("rax") prev_lo, - out("rdx") prev_hi, - in($rdi) dst, - // Do not use `preserves_flags` because CMPXCHG16B modifies the ZF flag. - options(nostack), - ) - }; - } - #[cfg(target_pointer_width = "32")] - cmpxchg16b!("edi"); - #[cfg(target_pointer_width = "64")] - cmpxchg16b!("rdi"); - U128 { pair: Pair { lo: prev_lo, hi: prev_hi } }.whole - } - } - }; -} - -atomic_rmw_cas_3! { - atomic_add_cmpxchg16b as atomic_add, - "mov rbx, rax", - "add rbx, rsi", - "mov rcx, rdx", - "adc rcx, r8", -} -atomic_rmw_cas_3! { - atomic_sub_cmpxchg16b as atomic_sub, - "mov rbx, rax", - "sub rbx, rsi", - "mov rcx, rdx", - "sbb rcx, r8", -} -atomic_rmw_cas_3! { - atomic_and_cmpxchg16b as atomic_and, - "mov rbx, rax", - "and rbx, rsi", - "mov rcx, rdx", - "and rcx, r8", -} -atomic_rmw_cas_3! { - atomic_nand_cmpxchg16b as atomic_nand, - "mov rbx, rax", - "and rbx, rsi", - "not rbx", - "mov rcx, rdx", - "and rcx, r8", - "not rcx", -} -atomic_rmw_cas_3! { - atomic_or_cmpxchg16b as atomic_or, - "mov rbx, rax", - "or rbx, rsi", - "mov rcx, rdx", - "or rcx, r8", -} -atomic_rmw_cas_3! { - atomic_xor_cmpxchg16b as atomic_xor, - "mov rbx, rax", - "xor rbx, rsi", - "mov rcx, rdx", - "xor rcx, r8", -} - -atomic_rmw_cas_2! { - atomic_not_cmpxchg16b as atomic_not, - "mov rbx, rax", - "not rbx", - "mov rcx, rdx", - "not rcx", -} -atomic_rmw_cas_2! { - atomic_neg_cmpxchg16b as atomic_neg, - "mov rbx, rax", - "neg rbx", - "mov rcx, 0", - "sbb rcx, rdx", -} - -atomic_rmw_cas_3! { - atomic_max_cmpxchg16b as atomic_max, - "cmp rsi, rax", - "mov rcx, r8", - "sbb rcx, rdx", - "mov rcx, r8", - "cmovl rcx, rdx", - "mov rbx, rsi", - "cmovl rbx, rax", -} -atomic_rmw_cas_3! { - atomic_umax_cmpxchg16b as atomic_umax, - "cmp rsi, rax", - "mov rcx, r8", - "sbb rcx, rdx", - "mov rcx, r8", - "cmovb rcx, rdx", - "mov rbx, rsi", - "cmovb rbx, rax", -} -atomic_rmw_cas_3! { - atomic_min_cmpxchg16b as atomic_min, - "cmp rsi, rax", - "mov rcx, r8", - "sbb rcx, rdx", - "mov rcx, r8", - "cmovge rcx, rdx", - "mov rbx, rsi", - "cmovge rbx, rax", -} -atomic_rmw_cas_3! { - atomic_umin_cmpxchg16b as atomic_umin, - "cmp rsi, rax", - "mov rcx, r8", - "sbb rcx, rdx", - "mov rcx, r8", - "cmovae rcx, rdx", - "mov rbx, rsi", - "cmovae rbx, rax", -} - -macro_rules! atomic_rmw_with_ifunc { - ( - unsafe fn $name:ident($($arg:tt)*) $(-> $ret_ty:ty)?; - cmpxchg16b = $cmpxchg16b_fn:ident; - fallback = $seqcst_fallback_fn:ident; - ) => { - #[cfg(not(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b", - )))] - #[inline] - unsafe fn $name($($arg)*, _order: Ordering) $(-> $ret_ty)? { - fn_alias! { - #[cfg_attr( - not(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b", - )), - target_feature(enable = "cmpxchg16b") - )] - unsafe fn($($arg)*) $(-> $ret_ty)?; - // cmpxchg16b is always SeqCst. - cmpxchg16b_seqcst_fn = $cmpxchg16b_fn(Ordering::SeqCst); - } - // SAFETY: the caller must uphold the safety contract. - // we only calls cmpxchg16b_fn if cmpxchg16b is available. - unsafe { - ifunc!(unsafe fn($($arg)*) $(-> $ret_ty)? { - if detect::detect().has_cmpxchg16b() { - cmpxchg16b_seqcst_fn - } else { - // Use SeqCst because cmpxchg16b is always SeqCst. - fallback::$seqcst_fallback_fn - } - }) - } - } - }; -} - -atomic_rmw_with_ifunc! { - unsafe fn atomic_swap(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_swap_cmpxchg16b; - fallback = atomic_swap_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_add(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_add_cmpxchg16b; - fallback = atomic_add_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_sub(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_sub_cmpxchg16b; - fallback = atomic_sub_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_and(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_and_cmpxchg16b; - fallback = atomic_and_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_nand(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_nand_cmpxchg16b; - fallback = atomic_nand_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_or(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_or_cmpxchg16b; - fallback = atomic_or_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_xor(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_xor_cmpxchg16b; - fallback = atomic_xor_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_max(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_max_cmpxchg16b; - fallback = atomic_max_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_umax(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_umax_cmpxchg16b; - fallback = atomic_umax_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_min(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_min_cmpxchg16b; - fallback = atomic_min_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_umin(dst: *mut u128, val: u128) -> u128; - cmpxchg16b = atomic_umin_cmpxchg16b; - fallback = atomic_umin_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_not(dst: *mut u128) -> u128; - cmpxchg16b = atomic_not_cmpxchg16b; - fallback = atomic_not_seqcst; -} -atomic_rmw_with_ifunc! { - unsafe fn atomic_neg(dst: *mut u128) -> u128; - cmpxchg16b = atomic_neg_cmpxchg16b; - fallback = atomic_neg_seqcst; -} - -#[inline] -fn is_lock_free() -> bool { - #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))] - { - // CMPXCHG16B is available at compile-time. - true - } - #[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))] - { - detect::detect().has_cmpxchg16b() - } -} -const IS_ALWAYS_LOCK_FREE: bool = - cfg!(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")); - -atomic128!(AtomicI128, i128, atomic_max, atomic_min); -atomic128!(AtomicU128, u128, atomic_umax, atomic_umin); - -#[allow(clippy::undocumented_unsafe_blocks, clippy::wildcard_imports)] -#[cfg(test)] -mod tests { - use super::*; - - test_atomic_int!(i128); - test_atomic_int!(u128); - - // load/store/swap implementation is not affected by signedness, so it is - // enough to test only unsigned types. - stress_test!(u128); -} |