diff options
author | Valentin Popov <valentin@popov.link> | 2024-07-19 15:37:58 +0300 |
---|---|---|
committer | Valentin Popov <valentin@popov.link> | 2024-07-19 15:37:58 +0300 |
commit | a990de90fe41456a23e58bd087d2f107d321f3a1 (patch) | |
tree | 15afc392522a9e85dc3332235e311b7d39352ea9 /vendor/portable-atomic/src/imp/atomic128/s390x.rs | |
parent | 3d48cd3f81164bbfc1a755dc1d4a9a02f98c8ddd (diff) | |
download | fparkan-a990de90fe41456a23e58bd087d2f107d321f3a1.tar.xz fparkan-a990de90fe41456a23e58bd087d2f107d321f3a1.zip |
Deleted vendor folder
Diffstat (limited to 'vendor/portable-atomic/src/imp/atomic128/s390x.rs')
-rw-r--r-- | vendor/portable-atomic/src/imp/atomic128/s390x.rs | 461 |
1 files changed, 0 insertions, 461 deletions
diff --git a/vendor/portable-atomic/src/imp/atomic128/s390x.rs b/vendor/portable-atomic/src/imp/atomic128/s390x.rs deleted file mode 100644 index 37c2063..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/s390x.rs +++ /dev/null @@ -1,461 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Atomic{I,U}128 implementation on s390x. -// -// s390x supports 128-bit atomic load/store/cmpxchg: -// https://github.com/llvm/llvm-project/commit/a11f63a952664f700f076fd754476a2b9eb158cc -// -// LLVM's minimal supported architecture level is z10: -// https://github.com/llvm/llvm-project/blob/llvmorg-17.0.0-rc2/llvm/lib/Target/SystemZ/SystemZProcessors.td) -// This does not appear to have changed since the current s390x backend was added in LLVM 3.3: -// https://github.com/llvm/llvm-project/commit/5f613dfd1f7edb0ae95d521b7107b582d9df5103#diff-cbaef692b3958312e80fd5507a7e2aff071f1acb086f10e8a96bc06a7bb289db -// -// Note: On Miri and ThreadSanitizer which do not support inline assembly, we don't use -// this module and use intrinsics.rs instead. -// -// Refs: -// - z/Architecture Principles of Operation https://publibfp.dhe.ibm.com/epubs/pdf/a227832d.pdf -// - z/Architecture Reference Summary https://www.ibm.com/support/pages/zarchitecture-reference-summary -// - atomic-maybe-uninit https://github.com/taiki-e/atomic-maybe-uninit -// -// Generated asm: -// - s390x https://godbolt.org/z/b11znnEh4 -// - s390x (z196) https://godbolt.org/z/s5n9PGcv6 -// - s390x (z15) https://godbolt.org/z/Wf49h7bPf - -include!("macros.rs"); - -use core::{arch::asm, sync::atomic::Ordering}; - -use crate::utils::{Pair, U128}; - -// Use distinct operands on z196 or later, otherwise split to lgr and $op. -#[cfg(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops"))] -macro_rules! distinct_op { - ($op:tt, $a0:tt, $a1:tt, $a2:tt) => { - concat!($op, "k ", $a0, ", ", $a1, ", ", $a2) - }; -} -#[cfg(not(any(target_feature = "distinct-ops", portable_atomic_target_feature = "distinct-ops")))] -macro_rules! distinct_op { - ($op:tt, $a0:tt, $a1:tt, $a2:tt) => { - concat!("lgr ", $a0, ", ", $a1, "\n", $op, " ", $a0, ", ", $a2) - }; -} - -// Use selgr$cond on z15 or later, otherwise split to locgr$cond and $op. -#[cfg(any( - target_feature = "miscellaneous-extensions-3", - portable_atomic_target_feature = "miscellaneous-extensions-3", -))] -#[cfg(any( - target_feature = "load-store-on-cond", - portable_atomic_target_feature = "load-store-on-cond", -))] -macro_rules! select_op { - ($cond:tt, $a0:tt, $a1:tt, $a2:tt) => { - concat!("selgr", $cond, " ", $a0, ", ", $a1, ", ", $a2) - }; -} -#[cfg(not(any( - target_feature = "miscellaneous-extensions-3", - portable_atomic_target_feature = "miscellaneous-extensions-3", -)))] -#[cfg(any( - target_feature = "load-store-on-cond", - portable_atomic_target_feature = "load-store-on-cond", -))] -macro_rules! select_op { - ($cond:tt, $a0:tt, $a1:tt, $a2:tt) => { - concat!("lgr ", $a0, ", ", $a2, "\n", "locgr", $cond, " ", $a0, ", ", $a1) - }; -} - -#[inline] -unsafe fn atomic_load(src: *mut u128, _order: Ordering) -> u128 { - debug_assert!(src as usize % 16 == 0); - - // SAFETY: the caller must uphold the safety contract. - unsafe { - // atomic load is always SeqCst. - let (out_hi, out_lo); - asm!( - "lpq %r0, 0({src})", - src = in(reg) ptr_reg!(src), - // Quadword atomic instructions work with even/odd pair of specified register and subsequent register. - out("r0") out_hi, - out("r1") out_lo, - options(nostack, preserves_flags), - ); - U128 { pair: Pair { hi: out_hi, lo: out_lo } }.whole - } -} - -#[inline] -unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) { - debug_assert!(dst as usize % 16 == 0); - - // SAFETY: the caller must uphold the safety contract. - unsafe { - let val = U128 { whole: val }; - macro_rules! atomic_store { - ($fence:tt) => { - asm!( - "stpq %r0, 0({dst})", - $fence, - dst = in(reg) ptr_reg!(dst), - // Quadword atomic instructions work with even/odd pair of specified register and subsequent register. - in("r0") val.pair.hi, - in("r1") val.pair.lo, - options(nostack, preserves_flags), - ) - }; - } - match order { - // Relaxed and Release stores are equivalent. - Ordering::Relaxed | Ordering::Release => atomic_store!(""), - // bcr 14,0 (fast-BCR-serialization) requires z196 or later. - #[cfg(any( - target_feature = "fast-serialization", - portable_atomic_target_feature = "fast-serialization", - ))] - Ordering::SeqCst => atomic_store!("bcr 14, 0"), - #[cfg(not(any( - target_feature = "fast-serialization", - portable_atomic_target_feature = "fast-serialization", - )))] - Ordering::SeqCst => atomic_store!("bcr 15, 0"), - _ => unreachable!("{:?}", order), - } - } -} - -#[inline] -unsafe fn atomic_compare_exchange( - dst: *mut u128, - old: u128, - new: u128, - _success: Ordering, - _failure: Ordering, -) -> Result<u128, u128> { - debug_assert!(dst as usize % 16 == 0); - - // SAFETY: the caller must uphold the safety contract. - let prev = unsafe { - // atomic CAS is always SeqCst. - let old = U128 { whole: old }; - let new = U128 { whole: new }; - let (prev_hi, prev_lo); - asm!( - "cdsg %r0, %r12, 0({dst})", - dst = in(reg) ptr_reg!(dst), - // Quadword atomic instructions work with even/odd pair of specified register and subsequent register. - inout("r0") old.pair.hi => prev_hi, - inout("r1") old.pair.lo => prev_lo, - in("r12") new.pair.hi, - in("r13") new.pair.lo, - // Do not use `preserves_flags` because CDSG modifies the condition code. - options(nostack), - ); - U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole - }; - if prev == old { - Ok(prev) - } else { - Err(prev) - } -} - -// cdsg is always strong. -use atomic_compare_exchange as atomic_compare_exchange_weak; - -#[cfg(not(any( - target_feature = "load-store-on-cond", - portable_atomic_target_feature = "load-store-on-cond", -)))] -#[inline(always)] -unsafe fn atomic_update<F>(dst: *mut u128, order: Ordering, mut f: F) -> u128 -where - F: FnMut(u128) -> u128, -{ - // SAFETY: the caller must uphold the safety contract. - unsafe { - // This is a private function and all instances of `f` only operate on the value - // loaded, so there is no need to synchronize the first load/failed CAS. - let mut prev = atomic_load(dst, Ordering::Relaxed); - loop { - let next = f(prev); - match atomic_compare_exchange_weak(dst, prev, next, order, Ordering::Relaxed) { - Ok(x) => return x, - Err(x) => prev = x, - } - } - } -} - -#[inline] -unsafe fn atomic_swap(dst: *mut u128, val: u128, _order: Ordering) -> u128 { - debug_assert!(dst as usize % 16 == 0); - - // SAFETY: the caller must uphold the safety contract. - // - // We could use atomic_update here, but using an inline assembly allows omitting - // the comparison of results and the storing/comparing of condition flags. - // - // Do not use atomic_rmw_cas_3 because it needs extra LGR to implement swap. - unsafe { - // atomic swap is always SeqCst. - let val = U128 { whole: val }; - let (mut prev_hi, mut prev_lo); - asm!( - "lpq %r0, 0({dst})", - "2:", - "cdsg %r0, %r12, 0({dst})", - "jl 2b", - dst = in(reg) ptr_reg!(dst), - // Quadword atomic instructions work with even/odd pair of specified register and subsequent register. - out("r0") prev_hi, - out("r1") prev_lo, - in("r12") val.pair.hi, - in("r13") val.pair.lo, - // Do not use `preserves_flags` because CDSG modifies the condition code. - options(nostack), - ); - U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole - } -} - -/// Atomic RMW by CAS loop (3 arguments) -/// `unsafe fn(dst: *mut u128, val: u128, order: Ordering) -> u128;` -/// -/// `$op` can use the following registers: -/// - val_hi/val_lo pair: val argument (read-only for `$op`) -/// - r0/r1 pair: previous value loaded (read-only for `$op`) -/// - r12/r13 pair: new value that will be stored -// We could use atomic_update here, but using an inline assembly allows omitting -// the comparison of results and the storing/comparing of condition flags. -macro_rules! atomic_rmw_cas_3 { - ($name:ident, [$($reg:tt)*], $($op:tt)*) => { - #[inline] - unsafe fn $name(dst: *mut u128, val: u128, _order: Ordering) -> u128 { - debug_assert!(dst as usize % 16 == 0); - // SAFETY: the caller must uphold the safety contract. - unsafe { - // atomic RMW is always SeqCst. - let val = U128 { whole: val }; - let (mut prev_hi, mut prev_lo); - asm!( - "lpq %r0, 0({dst})", - "2:", - $($op)* - "cdsg %r0, %r12, 0({dst})", - "jl 2b", - dst = in(reg) ptr_reg!(dst), - val_hi = in(reg) val.pair.hi, - val_lo = in(reg) val.pair.lo, - $($reg)* - // Quadword atomic instructions work with even/odd pair of specified register and subsequent register. - out("r0") prev_hi, - out("r1") prev_lo, - out("r12") _, - out("r13") _, - // Do not use `preserves_flags` because CDSG modifies the condition code. - options(nostack), - ); - U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole - } - } - }; -} -/// Atomic RMW by CAS loop (2 arguments) -/// `unsafe fn(dst: *mut u128, order: Ordering) -> u128;` -/// -/// `$op` can use the following registers: -/// - r0/r1 pair: previous value loaded (read-only for `$op`) -/// - r12/r13 pair: new value that will be stored -// We could use atomic_update here, but using an inline assembly allows omitting -// the comparison of results and the storing/comparing of condition flags. -macro_rules! atomic_rmw_cas_2 { - ($name:ident, $($op:tt)*) => { - #[inline] - unsafe fn $name(dst: *mut u128, _order: Ordering) -> u128 { - debug_assert!(dst as usize % 16 == 0); - // SAFETY: the caller must uphold the safety contract. - unsafe { - // atomic RMW is always SeqCst. - let (mut prev_hi, mut prev_lo); - asm!( - "lpq %r0, 0({dst})", - "2:", - $($op)* - "cdsg %r0, %r12, 0({dst})", - "jl 2b", - dst = in(reg) ptr_reg!(dst), - // Quadword atomic instructions work with even/odd pair of specified register and subsequent register. - out("r0") prev_hi, - out("r1") prev_lo, - out("r12") _, - out("r13") _, - // Do not use `preserves_flags` because CDSG modifies the condition code. - options(nostack), - ); - U128 { pair: Pair { hi: prev_hi, lo: prev_lo } }.whole - } - } - }; -} - -atomic_rmw_cas_3! { - atomic_add, [], - distinct_op!("algr", "%r13", "%r1", "{val_lo}"), - "lgr %r12, %r0", - "alcgr %r12, {val_hi}", -} -atomic_rmw_cas_3! { - atomic_sub, [], - distinct_op!("slgr", "%r13", "%r1", "{val_lo}"), - "lgr %r12, %r0", - "slbgr %r12, {val_hi}", -} -atomic_rmw_cas_3! { - atomic_and, [], - distinct_op!("ngr", "%r13", "%r1", "{val_lo}"), - distinct_op!("ngr", "%r12", "%r0", "{val_hi}"), -} - -// Use nngrk on z15 or later. -#[cfg(any( - target_feature = "miscellaneous-extensions-3", - portable_atomic_target_feature = "miscellaneous-extensions-3", -))] -atomic_rmw_cas_3! { - atomic_nand, [], - "nngrk %r13, %r1, {val_lo}", - "nngrk %r12, %r0, {val_hi}", -} -#[cfg(not(any( - target_feature = "miscellaneous-extensions-3", - portable_atomic_target_feature = "miscellaneous-extensions-3", -)))] -atomic_rmw_cas_3! { - atomic_nand, [], - distinct_op!("ngr", "%r13", "%r1", "{val_lo}"), - "xihf %r13, 4294967295", - "xilf %r13, 4294967295", - distinct_op!("ngr", "%r12", "%r0", "{val_hi}"), - "xihf %r12, 4294967295", - "xilf %r12, 4294967295", -} - -atomic_rmw_cas_3! { - atomic_or, [], - distinct_op!("ogr", "%r13", "%r1", "{val_lo}"), - distinct_op!("ogr", "%r12", "%r0", "{val_hi}"), -} -atomic_rmw_cas_3! { - atomic_xor, [], - distinct_op!("xgr", "%r13", "%r1", "{val_lo}"), - distinct_op!("xgr", "%r12", "%r0", "{val_hi}"), -} - -#[cfg(any( - target_feature = "load-store-on-cond", - portable_atomic_target_feature = "load-store-on-cond", -))] -atomic_rmw_cas_3! { - atomic_max, [], - "clgr %r1, {val_lo}", - select_op!("h", "%r12", "%r1", "{val_lo}"), - "cgr %r0, {val_hi}", - select_op!("h", "%r13", "%r1", "{val_lo}"), - "locgre %r13, %r12", - select_op!("h", "%r12", "%r0", "{val_hi}"), -} -#[cfg(any( - target_feature = "load-store-on-cond", - portable_atomic_target_feature = "load-store-on-cond", -))] -atomic_rmw_cas_3! { - atomic_umax, [tmp = out(reg) _,], - "clgr %r1, {val_lo}", - select_op!("h", "{tmp}", "%r1", "{val_lo}"), - "clgr %r0, {val_hi}", - select_op!("h", "%r12", "%r0", "{val_hi}"), - select_op!("h", "%r13", "%r1", "{val_lo}"), - "cgr %r0, {val_hi}", - "locgre %r13, {tmp}", -} -#[cfg(any( - target_feature = "load-store-on-cond", - portable_atomic_target_feature = "load-store-on-cond", -))] -atomic_rmw_cas_3! { - atomic_min, [], - "clgr %r1, {val_lo}", - select_op!("l", "%r12", "%r1", "{val_lo}"), - "cgr %r0, {val_hi}", - select_op!("l", "%r13", "%r1", "{val_lo}"), - "locgre %r13, %r12", - select_op!("l", "%r12", "%r0", "{val_hi}"), -} -#[cfg(any( - target_feature = "load-store-on-cond", - portable_atomic_target_feature = "load-store-on-cond", -))] -atomic_rmw_cas_3! { - atomic_umin, [tmp = out(reg) _,], - "clgr %r1, {val_lo}", - select_op!("l", "{tmp}", "%r1", "{val_lo}"), - "clgr %r0, {val_hi}", - select_op!("l", "%r12", "%r0", "{val_hi}"), - select_op!("l", "%r13", "%r1", "{val_lo}"), - "cgr %r0, {val_hi}", - "locgre %r13, {tmp}", -} -// We use atomic_update for atomic min/max on pre-z196 because -// z10 doesn't seem to have a good way to implement 128-bit min/max. -// loc{,g}r requires z196 or later. -// https://godbolt.org/z/j8KG9q5oq -#[cfg(not(any( - target_feature = "load-store-on-cond", - portable_atomic_target_feature = "load-store-on-cond", -)))] -atomic_rmw_by_atomic_update!(cmp); - -atomic_rmw_cas_2! { - atomic_not, - "lgr %r13, %r1", - "xihf %r13, 4294967295", - "xilf %r13, 4294967295", - "lgr %r12, %r0", - "xihf %r12, 4294967295", - "xilf %r12, 4294967295", -} -atomic_rmw_cas_2! { - atomic_neg, - "lghi %r13, 0", - "slgr %r13, %r1", - "lghi %r12, 0", - "slbgr %r12, %r0", -} - -#[inline] -const fn is_lock_free() -> bool { - IS_ALWAYS_LOCK_FREE -} -const IS_ALWAYS_LOCK_FREE: bool = true; - -atomic128!(AtomicI128, i128, atomic_max, atomic_min); -atomic128!(AtomicU128, u128, atomic_umax, atomic_umin); - -#[cfg(test)] -mod tests { - use super::*; - - test_atomic_int!(i128); - test_atomic_int!(u128); - - // load/store/swap implementation is not affected by signedness, so it is - // enough to test only unsigned types. - stress_test!(u128); -} |