diff options
Diffstat (limited to 'vendor/portable-atomic/src/imp/atomic128/detect')
7 files changed, 0 insertions, 2190 deletions
diff --git a/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_aa64reg.rs b/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_aa64reg.rs deleted file mode 100644 index 4cbdb51..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_aa64reg.rs +++ /dev/null @@ -1,628 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Run-time feature detection on aarch64 Linux/FreeBSD/NetBSD/OpenBSD by parsing system registers. -// -// As of nightly-2023-01-23, is_aarch64_feature_detected doesn't support run-time detection on NetBSD/OpenBSD. -// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/mod.rs -// https://github.com/rust-lang/stdarch/pull/1374 -// -// Refs: -// - https://developer.arm.com/documentation/ddi0601/latest/AArch64-Registers -// - https://www.kernel.org/doc/Documentation/arm64/cpu-feature-registers.txt -// - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/aarch64.rs -// -// Supported platforms: -// - Linux 4.11+ (emulate mrs instruction) -// https://github.com/torvalds/linux/commit/77c97b4ee21290f5f083173d957843b615abbff2 -// - FreeBSD 12.0+ (emulate mrs instruction) -// https://github.com/freebsd/freebsd-src/commit/398810619cb32abf349f8de23f29510b2ee0839b -// - NetBSD 9.0+ (through sysctl) -// https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb -// - OpenBSD 7.1+ (through sysctl) -// https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8 -// -// For now, this module is only used on NetBSD/OpenBSD. -// On Linux/FreeBSD, this module is test-only: -// - On Linux, this approach requires a higher kernel version than Rust supports, -// and also does not work with qemu-user (as of QEMU 7.2) and Valgrind. -// (Looking into HWCAP_CPUID in auxvec, it appears that Valgrind is setting it -// to false correctly, but qemu-user is setting it to true.) -// - On FreeBSD, this approach does not work on FreeBSD 12 on QEMU (confirmed on -// FreeBSD 12.{2,3,4}), and we got SIGILL (worked on FreeBSD 13 and 14). - -include!("common.rs"); - -#[cfg_attr(test, derive(Debug, PartialEq))] -struct AA64Reg { - aa64isar0: u64, - #[cfg(test)] - aa64isar1: u64, - // OpenBSD has an API to get this, but currently always returns 0. - // https://github.com/openbsd/src/blob/6a233889798dc3ecb18acc52dce1e57862af2957/sys/arch/arm64/arm64/machdep.c#L371-L377 - #[cfg_attr(target_os = "openbsd", cfg(test))] - aa64mmfr2: u64, -} - -#[cold] -fn _detect(info: &mut CpuInfo) { - let AA64Reg { - aa64isar0, - #[cfg(test)] - aa64isar1, - #[cfg_attr(target_os = "openbsd", cfg(test))] - aa64mmfr2, - } = imp::aa64reg(); - - // ID_AA64ISAR0_EL1, Instruction Set Attribute Register 0 - // https://developer.arm.com/documentation/ddi0601/2023-06/AArch64-Registers/ID-AA64ISAR0-EL1--AArch64-Instruction-Set-Attribute-Register-0?lang=en - let atomic = extract(aa64isar0, 23, 20); - if atomic >= 2 { - info.set(CpuInfo::HAS_LSE); - // we currently only use FEAT_LSE and FEAT_LSE2 in outline-atomics. - #[cfg(test)] - { - if atomic >= 3 { - info.set(CpuInfo::HAS_LSE128); - } - } - } - // we currently only use FEAT_LSE and FEAT_LSE2 in outline-atomics. - #[cfg(test)] - { - // ID_AA64ISAR1_EL1, Instruction Set Attribute Register 1 - // https://developer.arm.com/documentation/ddi0601/2023-06/AArch64-Registers/ID-AA64ISAR1-EL1--AArch64-Instruction-Set-Attribute-Register-1?lang=en - if extract(aa64isar1, 23, 20) >= 3 { - info.set(CpuInfo::HAS_RCPC3); - } - } - // OpenBSD has an API to get this, but currently always returns 0. - // https://github.com/openbsd/src/blob/6a233889798dc3ecb18acc52dce1e57862af2957/sys/arch/arm64/arm64/machdep.c#L371-L377 - #[cfg_attr(target_os = "openbsd", cfg(test))] - { - // ID_AA64MMFR2_EL1, AArch64 Memory Model Feature Register 2 - // https://developer.arm.com/documentation/ddi0601/2023-06/AArch64-Registers/ID-AA64MMFR2-EL1--AArch64-Memory-Model-Feature-Register-2?lang=en - if extract(aa64mmfr2, 35, 32) >= 1 { - info.set(CpuInfo::HAS_LSE2); - } - } -} - -fn extract(x: u64, high: usize, low: usize) -> u64 { - (x >> low) & ((1 << (high - low + 1)) - 1) -} - -#[cfg(not(any(target_os = "netbsd", target_os = "openbsd")))] -mod imp { - // This module is test-only. See parent module docs for details. - - #[cfg(not(portable_atomic_no_asm))] - use core::arch::asm; - - use super::AA64Reg; - - pub(super) fn aa64reg() -> AA64Reg { - // SAFETY: This is safe on FreeBSD 12.0+. FreeBSD 11 was EoL on 2021-09-30. - // Note that stdarch has been doing the same thing since before FreeBSD 11 was EoL. - // https://github.com/rust-lang/stdarch/pull/611 - unsafe { - let aa64isar0: u64; - asm!( - "mrs {0}, ID_AA64ISAR0_EL1", - out(reg) aa64isar0, - options(pure, nomem, nostack, preserves_flags) - ); - #[cfg(test)] - let aa64isar1: u64; - #[cfg(test)] - { - asm!( - "mrs {0}, ID_AA64ISAR1_EL1", - out(reg) aa64isar1, - options(pure, nomem, nostack, preserves_flags) - ); - } - let aa64mmfr2: u64; - asm!( - "mrs {0}, ID_AA64MMFR2_EL1", - out(reg) aa64mmfr2, - options(pure, nomem, nostack, preserves_flags) - ); - AA64Reg { - aa64isar0, - #[cfg(test)] - aa64isar1, - aa64mmfr2, - } - } - } -} -#[cfg(target_os = "netbsd")] -mod imp { - // NetBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl. - // https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb - // https://github.com/golang/sys/commit/ef9fd89ba245e184bdd308f7f2b4f3c551fa5b0f - - use core::ptr; - - use super::AA64Reg; - - // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47 - #[allow(non_camel_case_types)] - pub(super) mod ffi { - pub(crate) use super::super::c_types::{c_char, c_int, c_size_t, c_void}; - - extern "C" { - // Defined in sys/sysctl.h. - // https://man.netbsd.org/sysctl.3 - // https://github.com/NetBSD/src/blob/167403557cf60bed09a63fc84d941a1a4bd7d52e/sys/sys/sysctl.h - // https://github.com/rust-lang/libc/blob/0.2.139/src/unix/bsd/netbsdlike/netbsd/mod.rs#L2582 - pub(crate) fn sysctlbyname( - name: *const c_char, - old_p: *mut c_void, - old_len_p: *mut c_size_t, - new_p: *const c_void, - new_len: c_size_t, - ) -> c_int; - } - - // Defined in aarch64/armreg.h. - // https://github.com/NetBSD/src/blob/167403557cf60bed09a63fc84d941a1a4bd7d52e/sys/arch/aarch64/include/armreg.h#L1626 - #[derive(Clone, Copy)] - #[repr(C)] - pub(crate) struct aarch64_sysctl_cpu_id { - // NetBSD 9.0+ - // https://github.com/NetBSD/src/commit/0e9d25528729f7fea53e78275d1bc5039dfe8ffb - pub(crate) midr: u64, - pub(crate) revidr: u64, - pub(crate) mpidr: u64, - pub(crate) aa64dfr0: u64, - pub(crate) aa64dfr1: u64, - pub(crate) aa64isar0: u64, - pub(crate) aa64isar1: u64, - pub(crate) aa64mmfr0: u64, - pub(crate) aa64mmfr1: u64, - pub(crate) aa64mmfr2: u64, - pub(crate) aa64pfr0: u64, - pub(crate) aa64pfr1: u64, - pub(crate) aa64zfr0: u64, - pub(crate) mvfr0: u32, - pub(crate) mvfr1: u32, - pub(crate) mvfr2: u32, - // NetBSD 10.0+ - // https://github.com/NetBSD/src/commit/0c7bdc13f0e332cccec56e307f023b4888638973 - pub(crate) pad: u32, - pub(crate) clidr: u64, - pub(crate) ctr: u64, - } - } - - pub(super) unsafe fn sysctl_cpu_id(name: &[u8]) -> Option<AA64Reg> { - const OUT_LEN: ffi::c_size_t = - core::mem::size_of::<ffi::aarch64_sysctl_cpu_id>() as ffi::c_size_t; - - debug_assert_eq!(name.last(), Some(&0), "{:?}", name); - debug_assert_eq!(name.iter().filter(|&&v| v == 0).count(), 1, "{:?}", name); - - // SAFETY: all fields of aarch64_sysctl_cpu_id are zero-able and we use - // the result when machdep.cpuN.cpu_id sysctl was successful. - let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { core::mem::zeroed() }; - let mut out_len = OUT_LEN; - // SAFETY: - // - the caller must guarantee that `name` is ` machdep.cpuN.cpu_id` in a C string. - // - `out_len` does not exceed the size of the value at `buf`. - // - `sysctlbyname` is thread-safe. - let res = unsafe { - ffi::sysctlbyname( - name.as_ptr().cast::<ffi::c_char>(), - (&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::<ffi::c_void>(), - &mut out_len, - ptr::null_mut(), - 0, - ) - }; - if res != 0 { - return None; - } - Some(AA64Reg { - aa64isar0: buf.aa64isar0, - #[cfg(test)] - aa64isar1: buf.aa64isar1, - aa64mmfr2: buf.aa64mmfr2, - }) - } - - pub(super) fn aa64reg() -> AA64Reg { - // Get system registers for cpu0. - // If failed, returns default because machdep.cpuN.cpu_id sysctl is not available. - // machdep.cpuN.cpu_id sysctl was added on NetBSD 9.0 so it is not available on older versions. - // SAFETY: we passed a valid name in a C string. - // It is ok to check only cpu0, even if there are more CPUs. - // https://github.com/NetBSD/src/commit/bd9707e06ea7d21b5c24df6dfc14cb37c2819416 - // https://github.com/golang/sys/commit/ef9fd89ba245e184bdd308f7f2b4f3c551fa5b0f - match unsafe { sysctl_cpu_id(b"machdep.cpu0.cpu_id\0") } { - Some(cpu_id) => cpu_id, - None => AA64Reg { - aa64isar0: 0, - #[cfg(test)] - aa64isar1: 0, - aa64mmfr2: 0, - }, - } - } -} -#[cfg(target_os = "openbsd")] -mod imp { - // OpenBSD doesn't trap the mrs instruction, but exposes the system registers through sysctl. - // https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8 - // https://github.com/golang/go/commit/cd54ef1f61945459486e9eea2f016d99ef1da925 - - use core::ptr; - - use super::AA64Reg; - - // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47 - #[allow(non_camel_case_types)] - pub(super) mod ffi { - pub(crate) use super::super::c_types::{c_int, c_size_t, c_uint, c_void}; - - // Defined in sys/sysctl.h. - // https://github.com/openbsd/src/blob/72ccc03bd11da614f31f7ff76e3f6fce99bc1c79/sys/sys/sysctl.h#L82 - pub(crate) const CTL_MACHDEP: c_int = 7; - // Defined in machine/cpu.h. - // https://github.com/openbsd/src/blob/72ccc03bd11da614f31f7ff76e3f6fce99bc1c79/sys/arch/arm64/include/cpu.h#L25-L40 - pub(crate) const CPU_ID_AA64ISAR0: c_int = 2; - #[cfg(test)] - pub(crate) const CPU_ID_AA64ISAR1: c_int = 3; - // OpenBSD has an API to get this, but currently always returns 0. - // https://github.com/openbsd/src/blob/6a233889798dc3ecb18acc52dce1e57862af2957/sys/arch/arm64/arm64/machdep.c#L371-L377 - #[cfg(test)] - pub(crate) const CPU_ID_AA64MMFR2: c_int = 7; - - extern "C" { - // Defined in sys/sysctl.h. - // https://man.openbsd.org/sysctl.2 - // https://github.com/openbsd/src/blob/72ccc03bd11da614f31f7ff76e3f6fce99bc1c79/sys/sys/sysctl.h - // https://github.com/rust-lang/libc/blob/0.2.139/src/unix/bsd/netbsdlike/openbsd/mod.rs#L1817-L1824 - pub(crate) fn sysctl( - name: *const c_int, - name_len: c_uint, - old_p: *mut c_void, - old_len_p: *mut c_size_t, - new_p: *mut c_void, - new_len: c_size_t, - ) -> c_int; - } - } - - // ID_AA64ISAR0_EL1 and ID_AA64ISAR1_EL1 are supported on OpenBSD 7.1+. - // https://github.com/openbsd/src/commit/d335af936b9d7dd9cf655cae1ce19560c45de6c8 - // Others are supported on OpenBSD 7.3+. - // https://github.com/openbsd/src/commit/c7654cd65262d532212f65123ee3905ba200365c - // sysctl returns an unsupported error if operation is not supported, - // so we can safely use this function on older versions of OpenBSD. - pub(super) fn aa64reg() -> AA64Reg { - let aa64isar0 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR0]).unwrap_or(0); - #[cfg(test)] - let aa64isar1 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR1]).unwrap_or(0); - #[cfg(test)] - let aa64mmfr2 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64MMFR2]).unwrap_or(0); - AA64Reg { - aa64isar0, - #[cfg(test)] - aa64isar1, - #[cfg(test)] - aa64mmfr2, - } - } - - fn sysctl64(mib: &[ffi::c_int]) -> Option<u64> { - const OUT_LEN: ffi::c_size_t = core::mem::size_of::<u64>() as ffi::c_size_t; - let mut out = 0_u64; - let mut out_len = OUT_LEN; - #[allow(clippy::cast_possible_truncation)] - // SAFETY: - // - `mib.len()` does not exceed the size of `mib`. - // - `out_len` does not exceed the size of `out`. - // - `sysctl` is thread-safe. - let res = unsafe { - ffi::sysctl( - mib.as_ptr(), - mib.len() as ffi::c_uint, - (&mut out as *mut u64).cast::<ffi::c_void>(), - &mut out_len, - ptr::null_mut(), - 0, - ) - }; - if res == -1 { - return None; - } - debug_assert_eq!(out_len, OUT_LEN); - Some(out) - } -} - -#[allow( - clippy::alloc_instead_of_core, - clippy::std_instead_of_alloc, - clippy::std_instead_of_core, - clippy::undocumented_unsafe_blocks, - clippy::wildcard_imports -)] -#[cfg(test)] -mod tests { - use std::{ - process::Command, - string::{String, ToString}, - }; - - use super::*; - - #[test] - fn test_aa64reg() { - let AA64Reg { aa64isar0, aa64isar1, aa64mmfr2 } = imp::aa64reg(); - std::eprintln!("aa64isar0={}", aa64isar0); - std::eprintln!("aa64isar1={}", aa64isar1); - std::eprintln!("aa64mmfr2={}", aa64mmfr2); - if cfg!(target_os = "openbsd") { - let output = Command::new("sysctl").arg("machdep").output().unwrap(); - assert!(output.status.success()); - let stdout = String::from_utf8(output.stdout).unwrap(); - // OpenBSD 7.1+ - assert_eq!( - stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64isar0=")).unwrap_or("0"), - aa64isar0.to_string(), - ); - assert_eq!( - stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64isar1=")).unwrap_or("0"), - aa64isar1.to_string(), - ); - // OpenBSD 7.3+ - assert_eq!( - stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64mmfr2=")).unwrap_or("0"), - aa64mmfr2.to_string(), - ); - } - if detect().test(CpuInfo::HAS_LSE) { - let atomic = extract(aa64isar0, 23, 20); - if detect().test(CpuInfo::HAS_LSE128) { - assert_eq!(atomic, 3); - } else { - assert_eq!(atomic, 2); - } - } - if detect().test(CpuInfo::HAS_LSE2) { - assert_eq!(extract(aa64mmfr2, 35, 32), 1); - } - if detect().test(CpuInfo::HAS_RCPC3) { - assert_eq!(extract(aa64isar1, 23, 20), 3); - } - } - - #[allow(clippy::cast_possible_wrap)] - #[cfg(target_os = "netbsd")] - #[test] - fn test_netbsd() { - use c_types::*; - use core::{arch::asm, mem, ptr}; - use imp::ffi; - use test_helper::sys; - - // Call syscall using asm instead of libc. - // Note that NetBSD does not guarantee the stability of raw syscall as - // much as Linux does (It may actually be stable enough, though: https://lists.llvm.org/pipermail/llvm-dev/2019-June/133393.html). - // - // This is currently used only for testing. - unsafe fn sysctl_cpu_id_asm_syscall(name: &[&[u8]]) -> Result<AA64Reg, c_int> { - // https://github.com/golang/go/blob/4badad8d477ffd7a6b762c35bc69aed82faface7/src/syscall/asm_netbsd_arm64.s - #[inline] - unsafe fn sysctl( - name: *const c_int, - name_len: c_uint, - old_p: *mut c_void, - old_len_p: *mut c_size_t, - new_p: *const c_void, - new_len: c_size_t, - ) -> Result<c_int, c_int> { - #[allow(clippy::cast_possible_truncation)] - // SAFETY: the caller must uphold the safety contract. - unsafe { - let mut n = sys::SYS___sysctl as u64; - let r: i64; - asm!( - "svc 0", - "b.cc 2f", - "mov x17, x0", - "mov x0, #-1", - "2:", - inout("x17") n, - inout("x0") ptr_reg!(name) => r, - inout("x1") name_len as u64 => _, - in("x2") ptr_reg!(old_p), - in("x3") ptr_reg!(old_len_p), - in("x4") ptr_reg!(new_p), - in("x5") new_len as u64, - options(nostack), - ); - if r as c_int == -1 { - Err(n as c_int) - } else { - Ok(r as c_int) - } - } - } - - // https://github.com/golang/sys/blob/4badad8d477ffd7a6b762c35bc69aed82faface7/cpu/cpu_netbsd_arm64.go. - use std::{vec, vec::Vec}; - fn sysctl_nodes(mib: &mut Vec<i32>) -> Result<Vec<sys::sysctlnode>, i32> { - mib.push(sys::CTL_QUERY); - let mut q_node = sys::sysctlnode { - sysctl_flags: sys::SYSCTL_VERS_1, - ..unsafe { mem::zeroed() } - }; - let qp = (&mut q_node as *mut sys::sysctlnode).cast::<ffi::c_void>(); - let sz = mem::size_of::<sys::sysctlnode>(); - let mut olen = 0; - #[allow(clippy::cast_possible_truncation)] - unsafe { - sysctl(mib.as_ptr(), mib.len() as c_uint, ptr::null_mut(), &mut olen, qp, sz)?; - } - - let mut nodes = Vec::<sys::sysctlnode>::with_capacity(olen / sz); - let np = nodes.as_mut_ptr().cast::<ffi::c_void>(); - #[allow(clippy::cast_possible_truncation)] - unsafe { - sysctl(mib.as_ptr(), mib.len() as c_uint, np, &mut olen, qp, sz)?; - nodes.set_len(olen / sz); - } - - mib.pop(); // pop CTL_QUERY - Ok(nodes) - } - fn name_to_mib(parts: &[&[u8]]) -> Result<Vec<i32>, i32> { - let mut mib = vec![]; - for (part_no, &part) in parts.iter().enumerate() { - let nodes = sysctl_nodes(&mut mib)?; - for node in nodes { - let mut n = vec![]; - for b in node.sysctl_name { - if b != 0 { - n.push(b); - } - } - if n == part { - mib.push(node.sysctl_num); - break; - } - } - if mib.len() != part_no + 1 { - return Err(0); - } - } - - Ok(mib) - } - - const OUT_LEN: ffi::c_size_t = - core::mem::size_of::<ffi::aarch64_sysctl_cpu_id>() as ffi::c_size_t; - - let mib = name_to_mib(name)?; - - let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { core::mem::zeroed() }; - let mut out_len = OUT_LEN; - #[allow(clippy::cast_possible_truncation)] - unsafe { - sysctl( - mib.as_ptr(), - mib.len() as c_uint, - (&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::<ffi::c_void>(), - &mut out_len, - ptr::null_mut(), - 0, - )?; - } - Ok(AA64Reg { - aa64isar0: buf.aa64isar0, - #[cfg(test)] - aa64isar1: buf.aa64isar1, - #[cfg(test)] - aa64mmfr2: buf.aa64mmfr2, - }) - } - - unsafe { - assert_eq!( - imp::sysctl_cpu_id(b"machdep.cpu0.cpu_id\0").unwrap(), - sysctl_cpu_id_asm_syscall(&[b"machdep", b"cpu0", b"cpu_id"]).unwrap() - ); - } - } - - // Static assertions for FFI bindings. - // This checks that FFI bindings defined in this crate, FFI bindings defined - // in libc, and FFI bindings generated for the platform's latest header file - // using bindgen have compatible signatures (or the same values if constants). - // Since this is static assertion, we can detect problems with - // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh) - // without actually running tests on these platforms. - // See also tools/codegen/src/ffi.rs. - // TODO(codegen): auto-generate this test - #[cfg(target_os = "netbsd")] - #[allow( - clippy::cast_possible_wrap, - clippy::cast_sign_loss, - clippy::no_effect_underscore_binding, - clippy::used_underscore_binding - )] - const _: fn() = || { - use core::mem::size_of; - use imp::ffi; - use test_helper::{libc, sys}; - let mut _sysctlbyname: unsafe extern "C" fn( - *const ffi::c_char, - *mut ffi::c_void, - *mut ffi::c_size_t, - *const ffi::c_void, - ffi::c_size_t, - ) -> ffi::c_int = ffi::sysctlbyname; - _sysctlbyname = libc::sysctlbyname; - _sysctlbyname = sys::sysctlbyname; - // libc doesn't have this - // static_assert!( - // size_of::<ffi::aarch64_sysctl_cpu_id>() == size_of::<libc::aarch64_sysctl_cpu_id>() - // ); - static_assert!( - size_of::<ffi::aarch64_sysctl_cpu_id>() == size_of::<sys::aarch64_sysctl_cpu_id>() - ); - let ffi: ffi::aarch64_sysctl_cpu_id = unsafe { core::mem::zeroed() }; - let _ = sys::aarch64_sysctl_cpu_id { - ac_midr: ffi.midr, - ac_revidr: ffi.revidr, - ac_mpidr: ffi.mpidr, - ac_aa64dfr0: ffi.aa64dfr0, - ac_aa64dfr1: ffi.aa64dfr1, - ac_aa64isar0: ffi.aa64isar0, - ac_aa64isar1: ffi.aa64isar1, - ac_aa64mmfr0: ffi.aa64mmfr0, - ac_aa64mmfr1: ffi.aa64mmfr1, - ac_aa64mmfr2: ffi.aa64mmfr2, - ac_aa64pfr0: ffi.aa64pfr0, - ac_aa64pfr1: ffi.aa64pfr1, - ac_aa64zfr0: ffi.aa64zfr0, - ac_mvfr0: ffi.mvfr0, - ac_mvfr1: ffi.mvfr1, - ac_mvfr2: ffi.mvfr2, - ac_pad: ffi.pad, - ac_clidr: ffi.clidr, - ac_ctr: ffi.ctr, - }; - }; - #[cfg(target_os = "openbsd")] - #[allow( - clippy::cast_possible_wrap, - clippy::cast_sign_loss, - clippy::no_effect_underscore_binding - )] - const _: fn() = || { - use imp::ffi; - use test_helper::{libc, sys}; - let mut _sysctl: unsafe extern "C" fn( - *const ffi::c_int, - ffi::c_uint, - *mut ffi::c_void, - *mut ffi::c_size_t, - *mut ffi::c_void, - ffi::c_size_t, - ) -> ffi::c_int = ffi::sysctl; - _sysctl = libc::sysctl; - _sysctl = sys::sysctl; - static_assert!(ffi::CTL_MACHDEP == libc::CTL_MACHDEP); - static_assert!(ffi::CTL_MACHDEP == sys::CTL_MACHDEP as ffi::c_int); - // static_assert!(ffi::CPU_ID_AA64ISAR0 == libc::CPU_ID_AA64ISAR0); // libc doesn't have this - static_assert!(ffi::CPU_ID_AA64ISAR0 == sys::CPU_ID_AA64ISAR0 as ffi::c_int); - // static_assert!(ffi::CPU_ID_AA64ISAR1 == libc::CPU_ID_AA64ISAR1); // libc doesn't have this - static_assert!(ffi::CPU_ID_AA64ISAR1 == sys::CPU_ID_AA64ISAR1 as ffi::c_int); - // static_assert!(ffi::CPU_ID_AA64MMFR2 == libc::CPU_ID_AA64MMFR2); // libc doesn't have this - static_assert!(ffi::CPU_ID_AA64MMFR2 == sys::CPU_ID_AA64MMFR2 as ffi::c_int); - }; -} diff --git a/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_fuchsia.rs b/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_fuchsia.rs deleted file mode 100644 index 978418c..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_fuchsia.rs +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Run-time feature detection on aarch64 Fuchsia by using zx_system_get_features. -// -// As of nightly-2023-01-23, is_aarch64_feature_detected doesn't support run-time detection on Fuchsia. -// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/mod.rs -// -// Refs: -// - https://fuchsia.dev/fuchsia-src/reference/syscalls/system_get_features -// - https://github.com/llvm/llvm-project/commit/4e731abc55681751b5d736b613f7720e50eb1ad4 - -include!("common.rs"); - -#[allow(non_camel_case_types)] -mod ffi { - // https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/types.h - pub(crate) type zx_status_t = i32; - - // https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/errors.h - pub(crate) const ZX_OK: zx_status_t = 0; - // https://fuchsia.googlesource.com/fuchsia/+/refs/heads/main/zircon/system/public/zircon/features.h - pub(crate) const ZX_FEATURE_KIND_CPU: u32 = 0; - pub(crate) const ZX_ARM64_FEATURE_ISA_ATOMICS: u32 = 1 << 8; - - #[link(name = "zircon")] - extern "C" { - // https://fuchsia.dev/fuchsia-src/reference/syscalls/system_get_features - pub(crate) fn zx_system_get_features(kind: u32, features: *mut u32) -> zx_status_t; - } -} - -fn zx_system_get_features(kind: u32) -> u32 { - let mut out = 0_u32; - // SAFETY: the pointer is valid because we got it from a reference. - let res = unsafe { ffi::zx_system_get_features(kind, &mut out) }; - if res != ffi::ZX_OK { - return 0; - } - out -} - -#[cold] -fn _detect(info: &mut CpuInfo) { - let features = zx_system_get_features(ffi::ZX_FEATURE_KIND_CPU); - if features & ffi::ZX_ARM64_FEATURE_ISA_ATOMICS != 0 { - info.set(CpuInfo::HAS_LSE); - } -} - -#[allow( - clippy::alloc_instead_of_core, - clippy::std_instead_of_alloc, - clippy::std_instead_of_core, - clippy::undocumented_unsafe_blocks, - clippy::wildcard_imports -)] -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_fuchsia() { - let features = zx_system_get_features(ffi::ZX_FEATURE_KIND_CPU); - assert_ne!(features, 0); - std::eprintln!("features: {:b}", features); - } - - // Static assertions for FFI bindings. - // This checks that FFI bindings defined in this crate and FFI bindings - // generated for the platform's latest header file using bindgen have - // compatible signatures (or the same values if constants). - // Since this is static assertion, we can detect problems with - // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh) - // without actually running tests on these platforms. - // See also tools/codegen/src/ffi.rs. - // TODO(codegen): auto-generate this test - #[allow( - clippy::cast_possible_wrap, - clippy::cast_sign_loss, - clippy::cast_possible_truncation, - clippy::no_effect_underscore_binding - )] - const _: fn() = || { - use test_helper::sys; - // TODO(codegen): zx_system_get_features - let _: ffi::zx_status_t = 0 as sys::zx_status_t; - static_assert!(ffi::ZX_OK == sys::ZX_OK as ffi::zx_status_t); - static_assert!(ffi::ZX_FEATURE_KIND_CPU == sys::ZX_FEATURE_KIND_CPU); - static_assert!(ffi::ZX_ARM64_FEATURE_ISA_ATOMICS == sys::ZX_ARM64_FEATURE_ISA_ATOMICS); - }; -} diff --git a/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_macos.rs b/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_macos.rs deleted file mode 100644 index d6bf9d0..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_macos.rs +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Run-time feature detection on aarch64 macOS by using sysctl. -// -// This module is currently only enabled on tests because aarch64 macOS always supports FEAT_LSE and FEAT_LSE2. -// https://github.com/llvm/llvm-project/blob/llvmorg-17.0.0-rc2/llvm/include/llvm/TargetParser/AArch64TargetParser.h#L494 -// -// If macOS supporting Armv9.4-a becomes popular in the future, this module will -// be used to support outline-atomics for FEAT_LSE128/FEAT_LRCPC3. -// -// Refs: https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics -// -// Note that iOS doesn't support sysctl: -// - https://developer.apple.com/forums/thread/9440 -// - https://nabla-c0d3.github.io/blog/2015/06/16/ios9-security-privacy - -include!("common.rs"); - -use core::ptr; - -// core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47 -#[allow(non_camel_case_types)] -mod ffi { - pub(crate) use super::c_types::{c_char, c_int, c_size_t, c_void}; - - extern "C" { - // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname - // https://github.com/apple-oss-distributions/xnu/blob/5c2921b07a2480ab43ec66f5b9e41cb872bc554f/bsd/sys/sysctl.h - // https://github.com/rust-lang/libc/blob/0.2.139/src/unix/bsd/apple/mod.rs#L5167-L5173 - pub(crate) fn sysctlbyname( - name: *const c_char, - old_p: *mut c_void, - old_len_p: *mut c_size_t, - new_p: *mut c_void, - new_len: c_size_t, - ) -> c_int; - } -} - -unsafe fn sysctlbyname32(name: &[u8]) -> Option<u32> { - const OUT_LEN: ffi::c_size_t = core::mem::size_of::<u32>() as ffi::c_size_t; - - debug_assert_eq!(name.last(), Some(&0), "{:?}", name); - debug_assert_eq!(name.iter().filter(|&&v| v == 0).count(), 1, "{:?}", name); - - let mut out = 0_u32; - let mut out_len = OUT_LEN; - // SAFETY: - // - the caller must guarantee that `name` a valid C string. - // - `out_len` does not exceed the size of `out`. - // - `sysctlbyname` is thread-safe. - let res = unsafe { - ffi::sysctlbyname( - name.as_ptr().cast::<ffi::c_char>(), - (&mut out as *mut u32).cast::<ffi::c_void>(), - &mut out_len, - ptr::null_mut(), - 0, - ) - }; - if res != 0 { - return None; - } - debug_assert_eq!(out_len, OUT_LEN); - Some(out) -} - -#[cold] -fn _detect(info: &mut CpuInfo) { - // hw.optional.armv8_1_atomics is available on macOS 11+ (note: aarch64 support was added on macOS 11), - // hw.optional.arm.FEAT_* are only available on macOS 12+. - // Query both names in case future versions of macOS remove the old name. - // https://github.com/golang/go/commit/c15593197453b8bf90fc3a9080ba2afeaf7934ea - // https://github.com/google/boringssl/commit/91e0b11eba517d83b910b20fe3740eeb39ecb37e - // SAFETY: we passed a valid C string. - if unsafe { - sysctlbyname32(b"hw.optional.arm.FEAT_LSE\0").unwrap_or(0) != 0 - || sysctlbyname32(b"hw.optional.armv8_1_atomics\0").unwrap_or(0) != 0 - } { - info.set(CpuInfo::HAS_LSE); - } - // SAFETY: we passed a valid C string. - if unsafe { sysctlbyname32(b"hw.optional.arm.FEAT_LSE2\0").unwrap_or(0) != 0 } { - info.set(CpuInfo::HAS_LSE2); - } - // we currently only use FEAT_LSE and FEAT_LSE2 in outline-atomics. - #[cfg(test)] - { - // SAFETY: we passed a valid C string. - if unsafe { sysctlbyname32(b"hw.optional.arm.FEAT_LSE128\0").unwrap_or(0) != 0 } { - info.set(CpuInfo::HAS_LSE128); - } - // SAFETY: we passed a valid C string. - if unsafe { sysctlbyname32(b"hw.optional.arm.FEAT_LRCPC3\0").unwrap_or(0) != 0 } { - info.set(CpuInfo::HAS_RCPC3); - } - } -} - -#[allow( - clippy::alloc_instead_of_core, - clippy::std_instead_of_alloc, - clippy::std_instead_of_core, - clippy::undocumented_unsafe_blocks, - clippy::wildcard_imports -)] -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_macos() { - unsafe { - assert_eq!(sysctlbyname32(b"hw.optional.armv8_1_atomics\0"), Some(1)); - assert_eq!(sysctlbyname32(b"hw.optional.arm.FEAT_LSE\0"), Some(1)); - assert_eq!(sysctlbyname32(b"hw.optional.arm.FEAT_LSE2\0"), Some(1)); - assert_eq!(sysctlbyname32(b"hw.optional.arm.FEAT_LSE128\0"), None); - assert_eq!(std::io::Error::last_os_error().kind(), std::io::ErrorKind::NotFound); - assert_eq!(sysctlbyname32(b"hw.optional.arm.FEAT_LRCPC\0"), Some(1)); - assert_eq!(sysctlbyname32(b"hw.optional.arm.FEAT_LRCPC2\0"), Some(1)); - assert_eq!(sysctlbyname32(b"hw.optional.arm.FEAT_LRCPC3\0"), None); - assert_eq!(std::io::Error::last_os_error().kind(), std::io::ErrorKind::NotFound); - } - } - - // Static assertions for FFI bindings. - // This checks that FFI bindings defined in this crate, FFI bindings defined - // in libc, and FFI bindings generated for the platform's latest header file - // using bindgen have compatible signatures (or the same values if constants). - // Since this is static assertion, we can detect problems with - // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh) - // without actually running tests on these platforms. - // See also tools/codegen/src/ffi.rs. - // TODO(codegen): auto-generate this test - #[allow( - clippy::cast_possible_wrap, - clippy::cast_sign_loss, - clippy::no_effect_underscore_binding - )] - const _: fn() = || { - use test_helper::{libc, sys}; - let mut _sysctlbyname: unsafe extern "C" fn( - *const ffi::c_char, - *mut ffi::c_void, - *mut ffi::c_size_t, - *mut ffi::c_void, - ffi::c_size_t, - ) -> ffi::c_int = ffi::sysctlbyname; - _sysctlbyname = libc::sysctlbyname; - _sysctlbyname = sys::sysctlbyname; - }; -} diff --git a/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_windows.rs b/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_windows.rs deleted file mode 100644 index 6ace866..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/detect/aarch64_windows.rs +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Run-time feature detection on aarch64 Windows by using IsProcessorFeaturePresent. -// -// As of nightly-2023-01-23, is_aarch64_feature_detected doesn't support run-time detection of FEAT_LSE on Windows. -// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/windows/aarch64.rs -// https://github.com/rust-lang/stdarch/pull/1373 -// -// Refs: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent - -include!("common.rs"); - -// windows-sys requires Rust 1.56 -#[allow(clippy::upper_case_acronyms)] -mod ffi { - pub(crate) type DWORD = u32; - pub(crate) type BOOL = i32; - - pub(crate) const FALSE: BOOL = 0; - // Defined in winnt.h of Windows SDK. - pub(crate) const PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE: DWORD = 34; - - extern "system" { - // https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent - pub(crate) fn IsProcessorFeaturePresent(ProcessorFeature: DWORD) -> BOOL; - } -} - -#[cold] -fn _detect(info: &mut CpuInfo) { - // SAFETY: calling IsProcessorFeaturePresent is safe, and FALSE is also - // returned if the HAL does not support detection of the specified feature. - if unsafe { - ffi::IsProcessorFeaturePresent(ffi::PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE) != ffi::FALSE - } { - info.set(CpuInfo::HAS_LSE); - } -} - -#[allow( - clippy::alloc_instead_of_core, - clippy::std_instead_of_alloc, - clippy::std_instead_of_core, - clippy::undocumented_unsafe_blocks, - clippy::wildcard_imports -)] -#[cfg(test)] -mod tests { - use super::*; - - // Static assertions for FFI bindings. - // This checks that FFI bindings defined in this crate and FFI bindings defined - // in windows-sys have compatible signatures (or the same values if constants). - // Since this is static assertion, we can detect problems with - // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh) - // without actually running tests on these platforms. - // (Unlike libc, windows-sys programmatically generates bindings from Windows - // API metadata, so it should be enough to check compatibility with the - // windows-sys' signatures/values.) - // See also tools/codegen/src/ffi.rs. - // TODO(codegen): auto-generate this test - #[allow( - clippy::cast_possible_wrap, - clippy::cast_sign_loss, - clippy::cast_possible_truncation, - clippy::no_effect_underscore_binding - )] - const _: fn() = || { - use test_helper::windows_sys; - let _: ffi::DWORD = 0 as windows_sys::Win32::System::Threading::PROCESSOR_FEATURE_ID; - let _: ffi::BOOL = 0 as windows_sys::Win32::Foundation::BOOL; - let mut _sysctl: unsafe extern "system" fn(ffi::DWORD) -> ffi::BOOL = - ffi::IsProcessorFeaturePresent; - _sysctl = windows_sys::Win32::System::Threading::IsProcessorFeaturePresent; - static_assert!(ffi::FALSE == windows_sys::Win32::Foundation::FALSE); - static_assert!( - ffi::PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE - == windows_sys::Win32::System::Threading::PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE - ); - }; -} diff --git a/vendor/portable-atomic/src/imp/atomic128/detect/auxv.rs b/vendor/portable-atomic/src/imp/atomic128/detect/auxv.rs deleted file mode 100644 index 1be3095..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/detect/auxv.rs +++ /dev/null @@ -1,727 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Run-time feature detection on aarch64/powerpc64 Linux/Android/FreeBSD by parsing ELF auxiliary vectors. -// -// # Linux/Android -// -// As of nightly-2023-01-23, is_aarch64_feature_detected always uses dlsym by default -// on aarch64 Linux/Android, but on the following platforms, so we can safely assume -// getauxval is linked to the binary. -// -// - On glibc (*-linux-gnu*), [aarch64 support is available on glibc 2.17+](https://sourceware.org/legacy-ml/libc-announce/2012/msg00001.html) -// and is newer than [glibc 2.16 that added getauxval](https://sourceware.org/legacy-ml/libc-announce/2012/msg00000.html). -// - On musl (*-linux-musl*, *-linux-ohos*), [aarch64 support is available on musl 1.1.7+](https://git.musl-libc.org/cgit/musl/tree/WHATSNEW?h=v1.1.7#n1422) -// and is newer than [musl 1.1.0 that added getauxval](https://git.musl-libc.org/cgit/musl/tree/WHATSNEW?h=v1.1.0#n1197). -// https://github.com/rust-lang/rust/commit/9a04ae4997493e9260352064163285cddc43de3c -// - On bionic (*-android*), [64-bit architecture support is available on Android 5.0+ (API level 21+)](https://android-developers.googleblog.com/2014/10/whats-new-in-android-50-lollipop.html) -// and is newer than [Android 4.3 (API level 18) that added getauxval](https://github.com/aosp-mirror/platform_bionic/blob/d3ebc2f7c49a9893b114124d4a6b315f3a328764/libc/include/sys/auxv.h#L49). -// -// However, on musl with static linking, it seems that getauxval is not always available, independent of version requirements: https://github.com/rust-lang/rust/issues/89626 -// (That problem may have been fixed in https://github.com/rust-lang/rust/commit/9a04ae4997493e9260352064163285cddc43de3c, -// but even in the version containing that patch, [there is report](https://github.com/rust-lang/rust/issues/89626#issuecomment-1242636038) -// of the same error.) -// -// On other Linux targets, we cannot assume that getauxval is always available, so we don't enable -// outline-atomics by default (can be enabled by `--cfg portable_atomic_outline_atomics`). -// -// - On musl with static linking. See the above for more. -// Also, in this case, dlsym(getauxval) always returns null. -// - On uClibc-ng (*-linux-uclibc*, *-l4re-uclibc*), [uClibc-ng 1.0.43 (released in 2023-04-05) added getauxval](https://github.com/wbx-github/uclibc-ng/commit/d869bb1600942c01a77539128f9ba5b5b55ad647). -// - On Picolibc, [Picolibc 1.4.6 added getauxval stub](https://github.com/picolibc/picolibc#picolibc-version-146). -// -// See also https://github.com/rust-lang/stdarch/pull/1375 -// -// See tests::test_linux_like and aarch64_aa64reg.rs for (test-only) alternative implementations. -// -// # FreeBSD -// -// As of nightly-2023-01-23, is_aarch64_feature_detected always uses mrs on -// aarch64 FreeBSD. However, they do not work on FreeBSD 12 on QEMU (confirmed -// on FreeBSD 12.{2,3,4}), and we got SIGILL (worked on FreeBSD 13 and 14). -// -// So use elf_aux_info instead of mrs like compiler-rt does. -// https://man.freebsd.org/elf_aux_info(3) -// https://reviews.llvm.org/D109330 -// -// elf_aux_info is available on FreeBSD 12.0+ and 11.4+: -// https://github.com/freebsd/freebsd-src/commit/0b08ae2120cdd08c20a2b806e2fcef4d0a36c470 -// https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/sys/auxv.h -// On FreeBSD, [aarch64 support is available on FreeBSD 11.0+](https://www.freebsd.org/releases/11.0R/relnotes/#hardware-arm), -// but FreeBSD 11 (11.4) was EoL on 2021-09-30, and FreeBSD 11.3 was EoL on 2020-09-30: -// https://www.freebsd.org/security/unsupported -// See also https://github.com/rust-lang/stdarch/pull/611#issuecomment-445464613 -// -// See tests::test_freebsd and aarch64_aa64reg.rs for (test-only) alternative implementations. -// -// # PowerPC64 -// -// On PowerPC64, outline-atomics is currently disabled by default mainly for -// compatibility with older versions of operating systems -// (can be enabled by `--cfg portable_atomic_outline_atomics`). - -include!("common.rs"); - -use os::ffi; -#[cfg(any(target_os = "linux", target_os = "android"))] -mod os { - // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47 - #[cfg_attr(test, allow(dead_code))] - pub(super) mod ffi { - pub(crate) use super::super::c_types::c_ulong; - #[cfg(all(target_arch = "aarch64", target_os = "android"))] - pub(crate) use super::super::c_types::{c_char, c_int}; - - extern "C" { - // https://man7.org/linux/man-pages/man3/getauxval.3.html - // https://github.com/bminor/glibc/blob/801af9fafd4689337ebf27260aa115335a0cb2bc/misc/sys/auxv.h - // https://github.com/bminor/musl/blob/7d756e1c04de6eb3f2b3d3e1141a218bb329fcfb/include/sys/auxv.h - // https://github.com/wbx-github/uclibc-ng/blob/cdb07d2cd52af39feb425e6d36c02b30916b9f0a/include/sys/auxv.h - // https://github.com/aosp-mirror/platform_bionic/blob/d3ebc2f7c49a9893b114124d4a6b315f3a328764/libc/include/sys/auxv.h - // https://github.com/picolibc/picolibc/blob/7a8a58aeaa5946cb662577a518051091b691af3a/newlib/libc/picolib/getauxval.c - // https://github.com/rust-lang/libc/blob/0.2.139/src/unix/linux_like/linux/gnu/mod.rs#L1201 - // https://github.com/rust-lang/libc/blob/0.2.139/src/unix/linux_like/linux/musl/mod.rs#L744 - // https://github.com/rust-lang/libc/blob/0.2.139/src/unix/linux_like/android/b64/mod.rs#L333 - pub(crate) fn getauxval(type_: c_ulong) -> c_ulong; - - // Defined in sys/system_properties.h. - // https://github.com/aosp-mirror/platform_bionic/blob/d3ebc2f7c49a9893b114124d4a6b315f3a328764/libc/include/sys/system_properties.h - // https://github.com/rust-lang/libc/blob/0.2.139/src/unix/linux_like/android/mod.rs#L3471 - #[cfg(all(target_arch = "aarch64", target_os = "android"))] - pub(crate) fn __system_property_get(name: *const c_char, value: *mut c_char) -> c_int; - } - - // https://github.com/torvalds/linux/blob/v6.1/include/uapi/linux/auxvec.h - #[cfg(any(test, target_arch = "aarch64"))] - pub(crate) const AT_HWCAP: c_ulong = 16; - #[cfg(any(test, target_arch = "powerpc64"))] - pub(crate) const AT_HWCAP2: c_ulong = 26; - - // Defined in sys/system_properties.h. - // https://github.com/aosp-mirror/platform_bionic/blob/d3ebc2f7c49a9893b114124d4a6b315f3a328764/libc/include/sys/system_properties.h - #[cfg(all(target_arch = "aarch64", target_os = "android"))] - pub(crate) const PROP_VALUE_MAX: c_int = 92; - } - - pub(super) fn getauxval(type_: ffi::c_ulong) -> ffi::c_ulong { - #[cfg(all(target_arch = "aarch64", target_os = "android"))] - { - // Samsung Exynos 9810 has a bug that big and little cores have different - // ISAs. And on older Android (pre-9), the kernel incorrectly reports - // that features available only on some cores are available on all cores. - // https://reviews.llvm.org/D114523 - let mut arch = [0_u8; ffi::PROP_VALUE_MAX as usize]; - // SAFETY: we've passed a valid C string and a buffer with max length. - let len = unsafe { - ffi::__system_property_get( - b"ro.arch\0".as_ptr().cast::<ffi::c_char>(), - arch.as_mut_ptr().cast::<ffi::c_char>(), - ) - }; - // On Exynos, ro.arch is not available on Android 12+, but it is fine - // because Android 9+ includes the fix. - if len > 0 && arch.starts_with(b"exynos9810") { - return 0; - } - } - - // SAFETY: `getauxval` is thread-safe. See also the module level docs. - unsafe { ffi::getauxval(type_) } - } -} -#[cfg(target_os = "freebsd")] -mod os { - // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47 - #[cfg_attr(test, allow(dead_code))] - pub(super) mod ffi { - pub(crate) use super::super::c_types::{c_int, c_ulong, c_void}; - - extern "C" { - // Defined in sys/auxv.h. - // https://man.freebsd.org/elf_aux_info(3) - // https://github.com/freebsd/freebsd-src/blob/deb63adf945d446ed91a9d84124c71f15ae571d1/sys/sys/auxv.h - pub(crate) fn elf_aux_info(aux: c_int, buf: *mut c_void, buf_len: c_int) -> c_int; - } - - // Defined in sys/elf_common.h. - // https://github.com/freebsd/freebsd-src/blob/deb63adf945d446ed91a9d84124c71f15ae571d1/sys/sys/elf_common.h - #[cfg(any(test, target_arch = "aarch64"))] - pub(crate) const AT_HWCAP: c_int = 25; - #[cfg(any(test, target_arch = "powerpc64"))] - pub(crate) const AT_HWCAP2: c_int = 26; - } - - pub(super) fn getauxval(aux: ffi::c_int) -> ffi::c_ulong { - #[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)] - const OUT_LEN: ffi::c_int = core::mem::size_of::<ffi::c_ulong>() as ffi::c_int; - let mut out: ffi::c_ulong = 0; - // SAFETY: - // - the pointer is valid because we got it from a reference. - // - `OUT_LEN` is the same as the size of `out`. - // - `elf_aux_info` is thread-safe. - unsafe { - let res = ffi::elf_aux_info( - aux, - (&mut out as *mut ffi::c_ulong).cast::<ffi::c_void>(), - OUT_LEN, - ); - // If elf_aux_info fails, `out` will be left at zero (which is the proper default value). - debug_assert!(res == 0 || out == 0); - } - out - } -} - -// Basically, Linux and FreeBSD use the same hwcap values. -// FreeBSD supports a subset of the hwcap values supported by Linux. -use arch::_detect; -#[cfg(target_arch = "aarch64")] -mod arch { - use super::{ffi, os, CpuInfo}; - - // Linux - // https://github.com/torvalds/linux/blob/1c41041124bd14dd6610da256a3da4e5b74ce6b1/arch/arm64/include/uapi/asm/hwcap.h - // FreeBSD - // Defined in machine/elf.h. - // https://github.com/freebsd/freebsd-src/blob/deb63adf945d446ed91a9d84124c71f15ae571d1/sys/arm64/include/elf.h - // available on FreeBSD 13.0+ and 12.2+ - // https://github.com/freebsd/freebsd-src/blob/release/13.0.0/sys/arm64/include/elf.h - // https://github.com/freebsd/freebsd-src/blob/release/12.2.0/sys/arm64/include/elf.h - pub(super) const HWCAP_ATOMICS: ffi::c_ulong = 1 << 8; - pub(super) const HWCAP_USCAT: ffi::c_ulong = 1 << 25; - #[cfg(any(target_os = "linux", target_os = "android"))] - #[cfg(target_pointer_width = "64")] - #[cfg(test)] - pub(super) const HWCAP2_LRCPC3: ffi::c_ulong = 1 << 46; - #[cfg(any(target_os = "linux", target_os = "android"))] - #[cfg(target_pointer_width = "64")] - #[cfg(test)] - pub(super) const HWCAP2_LSE128: ffi::c_ulong = 1 << 47; - - #[cold] - pub(super) fn _detect(info: &mut CpuInfo) { - let hwcap = os::getauxval(ffi::AT_HWCAP); - - if hwcap & HWCAP_ATOMICS != 0 { - info.set(CpuInfo::HAS_LSE); - } - if hwcap & HWCAP_USCAT != 0 { - info.set(CpuInfo::HAS_LSE2); - } - #[cfg(any(target_os = "linux", target_os = "android"))] - #[cfg(target_pointer_width = "64")] - #[cfg(test)] - { - let hwcap2 = os::getauxval(ffi::AT_HWCAP2); - if hwcap2 & HWCAP2_LRCPC3 != 0 { - info.set(CpuInfo::HAS_RCPC3); - } - if hwcap2 & HWCAP2_LSE128 != 0 { - info.set(CpuInfo::HAS_LSE128); - } - } - } -} -#[cfg(target_arch = "powerpc64")] -mod arch { - use super::{ffi, os, CpuInfo}; - - // Linux - // https://github.com/torvalds/linux/blob/v6.1/arch/powerpc/include/uapi/asm/cputable.h - // FreeBSD - // Defined in machine/cpu.h. - // https://github.com/freebsd/freebsd-src/blob/deb63adf945d446ed91a9d84124c71f15ae571d1/sys/powerpc/include/cpu.h - // available on FreeBSD 11.0+ - // https://github.com/freebsd/freebsd-src/commit/b0bf7fcd298133457991b27625bbed766e612730 - pub(super) const PPC_FEATURE2_ARCH_2_07: ffi::c_ulong = 0x80000000; - - #[cold] - pub(super) fn _detect(info: &mut CpuInfo) { - let hwcap2 = os::getauxval(ffi::AT_HWCAP2); - - // power8 - if hwcap2 & PPC_FEATURE2_ARCH_2_07 != 0 { - info.set(CpuInfo::HAS_QUADWORD_ATOMICS); - } - } -} - -#[allow( - clippy::alloc_instead_of_core, - clippy::std_instead_of_alloc, - clippy::std_instead_of_core, - clippy::undocumented_unsafe_blocks, - clippy::wildcard_imports -)] -#[cfg(test)] -mod tests { - use super::*; - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[cfg(target_pointer_width = "64")] - #[test] - fn test_linux_like() { - use c_types::*; - use core::{arch::asm, mem}; - use std::vec; - use test_helper::{libc, sys}; - - // Linux kernel 6.4 has added a way to read auxv without depending on either libc or mrs trap. - // https://github.com/torvalds/linux/commit/ddc65971bb677aa9f6a4c21f76d3133e106f88eb - // - // This is currently used only for testing. - fn getauxval_pr_get_auxv(type_: ffi::c_ulong) -> Result<ffi::c_ulong, c_int> { - #[cfg(target_arch = "aarch64")] - unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> { - let r: i64; - unsafe { - asm!( - "svc 0", - in("x8") sys::__NR_prctl as u64, - inout("x0") sys::PR_GET_AUXV as u64 => r, - in("x1") ptr_reg!(out), - in("x2") len as u64, - // arg4 and arg5 must be zero. - in("x3") 0_u64, - in("x4") 0_u64, - options(nostack, preserves_flags) - ); - } - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - if (r as c_int) < 0 { - Err(r as c_int) - } else { - Ok(r as usize) - } - } - #[cfg(target_arch = "powerpc64")] - unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> { - let r: i64; - unsafe { - asm!( - "sc", - "bns+ 2f", - "neg %r3, %r3", - "2:", - inout("r0") sys::__NR_prctl as u64 => _, - inout("r3") sys::PR_GET_AUXV as u64 => r, - inout("r4") ptr_reg!(out) => _, - inout("r5") len as u64 => _, - // arg4 and arg5 must be zero. - inout("r6") 0_u64 => _, - inout("r7") 0_u64 => _, - out("r8") _, - out("r9") _, - out("r10") _, - out("r11") _, - out("r12") _, - out("cr0") _, - options(nostack, preserves_flags) - ); - } - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - if (r as c_int) < 0 { - Err(r as c_int) - } else { - Ok(r as usize) - } - } - - let mut auxv = vec![unsafe { mem::zeroed::<sys::Elf64_auxv_t>() }; 38]; - - let old_len = auxv.len() * mem::size_of::<sys::Elf64_auxv_t>(); - - // SAFETY: - // - `out_len` does not exceed the size of `auxv`. - let _len = unsafe { prctl_get_auxv(auxv.as_mut_ptr().cast::<c_void>(), old_len)? }; - - for aux in &auxv { - if aux.a_type == type_ { - // SAFETY: aux.a_un is #[repr(C)] union and all fields have - // the same size and can be safely transmuted to integers. - return Ok(unsafe { aux.a_un.a_val }); - } - } - Err(0) - } - - unsafe { - let mut u = mem::zeroed(); - assert_eq!(libc::uname(&mut u), 0); - let release = std::ffi::CStr::from_ptr(u.release.as_ptr()); - let release = core::str::from_utf8(release.to_bytes()).unwrap(); - let mut digits = release.split('.'); - let major = digits.next().unwrap().parse::<u32>().unwrap(); - let minor = digits.next().unwrap().parse::<u32>().unwrap(); - if (major, minor) < (6, 4) { - std::eprintln!("kernel version: {major}.{minor} (no pr_get_auxv)"); - assert_eq!(getauxval_pr_get_auxv(ffi::AT_HWCAP).unwrap_err(), -22); - assert_eq!(getauxval_pr_get_auxv(ffi::AT_HWCAP2).unwrap_err(), -22); - } else { - std::eprintln!("kernel version: {major}.{minor} (has pr_get_auxv)"); - assert_eq!( - os::getauxval(ffi::AT_HWCAP), - getauxval_pr_get_auxv(ffi::AT_HWCAP).unwrap() - ); - assert_eq!( - os::getauxval(ffi::AT_HWCAP2), - getauxval_pr_get_auxv(ffi::AT_HWCAP2).unwrap() - ); - } - } - } - - #[allow(clippy::cast_sign_loss)] - #[cfg(all(target_arch = "aarch64", target_os = "android"))] - #[test] - fn test_android() { - unsafe { - let mut arch = [1; ffi::PROP_VALUE_MAX as usize]; - let len = ffi::__system_property_get( - b"ro.arch\0".as_ptr().cast::<ffi::c_char>(), - arch.as_mut_ptr().cast::<ffi::c_char>(), - ); - assert!(len >= 0); - std::eprintln!("len={}", len); - std::eprintln!("arch={:?}", arch); - std::eprintln!( - "arch={:?}", - core::str::from_utf8(core::slice::from_raw_parts(arch.as_ptr(), len as usize)) - .unwrap() - ); - } - } - - #[allow(clippy::cast_possible_wrap)] - #[cfg(target_os = "freebsd")] - #[test] - fn test_freebsd() { - use c_types::*; - use core::{arch::asm, mem, ptr}; - use test_helper::sys; - - // This is almost equivalent to what elf_aux_info does. - // https://man.freebsd.org/elf_aux_info(3) - // On FreeBSD, [aarch64 support is available on FreeBSD 11.0+](https://www.freebsd.org/releases/11.0R/relnotes/#hardware-arm), - // but elf_aux_info is available on FreeBSD 12.0+ and 11.4+: - // https://github.com/freebsd/freebsd-src/commit/0b08ae2120cdd08c20a2b806e2fcef4d0a36c470 - // https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/sys/auxv.h - // so use sysctl instead of elf_aux_info. - // Note that FreeBSD 11 (11.4) was EoL on 2021-09-30, and FreeBSD 11.3 was EoL on 2020-09-30: - // https://www.freebsd.org/security/unsupported - // - // std_detect uses this way, but it appears to be somewhat incorrect - // (the type of arg4 of sysctl, auxv is smaller than AT_COUNT, etc.). - // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/freebsd/auxvec.rs#L52 - // - // This is currently used only for testing. - // If you want us to use this implementation for compatibility with the older FreeBSD - // version that came to EoL a few years ago, please open an issue. - fn getauxval_sysctl_libc(type_: ffi::c_int) -> ffi::c_ulong { - let mut auxv: [sys::Elf64_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() }; - - let mut len = core::mem::size_of_val(&auxv) as c_size_t; - - // SAFETY: calling getpid is safe. - let pid = unsafe { sys::getpid() }; - let mib = [ - sys::CTL_KERN as c_int, - sys::KERN_PROC as c_int, - sys::KERN_PROC_AUXV as c_int, - pid, - ]; - - #[allow(clippy::cast_possible_truncation)] - // SAFETY: - // - `mib.len()` does not exceed the size of `mib`. - // - `len` does not exceed the size of `auxv`. - // - `sysctl` is thread-safe. - let res = unsafe { - sys::sysctl( - mib.as_ptr(), - mib.len() as c_uint, - auxv.as_mut_ptr().cast::<c_void>(), - &mut len, - ptr::null_mut(), - 0, - ) - }; - - if res != -1 { - for aux in &auxv { - if aux.a_type == type_ as c_long { - // SAFETY: aux.a_un is #[repr(C)] union and all fields have - // the same size and can be safely transmuted to integers. - return unsafe { aux.a_un.a_val as c_ulong }; - } - } - } - 0 - } - // Similar to the above, but call syscall using asm instead of libc. - // Note that FreeBSD does not guarantee the stability of raw syscall as - // much as Linux does (It may actually be stable enough, though: - // https://lists.llvm.org/pipermail/llvm-dev/2019-June/133393.html, - // https://github.com/ziglang/zig/issues/16590). - // - // This is currently used only for testing. - fn getauxval_sysctl_asm_syscall(type_: ffi::c_int) -> Result<ffi::c_ulong, c_int> { - #[allow(non_camel_case_types)] - type pid_t = c_int; - - // https://github.com/freebsd/freebsd-src/blob/9888a79adad22ba06b5aff17d05abac0029c537a/lib/libc/aarch64/SYS.h - // https://github.com/golang/go/blob/4badad8d477ffd7a6b762c35bc69aed82faface7/src/syscall/asm_freebsd_arm64.s - #[cfg(target_arch = "aarch64")] - #[inline] - fn getpid() -> pid_t { - #[allow(clippy::cast_possible_truncation)] - // SAFETY: calling getpid is safe. - unsafe { - let n = sys::SYS_getpid; - let r: i64; - asm!( - "svc 0", - in("x8") n as u64, - out("x0") r, - options(nostack, readonly), - ); - r as pid_t - } - } - #[cfg(target_arch = "aarch64")] - #[inline] - unsafe fn sysctl( - name: *const c_int, - name_len: c_uint, - old_p: *mut c_void, - old_len_p: *mut c_size_t, - new_p: *const c_void, - new_len: c_size_t, - ) -> Result<c_int, c_int> { - #[allow(clippy::cast_possible_truncation)] - // SAFETY: the caller must uphold the safety contract. - unsafe { - let mut n = sys::SYS___sysctl as u64; - let r: i64; - asm!( - "svc 0", - "b.cc 2f", - "mov x8, x0", - "mov x0, #-1", - "2:", - inout("x8") n, - inout("x0") ptr_reg!(name) => r, - inout("x1") name_len as u64 => _, - in("x2") ptr_reg!(old_p), - in("x3") ptr_reg!(old_len_p), - in("x4") ptr_reg!(new_p), - in("x5") new_len as u64, - options(nostack), - ); - if r as c_int == -1 { - Err(n as c_int) - } else { - Ok(r as c_int) - } - } - } - - // https://github.com/freebsd/freebsd-src/blob/9888a79adad22ba06b5aff17d05abac0029c537a/lib/libc/powerpc64/SYS.h - #[cfg(target_arch = "powerpc64")] - #[inline] - fn getpid() -> pid_t { - #[allow(clippy::cast_possible_truncation)] - // SAFETY: calling getpid is safe. - unsafe { - let n = sys::SYS_getpid; - let r: i64; - asm!( - "sc", - inout("r0") n as u64 => _, - out("r3") r, - out("r4") _, - out("r5") _, - out("r6") _, - out("r7") _, - out("r8") _, - out("r9") _, - out("r10") _, - out("r11") _, - out("r12") _, - out("cr0") _, - options(nostack, preserves_flags, readonly), - ); - r as pid_t - } - } - #[cfg(target_arch = "powerpc64")] - #[inline] - unsafe fn sysctl( - name: *const c_int, - name_len: c_uint, - old_p: *mut c_void, - old_len_p: *mut c_size_t, - new_p: *const c_void, - new_len: c_size_t, - ) -> Result<c_int, c_int> { - #[allow(clippy::cast_possible_truncation)] - // SAFETY: the caller must uphold the safety contract. - unsafe { - let mut n = sys::SYS___sysctl as u64; - let r: i64; - asm!( - "sc", - "bns+ 2f", - "mr %r0, %r3", - "li %r3, -1", - "2:", - inout("r0") n, - inout("r3") ptr_reg!(name) => r, - inout("r4") name_len as u64 => _, - inout("r5") ptr_reg!(old_p) => _, - inout("r6") ptr_reg!(old_len_p) => _, - inout("r7") ptr_reg!(new_p) => _, - inout("r8") new_len as u64 => _, - out("r9") _, - out("r10") _, - out("r11") _, - out("r12") _, - out("cr0") _, - options(nostack, preserves_flags) - ); - if r as c_int == -1 { - Err(n as c_int) - } else { - Ok(r as c_int) - } - } - } - - let mut auxv: [sys::Elf64_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() }; - - let mut len = core::mem::size_of_val(&auxv) as c_size_t; - - let pid = getpid(); - let mib = [ - sys::CTL_KERN as c_int, - sys::KERN_PROC as c_int, - sys::KERN_PROC_AUXV as c_int, - pid, - ]; - - #[allow(clippy::cast_possible_truncation)] - // SAFETY: - // - `mib.len()` does not exceed the size of `mib`. - // - `len` does not exceed the size of `auxv`. - // - `sysctl` is thread-safe. - unsafe { - sysctl( - mib.as_ptr(), - mib.len() as c_uint, - auxv.as_mut_ptr().cast::<c_void>(), - &mut len, - ptr::null_mut(), - 0, - )?; - } - - for aux in &auxv { - if aux.a_type == type_ as c_long { - // SAFETY: aux.a_un is #[repr(C)] union and all fields have - // the same size and can be safely transmuted to integers. - return Ok(unsafe { aux.a_un.a_val as c_ulong }); - } - } - Err(0) - } - - assert_eq!(os::getauxval(ffi::AT_HWCAP), getauxval_sysctl_libc(ffi::AT_HWCAP)); - assert_eq!(os::getauxval(ffi::AT_HWCAP2), getauxval_sysctl_libc(ffi::AT_HWCAP2)); - assert_eq!( - os::getauxval(ffi::AT_HWCAP), - getauxval_sysctl_asm_syscall(ffi::AT_HWCAP).unwrap() - ); - assert_eq!( - os::getauxval(ffi::AT_HWCAP2), - // AT_HWCAP2 is only available on FreeBSD 13+, at least for aarch64. - getauxval_sysctl_asm_syscall(ffi::AT_HWCAP2).unwrap_or(0) - ); - } - - // Static assertions for FFI bindings. - // This checks that FFI bindings defined in this crate, FFI bindings defined - // in libc, and FFI bindings generated for the platform's latest header file - // using bindgen have compatible signatures (or the same values if constants). - // Since this is static assertion, we can detect problems with - // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh) - // without actually running tests on these platforms. - // See also tools/codegen/src/ffi.rs. - // TODO(codegen): auto-generate this test - #[allow( - clippy::cast_possible_wrap, - clippy::cast_sign_loss, - clippy::cast_possible_truncation, - clippy::no_effect_underscore_binding - )] - const _: fn() = || { - use test_helper::{libc, sys}; - #[cfg(not(target_os = "freebsd"))] - type AtType = ffi::c_ulong; - #[cfg(target_os = "freebsd")] - type AtType = ffi::c_int; - #[cfg(any(target_os = "linux", target_os = "android"))] - { - let mut _getauxval: unsafe extern "C" fn(ffi::c_ulong) -> ffi::c_ulong = ffi::getauxval; - _getauxval = libc::getauxval; - _getauxval = sys::getauxval; - } - #[cfg(all(target_arch = "aarch64", target_os = "android"))] - { - let mut ___system_property_get: unsafe extern "C" fn( - *const ffi::c_char, - *mut ffi::c_char, - ) -> ffi::c_int = ffi::__system_property_get; - ___system_property_get = libc::__system_property_get; - ___system_property_get = sys::__system_property_get; - static_assert!(ffi::PROP_VALUE_MAX == libc::PROP_VALUE_MAX); - static_assert!(ffi::PROP_VALUE_MAX == sys::PROP_VALUE_MAX as ffi::c_int); - } - #[cfg(target_os = "freebsd")] - { - let mut _elf_aux_info: unsafe extern "C" fn( - ffi::c_int, - *mut ffi::c_void, - ffi::c_int, - ) -> ffi::c_int = ffi::elf_aux_info; - _elf_aux_info = libc::elf_aux_info; - _elf_aux_info = sys::elf_aux_info; - } - #[cfg(not(target_os = "freebsd"))] // libc doesn't have this on FreeBSD - static_assert!(ffi::AT_HWCAP == libc::AT_HWCAP); - static_assert!(ffi::AT_HWCAP == sys::AT_HWCAP as AtType); - #[cfg(not(target_os = "freebsd"))] // libc doesn't have this on FreeBSD - static_assert!(ffi::AT_HWCAP2 == libc::AT_HWCAP2); - static_assert!(ffi::AT_HWCAP2 == sys::AT_HWCAP2 as AtType); - #[cfg(target_arch = "aarch64")] - { - // static_assert!(arch::HWCAP_ATOMICS == libc::HWCAP_ATOMICS); // libc doesn't have this - static_assert!(arch::HWCAP_ATOMICS == sys::HWCAP_ATOMICS as ffi::c_ulong); - // static_assert!(HWCAP_USCAT == libc::HWCAP_USCAT); // libc doesn't have this - static_assert!(arch::HWCAP_USCAT == sys::HWCAP_USCAT as ffi::c_ulong); - #[cfg(any(target_os = "linux", target_os = "android"))] - #[cfg(target_pointer_width = "64")] - { - // static_assert!(HWCAP2_LRCPC3 == libc::HWCAP2_LRCPC3); // libc doesn't have this - static_assert!(arch::HWCAP2_LRCPC3 == sys::HWCAP2_LRCPC3 as ffi::c_ulong); - // static_assert!(HWCAP2_LSE128 == libc::HWCAP2_LSE128); // libc doesn't have this - static_assert!(arch::HWCAP2_LSE128 == sys::HWCAP2_LSE128 as ffi::c_ulong); - } - } - #[cfg(target_arch = "powerpc64")] - { - // static_assert!(arch::PPC_FEATURE2_ARCH_2_07 == libc::PPC_FEATURE2_ARCH_2_07); // libc doesn't have this - static_assert!( - arch::PPC_FEATURE2_ARCH_2_07 == sys::PPC_FEATURE2_ARCH_2_07 as ffi::c_ulong - ); - } - }; -} diff --git a/vendor/portable-atomic/src/imp/atomic128/detect/common.rs b/vendor/portable-atomic/src/imp/atomic128/detect/common.rs deleted file mode 100644 index b87caa3..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/detect/common.rs +++ /dev/null @@ -1,395 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -#[derive(Clone, Copy)] -pub(crate) struct CpuInfo(u32); - -impl CpuInfo { - const INIT: u32 = 0; - - #[inline] - fn set(&mut self, bit: u32) { - self.0 = set(self.0, bit); - } - #[inline] - fn test(self, bit: u32) -> bool { - test(self.0, bit) - } -} - -#[inline] -fn set(x: u32, bit: u32) -> u32 { - x | 1 << bit -} -#[inline] -fn test(x: u32, bit: u32) -> bool { - x & (1 << bit) != 0 -} - -#[inline] -pub(crate) fn detect() -> CpuInfo { - use core::sync::atomic::{AtomicU32, Ordering}; - - static CACHE: AtomicU32 = AtomicU32::new(0); - let mut info = CpuInfo(CACHE.load(Ordering::Relaxed)); - if info.0 != 0 { - return info; - } - info.set(CpuInfo::INIT); - // Note: detect_false cfg is intended to make it easy for portable-atomic developers to - // test cases such as has_cmpxchg16b == false, has_lse == false, - // __kuser_helper_version < 5, etc., and is not a public API. - if !cfg!(portable_atomic_test_outline_atomics_detect_false) { - _detect(&mut info); - } - CACHE.store(info.0, Ordering::Relaxed); - info -} - -#[cfg(target_arch = "aarch64")] -impl CpuInfo { - /// Whether FEAT_LSE is available - const HAS_LSE: u32 = 1; - /// Whether FEAT_LSE2 is available - #[cfg_attr(not(test), allow(dead_code))] - const HAS_LSE2: u32 = 2; - /// Whether FEAT_LSE128 is available - // This is currently only used in tests. - #[cfg(test)] - const HAS_LSE128: u32 = 3; - /// Whether FEAT_LRCPC3 is available - // This is currently only used in tests. - #[cfg(test)] - const HAS_RCPC3: u32 = 4; - - #[cfg(any(test, not(any(target_feature = "lse", portable_atomic_target_feature = "lse"))))] - #[inline] - pub(crate) fn has_lse(self) -> bool { - self.test(CpuInfo::HAS_LSE) - } - #[cfg_attr(not(test), allow(dead_code))] - #[cfg(any(test, not(any(target_feature = "lse2", portable_atomic_target_feature = "lse2"))))] - #[inline] - pub(crate) fn has_lse2(self) -> bool { - self.test(CpuInfo::HAS_LSE2) - } - #[cfg(test)] - #[inline] - pub(crate) fn has_lse128(self) -> bool { - self.test(CpuInfo::HAS_LSE128) - } - #[cfg(test)] - #[inline] - pub(crate) fn has_rcpc3(self) -> bool { - self.test(CpuInfo::HAS_RCPC3) - } -} - -#[cfg(target_arch = "x86_64")] -impl CpuInfo { - /// Whether CMPXCHG16B is available - const HAS_CMPXCHG16B: u32 = 1; - /// Whether VMOVDQA is atomic - const HAS_VMOVDQA_ATOMIC: u32 = 2; - - #[cfg(any( - test, - not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")), - ))] - #[inline] - pub(crate) fn has_cmpxchg16b(self) -> bool { - self.test(CpuInfo::HAS_CMPXCHG16B) - } - #[inline] - pub(crate) fn has_vmovdqa_atomic(self) -> bool { - self.test(CpuInfo::HAS_VMOVDQA_ATOMIC) - } -} - -#[cfg(target_arch = "powerpc64")] -impl CpuInfo { - /// Whether lqarx and stqcx. instructions are available - const HAS_QUADWORD_ATOMICS: u32 = 1; - - #[cfg(any( - test, - not(any( - target_feature = "quadword-atomics", - portable_atomic_target_feature = "quadword-atomics", - )), - ))] - #[inline] - pub(crate) fn has_quadword_atomics(self) -> bool { - self.test(CpuInfo::HAS_QUADWORD_ATOMICS) - } -} - -// core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47 -#[cfg(any(target_arch = "aarch64", target_arch = "powerpc64"))] -#[cfg(not(windows))] -#[allow(dead_code, non_camel_case_types)] -mod c_types { - pub(crate) type c_void = core::ffi::c_void; - // c_{,u}int is {i,u}32 on non-16-bit architectures - // https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/ffi/mod.rs#L160 - // (16-bit architectures currently don't use this module) - pub(crate) type c_int = i32; - pub(crate) type c_uint = u32; - // c_{,u}long is {i,u}64 on non-Windows 64-bit targets, otherwise is {i,u}32 - // https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/ffi/mod.rs#L176 - // (Windows currently doesn't use this module - this module is cfg(not(windows))) - #[cfg(target_pointer_width = "64")] - pub(crate) type c_long = i64; - #[cfg(not(target_pointer_width = "64"))] - pub(crate) type c_long = i32; - #[cfg(target_pointer_width = "64")] - pub(crate) type c_ulong = u64; - #[cfg(not(target_pointer_width = "64"))] - pub(crate) type c_ulong = u32; - // c_size_t is currently always usize - // https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/ffi/mod.rs#L88 - pub(crate) type c_size_t = usize; - // c_char is u8 by default on most non-Apple/non-Windows ARM/PowerPC/RISC-V/s390x/Hexagon targets - // (Linux/Android/FreeBSD/NetBSD/OpenBSD/VxWorks/Fuchsia/QNX Neutrino/Horizon/AIX/z/OS) - // https://github.com/rust-lang/rust/blob/1.70.0/library/core/src/ffi/mod.rs#L104 - // https://github.com/llvm/llvm-project/blob/9734b2256d89cb4c61a4dbf4a3c3f3f942fe9b8c/lldb/source/Utility/ArchSpec.cpp#L712 - // RISC-V https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/HEAD/riscv-cc.adoc#cc-type-representations - // Hexagon https://lists.llvm.org/pipermail/llvm-dev/attachments/20190916/21516a52/attachment-0001.pdf - // AIX https://www.ibm.com/docs/en/xl-c-aix/13.1.2?topic=descriptions-qchars - // z/OS https://www.ibm.com/docs/en/zos/2.5.0?topic=specifiers-character-types - // (macOS is currently the only Apple target that uses this module, and Windows currently doesn't use this module) - #[cfg(not(target_os = "macos"))] - pub(crate) type c_char = u8; - // c_char is i8 on all Apple targets - #[cfg(target_os = "macos")] - pub(crate) type c_char = i8; - - // Static assertions for C type definitions. - #[cfg(test)] - const _: fn() = || { - use test_helper::{libc, sys}; - let _: c_int = 0 as std::os::raw::c_int; - let _: c_uint = 0 as std::os::raw::c_uint; - let _: c_long = 0 as std::os::raw::c_long; - let _: c_ulong = 0 as std::os::raw::c_ulong; - let _: c_size_t = 0 as libc::size_t; // std::os::raw::c_size_t is unstable - let _: c_char = 0 as std::os::raw::c_char; - let _: c_char = 0 as sys::c_char; - }; -} - -#[allow( - clippy::alloc_instead_of_core, - clippy::std_instead_of_alloc, - clippy::std_instead_of_core, - clippy::undocumented_unsafe_blocks, - clippy::wildcard_imports -)] -#[cfg(test)] -mod tests_common { - use super::*; - - #[test] - fn test_bit_flags() { - let mut x = CpuInfo(0); - #[cfg(target_arch = "aarch64")] - { - assert!(!x.test(CpuInfo::INIT)); - assert!(!x.test(CpuInfo::HAS_LSE)); - assert!(!x.test(CpuInfo::HAS_LSE2)); - assert!(!x.test(CpuInfo::HAS_LSE128)); - assert!(!x.test(CpuInfo::HAS_RCPC3)); - x.set(CpuInfo::INIT); - assert!(x.test(CpuInfo::INIT)); - assert!(!x.test(CpuInfo::HAS_LSE)); - assert!(!x.test(CpuInfo::HAS_LSE2)); - assert!(!x.test(CpuInfo::HAS_LSE128)); - assert!(!x.test(CpuInfo::HAS_RCPC3)); - x.set(CpuInfo::HAS_LSE); - assert!(x.test(CpuInfo::INIT)); - assert!(x.test(CpuInfo::HAS_LSE)); - assert!(!x.test(CpuInfo::HAS_LSE2)); - assert!(!x.test(CpuInfo::HAS_LSE128)); - assert!(!x.test(CpuInfo::HAS_RCPC3)); - x.set(CpuInfo::HAS_LSE2); - assert!(x.test(CpuInfo::INIT)); - assert!(x.test(CpuInfo::HAS_LSE)); - assert!(x.test(CpuInfo::HAS_LSE2)); - assert!(!x.test(CpuInfo::HAS_LSE128)); - assert!(!x.test(CpuInfo::HAS_RCPC3)); - x.set(CpuInfo::HAS_LSE128); - assert!(x.test(CpuInfo::INIT)); - assert!(x.test(CpuInfo::HAS_LSE)); - assert!(x.test(CpuInfo::HAS_LSE2)); - assert!(x.test(CpuInfo::HAS_LSE128)); - assert!(!x.test(CpuInfo::HAS_RCPC3)); - x.set(CpuInfo::HAS_RCPC3); - assert!(x.test(CpuInfo::INIT)); - assert!(x.test(CpuInfo::HAS_LSE)); - assert!(x.test(CpuInfo::HAS_LSE2)); - assert!(x.test(CpuInfo::HAS_LSE128)); - assert!(x.test(CpuInfo::HAS_RCPC3)); - } - #[cfg(target_arch = "x86_64")] - { - assert!(!x.test(CpuInfo::INIT)); - assert!(!x.test(CpuInfo::HAS_CMPXCHG16B)); - assert!(!x.test(CpuInfo::HAS_VMOVDQA_ATOMIC)); - x.set(CpuInfo::INIT); - assert!(x.test(CpuInfo::INIT)); - assert!(!x.test(CpuInfo::HAS_CMPXCHG16B)); - assert!(!x.test(CpuInfo::HAS_VMOVDQA_ATOMIC)); - x.set(CpuInfo::HAS_CMPXCHG16B); - assert!(x.test(CpuInfo::INIT)); - assert!(x.test(CpuInfo::HAS_CMPXCHG16B)); - assert!(!x.test(CpuInfo::HAS_VMOVDQA_ATOMIC)); - x.set(CpuInfo::HAS_VMOVDQA_ATOMIC); - assert!(x.test(CpuInfo::INIT)); - assert!(x.test(CpuInfo::HAS_CMPXCHG16B)); - assert!(x.test(CpuInfo::HAS_VMOVDQA_ATOMIC)); - } - #[cfg(target_arch = "powerpc64")] - { - assert!(!x.test(CpuInfo::INIT)); - assert!(!x.test(CpuInfo::HAS_QUADWORD_ATOMICS)); - x.set(CpuInfo::INIT); - assert!(x.test(CpuInfo::INIT)); - assert!(!x.test(CpuInfo::HAS_QUADWORD_ATOMICS)); - x.set(CpuInfo::HAS_QUADWORD_ATOMICS); - assert!(x.test(CpuInfo::INIT)); - assert!(x.test(CpuInfo::HAS_QUADWORD_ATOMICS)); - } - } - - #[test] - fn print_features() { - use std::{fmt::Write as _, io::Write, string::String}; - - let mut features = String::new(); - macro_rules! print_feature { - ($name:expr, $enabled:expr $(,)?) => {{ - let _ = writeln!(features, " {}: {}", $name, $enabled); - }}; - } - #[cfg(target_arch = "aarch64")] - { - features.push_str("run-time:\n"); - print_feature!("lse", detect().test(CpuInfo::HAS_LSE)); - print_feature!("lse2", detect().test(CpuInfo::HAS_LSE2)); - print_feature!("lse128", detect().test(CpuInfo::HAS_LSE128)); - print_feature!("rcpc3", detect().test(CpuInfo::HAS_RCPC3)); - features.push_str("compile-time:\n"); - print_feature!( - "lse", - cfg!(any(target_feature = "lse", portable_atomic_target_feature = "lse")), - ); - print_feature!( - "lse2", - cfg!(any(target_feature = "lse2", portable_atomic_target_feature = "lse2")), - ); - } - #[cfg(target_arch = "x86_64")] - { - features.push_str("run-time:\n"); - print_feature!("cmpxchg16b", detect().test(CpuInfo::HAS_CMPXCHG16B)); - print_feature!("vmovdqa-atomic", detect().test(CpuInfo::HAS_VMOVDQA_ATOMIC)); - features.push_str("compile-time:\n"); - print_feature!( - "cmpxchg16b", - cfg!(any( - target_feature = "cmpxchg16b", - portable_atomic_target_feature = "cmpxchg16b", - )), - ); - } - #[cfg(target_arch = "powerpc64")] - { - features.push_str("run-time:\n"); - print_feature!("quadword-atomics", detect().test(CpuInfo::HAS_QUADWORD_ATOMICS)); - features.push_str("compile-time:\n"); - print_feature!( - "quadword-atomics", - cfg!(any( - target_feature = "quadword-atomics", - portable_atomic_target_feature = "quadword-atomics", - )), - ); - } - let stdout = std::io::stderr(); - let mut stdout = stdout.lock(); - let _ = stdout.write_all(features.as_bytes()); - } - - #[cfg(target_arch = "x86_64")] - #[test] - #[cfg_attr(portable_atomic_test_outline_atomics_detect_false, ignore)] - fn test_detect() { - if detect().has_cmpxchg16b() { - assert!(detect().test(CpuInfo::HAS_CMPXCHG16B)); - } else { - assert!(!detect().test(CpuInfo::HAS_CMPXCHG16B)); - } - if detect().has_vmovdqa_atomic() { - assert!(detect().test(CpuInfo::HAS_VMOVDQA_ATOMIC)); - } else { - assert!(!detect().test(CpuInfo::HAS_VMOVDQA_ATOMIC)); - } - } - #[cfg(target_arch = "aarch64")] - #[test] - #[cfg_attr(portable_atomic_test_outline_atomics_detect_false, ignore)] - fn test_detect() { - let proc_cpuinfo = test_helper::cpuinfo::ProcCpuinfo::new(); - if detect().has_lse() { - assert!(detect().test(CpuInfo::HAS_LSE)); - if let Ok(proc_cpuinfo) = proc_cpuinfo { - assert!(proc_cpuinfo.lse); - } - } else { - assert!(!detect().test(CpuInfo::HAS_LSE)); - if let Ok(proc_cpuinfo) = proc_cpuinfo { - assert!(!proc_cpuinfo.lse); - } - } - if detect().has_lse2() { - assert!(detect().test(CpuInfo::HAS_LSE)); - assert!(detect().test(CpuInfo::HAS_LSE2)); - if let Ok(test_helper::cpuinfo::ProcCpuinfo { lse2: Some(lse2), .. }) = proc_cpuinfo { - assert!(lse2); - } - } else { - assert!(!detect().test(CpuInfo::HAS_LSE2)); - if let Ok(test_helper::cpuinfo::ProcCpuinfo { lse2: Some(lse2), .. }) = proc_cpuinfo { - assert!(!lse2); - } - } - if detect().has_lse128() { - assert!(detect().test(CpuInfo::HAS_LSE)); - assert!(detect().test(CpuInfo::HAS_LSE2)); - assert!(detect().test(CpuInfo::HAS_LSE128)); - } else { - assert!(!detect().test(CpuInfo::HAS_LSE128)); - } - if detect().has_rcpc3() { - assert!(detect().test(CpuInfo::HAS_RCPC3)); - } else { - assert!(!detect().test(CpuInfo::HAS_RCPC3)); - } - } - #[cfg(target_arch = "powerpc64")] - #[test] - #[cfg_attr(portable_atomic_test_outline_atomics_detect_false, ignore)] - fn test_detect() { - let proc_cpuinfo = test_helper::cpuinfo::ProcCpuinfo::new(); - if detect().has_quadword_atomics() { - assert!(detect().test(CpuInfo::HAS_QUADWORD_ATOMICS)); - if let Ok(proc_cpuinfo) = proc_cpuinfo { - assert!(proc_cpuinfo.power8); - } - } else { - assert!(!detect().test(CpuInfo::HAS_QUADWORD_ATOMICS)); - if let Ok(proc_cpuinfo) = proc_cpuinfo { - assert!(!proc_cpuinfo.power8); - } - } - } -} diff --git a/vendor/portable-atomic/src/imp/atomic128/detect/x86_64.rs b/vendor/portable-atomic/src/imp/atomic128/detect/x86_64.rs deleted file mode 100644 index 80eefed..0000000 --- a/vendor/portable-atomic/src/imp/atomic128/detect/x86_64.rs +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 OR MIT - -// Adapted from https://github.com/rust-lang/stdarch. - -#![cfg_attr(any(not(target_feature = "sse"), portable_atomic_sanitize_thread), allow(dead_code))] - -// Miri doesn't support inline assembly used in __cpuid: https://github.com/rust-lang/miri/issues/932 -// SGX doesn't support CPUID: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105 -#[cfg(any(target_env = "sgx", miri))] -compile_error!("internal error: this module is not supported on this environment"); - -include!("common.rs"); - -#[cfg(not(portable_atomic_no_asm))] -use core::arch::asm; -use core::arch::x86_64::{CpuidResult, _xgetbv}; - -// Workaround for https://github.com/rust-lang/rust/issues/101346 -// It is not clear if our use cases are affected, but we implement this just in case. -// -// Refs: -// - https://www.felixcloutier.com/x86/cpuid -// - https://en.wikipedia.org/wiki/CPUID -// - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs -unsafe fn __cpuid(leaf: u32) -> CpuidResult { - let eax; - let mut ebx; - let ecx; - let edx; - // SAFETY: the caller must guarantee that CPU supports `cpuid`. - unsafe { - asm!( - // rbx is reserved by LLVM - "mov {ebx_tmp:r}, rbx", - "cpuid", - "xchg {ebx_tmp:r}, rbx", // restore rbx - ebx_tmp = out(reg) ebx, - inout("eax") leaf => eax, - inout("ecx") 0 => ecx, - out("edx") edx, - options(nostack, preserves_flags), - ); - } - CpuidResult { eax, ebx, ecx, edx } -} - -// https://en.wikipedia.org/wiki/CPUID -const VENDOR_ID_INTEL: [u8; 12] = *b"GenuineIntel"; -const VENDOR_ID_AMD: [u8; 12] = *b"AuthenticAMD"; - -unsafe fn _vendor_id() -> [u8; 12] { - // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L40-L59 - // SAFETY: the caller must guarantee that CPU supports `cpuid`. - let CpuidResult { ebx, ecx, edx, .. } = unsafe { __cpuid(0) }; - let vendor_id: [[u8; 4]; 3] = [ebx.to_ne_bytes(), edx.to_ne_bytes(), ecx.to_ne_bytes()]; - // SAFETY: transmute is safe because `[u8; 12]` and `[[u8; 4]; 3]` has the same layout. - unsafe { core::mem::transmute(vendor_id) } -} - -#[cold] -fn _detect(info: &mut CpuInfo) { - // SAFETY: Calling `_vendor_id`` is safe because the CPU has `cpuid` support. - let vendor_id = unsafe { _vendor_id() }; - - // SAFETY: Calling `__cpuid`` is safe because the CPU has `cpuid` support. - let proc_info_ecx = unsafe { __cpuid(0x0000_0001_u32).ecx }; - - // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L111 - if test(proc_info_ecx, 13) { - info.set(CpuInfo::HAS_CMPXCHG16B); - } - - // VMOVDQA is atomic on Intel and AMD CPUs with AVX. - // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details. - if vendor_id == VENDOR_ID_INTEL || vendor_id == VENDOR_ID_AMD { - // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L131-L224 - let cpu_xsave = test(proc_info_ecx, 26); - if cpu_xsave { - let cpu_osxsave = test(proc_info_ecx, 27); - if cpu_osxsave { - // SAFETY: Calling `_xgetbv`` is safe because the CPU has `xsave` support - // and OS has set `osxsave`. - let xcr0 = unsafe { _xgetbv(0) }; - let os_avx_support = xcr0 & 6 == 6; - if os_avx_support && test(proc_info_ecx, 28) { - info.set(CpuInfo::HAS_VMOVDQA_ATOMIC); - } - } - } - } -} - -#[allow( - clippy::alloc_instead_of_core, - clippy::std_instead_of_alloc, - clippy::std_instead_of_core, - clippy::undocumented_unsafe_blocks, - clippy::wildcard_imports -)] -#[cfg(test)] -mod tests { - #[cfg(not(portable_atomic_test_outline_atomics_detect_false))] - use super::*; - - #[cfg(not(portable_atomic_test_outline_atomics_detect_false))] - #[test] - fn test_cpuid() { - assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), detect().has_cmpxchg16b()); - let vendor_id = unsafe { _vendor_id() }; - if vendor_id == VENDOR_ID_INTEL || vendor_id == VENDOR_ID_AMD { - assert_eq!(std::is_x86_feature_detected!("avx"), detect().has_vmovdqa_atomic()); - } else { - assert!(!detect().has_vmovdqa_atomic()); - } - } -} |