1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
// SPDX-License-Identifier: Apache-2.0 OR MIT
// Adapted from https://github.com/rust-lang/stdarch.
#![cfg_attr(any(not(target_feature = "sse"), portable_atomic_sanitize_thread), allow(dead_code))]
// Miri doesn't support inline assembly used in __cpuid: https://github.com/rust-lang/miri/issues/932
// SGX doesn't support CPUID: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105
#[cfg(any(target_env = "sgx", miri))]
compile_error!("internal error: this module is not supported on this environment");
include!("common.rs");
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use core::arch::x86_64::{CpuidResult, _xgetbv};
// Workaround for https://github.com/rust-lang/rust/issues/101346
// It is not clear if our use cases are affected, but we implement this just in case.
//
// Refs:
// - https://www.felixcloutier.com/x86/cpuid
// - https://en.wikipedia.org/wiki/CPUID
// - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs
unsafe fn __cpuid(leaf: u32) -> CpuidResult {
let eax;
let mut ebx;
let ecx;
let edx;
// SAFETY: the caller must guarantee that CPU supports `cpuid`.
unsafe {
asm!(
// rbx is reserved by LLVM
"mov {ebx_tmp:r}, rbx",
"cpuid",
"xchg {ebx_tmp:r}, rbx", // restore rbx
ebx_tmp = out(reg) ebx,
inout("eax") leaf => eax,
inout("ecx") 0 => ecx,
out("edx") edx,
options(nostack, preserves_flags),
);
}
CpuidResult { eax, ebx, ecx, edx }
}
// https://en.wikipedia.org/wiki/CPUID
const VENDOR_ID_INTEL: [u8; 12] = *b"GenuineIntel";
const VENDOR_ID_AMD: [u8; 12] = *b"AuthenticAMD";
unsafe fn _vendor_id() -> [u8; 12] {
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L40-L59
// SAFETY: the caller must guarantee that CPU supports `cpuid`.
let CpuidResult { ebx, ecx, edx, .. } = unsafe { __cpuid(0) };
let vendor_id: [[u8; 4]; 3] = [ebx.to_ne_bytes(), edx.to_ne_bytes(), ecx.to_ne_bytes()];
// SAFETY: transmute is safe because `[u8; 12]` and `[[u8; 4]; 3]` has the same layout.
unsafe { core::mem::transmute(vendor_id) }
}
#[cold]
fn _detect(info: &mut CpuInfo) {
// SAFETY: Calling `_vendor_id`` is safe because the CPU has `cpuid` support.
let vendor_id = unsafe { _vendor_id() };
// SAFETY: Calling `__cpuid`` is safe because the CPU has `cpuid` support.
let proc_info_ecx = unsafe { __cpuid(0x0000_0001_u32).ecx };
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L111
if test(proc_info_ecx, 13) {
info.set(CpuInfo::HAS_CMPXCHG16B);
}
// VMOVDQA is atomic on Intel and AMD CPUs with AVX.
// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details.
if vendor_id == VENDOR_ID_INTEL || vendor_id == VENDOR_ID_AMD {
// https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L131-L224
let cpu_xsave = test(proc_info_ecx, 26);
if cpu_xsave {
let cpu_osxsave = test(proc_info_ecx, 27);
if cpu_osxsave {
// SAFETY: Calling `_xgetbv`` is safe because the CPU has `xsave` support
// and OS has set `osxsave`.
let xcr0 = unsafe { _xgetbv(0) };
let os_avx_support = xcr0 & 6 == 6;
if os_avx_support && test(proc_info_ecx, 28) {
info.set(CpuInfo::HAS_VMOVDQA_ATOMIC);
}
}
}
}
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
#[cfg(not(portable_atomic_test_outline_atomics_detect_false))]
use super::*;
#[cfg(not(portable_atomic_test_outline_atomics_detect_false))]
#[test]
fn test_cpuid() {
assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), detect().has_cmpxchg16b());
let vendor_id = unsafe { _vendor_id() };
if vendor_id == VENDOR_ID_INTEL || vendor_id == VENDOR_ID_AMD {
assert_eq!(std::is_x86_feature_detected!("avx"), detect().has_vmovdqa_atomic());
} else {
assert!(!detect().has_vmovdqa_atomic());
}
}
}
|
