aboutsummaryrefslogtreecommitdiff
path: root/vendor/crc32fast/src/specialized
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/crc32fast/src/specialized')
-rw-r--r--vendor/crc32fast/src/specialized/aarch64.rs88
-rw-r--r--vendor/crc32fast/src/specialized/mod.rs37
-rw-r--r--vendor/crc32fast/src/specialized/pclmulqdq.rs225
3 files changed, 0 insertions, 350 deletions
diff --git a/vendor/crc32fast/src/specialized/aarch64.rs b/vendor/crc32fast/src/specialized/aarch64.rs
deleted file mode 100644
index 49de6b0..0000000
--- a/vendor/crc32fast/src/specialized/aarch64.rs
+++ /dev/null
@@ -1,88 +0,0 @@
-use std::arch::aarch64 as arch;
-
-#[derive(Clone)]
-pub struct State {
- state: u32,
-}
-
-impl State {
- pub fn new(state: u32) -> Option<Self> {
- if std::arch::is_aarch64_feature_detected!("crc") {
- // SAFETY: The conditions above ensure that all
- // required instructions are supported by the CPU.
- Some(Self { state })
- } else {
- None
- }
- }
-
- pub fn update(&mut self, buf: &[u8]) {
- // SAFETY: The `State::new` constructor ensures that all
- // required instructions are supported by the CPU.
- self.state = unsafe { calculate(self.state, buf) }
- }
-
- pub fn finalize(self) -> u32 {
- self.state
- }
-
- pub fn reset(&mut self) {
- self.state = 0;
- }
-
- pub fn combine(&mut self, other: u32, amount: u64) {
- self.state = ::combine::combine(self.state, other, amount);
- }
-}
-
-// target_feature is necessary to allow rustc to inline the crc32* wrappers
-#[target_feature(enable = "crc")]
-pub unsafe fn calculate(crc: u32, data: &[u8]) -> u32 {
- let mut c32 = !crc;
- let (pre_quad, quads, post_quad) = data.align_to::<u64>();
-
- c32 = pre_quad.iter().fold(c32, |acc, &b| arch::__crc32b(acc, b));
-
- // unrolling increases performance by a lot
- let mut quad_iter = quads.chunks_exact(8);
- for chunk in &mut quad_iter {
- c32 = arch::__crc32d(c32, chunk[0]);
- c32 = arch::__crc32d(c32, chunk[1]);
- c32 = arch::__crc32d(c32, chunk[2]);
- c32 = arch::__crc32d(c32, chunk[3]);
- c32 = arch::__crc32d(c32, chunk[4]);
- c32 = arch::__crc32d(c32, chunk[5]);
- c32 = arch::__crc32d(c32, chunk[6]);
- c32 = arch::__crc32d(c32, chunk[7]);
- }
- c32 = quad_iter
- .remainder()
- .iter()
- .fold(c32, |acc, &q| arch::__crc32d(acc, q));
-
- c32 = post_quad.iter().fold(c32, |acc, &b| arch::__crc32b(acc, b));
-
- !c32
-}
-
-#[cfg(test)]
-mod test {
- quickcheck! {
- fn check_against_baseline(init: u32, chunks: Vec<(Vec<u8>, usize)>) -> bool {
- let mut baseline = super::super::super::baseline::State::new(init);
- let mut aarch64 = super::State::new(init).expect("not supported");
- for (chunk, mut offset) in chunks {
- // simulate random alignments by offsetting the slice by up to 15 bytes
- offset &= 0xF;
- if chunk.len() <= offset {
- baseline.update(&chunk);
- aarch64.update(&chunk);
- } else {
- baseline.update(&chunk[offset..]);
- aarch64.update(&chunk[offset..]);
- }
- }
- aarch64.finalize() == baseline.finalize()
- }
- }
-}
diff --git a/vendor/crc32fast/src/specialized/mod.rs b/vendor/crc32fast/src/specialized/mod.rs
deleted file mode 100644
index 525a42c..0000000
--- a/vendor/crc32fast/src/specialized/mod.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-cfg_if! {
- if #[cfg(all(
- crc32fast_stdarchx86,
- target_feature = "sse2",
- any(target_arch = "x86", target_arch = "x86_64")
- ))] {
- mod pclmulqdq;
- pub use self::pclmulqdq::State;
- } else if #[cfg(all(feature = "nightly", target_arch = "aarch64"))] {
- mod aarch64;
- pub use self::aarch64::State;
- } else {
- #[derive(Clone)]
- pub enum State {}
- impl State {
- pub fn new(_: u32) -> Option<Self> {
- None
- }
-
- pub fn update(&mut self, _buf: &[u8]) {
- match *self {}
- }
-
- pub fn finalize(self) -> u32 {
- match self{}
- }
-
- pub fn reset(&mut self) {
- match *self {}
- }
-
- pub fn combine(&mut self, _other: u32, _amount: u64) {
- match *self {}
- }
- }
- }
-}
diff --git a/vendor/crc32fast/src/specialized/pclmulqdq.rs b/vendor/crc32fast/src/specialized/pclmulqdq.rs
deleted file mode 100644
index 84a60ca..0000000
--- a/vendor/crc32fast/src/specialized/pclmulqdq.rs
+++ /dev/null
@@ -1,225 +0,0 @@
-#[cfg(target_arch = "x86")]
-use core::arch::x86 as arch;
-#[cfg(target_arch = "x86_64")]
-use core::arch::x86_64 as arch;
-
-#[derive(Clone)]
-pub struct State {
- state: u32,
-}
-
-impl State {
- #[cfg(not(feature = "std"))]
- pub fn new(state: u32) -> Option<Self> {
- if cfg!(target_feature = "pclmulqdq")
- && cfg!(target_feature = "sse2")
- && cfg!(target_feature = "sse4.1")
- {
- // SAFETY: The conditions above ensure that all
- // required instructions are supported by the CPU.
- Some(Self { state })
- } else {
- None
- }
- }
-
- #[cfg(feature = "std")]
- pub fn new(state: u32) -> Option<Self> {
- if is_x86_feature_detected!("pclmulqdq")
- && is_x86_feature_detected!("sse2")
- && is_x86_feature_detected!("sse4.1")
- {
- // SAFETY: The conditions above ensure that all
- // required instructions are supported by the CPU.
- Some(Self { state })
- } else {
- None
- }
- }
-
- pub fn update(&mut self, buf: &[u8]) {
- // SAFETY: The `State::new` constructor ensures that all
- // required instructions are supported by the CPU.
- self.state = unsafe { calculate(self.state, buf) }
- }
-
- pub fn finalize(self) -> u32 {
- self.state
- }
-
- pub fn reset(&mut self) {
- self.state = 0;
- }
-
- pub fn combine(&mut self, other: u32, amount: u64) {
- self.state = ::combine::combine(self.state, other, amount);
- }
-}
-
-const K1: i64 = 0x154442bd4;
-const K2: i64 = 0x1c6e41596;
-const K3: i64 = 0x1751997d0;
-const K4: i64 = 0x0ccaa009e;
-const K5: i64 = 0x163cd6124;
-const K6: i64 = 0x1db710640;
-
-const P_X: i64 = 0x1DB710641;
-const U_PRIME: i64 = 0x1F7011641;
-
-#[cfg(feature = "std")]
-unsafe fn debug(s: &str, a: arch::__m128i) -> arch::__m128i {
- if false {
- union A {
- a: arch::__m128i,
- b: [u8; 16],
- }
- let x = A { a }.b;
- print!(" {:20} | ", s);
- for x in x.iter() {
- print!("{:02x} ", x);
- }
- println!();
- }
- return a;
-}
-
-#[cfg(not(feature = "std"))]
-unsafe fn debug(_s: &str, a: arch::__m128i) -> arch::__m128i {
- a
-}
-
-#[target_feature(enable = "pclmulqdq", enable = "sse2", enable = "sse4.1")]
-unsafe fn calculate(crc: u32, mut data: &[u8]) -> u32 {
- // In theory we can accelerate smaller chunks too, but for now just rely on
- // the fallback implementation as it's too much hassle and doesn't seem too
- // beneficial.
- if data.len() < 128 {
- return ::baseline::update_fast_16(crc, data);
- }
-
- // Step 1: fold by 4 loop
- let mut x3 = get(&mut data);
- let mut x2 = get(&mut data);
- let mut x1 = get(&mut data);
- let mut x0 = get(&mut data);
-
- // fold in our initial value, part of the incremental crc checksum
- x3 = arch::_mm_xor_si128(x3, arch::_mm_cvtsi32_si128(!crc as i32));
-
- let k1k2 = arch::_mm_set_epi64x(K2, K1);
- while data.len() >= 64 {
- x3 = reduce128(x3, get(&mut data), k1k2);
- x2 = reduce128(x2, get(&mut data), k1k2);
- x1 = reduce128(x1, get(&mut data), k1k2);
- x0 = reduce128(x0, get(&mut data), k1k2);
- }
-
- let k3k4 = arch::_mm_set_epi64x(K4, K3);
- let mut x = reduce128(x3, x2, k3k4);
- x = reduce128(x, x1, k3k4);
- x = reduce128(x, x0, k3k4);
-
- // Step 2: fold by 1 loop
- while data.len() >= 16 {
- x = reduce128(x, get(&mut data), k3k4);
- }
-
- debug("128 > 64 init", x);
-
- // Perform step 3, reduction from 128 bits to 64 bits. This is
- // significantly different from the paper and basically doesn't follow it
- // at all. It's not really clear why, but implementations of this algorithm
- // in Chrome/Linux diverge in the same way. It is beyond me why this is
- // different than the paper, maybe the paper has like errata or something?
- // Unclear.
- //
- // It's also not clear to me what's actually happening here and/or why, but
- // algebraically what's happening is:
- //
- // x = (x[0:63] • K4) ^ x[64:127] // 96 bit result
- // x = ((x[0:31] as u64) • K5) ^ x[32:95] // 64 bit result
- //
- // It's... not clear to me what's going on here. The paper itself is pretty
- // vague on this part but definitely uses different constants at least.
- // It's not clear to me, reading the paper, where the xor operations are
- // happening or why things are shifting around. This implementation...
- // appears to work though!
- drop(K6);
- let x = arch::_mm_xor_si128(
- arch::_mm_clmulepi64_si128(x, k3k4, 0x10),
- arch::_mm_srli_si128(x, 8),
- );
- let x = arch::_mm_xor_si128(
- arch::_mm_clmulepi64_si128(
- arch::_mm_and_si128(x, arch::_mm_set_epi32(0, 0, 0, !0)),
- arch::_mm_set_epi64x(0, K5),
- 0x00,
- ),
- arch::_mm_srli_si128(x, 4),
- );
- debug("128 > 64 xx", x);
-
- // Perform a Barrett reduction from our now 64 bits to 32 bits. The
- // algorithm for this is described at the end of the paper, and note that
- // this also implements the "bit reflected input" variant.
- let pu = arch::_mm_set_epi64x(U_PRIME, P_X);
-
- // T1(x) = ⌊(R(x) % x^32)⌋ • μ
- let t1 = arch::_mm_clmulepi64_si128(
- arch::_mm_and_si128(x, arch::_mm_set_epi32(0, 0, 0, !0)),
- pu,
- 0x10,
- );
- // T2(x) = ⌊(T1(x) % x^32)⌋ • P(x)
- let t2 = arch::_mm_clmulepi64_si128(
- arch::_mm_and_si128(t1, arch::_mm_set_epi32(0, 0, 0, !0)),
- pu,
- 0x00,
- );
- // We're doing the bit-reflected variant, so get the upper 32-bits of the
- // 64-bit result instead of the lower 32-bits.
- //
- // C(x) = R(x) ^ T2(x) / x^32
- let c = arch::_mm_extract_epi32(arch::_mm_xor_si128(x, t2), 1) as u32;
-
- if !data.is_empty() {
- ::baseline::update_fast_16(!c, data)
- } else {
- !c
- }
-}
-
-unsafe fn reduce128(a: arch::__m128i, b: arch::__m128i, keys: arch::__m128i) -> arch::__m128i {
- let t1 = arch::_mm_clmulepi64_si128(a, keys, 0x00);
- let t2 = arch::_mm_clmulepi64_si128(a, keys, 0x11);
- arch::_mm_xor_si128(arch::_mm_xor_si128(b, t1), t2)
-}
-
-unsafe fn get(a: &mut &[u8]) -> arch::__m128i {
- debug_assert!(a.len() >= 16);
- let r = arch::_mm_loadu_si128(a.as_ptr() as *const arch::__m128i);
- *a = &a[16..];
- return r;
-}
-
-#[cfg(test)]
-mod test {
- quickcheck! {
- fn check_against_baseline(init: u32, chunks: Vec<(Vec<u8>, usize)>) -> bool {
- let mut baseline = super::super::super::baseline::State::new(init);
- let mut pclmulqdq = super::State::new(init).expect("not supported");
- for (chunk, mut offset) in chunks {
- // simulate random alignments by offsetting the slice by up to 15 bytes
- offset &= 0xF;
- if chunk.len() <= offset {
- baseline.update(&chunk);
- pclmulqdq.update(&chunk);
- } else {
- baseline.update(&chunk[offset..]);
- pclmulqdq.update(&chunk[offset..]);
- }
- }
- pclmulqdq.finalize() == baseline.finalize()
- }
- }
-}
Copyright © 2024 Valentin A. Popov. All Rights Reverse Engineered.