crc64fast_nvme/pclmulqdq/
mod.rs
use std::{
fmt::Debug,
ops::{BitXor, BitXorAssign},
};
use super::table;
use self::arch::Simd;
#[cfg(not(feature = "fake-simd"))]
#[cfg_attr(target_arch = "x86_64", path = "x86_64/mod.rs")]
#[cfg_attr(target_arch = "aarch64", path = "aarch64.rs")]
#[cfg_attr(target_arch = "x86", path = "x86.rs")]
mod arch;
#[cfg(feature = "fake-simd")]
mod arch;
trait SimdExt: Copy + Debug + BitXor {
fn is_supported() -> bool;
unsafe fn new(high: u64, low: u64) -> Self;
unsafe fn fold_16(self, coeff: Self) -> Self;
unsafe fn fold_8(self, coeff: u64) -> Self;
unsafe fn barrett(self, poly: u64, mu: u64) -> u64;
}
impl PartialEq for Simd {
fn eq(&self, other: &Self) -> bool {
unsafe {
use std::mem::transmute;
let a: u128 = transmute(*self);
let b: u128 = transmute(*other);
a == b
}
}
}
impl Eq for Simd {}
impl BitXorAssign for Simd {
fn bitxor_assign(&mut self, other: Self) {
*self = *self ^ other;
}
}
pub fn get_update() -> super::UpdateFn {
#[cfg(feature = "vpclmulqdq")]
{
use arch::vpclmulqdq::*;
if Simd256::is_supported() {
return update_256_batch;
}
}
if Simd::is_supported() {
update_128_batch
} else {
table::update
}
}
unsafe fn update_128_batch(mut state: u64, bytes: &[u8]) -> u64 {
let (left, middle, right) = bytes.align_to::<[Simd; 8]>();
if let Some((first, rest)) = middle.split_first() {
state = table::update(state, left);
state = update_simd(state, first, rest);
table::update(state, right)
} else {
table::update(state, bytes)
}
}
#[cfg(feature = "vpclmulqdq")]
#[target_feature(enable = "avx2", enable = "vpclmulqdq")]
unsafe fn update_256_batch(mut state: u64, bytes: &[u8]) -> u64 {
use arch::vpclmulqdq::*;
let (left, middle, right) = bytes.align_to::<[[Simd256; 4]; 2]>();
if let Some((first, rest)) = middle.split_first() {
state = update_128_batch(state, left);
state = update_vpclmulqdq(state, first, rest);
update_128_batch(state, right)
} else {
update_128_batch(state, bytes)
}
}
#[cfg_attr(
any(target_arch = "x86", target_arch = "x86_64"),
target_feature(enable = "pclmulqdq", enable = "sse2", enable = "sse4.1")
)]
#[cfg_attr(
target_arch = "aarch64",
target_feature(enable = "aes", enable = "neon")
)]
unsafe fn update_simd(state: u64, first: &[Simd; 8], rest: &[[Simd; 8]]) -> u64 {
let mut x = *first;
x[0] ^= Simd::new(0, state);
let coeff = Simd::new(table::K_1023, table::K_1087);
for chunk in rest {
for (xi, yi) in x.iter_mut().zip(chunk.iter()) {
*xi = *yi ^ xi.fold_16(coeff);
}
}
fold_tail(x)
}
#[inline(always)]
unsafe fn fold_tail(x: [Simd; 8]) -> u64 {
let coeffs = [
Simd::new(table::K_895, table::K_959), Simd::new(table::K_767, table::K_831), Simd::new(table::K_639, table::K_703), Simd::new(table::K_511, table::K_575), Simd::new(table::K_383, table::K_447), Simd::new(table::K_255, table::K_319), Simd::new(table::K_127, table::K_191), ];
x.iter()
.zip(&coeffs)
.fold(x[7], |acc, (m, c)| acc ^ m.fold_16(*c))
.fold_8(table::K_127) .barrett(table::POLY, table::MU) }
#[test]
fn test_size_and_alignment() {
assert_eq!(std::mem::size_of::<Simd>(), 16);
assert_eq!(std::mem::align_of::<Simd>(), 16);
}
#[test]
fn test_new() {
unsafe {
let x = Simd::new(0xd7c8_11cf_e5c5_c792, 0x86e6_5c36_e68b_4804);
let y = Simd::new(0xd7c8_11cf_e5c5_c792, 0x86e6_5c36_e68b_4804);
let z = Simd::new(0xfa3e_0099_cd5e_d60d, 0xad71_9ee6_57d1_498e);
assert_eq!(x, y);
assert_ne!(x, z);
}
}
#[test]
fn test_xor() {
unsafe {
let x = Simd::new(0xe450_87f9_b031_0d47, 0x3d72_e92a_96c7_4c63);
let y = Simd::new(0x7ed8_ae0a_dfbd_89c0, 0x1c9b_dfaa_953e_0ef4);
let mut z = x ^ y;
assert_eq!(z, Simd::new(0x9a88_29f3_6f8c_8487, 0x21e9_3680_03f9_4297));
z ^= Simd::new(0x57a2_0f44_c005_b2ea, 0x7056_bde9_9303_aa51);
assert_eq!(z, Simd::new(0xcd2a_26b7_af89_366d, 0x51bf_8b69_90fa_e8c6));
}
}
#[test]
fn test_fold_16() {
unsafe {
let x = Simd::new(0xb5f1_2590_5645_0b6c, 0x333a_2c49_c361_9e21);
let f = x.fold_16(Simd::new(0xbecc_9dd9_038f_c366, 0x5ba9_365b_e2e9_5bf5));
assert_eq!(f, Simd::new(0x4f55_42df_ef35_1810, 0x0c03_5bd6_70fc_5abd));
}
}
#[test]
fn test_fold_8() {
unsafe {
let x = Simd::new(0x60c0_b48f_4a92_2003, 0x203c_f7bc_ad34_103b);
let f = x.fold_8(0x3e90_3688_ea71_f472);
assert_eq!(f, Simd::new(0x07d7_2761_4d16_56db, 0x2bc0_ed8a_a341_7665));
}
}
#[test]
fn test_barrett() {
unsafe {
let x = Simd::new(0x2606_e582_3406_9bae, 0x76cc_1105_0fef_6d68);
let b = x.barrett(0x435d_0f79_19a6_1445, 0x5817_6272_f8fa_b8d5);
assert_eq!(b, 0x5e4d_0253_942a_d95d);
}
}