diff --git a/Cargo.toml b/Cargo.toml index 72eda7f..8387cd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,6 @@ digest = "0.7" generic-array = "0.9" clear_on_drop = "=0.2.3" subtle = { version = "0.6", features = ["generic-impls"], default-features = false } -stdsimd = { version = "0.0.4", optional = true } serde = { version = "1.0", optional = true } rand = { version = "0.4", optional = true } @@ -54,7 +53,6 @@ digest = "0.7" generic-array = "0.9" clear_on_drop = "=0.2.3" subtle = { version = "0.6", features = ["generic-impls"], default-features = false } -stdsimd = { version = "0.0.4", optional = true } serde = { version = "1.0", optional = true } # Allowing rand to be optional during builds causes a build failure when compiling for no_std targets rand = { version = "0.4", optional = false } @@ -70,4 +68,4 @@ radix_51 = [] # Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation. precomputed_tables = [] # experimental avx2 support -avx2_backend = ["nightly", "stdsimd"] +avx2_backend = ["nightly"] diff --git a/build.rs b/build.rs index 60a9f7a..8331688 100644 --- a/build.rs +++ b/build.rs @@ -1,5 +1,6 @@ #![cfg_attr(feature = "nightly", feature(i128_type))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] +#![cfg_attr(all(feature = "nightly", feature = "yolocrypto"), feature(stdsimd))] #![allow(unused_variables)] #![allow(non_snake_case)] #![allow(dead_code)] @@ -23,8 +24,6 @@ use std::path::Path; // For instance, this shouldn't exist here at all, but it does. #[cfg(feature = "serde")] extern crate serde; -#[cfg(feature = "yolocrypto")] -extern crate stdsimd; // Macros come first! #[path = "src/macros.rs"] diff --git a/src/backend/avx2/constants.rs b/src/backend/avx2/constants.rs index 3fb77d8..e0479db 100644 --- a/src/backend/avx2/constants.rs +++ b/src/backend/avx2/constants.rs @@ -10,7 +10,7 @@ //! This module contains constants used by the AVX2 backend. -use stdsimd::simd::u32x8; +use core::simd::u32x8; use scalar_mul::window::OddLookupTable; use backend::avx2::field::FieldElement32x4; diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index c9da1ea..9341ee9 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -17,7 +17,7 @@ use core::convert::From; use core::ops::{Index, Add, Sub, Mul, Neg}; use core::borrow::Borrow; -use stdsimd::simd::{u32x8, i32x8}; +use core::simd::{IntoBits, u32x8, i32x8}; use subtle::ConditionallyAssignable; use subtle::Choice; @@ -28,10 +28,7 @@ use scalar_mul::window::{LookupTable, OddLookupTable}; use traits::Identity; -use backend::avx2::field::FieldElement32x4; - -use backend::avx2::field::{A_LANES, B_LANES, C_LANES, D_LANES, ALL_LANES}; -use backend::avx2::field::D_LANES64; +use backend::avx2::field::{D_LANES, Lanes, FieldElement32x4}; use backend::avx2; @@ -79,10 +76,10 @@ impl Identity for ExtendedPoint { impl ExtendedPoint { pub fn double(&self) -> ExtendedPoint { unsafe { - use stdsimd::vendor::_mm256_permute2x128_si256; - use stdsimd::vendor::_mm256_permutevar8x32_epi32; - use stdsimd::vendor::_mm256_blend_epi32; - use stdsimd::vendor::_mm256_shuffle_epi32; + use core::arch::x86_64::_mm256_permute2x128_si256; + use core::arch::x86_64::_mm256_permutevar8x32_epi32; + use core::arch::x86_64::_mm256_blend_epi32; + use core::arch::x86_64::_mm256_shuffle_epi32; let P = &self.0; @@ -96,18 +93,18 @@ impl ExtendedPoint { // and then adding. // Set t0 = (X1 Y1 X1 Y1) - t0.0[0] = _mm256_permute2x128_si256(P.0[0].into(), P.0[0].into(), 0b0000_0000).into(); - t0.0[1] = _mm256_permute2x128_si256(P.0[1].into(), P.0[1].into(), 0b0000_0000).into(); - t0.0[2] = _mm256_permute2x128_si256(P.0[2].into(), P.0[2].into(), 0b0000_0000).into(); - t0.0[3] = _mm256_permute2x128_si256(P.0[3].into(), P.0[3].into(), 0b0000_0000).into(); - t0.0[4] = _mm256_permute2x128_si256(P.0[4].into(), P.0[4].into(), 0b0000_0000).into(); + t0.0[0] = _mm256_permute2x128_si256(P.0[0].into_bits(), P.0[0].into_bits(), 0b0000_0000).into_bits(); + t0.0[1] = _mm256_permute2x128_si256(P.0[1].into_bits(), P.0[1].into_bits(), 0b0000_0000).into_bits(); + t0.0[2] = _mm256_permute2x128_si256(P.0[2].into_bits(), P.0[2].into_bits(), 0b0000_0000).into_bits(); + t0.0[3] = _mm256_permute2x128_si256(P.0[3].into_bits(), P.0[3].into_bits(), 0b0000_0000).into_bits(); + t0.0[4] = _mm256_permute2x128_si256(P.0[4].into_bits(), P.0[4].into_bits(), 0b0000_0000).into_bits(); // Set t1 = (Y1 X1 Y1 X1) - t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into(), 0b10_11_00_01).into(); - t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into(), 0b10_11_00_01).into(); - t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into(), 0b10_11_00_01).into(); - t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into(), 0b10_11_00_01).into(); - t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into(), 0b10_11_00_01).into(); + t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into_bits(), 0b10_11_00_01).into_bits(); + t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into_bits(), 0b10_11_00_01).into_bits(); + t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into_bits(), 0b10_11_00_01).into_bits(); + t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into_bits(), 0b10_11_00_01).into_bits(); + t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into_bits(), 0b10_11_00_01).into_bits(); // Set t0 = (X1+Y1 X1+Y1 X1+Y1 X1+Y1) t0.0[0] = t0.0[0] + t1.0[0]; @@ -118,19 +115,19 @@ impl ExtendedPoint { // Set t0 = (X1 Y1 Z1 X1+Y1) // why does this intrinsic take an i32 for the imm8 ??? - t0.0[0] = _mm256_blend_epi32(P.0[0].into(), t0.0[0].into(), D_LANES as i32).into(); - t0.0[1] = _mm256_blend_epi32(P.0[1].into(), t0.0[1].into(), D_LANES as i32).into(); - t0.0[2] = _mm256_blend_epi32(P.0[2].into(), t0.0[2].into(), D_LANES as i32).into(); - t0.0[3] = _mm256_blend_epi32(P.0[3].into(), t0.0[3].into(), D_LANES as i32).into(); - t0.0[4] = _mm256_blend_epi32(P.0[4].into(), t0.0[4].into(), D_LANES as i32).into(); + t0.0[0] = _mm256_blend_epi32(P.0[0].into_bits(), t0.0[0].into_bits(), D_LANES as i32).into_bits(); + t0.0[1] = _mm256_blend_epi32(P.0[1].into_bits(), t0.0[1].into_bits(), D_LANES as i32).into_bits(); + t0.0[2] = _mm256_blend_epi32(P.0[2].into_bits(), t0.0[2].into_bits(), D_LANES as i32).into_bits(); + t0.0[3] = _mm256_blend_epi32(P.0[3].into_bits(), t0.0[3].into_bits(), D_LANES as i32).into_bits(); + t0.0[4] = _mm256_blend_epi32(P.0[4].into_bits(), t0.0[4].into_bits(), D_LANES as i32).into_bits(); // Set t1 = t0^2, negating the D values - t1 = t0.square(D_LANES64); + t1 = t0.square_and_negate_D(); // Now t1 = (S1 S2 S3 -S4) - let c0 = u32x8::new(0,0,2,2,0,0,2,2); // (ABCD) -> (AAAA) - let c1 = u32x8::new(1,1,3,3,1,1,3,3); // (ABCD) -> (BBBB) + let c0 = u32x8::new(0,0,2,2,0,0,2,2).into_bits(); // (ABCD) -> (AAAA) + let c1 = u32x8::new(1,1,3,3,1,1,3,3).into_bits(); // (ABCD) -> (BBBB) // See discussion of bounds in the module-level documentation. // @@ -147,24 +144,24 @@ impl ExtendedPoint { // S5 S6 S8 S9 // for i in 0..5 { - let zero = i32x8::splat(0); - let S1 = _mm256_permutevar8x32_epi32(t1.0[i], c0); - let S2 = _mm256_permutevar8x32_epi32(t1.0[i], c1); - let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into(), 0b01010000).into(); + let zero = u32x8::splat(0).into_bits(); + let S1: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c0).into_bits(); + let S2: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c1).into_bits(); + let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into_bits(), 0b01010000).into_bits(); // tmp0 = (0 0 2*S3 -S4) - let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into(), t1.0[i].into(), 0b10100000).into(); + let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into_bits(), t1.0[i].into_bits(), 0b10100000).into_bits(); t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1; - t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into(), 0b10100101).into(); - t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into(), zero, 0b10100101).into(); + t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into_bits(), 0b10100101).into_bits(); + t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into_bits(), zero, 0b10100101).into_bits(); } - let c0 = u32x8::new(4,0,6,2,4,0,6,2); // (ABCD) -> (CACA) - let c1 = u32x8::new(5,1,7,3,1,5,3,7); // (ABCD) -> (DBBD) + let c0 = u32x8::new(4,0,6,2,4,0,6,2).into_bits(); // (ABCD) -> (CACA) + let c1 = u32x8::new(5,1,7,3,1,5,3,7).into_bits(); // (ABCD) -> (DBBD) for i in 0..5 { let tmp = t0.0[i]; - t0.0[i] = _mm256_permutevar8x32_epi32(tmp, c0); - t1.0[i] = _mm256_permutevar8x32_epi32(tmp, c1); + t0.0[i] = _mm256_permutevar8x32_epi32(tmp.into_bits(), c0).into_bits(); + t1.0[i] = _mm256_permutevar8x32_epi32(tmp.into_bits(), c1).into_bits(); } ExtendedPoint(&t0 * &t1) @@ -189,13 +186,13 @@ impl From for CachedPoint { let mut x = P.0; // x = (S2 S3 Z2 T2) - x.diff_sum(0b00001111); + x.diff_sum(Lanes::AB); // x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2) x.scale_by_curve_constants(); // x = (121666*S2 121666*S3 2*121666*Z2 -2*121665*T2) - x.negate(D_LANES); + x.negate_D(); CachedPoint(x) } @@ -231,7 +228,7 @@ impl<'a> Neg for &'a CachedPoint { fn neg(self) -> CachedPoint { let mut neg = *self; neg.0.swap_AB(); - neg.0.negate_lazy(D_LANES); + neg.0.negate_D_lazy(); neg } } @@ -242,12 +239,12 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { /// Uses a slight tweak of the parallel unified formulas of HWCD'08 fn add(self, other: &'b CachedPoint) -> ExtendedPoint { unsafe { - use stdsimd::vendor::_mm256_permutevar8x32_epi32; + use core::arch::x86_64::_mm256_permutevar8x32_epi32; let mut tmp = self.0; // tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1) - tmp.diff_sum(A_LANES | B_LANES); + tmp.diff_sum(Lanes::AB); // tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11) tmp = &tmp * &other.0; @@ -256,7 +253,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { tmp.swap_CD(); // tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) - tmp.diff_sum(ALL_LANES); + tmp.diff_sum(Lanes::ALL); let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) @@ -266,8 +263,8 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { let mut t0 = FieldElement32x4::zero(); let mut t1 = FieldElement32x4::zero(); for i in 0..5 { - t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c0); - t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c1); + t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i].into_bits(), c0.into_bits()).into_bits(); + t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i].into_bits(), c1.into_bits()).into_bits(); } // return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 125aa55..2af68b2 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -9,7 +9,6 @@ // - Henry de Valence //! 4-way vectorized 32bit field arithmetic using AVX2. -//! #![allow(bad_style)] @@ -26,13 +25,31 @@ pub const D_LANES64: u8 = 0b11_00_00_00; pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES; use core::ops::Mul; - -use stdsimd::simd::{u32x8, i32x8, u64x4}; +use core::simd::{IntoBits, u32x8, i32x8, u64x4}; use backend::u64::field::FieldElement64; - use backend::avx2::constants::{P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_16_LO, P_TIMES_16_HI}; +#[derive(Copy, Clone)] +pub enum Lanes { + AB, + CD, + ALL, +} + +#[inline(always)] +fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 { + unsafe { + use core::arch::x86_64::_mm256_blend_epi32; + + match control { + Lanes::AB => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32).into_bits(), + Lanes::CD => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32).into_bits(), + Lanes::ALL => _mm256_blend_epi32(x.into_bits(), y.into_bits(), ALL_LANES as i32).into_bits(), + } + } +} + /// A vector of four `FieldElements`, implemented using AVX2. #[derive(Clone, Copy, Debug)] pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]); @@ -50,7 +67,6 @@ impl ConditionallyAssignable for FieldElement32x4 { } } - impl FieldElement32x4 { pub(crate) fn split(&self) -> [FieldElement64; 4] { let mut out = [FieldElement64::zero(); 4]; @@ -108,28 +124,31 @@ impl FieldElement32x4 { return out; } - pub fn negate_lazy(&mut self, mask: u8) { - let mask = mask as i32; + /// Negate the \\(D\\) variable of \\((A,B,C,D)\\). + /// + /// Input limbs must be less than the limbs of \\(2p\\), i.e., freshly reduced. + pub fn negate_D_lazy(&mut self) { unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_2_LO - self.0[0]).into(), mask).into(); - self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_2_HI - self.0[1]).into(), mask).into(); - self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_2_HI - self.0[2]).into(), mask).into(); - self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_2_HI - self.0[3]).into(), mask).into(); - self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_2_HI - self.0[4]).into(), mask).into(); + use core::arch::x86_64::_mm256_blend_epi32; + self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_2_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits(); + self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_2_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits(); + self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_2_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits(); + self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_2_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits(); + self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_2_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits(); } } - /// Negate variables in lanes where mask is set - pub fn negate(&mut self, mask: u8) { - let mask = mask as i32; + /// Negate the \\(D\\) variable of \\((A,B,C,D)\\). + /// + /// Input limbs must be less than the limbs of \\(2p\\), i.e., freshly reduced. + pub fn negate_D(&mut self) { unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_16_LO - self.0[0]).into(), mask).into(); - self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_16_HI - self.0[1]).into(), mask).into(); - self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_16_HI - self.0[2]).into(), mask).into(); - self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_16_HI - self.0[3]).into(), mask).into(); - self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_16_HI - self.0[4]).into(), mask).into(); + use core::arch::x86_64::_mm256_blend_epi32; + self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_16_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits(); + self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_16_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits(); + self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_16_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits(); + self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_16_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits(); + self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_16_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits(); } self.reduce32(); } @@ -137,11 +156,11 @@ impl FieldElement32x4 { /// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)` pub fn swap_AB(&mut self) { unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - use stdsimd::vendor::_mm256_blend_epi32; + use core::arch::x86_64::_mm256_shuffle_epi32; + use core::arch::x86_64::_mm256_blend_epi32; for i in 0..5 { - let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01); - self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b00001111).into(); + let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01); + self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b00001111).into_bits(); } } } @@ -149,55 +168,61 @@ impl FieldElement32x4 { /// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` pub fn swap_CD(&mut self) { unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - use stdsimd::vendor::_mm256_blend_epi32; + use core::arch::x86_64::_mm256_shuffle_epi32; + use core::arch::x86_64::_mm256_blend_epi32; for i in 0..5 { - let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01); - self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b11110000).into(); + let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01); + self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b11110000).into_bits(); } } } /// Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)` according to `mask`. - pub fn diff_sum(&mut self, mask: u8) { - let mask = mask as i32; + /// + /// This is `#[inline(always)]` because the `mask` parameter should be an immediate. + #[inline(always)] + pub fn diff_sum(&mut self, control: Lanes) { unsafe { - use stdsimd::vendor::{_mm256_shuffle_epi32, _mm256_blend_epi32}; + use core::arch::x86_64::{_mm256_shuffle_epi32, _mm256_blend_epi32}; + + let shuffle = |v: u32x8| -> u32x8 { + _mm256_shuffle_epi32(v.into_bits(), 0b10_11_00_01).into_bits() + }; let x01 = self.0[0]; - let x01_shuf = _mm256_shuffle_epi32(x01.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x01_shuf = shuffle(x01); let v1 = (x01_shuf + P_TIMES_2_LO) - x01; let v2 = x01_shuf + x01; - let diffsum01 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[0] = _mm256_blend_epi32(x01.into(), diffsum01.into(), mask).into(); + let diffsum01 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[0] = blend_lanes(x01, diffsum01, control); let x23 = self.0[1]; - let x23_shuf = _mm256_shuffle_epi32(x23.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x23_shuf = shuffle(x23); let v1 = (x23_shuf + P_TIMES_2_HI) - x23; let v2 = x23_shuf + x23; - let diffsum23 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[1] = _mm256_blend_epi32(x23.into(), diffsum23.into(), mask).into(); + let diffsum23 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[1] = blend_lanes(x23, diffsum23, control); let x45 = self.0[2]; - let x45_shuf = _mm256_shuffle_epi32(x45.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x45_shuf = shuffle(x45); let v1 = (x45_shuf + P_TIMES_2_HI) - x45; let v2 = x45_shuf + x45; - let diffsum45 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[2] = _mm256_blend_epi32(x45.into(), diffsum45.into(), mask).into(); + let diffsum45 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[2] = blend_lanes(x45, diffsum45, control); let x67 = self.0[3]; - let x67_shuf = _mm256_shuffle_epi32(x67.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x67_shuf = shuffle(x67); let v1 = (x67_shuf + P_TIMES_2_HI) - x67; let v2 = x67_shuf + x67; - let diffsum67 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[3] = _mm256_blend_epi32(x67.into(), diffsum67.into(), mask).into(); + let diffsum67 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[3] = blend_lanes(x67, diffsum67, control); let x89 = self.0[4]; - let x89_shuf = _mm256_shuffle_epi32(x89.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x89_shuf = shuffle(x89); let v1 = (x89_shuf + P_TIMES_2_HI) - x89; let v2 = x89_shuf + x89; - let diffsum89 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[4] = _mm256_blend_epi32(x89.into(), diffsum89.into(), mask).into(); + let diffsum89 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[4] = blend_lanes(x89, diffsum89, control); } } @@ -208,30 +233,30 @@ impl FieldElement32x4 { pub fn scale_by_curve_constants(&mut self) { let mut b = [u64x4::splat(0); 10]; - let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); + let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); unsafe { - use stdsimd::vendor::_mm256_mul_epu32; + use core::arch::x86_64::_mm256_mul_epu32; let (b0, b1) = unpack_pair(self.0[0]); - b[0] = _mm256_mul_epu32(b0, consts); - b[1] = _mm256_mul_epu32(b1, consts); + b[0] = _mm256_mul_epu32(b0.into_bits(), consts.into_bits()).into_bits(); + b[1] = _mm256_mul_epu32(b1.into_bits(), consts.into_bits()).into_bits(); let (b2, b3) = unpack_pair(self.0[1]); - b[2] = _mm256_mul_epu32(b2, consts); - b[3] = _mm256_mul_epu32(b3, consts); + b[2] = _mm256_mul_epu32(b2.into_bits(), consts.into_bits()).into_bits(); + b[3] = _mm256_mul_epu32(b3.into_bits(), consts.into_bits()).into_bits(); let (b4, b5) = unpack_pair(self.0[2]); - b[4] = _mm256_mul_epu32(b4, consts); - b[5] = _mm256_mul_epu32(b5, consts); + b[4] = _mm256_mul_epu32(b4.into_bits(), consts.into_bits()).into_bits(); + b[5] = _mm256_mul_epu32(b5.into_bits(), consts.into_bits()).into_bits(); let (b6, b7) = unpack_pair(self.0[3]); - b[6] = _mm256_mul_epu32(b6, consts); - b[7] = _mm256_mul_epu32(b7, consts); + b[6] = _mm256_mul_epu32(b6.into_bits(), consts.into_bits()).into_bits(); + b[7] = _mm256_mul_epu32(b7.into_bits(), consts.into_bits()).into_bits(); let (b8, b9) = unpack_pair(self.0[4]); - b[8] = _mm256_mul_epu32(b8, consts); - b[9] = _mm256_mul_epu32(b9, consts); + b[8] = _mm256_mul_epu32(b8.into_bits(), consts.into_bits()).into_bits(); + b[9] = _mm256_mul_epu32(b9.into_bits(), consts.into_bits()).into_bits(); } *self = FieldElement32x4::reduce64(b); @@ -243,24 +268,25 @@ impl FieldElement32x4 { let masks = u32x8::new((1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1, (1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1); + /// Compute the carry-out of each coefficient of v using a left shift. let carry = |v: u32x8| -> u32x8 { unsafe { - use stdsimd::vendor::_mm256_srlv_epi32; - _mm256_srlv_epi32(v.into(), shifts).into() + use core::arch::x86_64::_mm256_srlv_epi32; + _mm256_srlv_epi32(v.into_bits(), shifts.into_bits()).into_bits() } }; let swap_lanes = |v: u32x8| -> u32x8 { unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - _mm256_shuffle_epi32(v.into(), 0b01_00_11_10).into() + use core::arch::x86_64::_mm256_shuffle_epi32; + _mm256_shuffle_epi32(v.into_bits(), 0b01_00_11_10).into_bits() } }; let combine = |v_lo: u32x8, v_hi: u32x8| -> u32x8 { unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - _mm256_blend_epi32(v_lo.into(), v_hi.into(), 0b11_00_11_00).into() + use core::arch::x86_64::_mm256_blend_epi32; + _mm256_blend_epi32(v_lo.into_bits(), v_hi.into_bits(), 0b11_00_11_00).into_bits() } }; @@ -282,11 +308,11 @@ impl FieldElement32x4 { // let c9_19: u32x8; unsafe { - use stdsimd::vendor::_mm256_mul_epu32; - use stdsimd::vendor::_mm256_shuffle_epi32; - let c9_spread: u32x8 = _mm256_shuffle_epi32(c98.into(), 0b11_01_10_00).into(); - let c9_19_spread: u32x8 = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into()).into(); - c9_19 = _mm256_shuffle_epi32(c9_19_spread.into(), 0b11_01_10_00).into(); + use core::arch::x86_64::_mm256_mul_epu32; + use core::arch::x86_64::_mm256_shuffle_epi32; + let c9_spread = _mm256_shuffle_epi32(c98.into_bits(), 0b11_01_10_00); + let c9_19_spread = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into_bits()); + c9_19 = _mm256_shuffle_epi32(c9_19_spread, 0b11_01_10_00).into_bits(); } v[0] = v[0] + c9_19; @@ -331,14 +357,14 @@ impl FieldElement32x4 { let c = z[9] >> 25; z[9] = z[9] & LOW_25_BITS; - let mut c0 = c & LOW_26_BITS; // c0 < 2^26; - let mut c1 = c >> 26; // c1 < 2^(39-26) = 2^13; + let mut c0: u64x4 = c & LOW_26_BITS; // c0 < 2^26; + let mut c1: u64x4 = c >> 26; // c1 < 2^(39-26) = 2^13; unsafe { - use stdsimd::vendor::_mm256_mul_epu32; - let x19 = u32x8::from(u64x4::splat(19)); - c0 = _mm256_mul_epu32(u32x8::from(c0), x19); // c0 < 2^30.25 - c1 = _mm256_mul_epu32(u32x8::from(c1), x19); // c1 < 2^17.25 + use core::arch::x86_64::_mm256_mul_epu32; + let x19 = u64x4::splat(19); + c0 = _mm256_mul_epu32(c0.into_bits(), x19.into_bits()).into_bits(); // c0 < 2^30.25 + c1 = _mm256_mul_epu32(c1.into_bits(), x19.into_bits()).into_bits(); // c1 < 2^17.25 } z[0] = z[0] + c0; // z0 < 2^26 + 2^30.25 < 2^30.33 @@ -348,11 +374,11 @@ impl FieldElement32x4 { // Now repack the [u64x4; 10] into a FieldElement32x4 FieldElement32x4([ - repack_pair(z[0].into(), z[1].into()), - repack_pair(z[2].into(), z[3].into()), - repack_pair(z[4].into(), z[5].into()), - repack_pair(z[6].into(), z[7].into()), - repack_pair(z[8].into(), z[9].into()), + repack_pair(z[0].into_bits(), z[1].into_bits()), + repack_pair(z[2].into_bits(), z[3].into_bits()), + repack_pair(z[4].into_bits(), z[5].into_bits()), + repack_pair(z[6].into_bits(), z[7].into_bits()), + repack_pair(z[8].into_bits(), z[9].into_bits()), ]) } } @@ -363,10 +389,10 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { let b: u32x8; let zero = i32x8::new(0,0,0,0,0,0,0,0); unsafe { - use stdsimd::vendor::_mm256_unpackhi_epi32; - use stdsimd::vendor::_mm256_unpacklo_epi32; - a = _mm256_unpacklo_epi32(src.as_i32x8(), zero).as_u32x8(); - b = _mm256_unpackhi_epi32(src.as_i32x8(), zero).as_u32x8(); + use core::arch::x86_64::_mm256_unpackhi_epi32; + use core::arch::x86_64::_mm256_unpacklo_epi32; + a = _mm256_unpacklo_epi32(src.into_bits(), zero.into_bits()).into_bits(); + b = _mm256_unpackhi_epi32(src.into_bits(), zero.into_bits()).into_bits(); } (a,b) } @@ -374,40 +400,43 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { #[inline(always)] pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - use stdsimd::vendor::_mm256_blend_epi32; + use core::arch::x86_64::_mm256_shuffle_epi32; + use core::arch::x86_64::_mm256_blend_epi32; // Input: x = (a0, 0, b0, 0, c0, 0, d0) // Input: y = (a1, 0, b1, 0, c1, 0, d1) - let x_shuffled = _mm256_shuffle_epi32(x.into(), 0b11_01_10_00); - let y_shuffled = _mm256_shuffle_epi32(y.into(), 0b10_00_11_01); + let x_shuffled = _mm256_shuffle_epi32(x.into_bits(), 0b11_01_10_00); + let y_shuffled = _mm256_shuffle_epi32(y.into_bits(), 0b10_00_11_01); // x' = (a0, b0, 0, 0, c0, d0, 0, 0) // y' = ( 0, 0, a1, b1, 0, 0, c1, d1) - return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).as_u32x8(); + return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).into_bits(); } } impl FieldElement32x4 { - /// Square this field element, then conditionally negate according to `neg_mask`; for instance, - /// `neg_mask = 0b11_00_00_00` negates the \\( D \\) value. + /// Square this field element, then conditionally negate according + /// to `neg_mask`. This parameter is hardcoded as `neg_mask = + /// D_LANES64` to negate the \\( D \\) value. /// /// # Precondition /// /// Limbs must be bounded by bit-excess \\( b < 2.0 \\). - pub fn square(&self, neg_mask: u8) -> FieldElement32x4 { + pub fn square_and_negate_D(&self) -> FieldElement32x4 { + let neg_mask = D_LANES64; + #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { - use stdsimd::vendor::_mm256_mul_epu32; - unsafe { _mm256_mul_epu32(x,y) } + use core::arch::x86_64::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } } #[inline(always)] fn m_lo(x: u32x8, y: u32x8) -> u32x8 { - use stdsimd::vendor::_mm256_mul_epu32; - unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } + use core::arch::x86_64::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } } let v19 = u32x8::new(19,0,19,0,19,0,19,0); @@ -444,14 +473,6 @@ impl FieldElement32x4 { let mut z8 = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1); let mut z9 = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5); - #[inline(always)] - fn mask_neg(x: u64x4, p: u64x4, mask: u8) -> u64x4 { - unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - _mm256_blend_epi32(x.into(), (p - x).into(), mask as i32).into() - } - } - // The biggest z_i is bounded as z_i < 249*2^(51 + 2*b); // if b < 1.5 we get z_i < 4485585228861014016. // @@ -472,16 +493,23 @@ impl FieldElement32x4 { let even_p37 = u64x4::splat(0x3ffffff << 37); let odd__p37 = u64x4::splat(0x1ffffff << 37); - z0 = mask_neg(z0, low__p37, neg_mask); - z1 = mask_neg(z1, odd__p37, neg_mask); - z2 = mask_neg(z2, even_p37, neg_mask); - z3 = mask_neg(z3, odd__p37, neg_mask); - z4 = mask_neg(z4, even_p37, neg_mask); - z5 = mask_neg(z5, odd__p37, neg_mask); - z6 = mask_neg(z6, even_p37, neg_mask); - z7 = mask_neg(z7, odd__p37, neg_mask); - z8 = mask_neg(z8, even_p37, neg_mask); - z9 = mask_neg(z9, odd__p37, neg_mask); + let negate_D = |x: u64x4, p: u64x4| -> u64x4 { + unsafe { + use core::arch::x86_64::_mm256_blend_epi32; + _mm256_blend_epi32(x.into_bits(), (p - x).into_bits(), D_LANES64 as i32).into_bits() + } + }; + + z0 = negate_D(z0, low__p37); + z1 = negate_D(z1, odd__p37); + z2 = negate_D(z2, even_p37); + z3 = negate_D(z3, odd__p37); + z4 = negate_D(z4, even_p37); + z5 = negate_D(z5, odd__p37); + z6 = negate_D(z6, even_p37); + z7 = negate_D(z7, odd__p37); + z8 = negate_D(z8, even_p37); + z9 = negate_D(z9, odd__p37); FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } @@ -490,17 +518,16 @@ impl FieldElement32x4 { impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { type Output = FieldElement32x4; fn mul(self, _rhs: &'b FieldElement32x4) -> FieldElement32x4 { - #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { - use stdsimd::vendor::_mm256_mul_epu32; - unsafe { _mm256_mul_epu32(x,y) } + use core::arch::x86_64::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } } #[inline(always)] fn m_lo(x: u32x8, y: u32x8) -> u32x8 { - use stdsimd::vendor::_mm256_mul_epu32; - unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } + use core::arch::x86_64::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } } let (x0, x1) = unpack_pair(self.0[0]); @@ -573,7 +600,7 @@ mod test { let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); - vec.diff_sum(0xff); + vec.diff_sum(Lanes::ALL); let result = vec.split(); @@ -583,13 +610,13 @@ mod test { assert_eq!(result[3], &x3 + &x2); let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); - vec.diff_sum(0b01011111); // leave D unchanged + vec.diff_sum(Lanes::AB); // leave C,D unchanged let result = vec.split(); assert_eq!(result[0], &x1 - &x0); assert_eq!(result[1], &x1 + &x0); - assert_eq!(result[2], &x3 - &x2); + assert_eq!(result[2], x2); assert_eq!(result[3], x3); } @@ -602,9 +629,7 @@ mod test { let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); - let neg_mask = 0b11_00_00_00; - - let result = vec.square(neg_mask).split(); + let result = vec.square_and_negate_D().split(); assert_eq!(result[0], &x0 * &x0); assert_eq!(result[1], &x1 * &x1); diff --git a/src/lib.rs b/src/lib.rs index ffe0009..0a8a1a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ #![cfg_attr(feature = "nightly", feature(i128_type))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "nightly", feature(external_doc))] +#![cfg_attr(all(feature = "nightly", feature = "yolocrypto"), feature(stdsimd))] // Refuse to compile if documentation is missing, but only on nightly. // @@ -40,9 +41,6 @@ extern crate alloc; extern crate clear_on_drop; -#[cfg(feature = "yolocrypto")] -extern crate stdsimd; - // The `Digest` trait is implemented using `generic_array`, so we need it // too. Hopefully we can eliminate `generic_array` from `Digest` once const // generics land.