mirror of
https://github.com/saymrwulf/curve25519-dalek-source.git
synced 2026-09-04 20:24:10 +00:00
Merge pull request #122 from hdevalence/feature/drop-stdsimd
Move from external `stdsimd` crate to `core::{arch, simd}` intrinsics
This commit is contained in:
Henry de Valence
GitHub
commit
46d8f9c45d
6 changed files with 199 additions and 187 deletions
|
|
@ -45,7 +45,6 @@ digest = "0.7"
|
|||
generic-array = "0.9"
|
||||
clear_on_drop = "=0.2.3"
|
||||
subtle = { version = "0.6", features = ["generic-impls"], default-features = false }
|
||||
stdsimd = { version = "0.0.4", optional = true }
|
||||
serde = { version = "1.0", optional = true }
|
||||
rand = { version = "0.4", optional = true }
|
||||
|
||||
|
|
@ -54,7 +53,6 @@ digest = "0.7"
|
|||
generic-array = "0.9"
|
||||
clear_on_drop = "=0.2.3"
|
||||
subtle = { version = "0.6", features = ["generic-impls"], default-features = false }
|
||||
stdsimd = { version = "0.0.4", optional = true }
|
||||
serde = { version = "1.0", optional = true }
|
||||
# Allowing rand to be optional during builds causes a build failure when compiling for no_std targets
|
||||
rand = { version = "0.4", optional = false }
|
||||
|
|
@ -70,4 +68,4 @@ radix_51 = []
|
|||
# Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation.
|
||||
precomputed_tables = []
|
||||
# experimental avx2 support
|
||||
avx2_backend = ["nightly", "stdsimd"]
|
||||
avx2_backend = ["nightly"]
|
||||
|
|
|
|||
3
build.rs
3
build.rs
|
|
@ -1,5 +1,6 @@
|
|||
#![cfg_attr(feature = "nightly", feature(i128_type))]
|
||||
#![cfg_attr(feature = "nightly", feature(cfg_target_feature))]
|
||||
#![cfg_attr(all(feature = "nightly", feature = "yolocrypto"), feature(stdsimd))]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(non_snake_case)]
|
||||
#![allow(dead_code)]
|
||||
|
|
@ -23,8 +24,6 @@ use std::path::Path;
|
|||
// For instance, this shouldn't exist here at all, but it does.
|
||||
#[cfg(feature = "serde")]
|
||||
extern crate serde;
|
||||
#[cfg(feature = "yolocrypto")]
|
||||
extern crate stdsimd;
|
||||
|
||||
// Macros come first!
|
||||
#[path = "src/macros.rs"]
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
|
||||
//! This module contains constants used by the AVX2 backend.
|
||||
|
||||
use stdsimd::simd::u32x8;
|
||||
use core::simd::u32x8;
|
||||
|
||||
use scalar_mul::window::OddLookupTable;
|
||||
use backend::avx2::field::FieldElement32x4;
|
||||
|
|
|
|||
|
|
@ -14,24 +14,19 @@
|
|||
#![allow(bad_style)]
|
||||
|
||||
use core::convert::From;
|
||||
use core::ops::{Index, Add, Sub, Mul, Neg};
|
||||
use core::borrow::Borrow;
|
||||
use core::ops::{Add, Sub, Neg};
|
||||
|
||||
use stdsimd::simd::{u32x8, i32x8};
|
||||
use core::simd::{IntoBits, u32x8};
|
||||
|
||||
use subtle::ConditionallyAssignable;
|
||||
use subtle::Choice;
|
||||
|
||||
use edwards;
|
||||
use scalar::Scalar;
|
||||
use scalar_mul::window::{LookupTable, OddLookupTable};
|
||||
|
||||
use traits::Identity;
|
||||
|
||||
use backend::avx2::field::FieldElement32x4;
|
||||
|
||||
use backend::avx2::field::{A_LANES, B_LANES, C_LANES, D_LANES, ALL_LANES};
|
||||
use backend::avx2::field::D_LANES64;
|
||||
use backend::avx2::field::{D_LANES, Lanes, FieldElement32x4};
|
||||
|
||||
use backend::avx2;
|
||||
|
||||
|
|
@ -79,10 +74,10 @@ impl Identity for ExtendedPoint {
|
|||
impl ExtendedPoint {
|
||||
pub fn double(&self) -> ExtendedPoint {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_permute2x128_si256;
|
||||
use stdsimd::vendor::_mm256_permutevar8x32_epi32;
|
||||
use stdsimd::vendor::_mm256_blend_epi32;
|
||||
use stdsimd::vendor::_mm256_shuffle_epi32;
|
||||
use core::arch::x86_64::_mm256_permute2x128_si256;
|
||||
use core::arch::x86_64::_mm256_permutevar8x32_epi32;
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
use core::arch::x86_64::_mm256_shuffle_epi32;
|
||||
|
||||
let P = &self.0;
|
||||
|
||||
|
|
@ -96,18 +91,18 @@ impl ExtendedPoint {
|
|||
// and then adding.
|
||||
|
||||
// Set t0 = (X1 Y1 X1 Y1)
|
||||
t0.0[0] = _mm256_permute2x128_si256(P.0[0].into(), P.0[0].into(), 0b0000_0000).into();
|
||||
t0.0[1] = _mm256_permute2x128_si256(P.0[1].into(), P.0[1].into(), 0b0000_0000).into();
|
||||
t0.0[2] = _mm256_permute2x128_si256(P.0[2].into(), P.0[2].into(), 0b0000_0000).into();
|
||||
t0.0[3] = _mm256_permute2x128_si256(P.0[3].into(), P.0[3].into(), 0b0000_0000).into();
|
||||
t0.0[4] = _mm256_permute2x128_si256(P.0[4].into(), P.0[4].into(), 0b0000_0000).into();
|
||||
t0.0[0] = _mm256_permute2x128_si256(P.0[0].into_bits(), P.0[0].into_bits(), 0b0000_0000).into_bits();
|
||||
t0.0[1] = _mm256_permute2x128_si256(P.0[1].into_bits(), P.0[1].into_bits(), 0b0000_0000).into_bits();
|
||||
t0.0[2] = _mm256_permute2x128_si256(P.0[2].into_bits(), P.0[2].into_bits(), 0b0000_0000).into_bits();
|
||||
t0.0[3] = _mm256_permute2x128_si256(P.0[3].into_bits(), P.0[3].into_bits(), 0b0000_0000).into_bits();
|
||||
t0.0[4] = _mm256_permute2x128_si256(P.0[4].into_bits(), P.0[4].into_bits(), 0b0000_0000).into_bits();
|
||||
|
||||
// Set t1 = (Y1 X1 Y1 X1)
|
||||
t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into(), 0b10_11_00_01).into();
|
||||
t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into(), 0b10_11_00_01).into();
|
||||
t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into(), 0b10_11_00_01).into();
|
||||
t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into(), 0b10_11_00_01).into();
|
||||
t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into(), 0b10_11_00_01).into();
|
||||
t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into_bits(), 0b10_11_00_01).into_bits();
|
||||
t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into_bits(), 0b10_11_00_01).into_bits();
|
||||
t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into_bits(), 0b10_11_00_01).into_bits();
|
||||
t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into_bits(), 0b10_11_00_01).into_bits();
|
||||
t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into_bits(), 0b10_11_00_01).into_bits();
|
||||
|
||||
// Set t0 = (X1+Y1 X1+Y1 X1+Y1 X1+Y1)
|
||||
t0.0[0] = t0.0[0] + t1.0[0];
|
||||
|
|
@ -118,19 +113,19 @@ impl ExtendedPoint {
|
|||
|
||||
// Set t0 = (X1 Y1 Z1 X1+Y1)
|
||||
// why does this intrinsic take an i32 for the imm8 ???
|
||||
t0.0[0] = _mm256_blend_epi32(P.0[0].into(), t0.0[0].into(), D_LANES as i32).into();
|
||||
t0.0[1] = _mm256_blend_epi32(P.0[1].into(), t0.0[1].into(), D_LANES as i32).into();
|
||||
t0.0[2] = _mm256_blend_epi32(P.0[2].into(), t0.0[2].into(), D_LANES as i32).into();
|
||||
t0.0[3] = _mm256_blend_epi32(P.0[3].into(), t0.0[3].into(), D_LANES as i32).into();
|
||||
t0.0[4] = _mm256_blend_epi32(P.0[4].into(), t0.0[4].into(), D_LANES as i32).into();
|
||||
t0.0[0] = _mm256_blend_epi32(P.0[0].into_bits(), t0.0[0].into_bits(), D_LANES as i32).into_bits();
|
||||
t0.0[1] = _mm256_blend_epi32(P.0[1].into_bits(), t0.0[1].into_bits(), D_LANES as i32).into_bits();
|
||||
t0.0[2] = _mm256_blend_epi32(P.0[2].into_bits(), t0.0[2].into_bits(), D_LANES as i32).into_bits();
|
||||
t0.0[3] = _mm256_blend_epi32(P.0[3].into_bits(), t0.0[3].into_bits(), D_LANES as i32).into_bits();
|
||||
t0.0[4] = _mm256_blend_epi32(P.0[4].into_bits(), t0.0[4].into_bits(), D_LANES as i32).into_bits();
|
||||
|
||||
// Set t1 = t0^2, negating the D values
|
||||
t1 = t0.square(D_LANES64);
|
||||
t1 = t0.square_and_negate_D();
|
||||
|
||||
// Now t1 = (S1 S2 S3 -S4)
|
||||
|
||||
let c0 = u32x8::new(0,0,2,2,0,0,2,2); // (ABCD) -> (AAAA)
|
||||
let c1 = u32x8::new(1,1,3,3,1,1,3,3); // (ABCD) -> (BBBB)
|
||||
let c0 = u32x8::new(0,0,2,2,0,0,2,2).into_bits(); // (ABCD) -> (AAAA)
|
||||
let c1 = u32x8::new(1,1,3,3,1,1,3,3).into_bits(); // (ABCD) -> (BBBB)
|
||||
|
||||
// See discussion of bounds in the module-level documentation.
|
||||
//
|
||||
|
|
@ -147,24 +142,24 @@ impl ExtendedPoint {
|
|||
// S5 S6 S8 S9
|
||||
//
|
||||
for i in 0..5 {
|
||||
let zero = i32x8::splat(0);
|
||||
let S1 = _mm256_permutevar8x32_epi32(t1.0[i], c0);
|
||||
let S2 = _mm256_permutevar8x32_epi32(t1.0[i], c1);
|
||||
let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into(), 0b01010000).into();
|
||||
let zero = u32x8::splat(0).into_bits();
|
||||
let S1: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c0).into_bits();
|
||||
let S2: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c1).into_bits();
|
||||
let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into_bits(), 0b01010000).into_bits();
|
||||
// tmp0 = (0 0 2*S3 -S4)
|
||||
let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into(), t1.0[i].into(), 0b10100000).into();
|
||||
let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into_bits(), t1.0[i].into_bits(), 0b10100000).into_bits();
|
||||
t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1;
|
||||
t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into(), 0b10100101).into();
|
||||
t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into(), zero, 0b10100101).into();
|
||||
t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into_bits(), 0b10100101).into_bits();
|
||||
t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into_bits(), zero, 0b10100101).into_bits();
|
||||
}
|
||||
|
||||
let c0 = u32x8::new(4,0,6,2,4,0,6,2); // (ABCD) -> (CACA)
|
||||
let c1 = u32x8::new(5,1,7,3,1,5,3,7); // (ABCD) -> (DBBD)
|
||||
let c0 = u32x8::new(4,0,6,2,4,0,6,2).into_bits(); // (ABCD) -> (CACA)
|
||||
let c1 = u32x8::new(5,1,7,3,1,5,3,7).into_bits(); // (ABCD) -> (DBBD)
|
||||
|
||||
for i in 0..5 {
|
||||
let tmp = t0.0[i];
|
||||
t0.0[i] = _mm256_permutevar8x32_epi32(tmp, c0);
|
||||
t1.0[i] = _mm256_permutevar8x32_epi32(tmp, c1);
|
||||
t0.0[i] = _mm256_permutevar8x32_epi32(tmp.into_bits(), c0).into_bits();
|
||||
t1.0[i] = _mm256_permutevar8x32_epi32(tmp.into_bits(), c1).into_bits();
|
||||
}
|
||||
|
||||
ExtendedPoint(&t0 * &t1)
|
||||
|
|
@ -189,13 +184,13 @@ impl From<ExtendedPoint> for CachedPoint {
|
|||
let mut x = P.0;
|
||||
|
||||
// x = (S2 S3 Z2 T2)
|
||||
x.diff_sum(0b00001111);
|
||||
x.diff_sum(Lanes::AB);
|
||||
|
||||
// x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2)
|
||||
x.scale_by_curve_constants();
|
||||
|
||||
// x = (121666*S2 121666*S3 2*121666*Z2 -2*121665*T2)
|
||||
x.negate(D_LANES);
|
||||
x.negate_D();
|
||||
|
||||
CachedPoint(x)
|
||||
}
|
||||
|
|
@ -231,7 +226,7 @@ impl<'a> Neg for &'a CachedPoint {
|
|||
fn neg(self) -> CachedPoint {
|
||||
let mut neg = *self;
|
||||
neg.0.swap_AB();
|
||||
neg.0.negate_lazy(D_LANES);
|
||||
neg.0.negate_D_lazy();
|
||||
neg
|
||||
}
|
||||
}
|
||||
|
|
@ -242,12 +237,12 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
|
|||
/// Uses a slight tweak of the parallel unified formulas of HWCD'08
|
||||
fn add(self, other: &'b CachedPoint) -> ExtendedPoint {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_permutevar8x32_epi32;
|
||||
use core::arch::x86_64::_mm256_permutevar8x32_epi32;
|
||||
|
||||
let mut tmp = self.0;
|
||||
|
||||
// tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1)
|
||||
tmp.diff_sum(A_LANES | B_LANES);
|
||||
tmp.diff_sum(Lanes::AB);
|
||||
|
||||
// tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11)
|
||||
tmp = &tmp * &other.0;
|
||||
|
|
@ -256,7 +251,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
|
|||
tmp.swap_CD();
|
||||
|
||||
// tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15)
|
||||
tmp.diff_sum(ALL_LANES);
|
||||
tmp.diff_sum(Lanes::ALL);
|
||||
|
||||
let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA)
|
||||
let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB)
|
||||
|
|
@ -266,8 +261,8 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
|
|||
let mut t0 = FieldElement32x4::zero();
|
||||
let mut t1 = FieldElement32x4::zero();
|
||||
for i in 0..5 {
|
||||
t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c0);
|
||||
t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c1);
|
||||
t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i].into_bits(), c0.into_bits()).into_bits();
|
||||
t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i].into_bits(), c1.into_bits()).into_bits();
|
||||
}
|
||||
|
||||
// return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3)
|
||||
|
|
@ -316,8 +311,6 @@ impl<'a> From<&'a edwards::EdwardsPoint> for OddLookupTable<CachedPoint> {
|
|||
mod test {
|
||||
use super::*;
|
||||
|
||||
use constants;
|
||||
|
||||
fn serial_add(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) -> edwards::EdwardsPoint {
|
||||
use backend::u64::field::FieldElement64;
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@
|
|||
// - Henry de Valence <hdevalence@hdevalence.ca>
|
||||
|
||||
//! 4-way vectorized 32bit field arithmetic using AVX2.
|
||||
//!
|
||||
|
||||
#![allow(bad_style)]
|
||||
|
||||
|
|
@ -26,13 +25,31 @@ pub const D_LANES64: u8 = 0b11_00_00_00;
|
|||
pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES;
|
||||
|
||||
use core::ops::Mul;
|
||||
|
||||
use stdsimd::simd::{u32x8, i32x8, u64x4};
|
||||
use core::simd::{IntoBits, u32x8, i32x8, u64x4};
|
||||
|
||||
use backend::u64::field::FieldElement64;
|
||||
|
||||
use backend::avx2::constants::{P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_16_LO, P_TIMES_16_HI};
|
||||
|
||||
#[derive(Copy, Clone)]
|
||||
pub enum Lanes {
|
||||
AB,
|
||||
CD,
|
||||
ALL,
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
|
||||
match control {
|
||||
Lanes::AB => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32).into_bits(),
|
||||
Lanes::CD => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32).into_bits(),
|
||||
Lanes::ALL => _mm256_blend_epi32(x.into_bits(), y.into_bits(), ALL_LANES as i32).into_bits(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A vector of four `FieldElements`, implemented using AVX2.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]);
|
||||
|
|
@ -50,7 +67,6 @@ impl ConditionallyAssignable for FieldElement32x4 {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
impl FieldElement32x4 {
|
||||
pub(crate) fn split(&self) -> [FieldElement64; 4] {
|
||||
let mut out = [FieldElement64::zero(); 4];
|
||||
|
|
@ -108,28 +124,31 @@ impl FieldElement32x4 {
|
|||
return out;
|
||||
}
|
||||
|
||||
pub fn negate_lazy(&mut self, mask: u8) {
|
||||
let mask = mask as i32;
|
||||
/// Negate the \\(D\\) variable of \\((A,B,C,D)\\).
|
||||
///
|
||||
/// Input limbs must be less than the limbs of \\(2p\\), i.e., freshly reduced.
|
||||
pub fn negate_D_lazy(&mut self) {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_blend_epi32;
|
||||
self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_2_LO - self.0[0]).into(), mask).into();
|
||||
self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_2_HI - self.0[1]).into(), mask).into();
|
||||
self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_2_HI - self.0[2]).into(), mask).into();
|
||||
self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_2_HI - self.0[3]).into(), mask).into();
|
||||
self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_2_HI - self.0[4]).into(), mask).into();
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_2_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits();
|
||||
self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_2_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits();
|
||||
self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_2_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits();
|
||||
self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_2_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits();
|
||||
self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_2_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits();
|
||||
}
|
||||
}
|
||||
|
||||
/// Negate variables in lanes where mask is set
|
||||
pub fn negate(&mut self, mask: u8) {
|
||||
let mask = mask as i32;
|
||||
/// Negate the \\(D\\) variable of \\((A,B,C,D)\\).
|
||||
///
|
||||
/// Input limbs must be less than the limbs of \\(2p\\), i.e., freshly reduced.
|
||||
pub fn negate_D(&mut self) {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_blend_epi32;
|
||||
self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_16_LO - self.0[0]).into(), mask).into();
|
||||
self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_16_HI - self.0[1]).into(), mask).into();
|
||||
self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_16_HI - self.0[2]).into(), mask).into();
|
||||
self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_16_HI - self.0[3]).into(), mask).into();
|
||||
self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_16_HI - self.0[4]).into(), mask).into();
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_16_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits();
|
||||
self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_16_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits();
|
||||
self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_16_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits();
|
||||
self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_16_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits();
|
||||
self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_16_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits();
|
||||
}
|
||||
self.reduce32();
|
||||
}
|
||||
|
|
@ -137,11 +156,11 @@ impl FieldElement32x4 {
|
|||
/// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)`
|
||||
pub fn swap_AB(&mut self) {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_shuffle_epi32;
|
||||
use stdsimd::vendor::_mm256_blend_epi32;
|
||||
use core::arch::x86_64::_mm256_shuffle_epi32;
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
for i in 0..5 {
|
||||
let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01);
|
||||
self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b00001111).into();
|
||||
let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01);
|
||||
self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b00001111).into_bits();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -149,55 +168,61 @@ impl FieldElement32x4 {
|
|||
/// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)`
|
||||
pub fn swap_CD(&mut self) {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_shuffle_epi32;
|
||||
use stdsimd::vendor::_mm256_blend_epi32;
|
||||
use core::arch::x86_64::_mm256_shuffle_epi32;
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
for i in 0..5 {
|
||||
let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01);
|
||||
self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b11110000).into();
|
||||
let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01);
|
||||
self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b11110000).into_bits();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)` according to `mask`.
|
||||
pub fn diff_sum(&mut self, mask: u8) {
|
||||
let mask = mask as i32;
|
||||
///
|
||||
/// This is `#[inline(always)]` because the `mask` parameter should be an immediate.
|
||||
#[inline(always)]
|
||||
pub fn diff_sum(&mut self, control: Lanes) {
|
||||
unsafe {
|
||||
use stdsimd::vendor::{_mm256_shuffle_epi32, _mm256_blend_epi32};
|
||||
use core::arch::x86_64::{_mm256_shuffle_epi32, _mm256_blend_epi32};
|
||||
|
||||
let shuffle = |v: u32x8| -> u32x8 {
|
||||
_mm256_shuffle_epi32(v.into_bits(), 0b10_11_00_01).into_bits()
|
||||
};
|
||||
|
||||
let x01 = self.0[0];
|
||||
let x01_shuf = _mm256_shuffle_epi32(x01.as_i32x8(), 0b10_11_00_01).as_u32x8();
|
||||
let x01_shuf = shuffle(x01);
|
||||
let v1 = (x01_shuf + P_TIMES_2_LO) - x01;
|
||||
let v2 = x01_shuf + x01;
|
||||
let diffsum01 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8();
|
||||
self.0[0] = _mm256_blend_epi32(x01.into(), diffsum01.into(), mask).into();
|
||||
let diffsum01 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
|
||||
self.0[0] = blend_lanes(x01, diffsum01, control);
|
||||
|
||||
let x23 = self.0[1];
|
||||
let x23_shuf = _mm256_shuffle_epi32(x23.as_i32x8(), 0b10_11_00_01).as_u32x8();
|
||||
let x23_shuf = shuffle(x23);
|
||||
let v1 = (x23_shuf + P_TIMES_2_HI) - x23;
|
||||
let v2 = x23_shuf + x23;
|
||||
let diffsum23 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8();
|
||||
self.0[1] = _mm256_blend_epi32(x23.into(), diffsum23.into(), mask).into();
|
||||
let diffsum23 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
|
||||
self.0[1] = blend_lanes(x23, diffsum23, control);
|
||||
|
||||
let x45 = self.0[2];
|
||||
let x45_shuf = _mm256_shuffle_epi32(x45.as_i32x8(), 0b10_11_00_01).as_u32x8();
|
||||
let x45_shuf = shuffle(x45);
|
||||
let v1 = (x45_shuf + P_TIMES_2_HI) - x45;
|
||||
let v2 = x45_shuf + x45;
|
||||
let diffsum45 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8();
|
||||
self.0[2] = _mm256_blend_epi32(x45.into(), diffsum45.into(), mask).into();
|
||||
let diffsum45 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
|
||||
self.0[2] = blend_lanes(x45, diffsum45, control);
|
||||
|
||||
let x67 = self.0[3];
|
||||
let x67_shuf = _mm256_shuffle_epi32(x67.as_i32x8(), 0b10_11_00_01).as_u32x8();
|
||||
let x67_shuf = shuffle(x67);
|
||||
let v1 = (x67_shuf + P_TIMES_2_HI) - x67;
|
||||
let v2 = x67_shuf + x67;
|
||||
let diffsum67 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8();
|
||||
self.0[3] = _mm256_blend_epi32(x67.into(), diffsum67.into(), mask).into();
|
||||
let diffsum67 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
|
||||
self.0[3] = blend_lanes(x67, diffsum67, control);
|
||||
|
||||
let x89 = self.0[4];
|
||||
let x89_shuf = _mm256_shuffle_epi32(x89.as_i32x8(), 0b10_11_00_01).as_u32x8();
|
||||
let x89_shuf = shuffle(x89);
|
||||
let v1 = (x89_shuf + P_TIMES_2_HI) - x89;
|
||||
let v2 = x89_shuf + x89;
|
||||
let diffsum89 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8();
|
||||
self.0[4] = _mm256_blend_epi32(x89.into(), diffsum89.into(), mask).into();
|
||||
let diffsum89 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
|
||||
self.0[4] = blend_lanes(x89, diffsum89, control);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -208,30 +233,30 @@ impl FieldElement32x4 {
|
|||
pub fn scale_by_curve_constants(&mut self) {
|
||||
let mut b = [u64x4::splat(0); 10];
|
||||
|
||||
let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0);
|
||||
let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0);
|
||||
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_mul_epu32;
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
|
||||
let (b0, b1) = unpack_pair(self.0[0]);
|
||||
b[0] = _mm256_mul_epu32(b0, consts);
|
||||
b[1] = _mm256_mul_epu32(b1, consts);
|
||||
b[0] = _mm256_mul_epu32(b0.into_bits(), consts.into_bits()).into_bits();
|
||||
b[1] = _mm256_mul_epu32(b1.into_bits(), consts.into_bits()).into_bits();
|
||||
|
||||
let (b2, b3) = unpack_pair(self.0[1]);
|
||||
b[2] = _mm256_mul_epu32(b2, consts);
|
||||
b[3] = _mm256_mul_epu32(b3, consts);
|
||||
b[2] = _mm256_mul_epu32(b2.into_bits(), consts.into_bits()).into_bits();
|
||||
b[3] = _mm256_mul_epu32(b3.into_bits(), consts.into_bits()).into_bits();
|
||||
|
||||
let (b4, b5) = unpack_pair(self.0[2]);
|
||||
b[4] = _mm256_mul_epu32(b4, consts);
|
||||
b[5] = _mm256_mul_epu32(b5, consts);
|
||||
b[4] = _mm256_mul_epu32(b4.into_bits(), consts.into_bits()).into_bits();
|
||||
b[5] = _mm256_mul_epu32(b5.into_bits(), consts.into_bits()).into_bits();
|
||||
|
||||
let (b6, b7) = unpack_pair(self.0[3]);
|
||||
b[6] = _mm256_mul_epu32(b6, consts);
|
||||
b[7] = _mm256_mul_epu32(b7, consts);
|
||||
b[6] = _mm256_mul_epu32(b6.into_bits(), consts.into_bits()).into_bits();
|
||||
b[7] = _mm256_mul_epu32(b7.into_bits(), consts.into_bits()).into_bits();
|
||||
|
||||
let (b8, b9) = unpack_pair(self.0[4]);
|
||||
b[8] = _mm256_mul_epu32(b8, consts);
|
||||
b[9] = _mm256_mul_epu32(b9, consts);
|
||||
b[8] = _mm256_mul_epu32(b8.into_bits(), consts.into_bits()).into_bits();
|
||||
b[9] = _mm256_mul_epu32(b9.into_bits(), consts.into_bits()).into_bits();
|
||||
}
|
||||
|
||||
*self = FieldElement32x4::reduce64(b);
|
||||
|
|
@ -245,22 +270,22 @@ impl FieldElement32x4 {
|
|||
|
||||
let carry = |v: u32x8| -> u32x8 {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_srlv_epi32;
|
||||
_mm256_srlv_epi32(v.into(), shifts).into()
|
||||
use core::arch::x86_64::_mm256_srlv_epi32;
|
||||
_mm256_srlv_epi32(v.into_bits(), shifts.into_bits()).into_bits()
|
||||
}
|
||||
};
|
||||
|
||||
let swap_lanes = |v: u32x8| -> u32x8 {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_shuffle_epi32;
|
||||
_mm256_shuffle_epi32(v.into(), 0b01_00_11_10).into()
|
||||
use core::arch::x86_64::_mm256_shuffle_epi32;
|
||||
_mm256_shuffle_epi32(v.into_bits(), 0b01_00_11_10).into_bits()
|
||||
}
|
||||
};
|
||||
|
||||
let combine = |v_lo: u32x8, v_hi: u32x8| -> u32x8 {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_blend_epi32;
|
||||
_mm256_blend_epi32(v_lo.into(), v_hi.into(), 0b11_00_11_00).into()
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
_mm256_blend_epi32(v_lo.into_bits(), v_hi.into_bits(), 0b11_00_11_00).into_bits()
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -282,11 +307,11 @@ impl FieldElement32x4 {
|
|||
//
|
||||
let c9_19: u32x8;
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_mul_epu32;
|
||||
use stdsimd::vendor::_mm256_shuffle_epi32;
|
||||
let c9_spread: u32x8 = _mm256_shuffle_epi32(c98.into(), 0b11_01_10_00).into();
|
||||
let c9_19_spread: u32x8 = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into()).into();
|
||||
c9_19 = _mm256_shuffle_epi32(c9_19_spread.into(), 0b11_01_10_00).into();
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
use core::arch::x86_64::_mm256_shuffle_epi32;
|
||||
let c9_spread = _mm256_shuffle_epi32(c98.into_bits(), 0b11_01_10_00);
|
||||
let c9_19_spread = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into_bits());
|
||||
c9_19 = _mm256_shuffle_epi32(c9_19_spread, 0b11_01_10_00).into_bits();
|
||||
}
|
||||
|
||||
v[0] = v[0] + c9_19;
|
||||
|
|
@ -331,14 +356,14 @@ impl FieldElement32x4 {
|
|||
|
||||
let c = z[9] >> 25;
|
||||
z[9] = z[9] & LOW_25_BITS;
|
||||
let mut c0 = c & LOW_26_BITS; // c0 < 2^26;
|
||||
let mut c1 = c >> 26; // c1 < 2^(39-26) = 2^13;
|
||||
let mut c0: u64x4 = c & LOW_26_BITS; // c0 < 2^26;
|
||||
let mut c1: u64x4 = c >> 26; // c1 < 2^(39-26) = 2^13;
|
||||
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_mul_epu32;
|
||||
let x19 = u32x8::from(u64x4::splat(19));
|
||||
c0 = _mm256_mul_epu32(u32x8::from(c0), x19); // c0 < 2^30.25
|
||||
c1 = _mm256_mul_epu32(u32x8::from(c1), x19); // c1 < 2^17.25
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
let x19 = u64x4::splat(19);
|
||||
c0 = _mm256_mul_epu32(c0.into_bits(), x19.into_bits()).into_bits(); // c0 < 2^30.25
|
||||
c1 = _mm256_mul_epu32(c1.into_bits(), x19.into_bits()).into_bits(); // c1 < 2^17.25
|
||||
}
|
||||
|
||||
z[0] = z[0] + c0; // z0 < 2^26 + 2^30.25 < 2^30.33
|
||||
|
|
@ -348,11 +373,11 @@ impl FieldElement32x4 {
|
|||
// Now repack the [u64x4; 10] into a FieldElement32x4
|
||||
|
||||
FieldElement32x4([
|
||||
repack_pair(z[0].into(), z[1].into()),
|
||||
repack_pair(z[2].into(), z[3].into()),
|
||||
repack_pair(z[4].into(), z[5].into()),
|
||||
repack_pair(z[6].into(), z[7].into()),
|
||||
repack_pair(z[8].into(), z[9].into()),
|
||||
repack_pair(z[0].into_bits(), z[1].into_bits()),
|
||||
repack_pair(z[2].into_bits(), z[3].into_bits()),
|
||||
repack_pair(z[4].into_bits(), z[5].into_bits()),
|
||||
repack_pair(z[6].into_bits(), z[7].into_bits()),
|
||||
repack_pair(z[8].into_bits(), z[9].into_bits()),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
|
@ -363,10 +388,10 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
|
|||
let b: u32x8;
|
||||
let zero = i32x8::new(0,0,0,0,0,0,0,0);
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_unpackhi_epi32;
|
||||
use stdsimd::vendor::_mm256_unpacklo_epi32;
|
||||
a = _mm256_unpacklo_epi32(src.as_i32x8(), zero).as_u32x8();
|
||||
b = _mm256_unpackhi_epi32(src.as_i32x8(), zero).as_u32x8();
|
||||
use core::arch::x86_64::_mm256_unpackhi_epi32;
|
||||
use core::arch::x86_64::_mm256_unpacklo_epi32;
|
||||
a = _mm256_unpacklo_epi32(src.into_bits(), zero.into_bits()).into_bits();
|
||||
b = _mm256_unpackhi_epi32(src.into_bits(), zero.into_bits()).into_bits();
|
||||
}
|
||||
(a,b)
|
||||
}
|
||||
|
|
@ -374,40 +399,43 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
|
|||
#[inline(always)]
|
||||
pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_shuffle_epi32;
|
||||
use stdsimd::vendor::_mm256_blend_epi32;
|
||||
use core::arch::x86_64::_mm256_shuffle_epi32;
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
|
||||
// Input: x = (a0, 0, b0, 0, c0, 0, d0)
|
||||
// Input: y = (a1, 0, b1, 0, c1, 0, d1)
|
||||
|
||||
let x_shuffled = _mm256_shuffle_epi32(x.into(), 0b11_01_10_00);
|
||||
let y_shuffled = _mm256_shuffle_epi32(y.into(), 0b10_00_11_01);
|
||||
let x_shuffled = _mm256_shuffle_epi32(x.into_bits(), 0b11_01_10_00);
|
||||
let y_shuffled = _mm256_shuffle_epi32(y.into_bits(), 0b10_00_11_01);
|
||||
|
||||
// x' = (a0, b0, 0, 0, c0, d0, 0, 0)
|
||||
// y' = ( 0, 0, a1, b1, 0, 0, c1, d1)
|
||||
|
||||
return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).as_u32x8();
|
||||
return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).into_bits();
|
||||
}
|
||||
}
|
||||
|
||||
impl FieldElement32x4 {
|
||||
/// Square this field element, then conditionally negate according to `neg_mask`; for instance,
|
||||
/// `neg_mask = 0b11_00_00_00` negates the \\( D \\) value.
|
||||
/// Square this field element, then conditionally negate according
|
||||
/// to `neg_mask`. This parameter is hardcoded as `neg_mask =
|
||||
/// D_LANES64` to negate the \\( D \\) value.
|
||||
///
|
||||
/// # Precondition
|
||||
///
|
||||
/// Limbs must be bounded by bit-excess \\( b < 2.0 \\).
|
||||
pub fn square(&self, neg_mask: u8) -> FieldElement32x4 {
|
||||
pub fn square_and_negate_D(&self) -> FieldElement32x4 {
|
||||
let neg_mask = D_LANES64;
|
||||
|
||||
#[inline(always)]
|
||||
fn m(x: u32x8, y: u32x8) -> u64x4 {
|
||||
use stdsimd::vendor::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x,y) }
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn m_lo(x: u32x8, y: u32x8) -> u32x8 {
|
||||
use stdsimd::vendor::_mm256_mul_epu32;
|
||||
unsafe { u32x8::from(_mm256_mul_epu32(x,y)) }
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() }
|
||||
}
|
||||
|
||||
let v19 = u32x8::new(19,0,19,0,19,0,19,0);
|
||||
|
|
@ -444,14 +472,6 @@ impl FieldElement32x4 {
|
|||
let mut z8 = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1);
|
||||
let mut z9 = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5);
|
||||
|
||||
#[inline(always)]
|
||||
fn mask_neg(x: u64x4, p: u64x4, mask: u8) -> u64x4 {
|
||||
unsafe {
|
||||
use stdsimd::vendor::_mm256_blend_epi32;
|
||||
_mm256_blend_epi32(x.into(), (p - x).into(), mask as i32).into()
|
||||
}
|
||||
}
|
||||
|
||||
// The biggest z_i is bounded as z_i < 249*2^(51 + 2*b);
|
||||
// if b < 1.5 we get z_i < 4485585228861014016.
|
||||
//
|
||||
|
|
@ -472,16 +492,23 @@ impl FieldElement32x4 {
|
|||
let even_p37 = u64x4::splat(0x3ffffff << 37);
|
||||
let odd__p37 = u64x4::splat(0x1ffffff << 37);
|
||||
|
||||
z0 = mask_neg(z0, low__p37, neg_mask);
|
||||
z1 = mask_neg(z1, odd__p37, neg_mask);
|
||||
z2 = mask_neg(z2, even_p37, neg_mask);
|
||||
z3 = mask_neg(z3, odd__p37, neg_mask);
|
||||
z4 = mask_neg(z4, even_p37, neg_mask);
|
||||
z5 = mask_neg(z5, odd__p37, neg_mask);
|
||||
z6 = mask_neg(z6, even_p37, neg_mask);
|
||||
z7 = mask_neg(z7, odd__p37, neg_mask);
|
||||
z8 = mask_neg(z8, even_p37, neg_mask);
|
||||
z9 = mask_neg(z9, odd__p37, neg_mask);
|
||||
let negate_D = |x: u64x4, p: u64x4| -> u64x4 {
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
_mm256_blend_epi32(x.into_bits(), (p - x).into_bits(), D_LANES64 as i32).into_bits()
|
||||
}
|
||||
};
|
||||
|
||||
z0 = negate_D(z0, low__p37);
|
||||
z1 = negate_D(z1, odd__p37);
|
||||
z2 = negate_D(z2, even_p37);
|
||||
z3 = negate_D(z3, odd__p37);
|
||||
z4 = negate_D(z4, even_p37);
|
||||
z5 = negate_D(z5, odd__p37);
|
||||
z6 = negate_D(z6, even_p37);
|
||||
z7 = negate_D(z7, odd__p37);
|
||||
z8 = negate_D(z8, even_p37);
|
||||
z9 = negate_D(z9, odd__p37);
|
||||
|
||||
FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
|
||||
}
|
||||
|
|
@ -490,17 +517,16 @@ impl FieldElement32x4 {
|
|||
impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 {
|
||||
type Output = FieldElement32x4;
|
||||
fn mul(self, _rhs: &'b FieldElement32x4) -> FieldElement32x4 {
|
||||
|
||||
#[inline(always)]
|
||||
fn m(x: u32x8, y: u32x8) -> u64x4 {
|
||||
use stdsimd::vendor::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x,y) }
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn m_lo(x: u32x8, y: u32x8) -> u32x8 {
|
||||
use stdsimd::vendor::_mm256_mul_epu32;
|
||||
unsafe { u32x8::from(_mm256_mul_epu32(x,y)) }
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() }
|
||||
}
|
||||
|
||||
let (x0, x1) = unpack_pair(self.0[0]);
|
||||
|
|
@ -573,7 +599,7 @@ mod test {
|
|||
let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]);
|
||||
|
||||
let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
|
||||
vec.diff_sum(0xff);
|
||||
vec.diff_sum(Lanes::ALL);
|
||||
|
||||
let result = vec.split();
|
||||
|
||||
|
|
@ -583,13 +609,13 @@ mod test {
|
|||
assert_eq!(result[3], &x3 + &x2);
|
||||
|
||||
let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
|
||||
vec.diff_sum(0b01011111); // leave D unchanged
|
||||
vec.diff_sum(Lanes::AB); // leave C,D unchanged
|
||||
|
||||
let result = vec.split();
|
||||
|
||||
assert_eq!(result[0], &x1 - &x0);
|
||||
assert_eq!(result[1], &x1 + &x0);
|
||||
assert_eq!(result[2], &x3 - &x2);
|
||||
assert_eq!(result[2], x2);
|
||||
assert_eq!(result[3], x3);
|
||||
}
|
||||
|
||||
|
|
@ -602,9 +628,7 @@ mod test {
|
|||
|
||||
let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
|
||||
|
||||
let neg_mask = 0b11_00_00_00;
|
||||
|
||||
let result = vec.square(neg_mask).split();
|
||||
let result = vec.square_and_negate_D().split();
|
||||
|
||||
assert_eq!(result[0], &x0 * &x0);
|
||||
assert_eq!(result[1], &x1 * &x1);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
#![cfg_attr(feature = "nightly", feature(i128_type))]
|
||||
#![cfg_attr(feature = "nightly", feature(cfg_target_feature))]
|
||||
#![cfg_attr(feature = "nightly", feature(external_doc))]
|
||||
#![cfg_attr(all(feature = "nightly", feature = "yolocrypto"), feature(stdsimd))]
|
||||
|
||||
// Refuse to compile if documentation is missing, but only on nightly.
|
||||
//
|
||||
|
|
@ -40,9 +41,6 @@ extern crate alloc;
|
|||
|
||||
extern crate clear_on_drop;
|
||||
|
||||
#[cfg(feature = "yolocrypto")]
|
||||
extern crate stdsimd;
|
||||
|
||||
// The `Digest` trait is implemented using `generic_array`, so we need it
|
||||
// too. Hopefully we can eliminate `generic_array` from `Digest` once const
|
||||
// generics land.
|
||||
|
|
|
|||
Loading…
Reference in a new issue