saymrwulf/curve25519-dalek-source
1
0
Fork 0
mirror of https://github.com/saymrwulf/curve25519-dalek-source.git synced 2026-09-06 20:41:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #122 from hdevalence/feature/drop-stdsimd

Browse source 
Move from external `stdsimd` crate to `core::{arch, simd}` intrinsics
This commit is contained in:
Henry de Valence 2018-04-04 11:06:23 -07:00 committed by GitHub
commit 46d8f9c45d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 199 additions and 187 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Cargo.toml
View file

@ -45,7 +45,6 @@ digest = "0.7"
generic-array = "0.9" generic-array = "0.9"
clear_on_drop = "=0.2.3" clear_on_drop = "=0.2.3"
subtle = { version = "0.6", features = ["generic-impls"], default-features = false } subtle = { version = "0.6", features = ["generic-impls"], default-features = false }
stdsimd = { version = "0.0.4", optional = true }
serde = { version = "1.0", optional = true } serde = { version = "1.0", optional = true }
rand = { version = "0.4", optional = true } rand = { version = "0.4", optional = true }
@ -54,7 +53,6 @@ digest = "0.7"
generic-array = "0.9" generic-array = "0.9"
clear_on_drop = "=0.2.3" clear_on_drop = "=0.2.3"
subtle = { version = "0.6", features = ["generic-impls"], default-features = false } subtle = { version = "0.6", features = ["generic-impls"], default-features = false }
stdsimd = { version = "0.0.4", optional = true }
serde = { version = "1.0", optional = true } serde = { version = "1.0", optional = true }
# Allowing rand to be optional during builds causes a build failure when compiling for no_std targets # Allowing rand to be optional during builds causes a build failure when compiling for no_std targets
rand = { version = "0.4", optional = false } rand = { version = "0.4", optional = false }
@ -70,4 +68,4 @@ radix_51 = []
# Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation. # Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation.
precomputed_tables = [] precomputed_tables = []
# experimental avx2 support # experimental avx2 support
avx2_backend = ["nightly", "stdsimd"] avx2_backend = ["nightly"]

3
build.rs
View file

@ -1,5 +1,6 @@
#![cfg_attr(feature = "nightly", feature(i128_type))] #![cfg_attr(feature = "nightly", feature(i128_type))]
#![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))]
#![cfg_attr(all(feature = "nightly", feature = "yolocrypto"), feature(stdsimd))]
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(non_snake_case)] #![allow(non_snake_case)]
#![allow(dead_code)] #![allow(dead_code)]
@ -23,8 +24,6 @@ use std::path::Path;
// For instance, this shouldn't exist here at all, but it does. // For instance, this shouldn't exist here at all, but it does.
#[cfg(feature = "serde")] #[cfg(feature = "serde")]
extern crate serde; extern crate serde;
#[cfg(feature = "yolocrypto")]
extern crate stdsimd;
// Macros come first! // Macros come first!
#[path = "src/macros.rs"] #[path = "src/macros.rs"]

2
src/backend/avx2/constants.rs
View file

@ -10,7 +10,7 @@
//! This module contains constants used by the AVX2 backend. //! This module contains constants used by the AVX2 backend.
use stdsimd::simd::u32x8; use core::simd::u32x8;
use scalar_mul::window::OddLookupTable; use scalar_mul::window::OddLookupTable;
use backend::avx2::field::FieldElement32x4; use backend::avx2::field::FieldElement32x4;

95
src/backend/avx2/edwards.rs
View file

@ -14,24 +14,19 @@
#![allow(bad_style)] #![allow(bad_style)]
use core::convert::From; use core::convert::From;
use core::ops::{Index, Add, Sub, Mul, Neg}; use core::ops::{Add, Sub, Neg};
use core::borrow::Borrow;
use stdsimd::simd::{u32x8, i32x8}; use core::simd::{IntoBits, u32x8};
use subtle::ConditionallyAssignable; use subtle::ConditionallyAssignable;
use subtle::Choice; use subtle::Choice;
use edwards; use edwards;
use scalar::Scalar;
use scalar_mul::window::{LookupTable, OddLookupTable}; use scalar_mul::window::{LookupTable, OddLookupTable};
use traits::Identity; use traits::Identity;
use backend::avx2::field::FieldElement32x4; use backend::avx2::field::{D_LANES, Lanes, FieldElement32x4};
use backend::avx2::field::{A_LANES, B_LANES, C_LANES, D_LANES, ALL_LANES};
use backend::avx2::field::D_LANES64;
use backend::avx2; use backend::avx2;
@ -79,10 +74,10 @@ impl Identity for ExtendedPoint {
impl ExtendedPoint { impl ExtendedPoint {
pub fn double(&self) -> ExtendedPoint { pub fn double(&self) -> ExtendedPoint {
unsafe { unsafe {
use stdsimd::vendor::_mm256_permute2x128_si256; use core::arch::x86_64::_mm256_permute2x128_si256;
use stdsimd::vendor::_mm256_permutevar8x32_epi32; use core::arch::x86_64::_mm256_permutevar8x32_epi32;
use stdsimd::vendor::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
use stdsimd::vendor::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
let P = &self.0; let P = &self.0;
@ -96,18 +91,18 @@ impl ExtendedPoint {
// and then adding. // and then adding.
// Set t0 = (X1 Y1 X1 Y1) // Set t0 = (X1 Y1 X1 Y1)
t0.0[0] = _mm256_permute2x128_si256(P.0[0].into(), P.0[0].into(), 0b0000_0000).into(); t0.0[0] = _mm256_permute2x128_si256(P.0[0].into_bits(), P.0[0].into_bits(), 0b0000_0000).into_bits();
t0.0[1] = _mm256_permute2x128_si256(P.0[1].into(), P.0[1].into(), 0b0000_0000).into(); t0.0[1] = _mm256_permute2x128_si256(P.0[1].into_bits(), P.0[1].into_bits(), 0b0000_0000).into_bits();
t0.0[2] = _mm256_permute2x128_si256(P.0[2].into(), P.0[2].into(), 0b0000_0000).into(); t0.0[2] = _mm256_permute2x128_si256(P.0[2].into_bits(), P.0[2].into_bits(), 0b0000_0000).into_bits();
t0.0[3] = _mm256_permute2x128_si256(P.0[3].into(), P.0[3].into(), 0b0000_0000).into(); t0.0[3] = _mm256_permute2x128_si256(P.0[3].into_bits(), P.0[3].into_bits(), 0b0000_0000).into_bits();
t0.0[4] = _mm256_permute2x128_si256(P.0[4].into(), P.0[4].into(), 0b0000_0000).into(); t0.0[4] = _mm256_permute2x128_si256(P.0[4].into_bits(), P.0[4].into_bits(), 0b0000_0000).into_bits();
// Set t1 = (Y1 X1 Y1 X1) // Set t1 = (Y1 X1 Y1 X1)
t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into(), 0b10_11_00_01).into(); t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into_bits(), 0b10_11_00_01).into_bits();
t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into(), 0b10_11_00_01).into(); t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into_bits(), 0b10_11_00_01).into_bits();
t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into(), 0b10_11_00_01).into(); t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into_bits(), 0b10_11_00_01).into_bits();
t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into(), 0b10_11_00_01).into(); t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into_bits(), 0b10_11_00_01).into_bits();
t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into(), 0b10_11_00_01).into(); t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into_bits(), 0b10_11_00_01).into_bits();
// Set t0 = (X1+Y1 X1+Y1 X1+Y1 X1+Y1) // Set t0 = (X1+Y1 X1+Y1 X1+Y1 X1+Y1)
t0.0[0] = t0.0[0] + t1.0[0]; t0.0[0] = t0.0[0] + t1.0[0];
@ -118,19 +113,19 @@ impl ExtendedPoint {
// Set t0 = (X1 Y1 Z1 X1+Y1) // Set t0 = (X1 Y1 Z1 X1+Y1)
// why does this intrinsic take an i32 for the imm8 ??? // why does this intrinsic take an i32 for the imm8 ???
t0.0[0] = _mm256_blend_epi32(P.0[0].into(), t0.0[0].into(), D_LANES as i32).into(); t0.0[0] = _mm256_blend_epi32(P.0[0].into_bits(), t0.0[0].into_bits(), D_LANES as i32).into_bits();
t0.0[1] = _mm256_blend_epi32(P.0[1].into(), t0.0[1].into(), D_LANES as i32).into(); t0.0[1] = _mm256_blend_epi32(P.0[1].into_bits(), t0.0[1].into_bits(), D_LANES as i32).into_bits();
t0.0[2] = _mm256_blend_epi32(P.0[2].into(), t0.0[2].into(), D_LANES as i32).into(); t0.0[2] = _mm256_blend_epi32(P.0[2].into_bits(), t0.0[2].into_bits(), D_LANES as i32).into_bits();
t0.0[3] = _mm256_blend_epi32(P.0[3].into(), t0.0[3].into(), D_LANES as i32).into(); t0.0[3] = _mm256_blend_epi32(P.0[3].into_bits(), t0.0[3].into_bits(), D_LANES as i32).into_bits();
t0.0[4] = _mm256_blend_epi32(P.0[4].into(), t0.0[4].into(), D_LANES as i32).into(); t0.0[4] = _mm256_blend_epi32(P.0[4].into_bits(), t0.0[4].into_bits(), D_LANES as i32).into_bits();
// Set t1 = t0^2, negating the D values // Set t1 = t0^2, negating the D values
t1 = t0.square(D_LANES64); t1 = t0.square_and_negate_D();
// Now t1 = (S1 S2 S3 -S4) // Now t1 = (S1 S2 S3 -S4)
let c0 = u32x8::new(0,0,2,2,0,0,2,2); // (ABCD) -> (AAAA) let c0 = u32x8::new(0,0,2,2,0,0,2,2).into_bits(); // (ABCD) -> (AAAA)
let c1 = u32x8::new(1,1,3,3,1,1,3,3); // (ABCD) -> (BBBB) let c1 = u32x8::new(1,1,3,3,1,1,3,3).into_bits(); // (ABCD) -> (BBBB)
// See discussion of bounds in the module-level documentation. // See discussion of bounds in the module-level documentation.
// //
@ -147,24 +142,24 @@ impl ExtendedPoint {
// S5 S6 S8 S9 // S5 S6 S8 S9
// //
for i in 0..5 { for i in 0..5 {
let zero = i32x8::splat(0); let zero = u32x8::splat(0).into_bits();
let S1 = _mm256_permutevar8x32_epi32(t1.0[i], c0); let S1: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c0).into_bits();
let S2 = _mm256_permutevar8x32_epi32(t1.0[i], c1); let S2: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c1).into_bits();
let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into(), 0b01010000).into(); let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into_bits(), 0b01010000).into_bits();
// tmp0 = (0 0 2*S3 -S4) // tmp0 = (0 0 2*S3 -S4)
let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into(), t1.0[i].into(), 0b10100000).into(); let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into_bits(), t1.0[i].into_bits(), 0b10100000).into_bits();
t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1; t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1;
t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into(), 0b10100101).into(); t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into_bits(), 0b10100101).into_bits();
t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into(), zero, 0b10100101).into(); t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into_bits(), zero, 0b10100101).into_bits();
} }
let c0 = u32x8::new(4,0,6,2,4,0,6,2); // (ABCD) -> (CACA) let c0 = u32x8::new(4,0,6,2,4,0,6,2).into_bits(); // (ABCD) -> (CACA)
let c1 = u32x8::new(5,1,7,3,1,5,3,7); // (ABCD) -> (DBBD) let c1 = u32x8::new(5,1,7,3,1,5,3,7).into_bits(); // (ABCD) -> (DBBD)
for i in 0..5 { for i in 0..5 {
let tmp = t0.0[i]; let tmp = t0.0[i];
t0.0[i] = _mm256_permutevar8x32_epi32(tmp, c0); t0.0[i] = _mm256_permutevar8x32_epi32(tmp.into_bits(), c0).into_bits();
t1.0[i] = _mm256_permutevar8x32_epi32(tmp, c1); t1.0[i] = _mm256_permutevar8x32_epi32(tmp.into_bits(), c1).into_bits();
} }
ExtendedPoint(&t0 * &t1) ExtendedPoint(&t0 * &t1)
@ -189,13 +184,13 @@ impl From<ExtendedPoint> for CachedPoint {
let mut x = P.0; let mut x = P.0;
// x = (S2 S3 Z2 T2) // x = (S2 S3 Z2 T2)
x.diff_sum(0b00001111); x.diff_sum(Lanes::AB);
// x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2) // x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2)
x.scale_by_curve_constants(); x.scale_by_curve_constants();
// x = (121666*S2 121666*S3 2*121666*Z2 -2*121665*T2) // x = (121666*S2 121666*S3 2*121666*Z2 -2*121665*T2)
x.negate(D_LANES); x.negate_D();
CachedPoint(x) CachedPoint(x)
} }
@ -231,7 +226,7 @@ impl<'a> Neg for &'a CachedPoint {
fn neg(self) -> CachedPoint { fn neg(self) -> CachedPoint {
let mut neg = *self; let mut neg = *self;
neg.0.swap_AB(); neg.0.swap_AB();
neg.0.negate_lazy(D_LANES); neg.0.negate_D_lazy();
neg neg
} }
} }
@ -242,12 +237,12 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
/// Uses a slight tweak of the parallel unified formulas of HWCD'08 /// Uses a slight tweak of the parallel unified formulas of HWCD'08
fn add(self, other: &'b CachedPoint) -> ExtendedPoint { fn add(self, other: &'b CachedPoint) -> ExtendedPoint {
unsafe { unsafe {
use stdsimd::vendor::_mm256_permutevar8x32_epi32; use core::arch::x86_64::_mm256_permutevar8x32_epi32;
let mut tmp = self.0; let mut tmp = self.0;
// tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1) // tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1)
tmp.diff_sum(A_LANES | B_LANES); tmp.diff_sum(Lanes::AB);
// tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11) // tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11)
tmp = &tmp * &other.0; tmp = &tmp * &other.0;
@ -256,7 +251,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
tmp.swap_CD(); tmp.swap_CD();
// tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) // tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15)
tmp.diff_sum(ALL_LANES); tmp.diff_sum(Lanes::ALL);
let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA)
let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB)
@ -266,8 +261,8 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
let mut t0 = FieldElement32x4::zero(); let mut t0 = FieldElement32x4::zero();
let mut t1 = FieldElement32x4::zero(); let mut t1 = FieldElement32x4::zero();
for i in 0..5 { for i in 0..5 {
t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c0); t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i].into_bits(), c0.into_bits()).into_bits();
t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c1); t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i].into_bits(), c1.into_bits()).into_bits();
} }
// return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) // return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3)
@ -316,8 +311,6 @@ impl<'a> From<&'a edwards::EdwardsPoint> for OddLookupTable<CachedPoint> {
mod test { mod test {
use super::*; use super::*;
use constants;
fn serial_add(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) -> edwards::EdwardsPoint { fn serial_add(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) -> edwards::EdwardsPoint {
use backend::u64::field::FieldElement64; use backend::u64::field::FieldElement64;

278
src/backend/avx2/field.rs
View file

@ -9,7 +9,6 @@
// - Henry de Valence <hdevalence@hdevalence.ca> // - Henry de Valence <hdevalence@hdevalence.ca>
//! 4-way vectorized 32bit field arithmetic using AVX2. //! 4-way vectorized 32bit field arithmetic using AVX2.
//!
#![allow(bad_style)] #![allow(bad_style)]
@ -26,13 +25,31 @@ pub const D_LANES64: u8 = 0b11_00_00_00;
pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES; pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES;
use core::ops::Mul; use core::ops::Mul;
use core::simd::{IntoBits, u32x8, i32x8, u64x4};
use stdsimd::simd::{u32x8, i32x8, u64x4};
use backend::u64::field::FieldElement64; use backend::u64::field::FieldElement64;
use backend::avx2::constants::{P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_16_LO, P_TIMES_16_HI}; use backend::avx2::constants::{P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_16_LO, P_TIMES_16_HI};
#[derive(Copy, Clone)]
pub enum Lanes {
AB,
CD,
ALL,
}
#[inline(always)]
fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
unsafe {
use core::arch::x86_64::_mm256_blend_epi32;
match control {
Lanes::AB => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32).into_bits(),
Lanes::CD => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32).into_bits(),
Lanes::ALL => _mm256_blend_epi32(x.into_bits(), y.into_bits(), ALL_LANES as i32).into_bits(),
}
}
}
/// A vector of four `FieldElements`, implemented using AVX2. /// A vector of four `FieldElements`, implemented using AVX2.
#[derive(Clone, Copy, Debug)] #[derive(Clone, Copy, Debug)]
pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]); pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]);
@ -50,7 +67,6 @@ impl ConditionallyAssignable for FieldElement32x4 {
} }
} }
impl FieldElement32x4 { impl FieldElement32x4 {
pub(crate) fn split(&self) -> [FieldElement64; 4] { pub(crate) fn split(&self) -> [FieldElement64; 4] {
let mut out = [FieldElement64::zero(); 4]; let mut out = [FieldElement64::zero(); 4];
@ -108,28 +124,31 @@ impl FieldElement32x4 {
return out; return out;
} }
pub fn negate_lazy(&mut self, mask: u8) { /// Negate the \\(D\\) variable of \\((A,B,C,D)\\).
let mask = mask as i32; ///
/// Input limbs must be less than the limbs of \\(2p\\), i.e., freshly reduced.
pub fn negate_D_lazy(&mut self) {
unsafe { unsafe {
use stdsimd::vendor::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_2_LO - self.0[0]).into(), mask).into(); self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_2_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits();
self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_2_HI - self.0[1]).into(), mask).into(); self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_2_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits();
self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_2_HI - self.0[2]).into(), mask).into(); self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_2_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits();
self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_2_HI - self.0[3]).into(), mask).into(); self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_2_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits();
self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_2_HI - self.0[4]).into(), mask).into(); self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_2_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits();
} }
} }
/// Negate variables in lanes where mask is set /// Negate the \\(D\\) variable of \\((A,B,C,D)\\).
pub fn negate(&mut self, mask: u8) { ///
let mask = mask as i32; /// Input limbs must be less than the limbs of \\(2p\\), i.e., freshly reduced.
pub fn negate_D(&mut self) {
unsafe { unsafe {
use stdsimd::vendor::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_16_LO - self.0[0]).into(), mask).into(); self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_16_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits();
self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_16_HI - self.0[1]).into(), mask).into(); self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_16_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits();
self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_16_HI - self.0[2]).into(), mask).into(); self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_16_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits();
self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_16_HI - self.0[3]).into(), mask).into(); self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_16_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits();
self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_16_HI - self.0[4]).into(), mask).into(); self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_16_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits();
} }
self.reduce32(); self.reduce32();
} }
@ -137,11 +156,11 @@ impl FieldElement32x4 {
/// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)` /// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)`
pub fn swap_AB(&mut self) { pub fn swap_AB(&mut self) {
unsafe { unsafe {
use stdsimd::vendor::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
use stdsimd::vendor::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
for i in 0..5 { for i in 0..5 {
let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01); let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01);
self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b00001111).into(); self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b00001111).into_bits();
} }
} }
} }
@ -149,55 +168,61 @@ impl FieldElement32x4 {
/// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` /// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)`
pub fn swap_CD(&mut self) { pub fn swap_CD(&mut self) {
unsafe { unsafe {
use stdsimd::vendor::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
use stdsimd::vendor::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
for i in 0..5 { for i in 0..5 {
let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01); let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01);
self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b11110000).into(); self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b11110000).into_bits();
} }
} }
} }
/// Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)` according to `mask`. /// Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)` according to `mask`.
pub fn diff_sum(&mut self, mask: u8) { ///
let mask = mask as i32; /// This is `#[inline(always)]` because the `mask` parameter should be an immediate.
#[inline(always)]
pub fn diff_sum(&mut self, control: Lanes) {
unsafe { unsafe {
use stdsimd::vendor::{_mm256_shuffle_epi32, _mm256_blend_epi32}; use core::arch::x86_64::{_mm256_shuffle_epi32, _mm256_blend_epi32};
let shuffle = |v: u32x8| -> u32x8 {
_mm256_shuffle_epi32(v.into_bits(), 0b10_11_00_01).into_bits()
};
let x01 = self.0[0]; let x01 = self.0[0];
let x01_shuf = _mm256_shuffle_epi32(x01.as_i32x8(), 0b10_11_00_01).as_u32x8(); let x01_shuf = shuffle(x01);
let v1 = (x01_shuf + P_TIMES_2_LO) - x01; let v1 = (x01_shuf + P_TIMES_2_LO) - x01;
let v2 = x01_shuf + x01; let v2 = x01_shuf + x01;
let diffsum01 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); let diffsum01 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[0] = _mm256_blend_epi32(x01.into(), diffsum01.into(), mask).into(); self.0[0] = blend_lanes(x01, diffsum01, control);
let x23 = self.0[1]; let x23 = self.0[1];
let x23_shuf = _mm256_shuffle_epi32(x23.as_i32x8(), 0b10_11_00_01).as_u32x8(); let x23_shuf = shuffle(x23);
let v1 = (x23_shuf + P_TIMES_2_HI) - x23; let v1 = (x23_shuf + P_TIMES_2_HI) - x23;
let v2 = x23_shuf + x23; let v2 = x23_shuf + x23;
let diffsum23 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); let diffsum23 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[1] = _mm256_blend_epi32(x23.into(), diffsum23.into(), mask).into(); self.0[1] = blend_lanes(x23, diffsum23, control);
let x45 = self.0[2]; let x45 = self.0[2];
let x45_shuf = _mm256_shuffle_epi32(x45.as_i32x8(), 0b10_11_00_01).as_u32x8(); let x45_shuf = shuffle(x45);
let v1 = (x45_shuf + P_TIMES_2_HI) - x45; let v1 = (x45_shuf + P_TIMES_2_HI) - x45;
let v2 = x45_shuf + x45; let v2 = x45_shuf + x45;
let diffsum45 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); let diffsum45 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[2] = _mm256_blend_epi32(x45.into(), diffsum45.into(), mask).into(); self.0[2] = blend_lanes(x45, diffsum45, control);
let x67 = self.0[3]; let x67 = self.0[3];
let x67_shuf = _mm256_shuffle_epi32(x67.as_i32x8(), 0b10_11_00_01).as_u32x8(); let x67_shuf = shuffle(x67);
let v1 = (x67_shuf + P_TIMES_2_HI) - x67; let v1 = (x67_shuf + P_TIMES_2_HI) - x67;
let v2 = x67_shuf + x67; let v2 = x67_shuf + x67;
let diffsum67 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); let diffsum67 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[3] = _mm256_blend_epi32(x67.into(), diffsum67.into(), mask).into(); self.0[3] = blend_lanes(x67, diffsum67, control);
let x89 = self.0[4]; let x89 = self.0[4];
let x89_shuf = _mm256_shuffle_epi32(x89.as_i32x8(), 0b10_11_00_01).as_u32x8(); let x89_shuf = shuffle(x89);
let v1 = (x89_shuf + P_TIMES_2_HI) - x89; let v1 = (x89_shuf + P_TIMES_2_HI) - x89;
let v2 = x89_shuf + x89; let v2 = x89_shuf + x89;
let diffsum89 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); let diffsum89 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[4] = _mm256_blend_epi32(x89.into(), diffsum89.into(), mask).into(); self.0[4] = blend_lanes(x89, diffsum89, control);
} }
} }
@ -208,30 +233,30 @@ impl FieldElement32x4 {
pub fn scale_by_curve_constants(&mut self) { pub fn scale_by_curve_constants(&mut self) {
let mut b = [u64x4::splat(0); 10]; let mut b = [u64x4::splat(0); 10];
let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0);
unsafe { unsafe {
use stdsimd::vendor::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
let (b0, b1) = unpack_pair(self.0[0]); let (b0, b1) = unpack_pair(self.0[0]);
b[0] = _mm256_mul_epu32(b0, consts); b[0] = _mm256_mul_epu32(b0.into_bits(), consts.into_bits()).into_bits();
b[1] = _mm256_mul_epu32(b1, consts); b[1] = _mm256_mul_epu32(b1.into_bits(), consts.into_bits()).into_bits();
let (b2, b3) = unpack_pair(self.0[1]); let (b2, b3) = unpack_pair(self.0[1]);
b[2] = _mm256_mul_epu32(b2, consts); b[2] = _mm256_mul_epu32(b2.into_bits(), consts.into_bits()).into_bits();
b[3] = _mm256_mul_epu32(b3, consts); b[3] = _mm256_mul_epu32(b3.into_bits(), consts.into_bits()).into_bits();
let (b4, b5) = unpack_pair(self.0[2]); let (b4, b5) = unpack_pair(self.0[2]);
b[4] = _mm256_mul_epu32(b4, consts); b[4] = _mm256_mul_epu32(b4.into_bits(), consts.into_bits()).into_bits();
b[5] = _mm256_mul_epu32(b5, consts); b[5] = _mm256_mul_epu32(b5.into_bits(), consts.into_bits()).into_bits();
let (b6, b7) = unpack_pair(self.0[3]); let (b6, b7) = unpack_pair(self.0[3]);
b[6] = _mm256_mul_epu32(b6, consts); b[6] = _mm256_mul_epu32(b6.into_bits(), consts.into_bits()).into_bits();
b[7] = _mm256_mul_epu32(b7, consts); b[7] = _mm256_mul_epu32(b7.into_bits(), consts.into_bits()).into_bits();
let (b8, b9) = unpack_pair(self.0[4]); let (b8, b9) = unpack_pair(self.0[4]);
b[8] = _mm256_mul_epu32(b8, consts); b[8] = _mm256_mul_epu32(b8.into_bits(), consts.into_bits()).into_bits();
b[9] = _mm256_mul_epu32(b9, consts); b[9] = _mm256_mul_epu32(b9.into_bits(), consts.into_bits()).into_bits();
} }
*self = FieldElement32x4::reduce64(b); *self = FieldElement32x4::reduce64(b);
@ -245,22 +270,22 @@ impl FieldElement32x4 {
let carry = |v: u32x8| -> u32x8 { let carry = |v: u32x8| -> u32x8 {
unsafe { unsafe {
use stdsimd::vendor::_mm256_srlv_epi32; use core::arch::x86_64::_mm256_srlv_epi32;
_mm256_srlv_epi32(v.into(), shifts).into() _mm256_srlv_epi32(v.into_bits(), shifts.into_bits()).into_bits()
} }
}; };
let swap_lanes = |v: u32x8| -> u32x8 { let swap_lanes = |v: u32x8| -> u32x8 {
unsafe { unsafe {
use stdsimd::vendor::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
_mm256_shuffle_epi32(v.into(), 0b01_00_11_10).into() _mm256_shuffle_epi32(v.into_bits(), 0b01_00_11_10).into_bits()
} }
}; };
let combine = |v_lo: u32x8, v_hi: u32x8| -> u32x8 { let combine = |v_lo: u32x8, v_hi: u32x8| -> u32x8 {
unsafe { unsafe {
use stdsimd::vendor::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
_mm256_blend_epi32(v_lo.into(), v_hi.into(), 0b11_00_11_00).into() _mm256_blend_epi32(v_lo.into_bits(), v_hi.into_bits(), 0b11_00_11_00).into_bits()
} }
}; };
@ -282,11 +307,11 @@ impl FieldElement32x4 {
// //
let c9_19: u32x8; let c9_19: u32x8;
unsafe { unsafe {
use stdsimd::vendor::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
use stdsimd::vendor::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
let c9_spread: u32x8 = _mm256_shuffle_epi32(c98.into(), 0b11_01_10_00).into(); let c9_spread = _mm256_shuffle_epi32(c98.into_bits(), 0b11_01_10_00);
let c9_19_spread: u32x8 = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into()).into(); let c9_19_spread = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into_bits());
c9_19 = _mm256_shuffle_epi32(c9_19_spread.into(), 0b11_01_10_00).into(); c9_19 = _mm256_shuffle_epi32(c9_19_spread, 0b11_01_10_00).into_bits();
} }
v[0] = v[0] + c9_19; v[0] = v[0] + c9_19;
@ -331,14 +356,14 @@ impl FieldElement32x4 {
let c = z[9] >> 25; let c = z[9] >> 25;
z[9] = z[9] & LOW_25_BITS; z[9] = z[9] & LOW_25_BITS;
let mut c0 = c & LOW_26_BITS; // c0 < 2^26; let mut c0: u64x4 = c & LOW_26_BITS; // c0 < 2^26;
let mut c1 = c >> 26; // c1 < 2^(39-26) = 2^13; let mut c1: u64x4 = c >> 26; // c1 < 2^(39-26) = 2^13;
unsafe { unsafe {
use stdsimd::vendor::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
let x19 = u32x8::from(u64x4::splat(19)); let x19 = u64x4::splat(19);
c0 = _mm256_mul_epu32(u32x8::from(c0), x19); // c0 < 2^30.25 c0 = _mm256_mul_epu32(c0.into_bits(), x19.into_bits()).into_bits(); // c0 < 2^30.25
c1 = _mm256_mul_epu32(u32x8::from(c1), x19); // c1 < 2^17.25 c1 = _mm256_mul_epu32(c1.into_bits(), x19.into_bits()).into_bits(); // c1 < 2^17.25
} }
z[0] = z[0] + c0; // z0 < 2^26 + 2^30.25 < 2^30.33 z[0] = z[0] + c0; // z0 < 2^26 + 2^30.25 < 2^30.33
@ -348,11 +373,11 @@ impl FieldElement32x4 {
// Now repack the [u64x4; 10] into a FieldElement32x4 // Now repack the [u64x4; 10] into a FieldElement32x4
FieldElement32x4([ FieldElement32x4([
repack_pair(z[0].into(), z[1].into()), repack_pair(z[0].into_bits(), z[1].into_bits()),
repack_pair(z[2].into(), z[3].into()), repack_pair(z[2].into_bits(), z[3].into_bits()),
repack_pair(z[4].into(), z[5].into()), repack_pair(z[4].into_bits(), z[5].into_bits()),
repack_pair(z[6].into(), z[7].into()), repack_pair(z[6].into_bits(), z[7].into_bits()),
repack_pair(z[8].into(), z[9].into()), repack_pair(z[8].into_bits(), z[9].into_bits()),
]) ])
} }
} }
@ -363,10 +388,10 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
let b: u32x8; let b: u32x8;
let zero = i32x8::new(0,0,0,0,0,0,0,0); let zero = i32x8::new(0,0,0,0,0,0,0,0);
unsafe { unsafe {
use stdsimd::vendor::_mm256_unpackhi_epi32; use core::arch::x86_64::_mm256_unpackhi_epi32;
use stdsimd::vendor::_mm256_unpacklo_epi32; use core::arch::x86_64::_mm256_unpacklo_epi32;
a = _mm256_unpacklo_epi32(src.as_i32x8(), zero).as_u32x8(); a = _mm256_unpacklo_epi32(src.into_bits(), zero.into_bits()).into_bits();
b = _mm256_unpackhi_epi32(src.as_i32x8(), zero).as_u32x8(); b = _mm256_unpackhi_epi32(src.into_bits(), zero.into_bits()).into_bits();
} }
(a,b) (a,b)
} }
@ -374,40 +399,43 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
#[inline(always)] #[inline(always)]
pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 {
unsafe { unsafe {
use stdsimd::vendor::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
use stdsimd::vendor::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
// Input: x = (a0, 0, b0, 0, c0, 0, d0) // Input: x = (a0, 0, b0, 0, c0, 0, d0)
// Input: y = (a1, 0, b1, 0, c1, 0, d1) // Input: y = (a1, 0, b1, 0, c1, 0, d1)
let x_shuffled = _mm256_shuffle_epi32(x.into(), 0b11_01_10_00); let x_shuffled = _mm256_shuffle_epi32(x.into_bits(), 0b11_01_10_00);
let y_shuffled = _mm256_shuffle_epi32(y.into(), 0b10_00_11_01); let y_shuffled = _mm256_shuffle_epi32(y.into_bits(), 0b10_00_11_01);
// x' = (a0, b0, 0, 0, c0, d0, 0, 0) // x' = (a0, b0, 0, 0, c0, d0, 0, 0)
// y' = ( 0, 0, a1, b1, 0, 0, c1, d1) // y' = ( 0, 0, a1, b1, 0, 0, c1, d1)
return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).as_u32x8(); return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).into_bits();
} }
} }
impl FieldElement32x4 { impl FieldElement32x4 {
/// Square this field element, then conditionally negate according to `neg_mask`; for instance, /// Square this field element, then conditionally negate according
/// `neg_mask = 0b11_00_00_00` negates the \\( D \\) value. /// to `neg_mask`. This parameter is hardcoded as `neg_mask =
/// D_LANES64` to negate the \\( D \\) value.
/// ///
/// # Precondition /// # Precondition
/// ///
/// Limbs must be bounded by bit-excess \\( b < 2.0 \\). /// Limbs must be bounded by bit-excess \\( b < 2.0 \\).
pub fn square(&self, neg_mask: u8) -> FieldElement32x4 { pub fn square_and_negate_D(&self) -> FieldElement32x4 {
let neg_mask = D_LANES64;
#[inline(always)] #[inline(always)]
fn m(x: u32x8, y: u32x8) -> u64x4 { fn m(x: u32x8, y: u32x8) -> u64x4 {
use stdsimd::vendor::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
unsafe { _mm256_mul_epu32(x,y) } unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() }
} }
#[inline(always)] #[inline(always)]
fn m_lo(x: u32x8, y: u32x8) -> u32x8 { fn m_lo(x: u32x8, y: u32x8) -> u32x8 {
use stdsimd::vendor::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() }
} }
let v19 = u32x8::new(19,0,19,0,19,0,19,0); let v19 = u32x8::new(19,0,19,0,19,0,19,0);
@ -444,14 +472,6 @@ impl FieldElement32x4 {
let mut z8 = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1); let mut z8 = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1);
let mut z9 = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5); let mut z9 = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5);
#[inline(always)]
fn mask_neg(x: u64x4, p: u64x4, mask: u8) -> u64x4 {
unsafe {
use stdsimd::vendor::_mm256_blend_epi32;
_mm256_blend_epi32(x.into(), (p - x).into(), mask as i32).into()
}
}
// The biggest z_i is bounded as z_i < 249*2^(51 + 2*b); // The biggest z_i is bounded as z_i < 249*2^(51 + 2*b);
// if b < 1.5 we get z_i < 4485585228861014016. // if b < 1.5 we get z_i < 4485585228861014016.
// //
@ -472,16 +492,23 @@ impl FieldElement32x4 {
let even_p37 = u64x4::splat(0x3ffffff << 37); let even_p37 = u64x4::splat(0x3ffffff << 37);
let odd__p37 = u64x4::splat(0x1ffffff << 37); let odd__p37 = u64x4::splat(0x1ffffff << 37);
z0 = mask_neg(z0, low__p37, neg_mask); let negate_D = |x: u64x4, p: u64x4| -> u64x4 {
z1 = mask_neg(z1, odd__p37, neg_mask); unsafe {
z2 = mask_neg(z2, even_p37, neg_mask); use core::arch::x86_64::_mm256_blend_epi32;
z3 = mask_neg(z3, odd__p37, neg_mask); _mm256_blend_epi32(x.into_bits(), (p - x).into_bits(), D_LANES64 as i32).into_bits()
z4 = mask_neg(z4, even_p37, neg_mask); }
z5 = mask_neg(z5, odd__p37, neg_mask); };
z6 = mask_neg(z6, even_p37, neg_mask);
z7 = mask_neg(z7, odd__p37, neg_mask); z0 = negate_D(z0, low__p37);
z8 = mask_neg(z8, even_p37, neg_mask); z1 = negate_D(z1, odd__p37);
z9 = mask_neg(z9, odd__p37, neg_mask); z2 = negate_D(z2, even_p37);
z3 = negate_D(z3, odd__p37);
z4 = negate_D(z4, even_p37);
z5 = negate_D(z5, odd__p37);
z6 = negate_D(z6, even_p37);
z7 = negate_D(z7, odd__p37);
z8 = negate_D(z8, even_p37);
z9 = negate_D(z9, odd__p37);
FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
} }
@ -490,17 +517,16 @@ impl FieldElement32x4 {
impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 {
type Output = FieldElement32x4; type Output = FieldElement32x4;
fn mul(self, _rhs: &'b FieldElement32x4) -> FieldElement32x4 { fn mul(self, _rhs: &'b FieldElement32x4) -> FieldElement32x4 {
#[inline(always)] #[inline(always)]
fn m(x: u32x8, y: u32x8) -> u64x4 { fn m(x: u32x8, y: u32x8) -> u64x4 {
use stdsimd::vendor::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
unsafe { _mm256_mul_epu32(x,y) } unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() }
} }
#[inline(always)] #[inline(always)]
fn m_lo(x: u32x8, y: u32x8) -> u32x8 { fn m_lo(x: u32x8, y: u32x8) -> u32x8 {
use stdsimd::vendor::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() }
} }
let (x0, x1) = unpack_pair(self.0[0]); let (x0, x1) = unpack_pair(self.0[0]);
@ -573,7 +599,7 @@ mod test {
let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]);
let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
vec.diff_sum(0xff); vec.diff_sum(Lanes::ALL);
let result = vec.split(); let result = vec.split();
@ -583,13 +609,13 @@ mod test {
assert_eq!(result[3], &x3 + &x2); assert_eq!(result[3], &x3 + &x2);
let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
vec.diff_sum(0b01011111); // leave D unchanged vec.diff_sum(Lanes::AB); // leave C,D unchanged
let result = vec.split(); let result = vec.split();
assert_eq!(result[0], &x1 - &x0); assert_eq!(result[0], &x1 - &x0);
assert_eq!(result[1], &x1 + &x0); assert_eq!(result[1], &x1 + &x0);
assert_eq!(result[2], &x3 - &x2); assert_eq!(result[2], x2);
assert_eq!(result[3], x3); assert_eq!(result[3], x3);
} }
@ -602,9 +628,7 @@ mod test {
let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
let neg_mask = 0b11_00_00_00; let result = vec.square_and_negate_D().split();
let result = vec.square(neg_mask).split();
assert_eq!(result[0], &x0 * &x0); assert_eq!(result[0], &x0 * &x0);
assert_eq!(result[1], &x1 * &x1); assert_eq!(result[1], &x1 * &x1);

4
src/lib.rs
View file

@ -15,6 +15,7 @@
#![cfg_attr(feature = "nightly", feature(i128_type))] #![cfg_attr(feature = "nightly", feature(i128_type))]
#![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))]
#![cfg_attr(feature = "nightly", feature(external_doc))] #![cfg_attr(feature = "nightly", feature(external_doc))]
#![cfg_attr(all(feature = "nightly", feature = "yolocrypto"), feature(stdsimd))]
// Refuse to compile if documentation is missing, but only on nightly. // Refuse to compile if documentation is missing, but only on nightly.
// //
@ -40,9 +41,6 @@ extern crate alloc;
extern crate clear_on_drop; extern crate clear_on_drop;
#[cfg(feature = "yolocrypto")]
extern crate stdsimd;
// The `Digest` trait is implemented using `generic_array`, so we need it // The `Digest` trait is implemented using `generic_array`, so we need it
// too. Hopefully we can eliminate `generic_array` from `Digest` once const // too. Hopefully we can eliminate `generic_array` from `Digest` once const
// generics land. // generics land.