saymrwulf/curve25519-dalek-source
1
0
Fork 0
mirror of https://github.com/saymrwulf/curve25519-dalek-source.git synced 2026-09-06 20:41:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

some rustfmt changes

Browse source
This commit is contained in:
Henry de Valence 2018-06-14 14:03:11 -07:00
parent d29618ca2e
commit bd1e3c5f3e
2 changed files with 178 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
src/backend/avx2/edwards.rs
View file

@ -14,19 +14,19 @@
#![allow(bad_style)] #![allow(bad_style)]
use core::convert::From; use core::convert::From;
use core::ops::{Add, Sub, Neg}; use core::ops::{Add, Neg, Sub};
use core::simd::{IntoBits, u32x8}; use core::simd::{u32x8, IntoBits};
use subtle::ConditionallyAssignable;
use subtle::Choice; use subtle::Choice;
use subtle::ConditionallyAssignable;
use edwards; use edwards;
use scalar_mul::window::{LookupTable, NafLookupTable5, NafLookupTable8}; use scalar_mul::window::{LookupTable, NafLookupTable5, NafLookupTable8};
use traits::Identity; use traits::Identity;
use backend::avx2::field::{D_LANES, Lanes, FieldElement32x4}; use backend::avx2::field::{FieldElement32x4, Lanes, D_LANES};
use backend::avx2; use backend::avx2;
@ -43,7 +43,12 @@ impl From<edwards::EdwardsPoint> for ExtendedPoint {
impl From<ExtendedPoint> for edwards::EdwardsPoint { impl From<ExtendedPoint> for edwards::EdwardsPoint {
fn from(P: ExtendedPoint) -> edwards::EdwardsPoint { fn from(P: ExtendedPoint) -> edwards::EdwardsPoint {
let tmp = P.0.split(); let tmp = P.0.split();
edwards::EdwardsPoint{X: tmp[0], Y: tmp[1], Z: tmp[2], T: tmp[3]} edwards::EdwardsPoint {
X: tmp[0],
Y: tmp[1],
Z: tmp[2],
T: tmp[3],
}
} }
} }
@ -62,7 +67,7 @@ impl Default for ExtendedPoint {
impl Identity for ExtendedPoint { impl Identity for ExtendedPoint {
fn identity() -> ExtendedPoint { fn identity() -> ExtendedPoint {
ExtendedPoint(FieldElement32x4([ ExtendedPoint(FieldElement32x4([
u32x8::new(0,1,0,0,1,0,0,0), u32x8::new(0, 1, 0, 0, 1, 0, 0, 0),
u32x8::splat(0), u32x8::splat(0),
u32x8::splat(0), u32x8::splat(0),
u32x8::splat(0), u32x8::splat(0),
@ -74,9 +79,9 @@ impl Identity for ExtendedPoint {
impl ExtendedPoint { impl ExtendedPoint {
pub fn double(&self) -> ExtendedPoint { pub fn double(&self) -> ExtendedPoint {
unsafe { unsafe {
use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_permute2x128_si256; use core::arch::x86_64::_mm256_permute2x128_si256;
use core::arch::x86_64::_mm256_permutevar8x32_epi32; use core::arch::x86_64::_mm256_permutevar8x32_epi32;
use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
let P = &self.0; let P = &self.0;
@ -140,17 +145,22 @@ impl ExtendedPoint {
// - | | S2 | S2 | | // - | | S2 | S2 | |
// ======================= // =======================
// S5 S6 S8 S9 // S5 S6 S8 S9
//
for i in 0..5 { for i in 0..5 {
let zero = u32x8::splat(0).into_bits(); let zero = u32x8::splat(0).into_bits();
let S1: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c0).into_bits(); let S1: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c0).into_bits();
let S2: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c1).into_bits(); let S2: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c1).into_bits();
let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into_bits(), 0b01010000).into_bits(); let S3_2: u32x8 =
_mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into_bits(), 0b01010000)
.into_bits();
// tmp0 = (0 0 2*S3 -S4) // tmp0 = (0 0 2*S3 -S4)
let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into_bits(), t1.0[i].into_bits(), 0b10100000).into_bits(); let tmp0: u32x8 =
_mm256_blend_epi32(S3_2.into_bits(), t1.0[i].into_bits(), 0b10100000)
.into_bits();
t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1; t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1;
let S2_pos: u32x8 = _mm256_blend_epi32(zero, S2.into_bits(), 0b10100101).into_bits(); let S2_pos: u32x8 =
let S2_neg: u32x8 = _mm256_blend_epi32(S2.into_bits(), zero, 0b10100101).into_bits(); _mm256_blend_epi32(zero, S2.into_bits(), 0b10100101).into_bits();
let S2_neg: u32x8 =
_mm256_blend_epi32(S2.into_bits(), zero, 0b10100101).into_bits();
t0.0[i] = t0.0[i] + S2_pos; t0.0[i] = t0.0[i] + S2_pos;
t0.0[i] = t0.0[i] - S2_neg; t0.0[i] = t0.0[i] - S2_neg;
} }
@ -290,7 +300,7 @@ impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable<CachedPoint> {
let P = ExtendedPoint::from(*point); let P = ExtendedPoint::from(*point);
let mut points = [CachedPoint::from(P); 8]; let mut points = [CachedPoint::from(P); 8];
for i in 0..7 { for i in 0..7 {
points[i+1] = (&P + &points[i]).into(); points[i + 1] = (&P + &points[i]).into();
} }
LookupTable(points) LookupTable(points)
} }
@ -335,23 +345,23 @@ mod test {
macro_rules! print_var { macro_rules! print_var {
($x:ident) => { ($x:ident) => {
println!("{} = {:?}", stringify!($x), $x.to_bytes()); println!("{} = {:?}", stringify!($x), $x.to_bytes());
} };
} }
let S0 = &Y1 - &X1; // R1 let S0 = &Y1 - &X1; // R1
let S1 = &Y1 + &X1; // R3 let S1 = &Y1 + &X1; // R3
let S2 = &Y2 - &X2; // R2 let S2 = &Y2 - &X2; // R2
let S3 = &Y2 + &X2; // R4 let S3 = &Y2 + &X2; // R4
print_var!(S0); print_var!(S0);
print_var!(S1); print_var!(S1);
print_var!(S2); print_var!(S2);
print_var!(S3); print_var!(S3);
println!(""); println!("");
let S4 = &S0 * &S2; // R5 = R1 * R2 let S4 = &S0 * &S2; // R5 = R1 * R2
let S5 = &S1 * &S3; // R6 = R3 * R4 let S5 = &S1 * &S3; // R6 = R3 * R4
let S6 = &Z1 * &Z2; // R8 let S6 = &Z1 * &Z2; // R8
let S7 = &T1 * &T2; // R7 let S7 = &T1 * &T2; // R7
print_var!(S4); print_var!(S4);
print_var!(S5); print_var!(S5);
print_var!(S6); print_var!(S6);
@ -362,8 +372,8 @@ mod test {
let S9 = &S5 * &FieldElement64([ 121666,0,0,0,0]); // R6 let S9 = &S5 * &FieldElement64([ 121666,0,0,0,0]); // R6
let S10 = &S6 * &FieldElement64([2*121666,0,0,0,0]); // R8 let S10 = &S6 * &FieldElement64([2*121666,0,0,0,0]); // R8
let S11 = &S7 * &(-&FieldElement64([2*121665,0,0,0,0])); // R7 let S11 = &S7 * &(-&FieldElement64([2*121665,0,0,0,0])); // R7
print_var!(S8 ); print_var!(S8);
print_var!(S9 ); print_var!(S9);
print_var!(S10); print_var!(S10);
print_var!(S11); print_var!(S11);
println!(""); println!("");
@ -378,12 +388,17 @@ mod test {
print_var!(S15); print_var!(S15);
println!(""); println!("");
let X3 = &S12 * &S14; // R1 * R2 let X3 = &S12 * &S14; // R1 * R2
let Y3 = &S15 * &S13; // R3 * R4 let Y3 = &S15 * &S13; // R3 * R4
let Z3 = &S15 * &S14; // R2 * R3 let Z3 = &S15 * &S14; // R2 * R3
let T3 = &S12 * &S13; // R1 * R4 let T3 = &S12 * &S13; // R1 * R4
edwards::EdwardsPoint{X: X3, Y: Y3, Z: Z3, T: T3} edwards::EdwardsPoint {
X: X3,
Y: Y3,
Z: Z3,
T: T3,
}
} }
fn addition_test_helper(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) { fn addition_test_helper(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) {
@ -442,10 +457,10 @@ mod test {
macro_rules! print_var { macro_rules! print_var {
($x:ident) => { ($x:ident) => {
println!("{} = {:?}", stringify!($x), $x.to_bytes()); println!("{} = {:?}", stringify!($x), $x.to_bytes());
} };
} }
let S0 = &X1 + &Y1; // R1 let S0 = &X1 + &Y1; // R1
print_var!(S0); print_var!(S0);
println!(""); println!("");
@ -476,7 +491,12 @@ mod test {
let Z3 = &S8 * &S6; let Z3 = &S8 * &S6;
let T3 = &S5 * &S9; let T3 = &S5 * &S9;
edwards::EdwardsPoint{X: X3, Y: Y3, Z: Z3, T: T3} edwards::EdwardsPoint {
X: X3,
Y: Y3,
Z: Z3,
T: T3,
}
} }
fn doubling_test_helper(P: edwards::EdwardsPoint) { fn doubling_test_helper(P: edwards::EdwardsPoint) {

186
src/backend/avx2/field.rs
View file

@ -24,11 +24,11 @@ pub const D_LANES64: u8 = 0b11_00_00_00;
pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES; pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES;
use core::ops::Mul; use core::ops::{Add, Mul};
use core::simd::{IntoBits, u32x8, i32x8, u64x4}; use core::simd::{i32x8, u32x8, u64x4, IntoBits};
use backend::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO};
use backend::u64::field::FieldElement64; use backend::u64::field::FieldElement64;
use backend::avx2::constants::{P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_16_LO, P_TIMES_16_HI};
#[derive(Copy, Clone)] #[derive(Copy, Clone)]
pub enum Lanes { pub enum Lanes {
@ -43,9 +43,23 @@ fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
match control { match control {
Lanes::AB => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32).into_bits(), Lanes::C => {
Lanes::CD => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32).into_bits(), _mm256_blend_epi32(x.into_bits(), y.into_bits(), C_LANES as i32).into_bits()
Lanes::ALL => _mm256_blend_epi32(x.into_bits(), y.into_bits(), ALL_LANES as i32).into_bits(), }
Lanes::D => {
_mm256_blend_epi32(x.into_bits(), y.into_bits(), D_LANES as i32).into_bits()
}
Lanes::AB => {
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32)
.into_bits()
}
Lanes::CD => {
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32)
.into_bits()
}
Lanes::ALL => {
_mm256_blend_epi32(x.into_bits(), y.into_bits(), ALL_LANES as i32).into_bits()
}
} }
} }
} }
@ -54,8 +68,8 @@ fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
#[derive(Clone, Copy, Debug)] #[derive(Clone, Copy, Debug)]
pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]); pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]);
use subtle::ConditionallyAssignable;
use subtle::Choice; use subtle::Choice;
use subtle::ConditionallyAssignable;
impl ConditionallyAssignable for FieldElement32x4 { impl ConditionallyAssignable for FieldElement32x4 {
fn conditional_assign(&mut self, other: &FieldElement32x4, choice: Choice) { fn conditional_assign(&mut self, other: &FieldElement32x4, choice: Choice) {
@ -71,7 +85,6 @@ impl FieldElement32x4 {
pub(crate) fn split(&self) -> [FieldElement64; 4] { pub(crate) fn split(&self) -> [FieldElement64; 4] {
let mut out = [FieldElement64::zero(); 4]; let mut out = [FieldElement64::zero(); 4];
for i in 0..5 { for i in 0..5 {
let a_2i = self.0[i].extract(0) as u64; // let a_2i = self.0[i].extract(0) as u64; //
let b_2i = self.0[i].extract(1) as u64; // let b_2i = self.0[i].extract(1) as u64; //
let a_2i_1 = self.0[i].extract(2) as u64; // `. let a_2i_1 = self.0[i].extract(2) as u64; // `.
@ -91,11 +104,11 @@ impl FieldElement32x4 {
} }
pub fn zero() -> FieldElement32x4 { pub fn zero() -> FieldElement32x4 {
FieldElement32x4([u32x8::splat(0);5]) FieldElement32x4([u32x8::splat(0); 5])
} }
pub fn splat(x: &FieldElement64) -> FieldElement32x4 { pub fn splat(x: &FieldElement64) -> FieldElement32x4 {
FieldElement32x4::new(x,x,x,x) FieldElement32x4::new(x, x, x, x)
} }
pub fn new( pub fn new(
@ -130,11 +143,31 @@ impl FieldElement32x4 {
pub fn negate_D_lazy(&mut self) { pub fn negate_D_lazy(&mut self) {
unsafe { unsafe {
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_2_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits(); self.0[0] = _mm256_blend_epi32(
self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_2_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits(); self.0[0].into_bits(),
self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_2_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits(); (P_TIMES_2_LO - self.0[0]).into_bits(),
self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_2_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits(); D_LANES as i32,
self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_2_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits(); ).into_bits();
self.0[1] = _mm256_blend_epi32(
self.0[1].into_bits(),
(P_TIMES_2_HI - self.0[1]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[2] = _mm256_blend_epi32(
self.0[2].into_bits(),
(P_TIMES_2_HI - self.0[2]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[3] = _mm256_blend_epi32(
self.0[3].into_bits(),
(P_TIMES_2_HI - self.0[3]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[4] = _mm256_blend_epi32(
self.0[4].into_bits(),
(P_TIMES_2_HI - self.0[4]).into_bits(),
D_LANES as i32,
).into_bits();
} }
} }
@ -144,11 +177,31 @@ impl FieldElement32x4 {
pub fn negate_D(&mut self) { pub fn negate_D(&mut self) {
unsafe { unsafe {
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_16_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits(); self.0[0] = _mm256_blend_epi32(
self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_16_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits(); self.0[0].into_bits(),
self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_16_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits(); (P_TIMES_16_LO - self.0[0]).into_bits(),
self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_16_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits(); D_LANES as i32,
self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_16_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits(); ).into_bits();
self.0[1] = _mm256_blend_epi32(
self.0[1].into_bits(),
(P_TIMES_16_HI - self.0[1]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[2] = _mm256_blend_epi32(
self.0[2].into_bits(),
(P_TIMES_16_HI - self.0[2]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[3] = _mm256_blend_epi32(
self.0[3].into_bits(),
(P_TIMES_16_HI - self.0[3]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[4] = _mm256_blend_epi32(
self.0[4].into_bits(),
(P_TIMES_16_HI - self.0[4]).into_bits(),
D_LANES as i32,
).into_bits();
} }
self.reduce32(); self.reduce32();
} }
@ -156,11 +209,12 @@ impl FieldElement32x4 {
/// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)` /// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)`
pub fn swap_AB(&mut self) { pub fn swap_AB(&mut self) {
unsafe { unsafe {
use core::arch::x86_64::_mm256_shuffle_epi32;
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32;
for i in 0..5 { for i in 0..5 {
let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01); let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01);
self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b00001111).into_bits(); self.0[i] =
_mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b00001111).into_bits();
} }
} }
} }
@ -168,11 +222,12 @@ impl FieldElement32x4 {
/// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` /// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)`
pub fn swap_CD(&mut self) { pub fn swap_CD(&mut self) {
unsafe { unsafe {
use core::arch::x86_64::_mm256_shuffle_epi32;
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32;
for i in 0..5 { for i in 0..5 {
let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01); let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01);
self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b11110000).into_bits(); self.0[i] =
_mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b11110000).into_bits();
} }
} }
} }
@ -183,7 +238,7 @@ impl FieldElement32x4 {
#[inline(always)] #[inline(always)]
pub fn diff_sum(&mut self, control: Lanes) { pub fn diff_sum(&mut self, control: Lanes) {
unsafe { unsafe {
use core::arch::x86_64::{_mm256_shuffle_epi32, _mm256_blend_epi32}; use core::arch::x86_64::{_mm256_blend_epi32, _mm256_shuffle_epi32};
let shuffle = |v: u32x8| -> u32x8 { let shuffle = |v: u32x8| -> u32x8 {
_mm256_shuffle_epi32(v.into_bits(), 0b10_11_00_01).into_bits() _mm256_shuffle_epi32(v.into_bits(), 0b10_11_00_01).into_bits()
@ -192,36 +247,41 @@ impl FieldElement32x4 {
let x01 = self.0[0]; let x01 = self.0[0];
let x01_shuf = shuffle(x01); let x01_shuf = shuffle(x01);
let v1 = (x01_shuf + P_TIMES_2_LO) - x01; let v1 = (x01_shuf + P_TIMES_2_LO) - x01;
let v2 = x01_shuf + x01; let v2 = x01_shuf + x01;
let diffsum01 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum01 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[0] = blend_lanes(x01, diffsum01, control); self.0[0] = blend_lanes(x01, diffsum01, control);
let x23 = self.0[1]; let x23 = self.0[1];
let x23_shuf = shuffle(x23); let x23_shuf = shuffle(x23);
let v1 = (x23_shuf + P_TIMES_2_HI) - x23; let v1 = (x23_shuf + P_TIMES_2_HI) - x23;
let v2 = x23_shuf + x23; let v2 = x23_shuf + x23;
let diffsum23 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum23 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[1] = blend_lanes(x23, diffsum23, control); self.0[1] = blend_lanes(x23, diffsum23, control);
let x45 = self.0[2]; let x45 = self.0[2];
let x45_shuf = shuffle(x45); let x45_shuf = shuffle(x45);
let v1 = (x45_shuf + P_TIMES_2_HI) - x45; let v1 = (x45_shuf + P_TIMES_2_HI) - x45;
let v2 = x45_shuf + x45; let v2 = x45_shuf + x45;
let diffsum45 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum45 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[2] = blend_lanes(x45, diffsum45, control); self.0[2] = blend_lanes(x45, diffsum45, control);
let x67 = self.0[3]; let x67 = self.0[3];
let x67_shuf = shuffle(x67); let x67_shuf = shuffle(x67);
let v1 = (x67_shuf + P_TIMES_2_HI) - x67; let v1 = (x67_shuf + P_TIMES_2_HI) - x67;
let v2 = x67_shuf + x67; let v2 = x67_shuf + x67;
let diffsum67 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum67 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[3] = blend_lanes(x67, diffsum67, control); self.0[3] = blend_lanes(x67, diffsum67, control);
let x89 = self.0[4]; let x89 = self.0[4];
let x89_shuf = shuffle(x89); let x89_shuf = shuffle(x89);
let v1 = (x89_shuf + P_TIMES_2_HI) - x89; let v1 = (x89_shuf + P_TIMES_2_HI) - x89;
let v2 = x89_shuf + x89; let v2 = x89_shuf + x89;
let diffsum89 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum89 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[4] = blend_lanes(x89, diffsum89, control); self.0[4] = blend_lanes(x89, diffsum89, control);
} }
} }
@ -233,7 +293,7 @@ impl FieldElement32x4 {
pub fn scale_by_curve_constants(&mut self) { pub fn scale_by_curve_constants(&mut self) {
let mut b = [u64x4::splat(0); 10]; let mut b = [u64x4::splat(0); 10];
let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); let consts = u32x8::new(121666, 0, 121666, 0, 2 * 121666, 0, 2 * 121665, 0);
unsafe { unsafe {
use core::arch::x86_64::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
@ -263,10 +323,17 @@ impl FieldElement32x4 {
} }
pub fn reduce32(&mut self) { pub fn reduce32(&mut self) {
let shifts = i32x8::new(26, 26, 25, 25, 26, 26, 25, 25);
let shifts = i32x8::new(26,26,25,25,26,26,25,25); let masks = u32x8::new(
let masks = u32x8::new((1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1, (1 << 26) - 1,
(1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1); (1 << 26) - 1,
(1 << 25) - 1,
(1 << 25) - 1,
(1 << 26) - 1,
(1 << 26) - 1,
(1 << 25) - 1,
(1 << 25) - 1,
);
let carry = |v: u32x8| -> u32x8 { let carry = |v: u32x8| -> u32x8 {
unsafe { unsafe {
@ -319,19 +386,19 @@ impl FieldElement32x4 {
pub fn reduce64(mut z: [u64x4; 10]) -> FieldElement32x4 { pub fn reduce64(mut z: [u64x4; 10]) -> FieldElement32x4 {
// These aren't const because splat isn't a const fn // These aren't const because splat isn't a const fn
let LOW_25_BITS: u64x4 = u64x4::splat((1<<25)-1); let LOW_25_BITS: u64x4 = u64x4::splat((1 << 25) - 1);
let LOW_26_BITS: u64x4 = u64x4::splat((1<<26)-1); let LOW_26_BITS: u64x4 = u64x4::splat((1 << 26) - 1);
// Carry the value from limb i = 0..8 to limb i+1 // Carry the value from limb i = 0..8 to limb i+1
let carry = |z: &mut [u64x4; 10], i: usize| { let carry = |z: &mut [u64x4; 10], i: usize| {
debug_assert!(i < 9); debug_assert!(i < 9);
if i % 2 == 0 { if i % 2 == 0 {
// Even limbs have 26 bits // Even limbs have 26 bits
z[i+1] = z[i+1] + (z[i] >> 26); z[i + 1] = z[i + 1] + (z[i] >> 26);
z[i] = z[i] & LOW_26_BITS; z[i] = z[i] & LOW_26_BITS;
} else { } else {
// Odd limbs have 25 bits // Odd limbs have 25 bits
z[i+1] = z[i+1] + (z[i] >> 25); z[i + 1] = z[i + 1] + (z[i] >> 25);
z[i] = z[i] & LOW_25_BITS; z[i] = z[i] & LOW_25_BITS;
} }
}; };
@ -386,21 +453,21 @@ impl FieldElement32x4 {
pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
let a: u32x8; let a: u32x8;
let b: u32x8; let b: u32x8;
let zero = i32x8::new(0,0,0,0,0,0,0,0); let zero = i32x8::new(0, 0, 0, 0, 0, 0, 0, 0);
unsafe { unsafe {
use core::arch::x86_64::_mm256_unpackhi_epi32; use core::arch::x86_64::_mm256_unpackhi_epi32;
use core::arch::x86_64::_mm256_unpacklo_epi32; use core::arch::x86_64::_mm256_unpacklo_epi32;
a = _mm256_unpacklo_epi32(src.into_bits(), zero.into_bits()).into_bits(); a = _mm256_unpacklo_epi32(src.into_bits(), zero.into_bits()).into_bits();
b = _mm256_unpackhi_epi32(src.into_bits(), zero.into_bits()).into_bits(); b = _mm256_unpackhi_epi32(src.into_bits(), zero.into_bits()).into_bits();
} }
(a,b) (a, b)
} }
#[inline(always)] #[inline(always)]
pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 {
unsafe { unsafe {
use core::arch::x86_64::_mm256_shuffle_epi32;
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32;
// Input: x = (a0, 0, b0, 0, c0, 0, d0) // Input: x = (a0, 0, b0, 0, c0, 0, d0)
// Input: y = (a1, 0, b1, 0, c1, 0, d1) // Input: y = (a1, 0, b1, 0, c1, 0, d1)
@ -429,16 +496,16 @@ impl FieldElement32x4 {
#[inline(always)] #[inline(always)]
fn m(x: u32x8, y: u32x8) -> u64x4 { fn m(x: u32x8, y: u32x8) -> u64x4 {
use core::arch::x86_64::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } unsafe { _mm256_mul_epu32(x.into_bits(), y.into_bits()).into_bits() }
} }
#[inline(always)] #[inline(always)]
fn m_lo(x: u32x8, y: u32x8) -> u32x8 { fn m_lo(x: u32x8, y: u32x8) -> u32x8 {
use core::arch::x86_64::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } unsafe { _mm256_mul_epu32(x.into_bits(), y.into_bits()).into_bits() }
} }
let v19 = u32x8::new(19,0,19,0,19,0,19,0); let v19 = u32x8::new(19, 0, 19, 0, 19, 0, 19, 0);
let (x0, x1) = unpack_pair(self.0[0]); let (x0, x1) = unpack_pair(self.0[0]);
let (x2, x3) = unpack_pair(self.0[1]); let (x2, x3) = unpack_pair(self.0[1]);
@ -520,13 +587,13 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 {
#[inline(always)] #[inline(always)]
fn m(x: u32x8, y: u32x8) -> u64x4 { fn m(x: u32x8, y: u32x8) -> u64x4 {
use core::arch::x86_64::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } unsafe { _mm256_mul_epu32(x.into_bits(), y.into_bits()).into_bits() }
} }
#[inline(always)] #[inline(always)]
fn m_lo(x: u32x8, y: u32x8) -> u32x8 { fn m_lo(x: u32x8, y: u32x8) -> u32x8 {
use core::arch::x86_64::_mm256_mul_epu32; use core::arch::x86_64::_mm256_mul_epu32;
unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } unsafe { _mm256_mul_epu32(x.into_bits(), y.into_bits()).into_bits() }
} }
let (x0, x1) = unpack_pair(self.0[0]); let (x0, x1) = unpack_pair(self.0[0]);
@ -541,7 +608,7 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 {
let (y6, y7) = unpack_pair(_rhs.0[3]); let (y6, y7) = unpack_pair(_rhs.0[3]);
let (y8, y9) = unpack_pair(_rhs.0[4]); let (y8, y9) = unpack_pair(_rhs.0[4]);
let v19 = u32x8::new(19,0,19,0,19,0,19,0); let v19 = u32x8::new(19, 0, 19, 0, 19, 0, 19, 0);
let y1_19 = m_lo(v19, y1); // This fits in a u32 let y1_19 = m_lo(v19, y1); // This fits in a u32
let y2_19 = m_lo(v19, y2); // iff 26 + b + lg(19) < 32 let y2_19 = m_lo(v19, y2); // iff 26 + b + lg(19) < 32
@ -585,10 +652,10 @@ mod test {
x.scale_by_curve_constants(); x.scale_by_curve_constants();
let xs = x.split(); let xs = x.split();
assert_eq!(xs[0], FieldElement64([ 121666,0,0,0,0])); assert_eq!(xs[0], FieldElement64([121666, 0, 0, 0, 0]));
assert_eq!(xs[1], FieldElement64([ 121666,0,0,0,0])); assert_eq!(xs[1], FieldElement64([121666, 0, 0, 0, 0]));
assert_eq!(xs[2], FieldElement64([2*121666,0,0,0,0])); assert_eq!(xs[2], FieldElement64([2 * 121666, 0, 0, 0, 0]));
assert_eq!(xs[3], FieldElement64([2*121665,0,0,0,0])); assert_eq!(xs[3], FieldElement64([2 * 121665, 0, 0, 0, 0]));
} }
#[test] #[test]
@ -636,7 +703,6 @@ mod test {
assert_eq!(result[3], -&(&x3 * &x3)); assert_eq!(result[3], -&(&x3 * &x3));
} }
#[test] #[test]
fn multiply_vs_serial() { fn multiply_vs_serial() {
let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]);
@ -666,7 +732,7 @@ mod test {
let src = vec.0[0]; let src = vec.0[0];
let (a,b) = unpack_pair(src); let (a, b) = unpack_pair(src);
let expected_a = u32x8::new(10000, 0, 10100, 0, 10200, 0, 10300, 0); let expected_a = u32x8::new(10000, 0, 10100, 0, 10200, 0, 10300, 0);
let expected_b = u32x8::new(10001, 0, 10101, 0, 10201, 0, 10301, 0); let expected_b = u32x8::new(10001, 0, 10101, 0, 10201, 0, 10301, 0);
@ -674,7 +740,7 @@ mod test {
assert_eq!(a, expected_a); assert_eq!(a, expected_a);
assert_eq!(b, expected_b); assert_eq!(b, expected_b);
let expected_src = repack_pair(a,b); let expected_src = repack_pair(a, b);
assert_eq!(src, expected_src); assert_eq!(src, expected_src);
} }