saymrwulf/curve25519-dalek-source
1
0
Fork 0
mirror of https://github.com/saymrwulf/curve25519-dalek-source.git synced 2026-09-06 20:41:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

some rustfmt changes

Browse source
This commit is contained in:
Henry de Valence 2018-06-14 14:03:11 -07:00
parent d29618ca2e
commit bd1e3c5f3e
2 changed files with 178 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
src/backend/avx2/edwards.rs
View file

@ -14,19 +14,19 @@
#![allow(bad_style)] #![allow(bad_style)]
use core::convert::From; use core::convert::From;
use core::ops::{Add, Sub, Neg}; use core::ops::{Add, Neg, Sub};
use core::simd::{IntoBits, u32x8}; use core::simd::{u32x8, IntoBits};
use subtle::ConditionallyAssignable;
use subtle::Choice; use subtle::Choice;
use subtle::ConditionallyAssignable;
use edwards; use edwards;
use scalar_mul::window::{LookupTable, NafLookupTable5, NafLookupTable8}; use scalar_mul::window::{LookupTable, NafLookupTable5, NafLookupTable8};
use traits::Identity; use traits::Identity;
use backend::avx2::field::{D_LANES, Lanes, FieldElement32x4}; use backend::avx2::field::{FieldElement32x4, Lanes, D_LANES};
use backend::avx2; use backend::avx2;
@ -43,7 +43,12 @@ impl From<edwards::EdwardsPoint> for ExtendedPoint {
impl From<ExtendedPoint> for edwards::EdwardsPoint { impl From<ExtendedPoint> for edwards::EdwardsPoint {
fn from(P: ExtendedPoint) -> edwards::EdwardsPoint { fn from(P: ExtendedPoint) -> edwards::EdwardsPoint {
let tmp = P.0.split(); let tmp = P.0.split();
edwards::EdwardsPoint{X: tmp[0], Y: tmp[1], Z: tmp[2], T: tmp[3]} edwards::EdwardsPoint {
X: tmp[0],
Y: tmp[1],
Z: tmp[2],
T: tmp[3],
}
} }
} }
@ -74,9 +79,9 @@ impl Identity for ExtendedPoint {
impl ExtendedPoint { impl ExtendedPoint {
pub fn double(&self) -> ExtendedPoint { pub fn double(&self) -> ExtendedPoint {
unsafe { unsafe {
use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_permute2x128_si256; use core::arch::x86_64::_mm256_permute2x128_si256;
use core::arch::x86_64::_mm256_permutevar8x32_epi32; use core::arch::x86_64::_mm256_permutevar8x32_epi32;
use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
let P = &self.0; let P = &self.0;
@ -140,17 +145,22 @@ impl ExtendedPoint {
// - | | S2 | S2 | | // - | | S2 | S2 | |
// ======================= // =======================
// S5 S6 S8 S9 // S5 S6 S8 S9
//
for i in 0..5 { for i in 0..5 {
let zero = u32x8::splat(0).into_bits(); let zero = u32x8::splat(0).into_bits();
let S1: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c0).into_bits(); let S1: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c0).into_bits();
let S2: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c1).into_bits(); let S2: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c1).into_bits();
let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into_bits(), 0b01010000).into_bits(); let S3_2: u32x8 =
_mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into_bits(), 0b01010000)
.into_bits();
// tmp0 = (0 0 2*S3 -S4) // tmp0 = (0 0 2*S3 -S4)
let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into_bits(), t1.0[i].into_bits(), 0b10100000).into_bits(); let tmp0: u32x8 =
_mm256_blend_epi32(S3_2.into_bits(), t1.0[i].into_bits(), 0b10100000)
.into_bits();
t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1; t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1;
let S2_pos: u32x8 = _mm256_blend_epi32(zero, S2.into_bits(), 0b10100101).into_bits(); let S2_pos: u32x8 =
let S2_neg: u32x8 = _mm256_blend_epi32(S2.into_bits(), zero, 0b10100101).into_bits(); _mm256_blend_epi32(zero, S2.into_bits(), 0b10100101).into_bits();
let S2_neg: u32x8 =
_mm256_blend_epi32(S2.into_bits(), zero, 0b10100101).into_bits();
t0.0[i] = t0.0[i] + S2_pos; t0.0[i] = t0.0[i] + S2_pos;
t0.0[i] = t0.0[i] - S2_neg; t0.0[i] = t0.0[i] - S2_neg;
} }
@ -335,7 +345,7 @@ mod test {
macro_rules! print_var { macro_rules! print_var {
($x:ident) => { ($x:ident) => {
println!("{} = {:?}", stringify!($x), $x.to_bytes()); println!("{} = {:?}", stringify!($x), $x.to_bytes());
} };
} }
let S0 = &Y1 - &X1; // R1 let S0 = &Y1 - &X1; // R1
@ -383,7 +393,12 @@ mod test {
let Z3 = &S15 * &S14; // R2 * R3 let Z3 = &S15 * &S14; // R2 * R3
let T3 = &S12 * &S13; // R1 * R4 let T3 = &S12 * &S13; // R1 * R4
edwards::EdwardsPoint{X: X3, Y: Y3, Z: Z3, T: T3} edwards::EdwardsPoint {
X: X3,
Y: Y3,
Z: Z3,
T: T3,
}
} }
fn addition_test_helper(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) { fn addition_test_helper(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) {
@ -442,7 +457,7 @@ mod test {
macro_rules! print_var { macro_rules! print_var {
($x:ident) => { ($x:ident) => {
println!("{} = {:?}", stringify!($x), $x.to_bytes()); println!("{} = {:?}", stringify!($x), $x.to_bytes());
} };
} }
let S0 = &X1 + &Y1; // R1 let S0 = &X1 + &Y1; // R1
@ -476,7 +491,12 @@ mod test {
let Z3 = &S8 * &S6; let Z3 = &S8 * &S6;
let T3 = &S5 * &S9; let T3 = &S5 * &S9;
edwards::EdwardsPoint{X: X3, Y: Y3, Z: Z3, T: T3} edwards::EdwardsPoint {
X: X3,
Y: Y3,
Z: Z3,
T: T3,
}
} }
fn doubling_test_helper(P: edwards::EdwardsPoint) { fn doubling_test_helper(P: edwards::EdwardsPoint) {

132
src/backend/avx2/field.rs
View file

@ -24,11 +24,11 @@ pub const D_LANES64: u8 = 0b11_00_00_00;
pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES; pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES;
use core::ops::Mul; use core::ops::{Add, Mul};
use core::simd::{IntoBits, u32x8, i32x8, u64x4}; use core::simd::{i32x8, u32x8, u64x4, IntoBits};
use backend::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO};
use backend::u64::field::FieldElement64; use backend::u64::field::FieldElement64;
use backend::avx2::constants::{P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_16_LO, P_TIMES_16_HI};
#[derive(Copy, Clone)] #[derive(Copy, Clone)]
pub enum Lanes { pub enum Lanes {
@ -43,9 +43,23 @@ fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
match control { match control {
Lanes::AB => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32).into_bits(), Lanes::C => {
Lanes::CD => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32).into_bits(), _mm256_blend_epi32(x.into_bits(), y.into_bits(), C_LANES as i32).into_bits()
Lanes::ALL => _mm256_blend_epi32(x.into_bits(), y.into_bits(), ALL_LANES as i32).into_bits(), }
Lanes::D => {
_mm256_blend_epi32(x.into_bits(), y.into_bits(), D_LANES as i32).into_bits()
}
Lanes::AB => {
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32)
.into_bits()
}
Lanes::CD => {
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32)
.into_bits()
}
Lanes::ALL => {
_mm256_blend_epi32(x.into_bits(), y.into_bits(), ALL_LANES as i32).into_bits()
}
} }
} }
} }
@ -54,8 +68,8 @@ fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
#[derive(Clone, Copy, Debug)] #[derive(Clone, Copy, Debug)]
pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]); pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]);
use subtle::ConditionallyAssignable;
use subtle::Choice; use subtle::Choice;
use subtle::ConditionallyAssignable;
impl ConditionallyAssignable for FieldElement32x4 { impl ConditionallyAssignable for FieldElement32x4 {
fn conditional_assign(&mut self, other: &FieldElement32x4, choice: Choice) { fn conditional_assign(&mut self, other: &FieldElement32x4, choice: Choice) {
@ -71,7 +85,6 @@ impl FieldElement32x4 {
pub(crate) fn split(&self) -> [FieldElement64; 4] { pub(crate) fn split(&self) -> [FieldElement64; 4] {
let mut out = [FieldElement64::zero(); 4]; let mut out = [FieldElement64::zero(); 4];
for i in 0..5 { for i in 0..5 {
let a_2i = self.0[i].extract(0) as u64; // let a_2i = self.0[i].extract(0) as u64; //
let b_2i = self.0[i].extract(1) as u64; // let b_2i = self.0[i].extract(1) as u64; //
let a_2i_1 = self.0[i].extract(2) as u64; // `. let a_2i_1 = self.0[i].extract(2) as u64; // `.
@ -130,11 +143,31 @@ impl FieldElement32x4 {
pub fn negate_D_lazy(&mut self) { pub fn negate_D_lazy(&mut self) {
unsafe { unsafe {
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_2_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits(); self.0[0] = _mm256_blend_epi32(
self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_2_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits(); self.0[0].into_bits(),
self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_2_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits(); (P_TIMES_2_LO - self.0[0]).into_bits(),
self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_2_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits(); D_LANES as i32,
self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_2_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits(); ).into_bits();
self.0[1] = _mm256_blend_epi32(
self.0[1].into_bits(),
(P_TIMES_2_HI - self.0[1]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[2] = _mm256_blend_epi32(
self.0[2].into_bits(),
(P_TIMES_2_HI - self.0[2]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[3] = _mm256_blend_epi32(
self.0[3].into_bits(),
(P_TIMES_2_HI - self.0[3]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[4] = _mm256_blend_epi32(
self.0[4].into_bits(),
(P_TIMES_2_HI - self.0[4]).into_bits(),
D_LANES as i32,
).into_bits();
} }
} }
@ -144,11 +177,31 @@ impl FieldElement32x4 {
pub fn negate_D(&mut self) { pub fn negate_D(&mut self) {
unsafe { unsafe {
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_16_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits(); self.0[0] = _mm256_blend_epi32(
self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_16_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits(); self.0[0].into_bits(),
self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_16_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits(); (P_TIMES_16_LO - self.0[0]).into_bits(),
self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_16_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits(); D_LANES as i32,
self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_16_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits(); ).into_bits();
self.0[1] = _mm256_blend_epi32(
self.0[1].into_bits(),
(P_TIMES_16_HI - self.0[1]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[2] = _mm256_blend_epi32(
self.0[2].into_bits(),
(P_TIMES_16_HI - self.0[2]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[3] = _mm256_blend_epi32(
self.0[3].into_bits(),
(P_TIMES_16_HI - self.0[3]).into_bits(),
D_LANES as i32,
).into_bits();
self.0[4] = _mm256_blend_epi32(
self.0[4].into_bits(),
(P_TIMES_16_HI - self.0[4]).into_bits(),
D_LANES as i32,
).into_bits();
} }
self.reduce32(); self.reduce32();
} }
@ -156,11 +209,12 @@ impl FieldElement32x4 {
/// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)` /// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)`
pub fn swap_AB(&mut self) { pub fn swap_AB(&mut self) {
unsafe { unsafe {
use core::arch::x86_64::_mm256_shuffle_epi32;
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32;
for i in 0..5 { for i in 0..5 {
let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01); let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01);
self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b00001111).into_bits(); self.0[i] =
_mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b00001111).into_bits();
} }
} }
} }
@ -168,11 +222,12 @@ impl FieldElement32x4 {
/// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` /// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)`
pub fn swap_CD(&mut self) { pub fn swap_CD(&mut self) {
unsafe { unsafe {
use core::arch::x86_64::_mm256_shuffle_epi32;
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32;
for i in 0..5 { for i in 0..5 {
let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01); let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01);
self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b11110000).into_bits(); self.0[i] =
_mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b11110000).into_bits();
} }
} }
} }
@ -183,7 +238,7 @@ impl FieldElement32x4 {
#[inline(always)] #[inline(always)]
pub fn diff_sum(&mut self, control: Lanes) { pub fn diff_sum(&mut self, control: Lanes) {
unsafe { unsafe {
use core::arch::x86_64::{_mm256_shuffle_epi32, _mm256_blend_epi32}; use core::arch::x86_64::{_mm256_blend_epi32, _mm256_shuffle_epi32};
let shuffle = |v: u32x8| -> u32x8 { let shuffle = |v: u32x8| -> u32x8 {
_mm256_shuffle_epi32(v.into_bits(), 0b10_11_00_01).into_bits() _mm256_shuffle_epi32(v.into_bits(), 0b10_11_00_01).into_bits()
@ -193,35 +248,40 @@ impl FieldElement32x4 {
let x01_shuf = shuffle(x01); let x01_shuf = shuffle(x01);
let v1 = (x01_shuf + P_TIMES_2_LO) - x01; let v1 = (x01_shuf + P_TIMES_2_LO) - x01;
let v2 = x01_shuf + x01; let v2 = x01_shuf + x01;
let diffsum01 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum01 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[0] = blend_lanes(x01, diffsum01, control); self.0[0] = blend_lanes(x01, diffsum01, control);
let x23 = self.0[1]; let x23 = self.0[1];
let x23_shuf = shuffle(x23); let x23_shuf = shuffle(x23);
let v1 = (x23_shuf + P_TIMES_2_HI) - x23; let v1 = (x23_shuf + P_TIMES_2_HI) - x23;
let v2 = x23_shuf + x23; let v2 = x23_shuf + x23;
let diffsum23 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum23 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[1] = blend_lanes(x23, diffsum23, control); self.0[1] = blend_lanes(x23, diffsum23, control);
let x45 = self.0[2]; let x45 = self.0[2];
let x45_shuf = shuffle(x45); let x45_shuf = shuffle(x45);
let v1 = (x45_shuf + P_TIMES_2_HI) - x45; let v1 = (x45_shuf + P_TIMES_2_HI) - x45;
let v2 = x45_shuf + x45; let v2 = x45_shuf + x45;
let diffsum45 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum45 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[2] = blend_lanes(x45, diffsum45, control); self.0[2] = blend_lanes(x45, diffsum45, control);
let x67 = self.0[3]; let x67 = self.0[3];
let x67_shuf = shuffle(x67); let x67_shuf = shuffle(x67);
let v1 = (x67_shuf + P_TIMES_2_HI) - x67; let v1 = (x67_shuf + P_TIMES_2_HI) - x67;
let v2 = x67_shuf + x67; let v2 = x67_shuf + x67;
let diffsum67 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum67 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[3] = blend_lanes(x67, diffsum67, control); self.0[3] = blend_lanes(x67, diffsum67, control);
let x89 = self.0[4]; let x89 = self.0[4];
let x89_shuf = shuffle(x89); let x89_shuf = shuffle(x89);
let v1 = (x89_shuf + P_TIMES_2_HI) - x89; let v1 = (x89_shuf + P_TIMES_2_HI) - x89;
let v2 = x89_shuf + x89; let v2 = x89_shuf + x89;
let diffsum89 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); let diffsum89 =
_mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits();
self.0[4] = blend_lanes(x89, diffsum89, control); self.0[4] = blend_lanes(x89, diffsum89, control);
} }
} }
@ -263,10 +323,17 @@ impl FieldElement32x4 {
} }
pub fn reduce32(&mut self) { pub fn reduce32(&mut self) {
let shifts = i32x8::new(26, 26, 25, 25, 26, 26, 25, 25); let shifts = i32x8::new(26, 26, 25, 25, 26, 26, 25, 25);
let masks = u32x8::new((1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1, let masks = u32x8::new(
(1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1); (1 << 26) - 1,
(1 << 26) - 1,
(1 << 25) - 1,
(1 << 25) - 1,
(1 << 26) - 1,
(1 << 26) - 1,
(1 << 25) - 1,
(1 << 25) - 1,
);
let carry = |v: u32x8| -> u32x8 { let carry = |v: u32x8| -> u32x8 {
unsafe { unsafe {
@ -399,8 +466,8 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
#[inline(always)] #[inline(always)]
pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 {
unsafe { unsafe {
use core::arch::x86_64::_mm256_shuffle_epi32;
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32;
// Input: x = (a0, 0, b0, 0, c0, 0, d0) // Input: x = (a0, 0, b0, 0, c0, 0, d0)
// Input: y = (a1, 0, b1, 0, c1, 0, d1) // Input: y = (a1, 0, b1, 0, c1, 0, d1)
@ -636,7 +703,6 @@ mod test {
assert_eq!(result[3], -&(&x3 * &x3)); assert_eq!(result[3], -&(&x3 * &x3));
} }
#[test] #[test]
fn multiply_vs_serial() { fn multiply_vs_serial() {
let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]);