mirror of
https://github.com/saymrwulf/risc0-curve25519-dalek-source.git
synced 2026-09-09 20:50:40 +00:00
Move blend_lanes into the blend function
This commit is contained in:
Henry de Valence
parent
64b1b481ba
commit
4dc219910a
1 changed files with 73 additions and 73 deletions
|
|
@ -46,6 +46,97 @@ pub enum Lanes {
|
||||||
ABCD,
|
ABCD,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The `Shuffle` enum represents a shuffle of a `FieldElement32x4`.
|
||||||
|
#[derive(Copy, Clone, Debug)]
|
||||||
|
pub enum Shuffle {
|
||||||
|
AAAA,
|
||||||
|
BBBB,
|
||||||
|
CACA,
|
||||||
|
DBBD,
|
||||||
|
ADDA,
|
||||||
|
CBCB,
|
||||||
|
ABAB,
|
||||||
|
BADC,
|
||||||
|
BACD,
|
||||||
|
ABDC,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A vector of four `FieldElements`, implemented using AVX2.
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]);
|
||||||
|
|
||||||
|
use subtle::Choice;
|
||||||
|
use subtle::ConditionallyAssignable;
|
||||||
|
|
||||||
|
impl ConditionallyAssignable for FieldElement32x4 {
|
||||||
|
fn conditional_assign(&mut self, other: &FieldElement32x4, choice: Choice) {
|
||||||
|
let mask = (-(choice.unwrap_u8() as i32)) as u32;
|
||||||
|
let mask_vec = u32x8::splat(mask);
|
||||||
|
for i in 0..5 {
|
||||||
|
self.0[i] = self.0[i] ^ (mask_vec & (self.0[i] ^ other.0[i]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FieldElement32x4 {
|
||||||
|
pub(crate) fn split(&self) -> [FieldElement64; 4] {
|
||||||
|
let mut out = [FieldElement64::zero(); 4];
|
||||||
|
for i in 0..5 {
|
||||||
|
let a_2i = self.0[i].extract(0) as u64; //
|
||||||
|
let b_2i = self.0[i].extract(1) as u64; //
|
||||||
|
let a_2i_1 = self.0[i].extract(2) as u64; // `.
|
||||||
|
let b_2i_1 = self.0[i].extract(3) as u64; // | pre-swapped to avoid
|
||||||
|
let c_2i = self.0[i].extract(4) as u64; // | a cross lane shuffle
|
||||||
|
let d_2i = self.0[i].extract(5) as u64; // .'
|
||||||
|
let c_2i_1 = self.0[i].extract(6) as u64; //
|
||||||
|
let d_2i_1 = self.0[i].extract(7) as u64; //
|
||||||
|
|
||||||
|
out[0].0[i] = a_2i + (a_2i_1 << 26);
|
||||||
|
out[1].0[i] = b_2i + (b_2i_1 << 26);
|
||||||
|
out[2].0[i] = c_2i + (c_2i_1 << 26);
|
||||||
|
out[3].0[i] = d_2i + (d_2i_1 << 26);
|
||||||
|
}
|
||||||
|
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn shuffle(&self, control: Shuffle) -> FieldElement32x4 {
|
||||||
|
#[inline(always)]
|
||||||
|
fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 {
|
||||||
|
unsafe {
|
||||||
|
use core::arch::x86_64::_mm256_permutevar8x32_epi32;
|
||||||
|
|
||||||
|
let c: u32x8 = match control {
|
||||||
|
Shuffle::AAAA => u32x8::new(0, 0, 2, 2, 0, 0, 2, 2),
|
||||||
|
Shuffle::BBBB => u32x8::new(1, 1, 3, 3, 1, 1, 3, 3),
|
||||||
|
Shuffle::CACA => u32x8::new(4, 0, 6, 2, 4, 0, 6, 2),
|
||||||
|
Shuffle::DBBD => u32x8::new(5, 1, 7, 3, 1, 5, 3, 7),
|
||||||
|
Shuffle::ADDA => u32x8::new(0, 5, 2, 7, 5, 0, 7, 2),
|
||||||
|
Shuffle::CBCB => u32x8::new(4, 1, 6, 3, 4, 1, 6, 3),
|
||||||
|
Shuffle::ABAB => u32x8::new(0, 1, 2, 3, 0, 1, 2, 3),
|
||||||
|
Shuffle::BADC => u32x8::new(1, 0, 3, 2, 5, 4, 7, 6),
|
||||||
|
Shuffle::BACD => u32x8::new(1, 0, 3, 2, 4, 5, 6, 7),
|
||||||
|
Shuffle::ABDC => u32x8::new(0, 1, 2, 3, 5, 4, 7, 6),
|
||||||
|
};
|
||||||
|
// Note that this gets turned into a generic LLVM
|
||||||
|
// shuffle-by-constants, which can be lowered to a simpler
|
||||||
|
// instruction than a generic permute.
|
||||||
|
_mm256_permutevar8x32_epi32(x.into_bits(), c.into_bits()).into_bits()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FieldElement32x4([
|
||||||
|
shuffle_lanes(self.0[0], control),
|
||||||
|
shuffle_lanes(self.0[1], control),
|
||||||
|
shuffle_lanes(self.0[2], control),
|
||||||
|
shuffle_lanes(self.0[3], control),
|
||||||
|
shuffle_lanes(self.0[4], control),
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn blend(&self, b: FieldElement32x4, control: Lanes) -> FieldElement32x4 {
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
|
fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
|
||||||
unsafe {
|
unsafe {
|
||||||
|
|
@ -107,62 +198,6 @@ fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The `Shuffle` enum represents a shuffle of a `FieldElement32x4`.
|
|
||||||
#[derive(Copy, Clone, Debug)]
|
|
||||||
pub enum Shuffle {
|
|
||||||
AAAA,
|
|
||||||
BBBB,
|
|
||||||
CACA,
|
|
||||||
DBBD,
|
|
||||||
ADDA,
|
|
||||||
CBCB,
|
|
||||||
ABAB,
|
|
||||||
BADC,
|
|
||||||
BACD,
|
|
||||||
ABDC,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A vector of four `FieldElements`, implemented using AVX2.
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
|
||||||
pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]);
|
|
||||||
|
|
||||||
use subtle::Choice;
|
|
||||||
use subtle::ConditionallyAssignable;
|
|
||||||
|
|
||||||
impl ConditionallyAssignable for FieldElement32x4 {
|
|
||||||
fn conditional_assign(&mut self, other: &FieldElement32x4, choice: Choice) {
|
|
||||||
let mask = (-(choice.unwrap_u8() as i32)) as u32;
|
|
||||||
let mask_vec = u32x8::splat(mask);
|
|
||||||
for i in 0..5 {
|
|
||||||
self.0[i] = self.0[i] ^ (mask_vec & (self.0[i] ^ other.0[i]));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl FieldElement32x4 {
|
|
||||||
pub(crate) fn split(&self) -> [FieldElement64; 4] {
|
|
||||||
let mut out = [FieldElement64::zero(); 4];
|
|
||||||
for i in 0..5 {
|
|
||||||
let a_2i = self.0[i].extract(0) as u64; //
|
|
||||||
let b_2i = self.0[i].extract(1) as u64; //
|
|
||||||
let a_2i_1 = self.0[i].extract(2) as u64; // `.
|
|
||||||
let b_2i_1 = self.0[i].extract(3) as u64; // | pre-swapped to avoid
|
|
||||||
let c_2i = self.0[i].extract(4) as u64; // | a cross lane shuffle
|
|
||||||
let d_2i = self.0[i].extract(5) as u64; // .'
|
|
||||||
let c_2i_1 = self.0[i].extract(6) as u64; //
|
|
||||||
let d_2i_1 = self.0[i].extract(7) as u64; //
|
|
||||||
|
|
||||||
out[0].0[i] = a_2i + (a_2i_1 << 26);
|
|
||||||
out[1].0[i] = b_2i + (b_2i_1 << 26);
|
|
||||||
out[2].0[i] = c_2i + (c_2i_1 << 26);
|
|
||||||
out[3].0[i] = d_2i + (d_2i_1 << 26);
|
|
||||||
}
|
|
||||||
|
|
||||||
out
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
|
||||||
pub fn blend(&self, b: FieldElement32x4, control: Lanes) -> FieldElement32x4 {
|
|
||||||
FieldElement32x4([
|
FieldElement32x4([
|
||||||
blend_lanes(self.0[0], b.0[0], control),
|
blend_lanes(self.0[0], b.0[0], control),
|
||||||
blend_lanes(self.0[1], b.0[1], control),
|
blend_lanes(self.0[1], b.0[1], control),
|
||||||
|
|
@ -172,41 +207,6 @@ impl FieldElement32x4 {
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline(always)]
|
|
||||||
pub fn shuffle(&self, control: Shuffle) -> FieldElement32x4 {
|
|
||||||
#[inline(always)]
|
|
||||||
fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 {
|
|
||||||
unsafe {
|
|
||||||
use core::arch::x86_64::_mm256_permutevar8x32_epi32;
|
|
||||||
|
|
||||||
let c: u32x8 = match control {
|
|
||||||
Shuffle::AAAA => u32x8::new(0, 0, 2, 2, 0, 0, 2, 2),
|
|
||||||
Shuffle::BBBB => u32x8::new(1, 1, 3, 3, 1, 1, 3, 3),
|
|
||||||
Shuffle::CACA => u32x8::new(4, 0, 6, 2, 4, 0, 6, 2),
|
|
||||||
Shuffle::DBBD => u32x8::new(5, 1, 7, 3, 1, 5, 3, 7),
|
|
||||||
Shuffle::ADDA => u32x8::new(0, 5, 2, 7, 5, 0, 7, 2),
|
|
||||||
Shuffle::CBCB => u32x8::new(4, 1, 6, 3, 4, 1, 6, 3),
|
|
||||||
Shuffle::ABAB => u32x8::new(0, 1, 2, 3, 0, 1, 2, 3),
|
|
||||||
Shuffle::BADC => u32x8::new(1, 0, 3, 2, 5, 4, 7, 6),
|
|
||||||
Shuffle::BACD => u32x8::new(1, 0, 3, 2, 4, 5, 6, 7),
|
|
||||||
Shuffle::ABDC => u32x8::new(0, 1, 2, 3, 5, 4, 7, 6),
|
|
||||||
};
|
|
||||||
// Note that this gets turned into a generic LLVM
|
|
||||||
// shuffle-by-constants, which can be lowered to a simpler
|
|
||||||
// instruction than a generic permute.
|
|
||||||
_mm256_permutevar8x32_epi32(x.into_bits(), c.into_bits()).into_bits()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
FieldElement32x4([
|
|
||||||
shuffle_lanes(self.0[0], control),
|
|
||||||
shuffle_lanes(self.0[1], control),
|
|
||||||
shuffle_lanes(self.0[2], control),
|
|
||||||
shuffle_lanes(self.0[3], control),
|
|
||||||
shuffle_lanes(self.0[4], control),
|
|
||||||
])
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn zero() -> FieldElement32x4 {
|
pub fn zero() -> FieldElement32x4 {
|
||||||
FieldElement32x4([u32x8::splat(0); 5])
|
FieldElement32x4([u32x8::splat(0); 5])
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue