mirror of
https://github.com/saymrwulf/risc0-curve25519-dalek-source.git
synced 2026-09-10 21:00:41 +00:00
Move blend_lanes into the blend function
This commit is contained in:
Henry de Valence
parent
64b1b481ba
commit
4dc219910a
1 changed files with 73 additions and 73 deletions
|
|
@ -46,8 +46,99 @@ pub enum Lanes {
|
|||
ABCD,
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
|
||||
/// The `Shuffle` enum represents a shuffle of a `FieldElement32x4`.
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub enum Shuffle {
|
||||
AAAA,
|
||||
BBBB,
|
||||
CACA,
|
||||
DBBD,
|
||||
ADDA,
|
||||
CBCB,
|
||||
ABAB,
|
||||
BADC,
|
||||
BACD,
|
||||
ABDC,
|
||||
}
|
||||
|
||||
/// A vector of four `FieldElements`, implemented using AVX2.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]);
|
||||
|
||||
use subtle::Choice;
|
||||
use subtle::ConditionallyAssignable;
|
||||
|
||||
impl ConditionallyAssignable for FieldElement32x4 {
|
||||
fn conditional_assign(&mut self, other: &FieldElement32x4, choice: Choice) {
|
||||
let mask = (-(choice.unwrap_u8() as i32)) as u32;
|
||||
let mask_vec = u32x8::splat(mask);
|
||||
for i in 0..5 {
|
||||
self.0[i] = self.0[i] ^ (mask_vec & (self.0[i] ^ other.0[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FieldElement32x4 {
|
||||
pub(crate) fn split(&self) -> [FieldElement64; 4] {
|
||||
let mut out = [FieldElement64::zero(); 4];
|
||||
for i in 0..5 {
|
||||
let a_2i = self.0[i].extract(0) as u64; //
|
||||
let b_2i = self.0[i].extract(1) as u64; //
|
||||
let a_2i_1 = self.0[i].extract(2) as u64; // `.
|
||||
let b_2i_1 = self.0[i].extract(3) as u64; // | pre-swapped to avoid
|
||||
let c_2i = self.0[i].extract(4) as u64; // | a cross lane shuffle
|
||||
let d_2i = self.0[i].extract(5) as u64; // .'
|
||||
let c_2i_1 = self.0[i].extract(6) as u64; //
|
||||
let d_2i_1 = self.0[i].extract(7) as u64; //
|
||||
|
||||
out[0].0[i] = a_2i + (a_2i_1 << 26);
|
||||
out[1].0[i] = b_2i + (b_2i_1 << 26);
|
||||
out[2].0[i] = c_2i + (c_2i_1 << 26);
|
||||
out[3].0[i] = d_2i + (d_2i_1 << 26);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn shuffle(&self, control: Shuffle) -> FieldElement32x4 {
|
||||
#[inline(always)]
|
||||
fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 {
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_permutevar8x32_epi32;
|
||||
|
||||
let c: u32x8 = match control {
|
||||
Shuffle::AAAA => u32x8::new(0, 0, 2, 2, 0, 0, 2, 2),
|
||||
Shuffle::BBBB => u32x8::new(1, 1, 3, 3, 1, 1, 3, 3),
|
||||
Shuffle::CACA => u32x8::new(4, 0, 6, 2, 4, 0, 6, 2),
|
||||
Shuffle::DBBD => u32x8::new(5, 1, 7, 3, 1, 5, 3, 7),
|
||||
Shuffle::ADDA => u32x8::new(0, 5, 2, 7, 5, 0, 7, 2),
|
||||
Shuffle::CBCB => u32x8::new(4, 1, 6, 3, 4, 1, 6, 3),
|
||||
Shuffle::ABAB => u32x8::new(0, 1, 2, 3, 0, 1, 2, 3),
|
||||
Shuffle::BADC => u32x8::new(1, 0, 3, 2, 5, 4, 7, 6),
|
||||
Shuffle::BACD => u32x8::new(1, 0, 3, 2, 4, 5, 6, 7),
|
||||
Shuffle::ABDC => u32x8::new(0, 1, 2, 3, 5, 4, 7, 6),
|
||||
};
|
||||
// Note that this gets turned into a generic LLVM
|
||||
// shuffle-by-constants, which can be lowered to a simpler
|
||||
// instruction than a generic permute.
|
||||
_mm256_permutevar8x32_epi32(x.into_bits(), c.into_bits()).into_bits()
|
||||
}
|
||||
}
|
||||
|
||||
FieldElement32x4([
|
||||
shuffle_lanes(self.0[0], control),
|
||||
shuffle_lanes(self.0[1], control),
|
||||
shuffle_lanes(self.0[2], control),
|
||||
shuffle_lanes(self.0[3], control),
|
||||
shuffle_lanes(self.0[4], control),
|
||||
])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn blend(&self, b: FieldElement32x4, control: Lanes) -> FieldElement32x4 {
|
||||
#[inline(always)]
|
||||
fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
|
||||
|
|
@ -105,64 +196,8 @@ fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
|
|||
).into_bits(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The `Shuffle` enum represents a shuffle of a `FieldElement32x4`.
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
pub enum Shuffle {
|
||||
AAAA,
|
||||
BBBB,
|
||||
CACA,
|
||||
DBBD,
|
||||
ADDA,
|
||||
CBCB,
|
||||
ABAB,
|
||||
BADC,
|
||||
BACD,
|
||||
ABDC,
|
||||
}
|
||||
|
||||
/// A vector of four `FieldElements`, implemented using AVX2.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]);
|
||||
|
||||
use subtle::Choice;
|
||||
use subtle::ConditionallyAssignable;
|
||||
|
||||
impl ConditionallyAssignable for FieldElement32x4 {
|
||||
fn conditional_assign(&mut self, other: &FieldElement32x4, choice: Choice) {
|
||||
let mask = (-(choice.unwrap_u8() as i32)) as u32;
|
||||
let mask_vec = u32x8::splat(mask);
|
||||
for i in 0..5 {
|
||||
self.0[i] = self.0[i] ^ (mask_vec & (self.0[i] ^ other.0[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FieldElement32x4 {
|
||||
pub(crate) fn split(&self) -> [FieldElement64; 4] {
|
||||
let mut out = [FieldElement64::zero(); 4];
|
||||
for i in 0..5 {
|
||||
let a_2i = self.0[i].extract(0) as u64; //
|
||||
let b_2i = self.0[i].extract(1) as u64; //
|
||||
let a_2i_1 = self.0[i].extract(2) as u64; // `.
|
||||
let b_2i_1 = self.0[i].extract(3) as u64; // | pre-swapped to avoid
|
||||
let c_2i = self.0[i].extract(4) as u64; // | a cross lane shuffle
|
||||
let d_2i = self.0[i].extract(5) as u64; // .'
|
||||
let c_2i_1 = self.0[i].extract(6) as u64; //
|
||||
let d_2i_1 = self.0[i].extract(7) as u64; //
|
||||
|
||||
out[0].0[i] = a_2i + (a_2i_1 << 26);
|
||||
out[1].0[i] = b_2i + (b_2i_1 << 26);
|
||||
out[2].0[i] = c_2i + (c_2i_1 << 26);
|
||||
out[3].0[i] = d_2i + (d_2i_1 << 26);
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn blend(&self, b: FieldElement32x4, control: Lanes) -> FieldElement32x4 {
|
||||
FieldElement32x4([
|
||||
blend_lanes(self.0[0], b.0[0], control),
|
||||
blend_lanes(self.0[1], b.0[1], control),
|
||||
|
|
@ -172,41 +207,6 @@ impl FieldElement32x4 {
|
|||
])
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn shuffle(&self, control: Shuffle) -> FieldElement32x4 {
|
||||
#[inline(always)]
|
||||
fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 {
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_permutevar8x32_epi32;
|
||||
|
||||
let c: u32x8 = match control {
|
||||
Shuffle::AAAA => u32x8::new(0, 0, 2, 2, 0, 0, 2, 2),
|
||||
Shuffle::BBBB => u32x8::new(1, 1, 3, 3, 1, 1, 3, 3),
|
||||
Shuffle::CACA => u32x8::new(4, 0, 6, 2, 4, 0, 6, 2),
|
||||
Shuffle::DBBD => u32x8::new(5, 1, 7, 3, 1, 5, 3, 7),
|
||||
Shuffle::ADDA => u32x8::new(0, 5, 2, 7, 5, 0, 7, 2),
|
||||
Shuffle::CBCB => u32x8::new(4, 1, 6, 3, 4, 1, 6, 3),
|
||||
Shuffle::ABAB => u32x8::new(0, 1, 2, 3, 0, 1, 2, 3),
|
||||
Shuffle::BADC => u32x8::new(1, 0, 3, 2, 5, 4, 7, 6),
|
||||
Shuffle::BACD => u32x8::new(1, 0, 3, 2, 4, 5, 6, 7),
|
||||
Shuffle::ABDC => u32x8::new(0, 1, 2, 3, 5, 4, 7, 6),
|
||||
};
|
||||
// Note that this gets turned into a generic LLVM
|
||||
// shuffle-by-constants, which can be lowered to a simpler
|
||||
// instruction than a generic permute.
|
||||
_mm256_permutevar8x32_epi32(x.into_bits(), c.into_bits()).into_bits()
|
||||
}
|
||||
}
|
||||
|
||||
FieldElement32x4([
|
||||
shuffle_lanes(self.0[0], control),
|
||||
shuffle_lanes(self.0[1], control),
|
||||
shuffle_lanes(self.0[2], control),
|
||||
shuffle_lanes(self.0[3], control),
|
||||
shuffle_lanes(self.0[4], control),
|
||||
])
|
||||
}
|
||||
|
||||
pub fn zero() -> FieldElement32x4 {
|
||||
FieldElement32x4([u32x8::splat(0); 5])
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue