saymrwulf/curve25519-dalek-source
1
0
Fork 0
mirror of https://github.com/saymrwulf/curve25519-dalek-source.git synced 2026-09-09 21:10:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert "(work-in-progress) Partially remove unsafe_target_feature"

Browse source 
This reverts commit c67e430cfd.
This commit is contained in:
Jan Bujak 2023-06-05 07:38:55 +00:00
parent c67e430cfd
commit 94247a79d1
No known key found for this signature in database
GPG key ID: 3B438F83D43341D4
3 changed files with 325 additions and 399 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

157
src/backend/vector/avx2/field.rs
View file

@ -48,6 +48,8 @@ use crate::backend::vector::avx2::constants::{
P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO, P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO,
}; };
use unsafe_target_feature::unsafe_target_feature;
/// Unpack 32-bit lanes into 64-bit lanes: /// Unpack 32-bit lanes into 64-bit lanes:
/// ```ascii,no_run /// ```ascii,no_run
/// (a0, b0, a1, b1, c0, d0, c1, d1) /// (a0, b0, a1, b1, c0, d0, c1, d1)
@ -57,9 +59,9 @@ use crate::backend::vector::avx2::constants::{
/// (a0, 0, b0, 0, c0, 0, d0, 0) /// (a0, 0, b0, 0, c0, 0, d0, 0)
/// (a1, 0, b1, 0, c1, 0, d1, 0) /// (a1, 0, b1, 0, c1, 0, d1, 0)
/// ``` /// ```
#[target_feature(enable = "avx2")] #[unsafe_target_feature("avx2")]
#[inline] #[inline(always)]
unsafe fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
let a: u32x8; let a: u32x8;
let b: u32x8; let b: u32x8;
let zero = u32x8::splat(0); let zero = u32x8::splat(0);
@ -81,9 +83,9 @@ unsafe fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
/// ```ascii,no_run /// ```ascii,no_run
/// (a0, b0, a1, b1, c0, d0, c1, d1) /// (a0, b0, a1, b1, c0, d0, c1, d1)
/// ``` /// ```
#[target_feature(enable = "avx2")] #[unsafe_target_feature("avx2")]
#[inline] #[inline(always)]
unsafe fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { fn repack_pair(x: u32x8, y: u32x8) -> u32x8 {
unsafe { unsafe {
use core::arch::x86_64::_mm256_blend_epi32; use core::arch::x86_64::_mm256_blend_epi32;
use core::arch::x86_64::_mm256_shuffle_epi32; use core::arch::x86_64::_mm256_shuffle_epi32;
@ -153,18 +155,12 @@ pub struct FieldElement2625x4(pub(crate) [u32x8; 5]);
use subtle::Choice; use subtle::Choice;
use subtle::ConditionallySelectable; use subtle::ConditionallySelectable;
#[unsafe_target_feature("avx2")]
impl ConditionallySelectable for FieldElement2625x4 { impl ConditionallySelectable for FieldElement2625x4 {
#[inline(always)]
fn conditional_select( fn conditional_select(
a: &FieldElement2625x4, a: &FieldElement2625x4,
b: &FieldElement2625x4, b: &FieldElement2625x4,
choice: Choice, choice: Choice,
) -> FieldElement2625x4 {
#[target_feature(enable = "avx2")]
unsafe fn inner(
a: &FieldElement2625x4,
b: &FieldElement2625x4,
choice: Choice,
) -> FieldElement2625x4 { ) -> FieldElement2625x4 {
let mask = (-(choice.unwrap_u8() as i32)) as u32; let mask = (-(choice.unwrap_u8() as i32)) as u32;
let mask_vec = u32x8::splat(mask); let mask_vec = u32x8::splat(mask);
@ -177,38 +173,25 @@ impl ConditionallySelectable for FieldElement2625x4 {
]) ])
} }
unsafe { inner(a, b, choice) }
}
#[inline(always)]
fn conditional_assign(&mut self, other: &FieldElement2625x4, choice: Choice) { fn conditional_assign(&mut self, other: &FieldElement2625x4, choice: Choice) {
#[target_feature(enable = "avx2")]
unsafe fn inner(
itself: &mut FieldElement2625x4,
other: &FieldElement2625x4,
choice: Choice,
) {
let mask = (-(choice.unwrap_u8() as i32)) as u32; let mask = (-(choice.unwrap_u8() as i32)) as u32;
let mask_vec = u32x8::splat(mask); let mask_vec = u32x8::splat(mask);
itself.0[0] ^= mask_vec & (itself.0[0] ^ other.0[0]); self.0[0] ^= mask_vec & (self.0[0] ^ other.0[0]);
itself.0[1] ^= mask_vec & (itself.0[1] ^ other.0[1]); self.0[1] ^= mask_vec & (self.0[1] ^ other.0[1]);
itself.0[2] ^= mask_vec & (itself.0[2] ^ other.0[2]); self.0[2] ^= mask_vec & (self.0[2] ^ other.0[2]);
itself.0[3] ^= mask_vec & (itself.0[3] ^ other.0[3]); self.0[3] ^= mask_vec & (self.0[3] ^ other.0[3]);
itself.0[4] ^= mask_vec & (itself.0[4] ^ other.0[4]); self.0[4] ^= mask_vec & (self.0[4] ^ other.0[4]);
}
unsafe { inner(self, other, choice) }
} }
} }
#[unsafe_target_feature("avx2")]
impl FieldElement2625x4 { impl FieldElement2625x4 {
pub const ZERO: FieldElement2625x4 = FieldElement2625x4([u32x8::splat_const::<0>(); 5]); pub const ZERO: FieldElement2625x4 = FieldElement2625x4([u32x8::splat_const::<0>(); 5]);
/// Split this vector into an array of four (serial) field /// Split this vector into an array of four (serial) field
/// elements. /// elements.
#[rustfmt::skip] // keep alignment of extracted lanes #[rustfmt::skip] // keep alignment of extracted lanes
#[target_feature(enable = "avx2")] pub fn split(&self) -> [FieldElement51; 4] {
pub unsafe fn split(&self) -> [FieldElement51; 4] {
let mut out = [FieldElement51::ZERO; 4]; let mut out = [FieldElement51::ZERO; 4];
for i in 0..5 { for i in 0..5 {
let a_2i = self.0[i].extract::<0>() as u64; // let a_2i = self.0[i].extract::<0>() as u64; //
@ -235,8 +218,7 @@ impl FieldElement2625x4 {
/// that when this function is inlined, LLVM is able to lower the /// that when this function is inlined, LLVM is able to lower the
/// shuffle using an immediate. /// shuffle using an immediate.
#[inline] #[inline]
#[target_feature(enable = "avx2")] pub fn shuffle(&self, control: Shuffle) -> FieldElement2625x4 {
pub unsafe fn shuffle(&self, control: Shuffle) -> FieldElement2625x4 {
#[inline(always)] #[inline(always)]
fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 { fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 {
unsafe { unsafe {
@ -276,8 +258,7 @@ impl FieldElement2625x4 {
/// that this function can be inlined and LLVM can lower it to a /// that this function can be inlined and LLVM can lower it to a
/// blend instruction using an immediate. /// blend instruction using an immediate.
#[inline] #[inline]
#[target_feature(enable = "avx2")] pub fn blend(&self, other: FieldElement2625x4, control: Lanes) -> FieldElement2625x4 {
pub unsafe fn blend(&self, other: FieldElement2625x4, control: Lanes) -> FieldElement2625x4 {
#[inline(always)] #[inline(always)]
fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 { fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
unsafe { unsafe {
@ -341,8 +322,7 @@ impl FieldElement2625x4 {
} }
/// Convenience wrapper around `new(x,x,x,x)`. /// Convenience wrapper around `new(x,x,x,x)`.
#[target_feature(enable = "avx2")] pub fn splat(x: &FieldElement51) -> FieldElement2625x4 {
pub unsafe fn splat(x: &FieldElement51) -> FieldElement2625x4 {
FieldElement2625x4::new(x, x, x, x) FieldElement2625x4::new(x, x, x, x)
} }
@ -352,8 +332,7 @@ impl FieldElement2625x4 {
/// ///
/// The resulting `FieldElement2625x4` is bounded with \\( b < 0.0002 \\). /// The resulting `FieldElement2625x4` is bounded with \\( b < 0.0002 \\).
#[rustfmt::skip] // keep alignment of computed lanes #[rustfmt::skip] // keep alignment of computed lanes
#[target_feature(enable = "avx2")] pub fn new(
pub unsafe fn new(
x0: &FieldElement51, x0: &FieldElement51,
x1: &FieldElement51, x1: &FieldElement51,
x2: &FieldElement51, x2: &FieldElement51,
@ -392,8 +371,7 @@ impl FieldElement2625x4 {
/// ///
/// The coefficients of the result are bounded with \\( b < 1 \\). /// The coefficients of the result are bounded with \\( b < 1 \\).
#[inline] #[inline]
#[target_feature(enable = "avx2")] pub fn negate_lazy(&self) -> FieldElement2625x4 {
pub unsafe fn negate_lazy(&self) -> FieldElement2625x4 {
// The limbs of self are bounded with b < 0.999, while the // The limbs of self are bounded with b < 0.999, while the
// smallest limb of 2*p is 67108845 > 2^{26+0.9999}, so // smallest limb of 2*p is 67108845 > 2^{26+0.9999}, so
// underflows are not possible. // underflows are not possible.
@ -416,8 +394,7 @@ impl FieldElement2625x4 {
/// ///
/// The coefficients of the result are bounded with \\( b < 1.6 \\). /// The coefficients of the result are bounded with \\( b < 1.6 \\).
#[inline] #[inline]
#[target_feature(enable = "avx2")] pub fn diff_sum(&self) -> FieldElement2625x4 {
pub unsafe fn diff_sum(&self) -> FieldElement2625x4 {
// tmp1 = (B, A, D, C) // tmp1 = (B, A, D, C)
let tmp1 = self.shuffle(Shuffle::BADC); let tmp1 = self.shuffle(Shuffle::BADC);
// tmp2 = (-A, B, -C, D) // tmp2 = (-A, B, -C, D)
@ -432,8 +409,7 @@ impl FieldElement2625x4 {
/// ///
/// The coefficients of the result are bounded with \\( b < 0.0002 \\). /// The coefficients of the result are bounded with \\( b < 0.0002 \\).
#[inline] #[inline]
#[target_feature(enable = "avx2")] pub fn reduce(&self) -> FieldElement2625x4 {
pub unsafe fn reduce(&self) -> FieldElement2625x4 {
let shifts = u32x8::new(26, 26, 25, 25, 26, 26, 25, 25); let shifts = u32x8::new(26, 26, 25, 25, 26, 26, 25, 25);
let masks = u32x8::new( let masks = u32x8::new(
(1 << 26) - 1, (1 << 26) - 1,
@ -542,8 +518,7 @@ impl FieldElement2625x4 {
/// The coefficients of the result are bounded with \\( b < 0.007 \\). /// The coefficients of the result are bounded with \\( b < 0.007 \\).
#[inline] #[inline]
#[rustfmt::skip] // keep alignment of carry chain #[rustfmt::skip] // keep alignment of carry chain
#[target_feature(enable = "avx2")] fn reduce64(mut z: [u64x4; 10]) -> FieldElement2625x4 {
unsafe fn reduce64(mut z: [u64x4; 10]) -> FieldElement2625x4 {
// These aren't const because splat isn't a const fn // These aren't const because splat isn't a const fn
let LOW_25_BITS: u64x4 = u64x4::splat((1 << 25) - 1); let LOW_25_BITS: u64x4 = u64x4::splat((1 << 25) - 1);
let LOW_26_BITS: u64x4 = u64x4::splat((1 << 26) - 1); let LOW_26_BITS: u64x4 = u64x4::splat((1 << 26) - 1);
@ -619,8 +594,7 @@ impl FieldElement2625x4 {
/// ///
/// The coefficients of the result are bounded with \\( b < 0.007 \\). /// The coefficients of the result are bounded with \\( b < 0.007 \\).
#[rustfmt::skip] // keep alignment of z* calculations #[rustfmt::skip] // keep alignment of z* calculations
#[target_feature(enable = "avx2")] pub fn square_and_negate_D(&self) -> FieldElement2625x4 {
pub unsafe fn square_and_negate_D(&self) -> FieldElement2625x4 {
#[inline(always)] #[inline(always)]
fn m(x: u32x8, y: u32x8) -> u64x4 { fn m(x: u32x8, y: u32x8) -> u64x4 {
x.mul32(y) x.mul32(y)
@ -707,6 +681,7 @@ impl FieldElement2625x4 {
} }
} }
#[unsafe_target_feature("avx2")]
impl Neg for FieldElement2625x4 { impl Neg for FieldElement2625x4 {
type Output = FieldElement2625x4; type Output = FieldElement2625x4;
@ -722,46 +697,36 @@ impl Neg for FieldElement2625x4 {
/// # Postconditions /// # Postconditions
/// ///
/// The coefficients of the result are bounded with \\( b < 0.0002 \\). /// The coefficients of the result are bounded with \\( b < 0.0002 \\).
#[inline(always)]
fn neg(self) -> FieldElement2625x4 {
#[inline] #[inline]
#[target_feature(enable = "avx2")] fn neg(self) -> FieldElement2625x4 {
unsafe fn inner(itself: FieldElement2625x4) -> FieldElement2625x4 {
FieldElement2625x4([ FieldElement2625x4([
P_TIMES_16_LO - itself.0[0], P_TIMES_16_LO - self.0[0],
P_TIMES_16_HI - itself.0[1], P_TIMES_16_HI - self.0[1],
P_TIMES_16_HI - itself.0[2], P_TIMES_16_HI - self.0[2],
P_TIMES_16_HI - itself.0[3], P_TIMES_16_HI - self.0[3],
P_TIMES_16_HI - itself.0[4], P_TIMES_16_HI - self.0[4],
]) ])
.reduce() .reduce()
} }
unsafe { inner(self) }
}
} }
#[unsafe_target_feature("avx2")]
impl Add<FieldElement2625x4> for FieldElement2625x4 { impl Add<FieldElement2625x4> for FieldElement2625x4 {
type Output = FieldElement2625x4; type Output = FieldElement2625x4;
/// Add two `FieldElement2625x4`s, without performing a reduction. /// Add two `FieldElement2625x4`s, without performing a reduction.
#[inline(always)]
fn add(self, rhs: FieldElement2625x4) -> FieldElement2625x4 {
#[inline] #[inline]
#[target_feature(enable = "avx2")] fn add(self, rhs: FieldElement2625x4) -> FieldElement2625x4 {
unsafe fn inner(itself: FieldElement2625x4, rhs: FieldElement2625x4) -> FieldElement2625x4 {
FieldElement2625x4([ FieldElement2625x4([
itself.0[0] + rhs.0[0], self.0[0] + rhs.0[0],
itself.0[1] + rhs.0[1], self.0[1] + rhs.0[1],
itself.0[2] + rhs.0[2], self.0[2] + rhs.0[2],
itself.0[3] + rhs.0[3], self.0[3] + rhs.0[3],
itself.0[4] + rhs.0[4], self.0[4] + rhs.0[4],
]) ])
} }
unsafe { inner(self, rhs) }
}
} }
#[unsafe_target_feature("avx2")]
impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 { impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 {
type Output = FieldElement2625x4; type Output = FieldElement2625x4;
/// Perform a multiplication by a vector of small constants. /// Perform a multiplication by a vector of small constants.
@ -769,21 +734,15 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 {
/// # Postconditions /// # Postconditions
/// ///
/// The coefficients of the result are bounded with \\( b < 0.007 \\). /// The coefficients of the result are bounded with \\( b < 0.007 \\).
#[inline(always)]
fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement2625x4 {
#[inline] #[inline]
#[target_feature(enable = "avx2")] fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement2625x4 {
unsafe fn inner(
itself: FieldElement2625x4,
scalars: (u32, u32, u32, u32),
) -> FieldElement2625x4 {
let consts = u32x8::new(scalars.0, 0, scalars.1, 0, scalars.2, 0, scalars.3, 0); let consts = u32x8::new(scalars.0, 0, scalars.1, 0, scalars.2, 0, scalars.3, 0);
let (b0, b1) = unpack_pair(itself.0[0]); let (b0, b1) = unpack_pair(self.0[0]);
let (b2, b3) = unpack_pair(itself.0[1]); let (b2, b3) = unpack_pair(self.0[1]);
let (b4, b5) = unpack_pair(itself.0[2]); let (b4, b5) = unpack_pair(self.0[2]);
let (b6, b7) = unpack_pair(itself.0[3]); let (b6, b7) = unpack_pair(self.0[3]);
let (b8, b9) = unpack_pair(itself.0[4]); let (b8, b9) = unpack_pair(self.0[4]);
FieldElement2625x4::reduce64([ FieldElement2625x4::reduce64([
b0.mul32(consts), b0.mul32(consts),
@ -798,11 +757,9 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 {
b9.mul32(consts), b9.mul32(consts),
]) ])
} }
unsafe { inner(self, scalars) }
}
} }
#[unsafe_target_feature("avx2")]
impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 { impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 {
type Output = FieldElement2625x4; type Output = FieldElement2625x4;
/// Multiply `self` by `rhs`. /// Multiply `self` by `rhs`.
@ -818,11 +775,8 @@ impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 {
/// The coefficients of the result are bounded with \\( b < 0.007 \\). /// The coefficients of the result are bounded with \\( b < 0.007 \\).
/// ///
#[rustfmt::skip] // keep alignment of z* calculations #[rustfmt::skip] // keep alignment of z* calculations
#[inline(always)]
fn mul(self, rhs: &'b FieldElement2625x4) -> FieldElement2625x4 {
#[inline] #[inline]
#[target_feature(enable = "avx2")] fn mul(self, rhs: &'b FieldElement2625x4) -> FieldElement2625x4 {
unsafe fn inner<'a, 'b>(itself: &'a FieldElement2625x4, rhs: &'b FieldElement2625x4) -> FieldElement2625x4 {
#[inline(always)] #[inline(always)]
fn m(x: u32x8, y: u32x8) -> u64x4 { fn m(x: u32x8, y: u32x8) -> u64x4 {
x.mul32(y) x.mul32(y)
@ -833,11 +787,11 @@ impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 {
x.mul32(y).into() x.mul32(y).into()
} }
let (x0, x1) = unpack_pair(itself.0[0]); let (x0, x1) = unpack_pair(self.0[0]);
let (x2, x3) = unpack_pair(itself.0[1]); let (x2, x3) = unpack_pair(self.0[1]);
let (x4, x5) = unpack_pair(itself.0[2]); let (x4, x5) = unpack_pair(self.0[2]);
let (x6, x7) = unpack_pair(itself.0[3]); let (x6, x7) = unpack_pair(self.0[3]);
let (x8, x9) = unpack_pair(itself.0[4]); let (x8, x9) = unpack_pair(self.0[4]);
let (y0, y1) = unpack_pair(rhs.0[0]); let (y0, y1) = unpack_pair(rhs.0[0]);
let (y2, y3) = unpack_pair(rhs.0[1]); let (y2, y3) = unpack_pair(rhs.0[1]);
@ -914,11 +868,6 @@ impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 {
// looser bound on b_x. // looser bound on b_x.
FieldElement2625x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) FieldElement2625x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
} }
unsafe {
inner(self, rhs)
}
}
} }
#[cfg(target_feature = "avx2")] #[cfg(target_feature = "avx2")]

25
src/backend/vector/packed_simd.rs
View file

@ -252,9 +252,9 @@ impl u64x4 {
} }
/// Constructs a new instance. /// Constructs a new instance.
#[target_feature(enable = "avx2")] #[unsafe_target_feature("avx2")]
#[inline] #[inline]
pub unsafe fn new(x0: u64, x1: u64, x2: u64, x3: u64) -> u64x4 { pub fn new(x0: u64, x1: u64, x2: u64, x3: u64) -> u64x4 {
unsafe { unsafe {
// _mm256_set_epi64 sets the underlying vector in reverse order of the args // _mm256_set_epi64 sets the underlying vector in reverse order of the args
u64x4(core::arch::x86_64::_mm256_set_epi64x( u64x4(core::arch::x86_64::_mm256_set_epi64x(
@ -264,9 +264,9 @@ impl u64x4 {
} }
/// Constructs a new instance with all of the elements initialized to the given value. /// Constructs a new instance with all of the elements initialized to the given value.
#[target_feature(enable = "avx2")] #[unsafe_target_feature("avx2")]
#[inline] #[inline]
pub unsafe fn splat(x: u64) -> u64x4 { pub fn splat(x: u64) -> u64x4 {
unsafe { u64x4(core::arch::x86_64::_mm256_set1_epi64x(x as i64)) } unsafe { u64x4(core::arch::x86_64::_mm256_set1_epi64x(x as i64)) }
} }
} }
@ -303,18 +303,9 @@ impl u32x8 {
/// Constructs a new instance. /// Constructs a new instance.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2")] #[unsafe_target_feature("avx2")]
#[inline] #[inline]
pub unsafe fn new( pub fn new(x0: u32, x1: u32, x2: u32, x3: u32, x4: u32, x5: u32, x6: u32, x7: u32) -> u32x8 {
x0: u32,
x1: u32,
x2: u32,
x3: u32,
x4: u32,
x5: u32,
x6: u32,
x7: u32,
) -> u32x8 {
unsafe { unsafe {
// _mm256_set_epi32 sets the underlying vector in reverse order of the args // _mm256_set_epi32 sets the underlying vector in reverse order of the args
u32x8(core::arch::x86_64::_mm256_set_epi32( u32x8(core::arch::x86_64::_mm256_set_epi32(
@ -325,9 +316,9 @@ impl u32x8 {
} }
/// Constructs a new instance with all of the elements initialized to the given value. /// Constructs a new instance with all of the elements initialized to the given value.
#[target_feature(enable = "avx2")] #[unsafe_target_feature("avx2")]
#[inline] #[inline]
pub unsafe fn splat(x: u32) -> u32x8 { pub fn splat(x: u32) -> u32x8 {
unsafe { u32x8(core::arch::x86_64::_mm256_set1_epi32(x as i32)) } unsafe { u32x8(core::arch::x86_64::_mm256_set1_epi32(x as i32)) }
} }
} }

40
src/backend/vector/scalar_mul/pippenger.rs
View file

@ -9,15 +9,22 @@
#![allow(non_snake_case)] #![allow(non_snake_case)]
macro_rules! implement { #[unsafe_target_feature::unsafe_target_feature_specialize(
($module:ident, $backend_module:ident, $features:expr) => { conditional("avx2", feature = "simd_avx2"),
pub mod $module { conditional("avx512ifma,avx512vl", all(feature = "simd_avx512", nightly))
)]
pub mod spec {
use alloc::vec::Vec; use alloc::vec::Vec;
use core::borrow::Borrow; use core::borrow::Borrow;
use core::cmp::Ordering; use core::cmp::Ordering;
use crate::backend::vector::$backend_module::{CachedPoint, ExtendedPoint}; #[for_target_feature("avx2")]
use crate::backend::vector::avx2::{CachedPoint, ExtendedPoint};
#[for_target_feature("avx512ifma")]
use crate::backend::vector::ifma::{CachedPoint, ExtendedPoint};
use crate::edwards::EdwardsPoint; use crate::edwards::EdwardsPoint;
use crate::scalar::Scalar; use crate::scalar::Scalar;
@ -31,15 +38,7 @@ macro_rules! implement {
impl VartimeMultiscalarMul for Pippenger { impl VartimeMultiscalarMul for Pippenger {
type Point = EdwardsPoint; type Point = EdwardsPoint;
#[inline(always)]
fn optional_multiscalar_mul<I, J>(scalars: I, points: J) -> Option<EdwardsPoint> fn optional_multiscalar_mul<I, J>(scalars: I, points: J) -> Option<EdwardsPoint>
where
I: IntoIterator,
I::Item: Borrow<Scalar>,
J: IntoIterator<Item = Option<EdwardsPoint>>,
{
#[target_feature(enable = $features)]
unsafe fn inner<I, J>(scalars: I, points: J) -> Option<EdwardsPoint>
where where
I: IntoIterator, I: IntoIterator,
I::Item: Borrow<Scalar>, I::Item: Borrow<Scalar>,
@ -117,8 +116,7 @@ macro_rules! implement {
for i in (0..(buckets_count - 1)).rev() { for i in (0..(buckets_count - 1)).rev() {
buckets_intermediate_sum = buckets_intermediate_sum =
&buckets_intermediate_sum + &CachedPoint::from(buckets[i]); &buckets_intermediate_sum + &CachedPoint::from(buckets[i]);
buckets_sum = buckets_sum = &buckets_sum + &CachedPoint::from(buckets_intermediate_sum);
&buckets_sum + &CachedPoint::from(buckets_intermediate_sum);
} }
buckets_sum buckets_sum
@ -136,12 +134,9 @@ macro_rules! implement {
.into(), .into(),
) )
} }
unsafe { inner(scalars, points) }
}
} }
#[cfg(test)] #[cfg(test)]
#[cfg(target_feature = $features)]
mod test { mod test {
#[test] #[test]
fn test_vartime_pippenger() { fn test_vartime_pippenger() {
@ -171,8 +166,7 @@ macro_rules! implement {
let points = &points[0..n].to_vec(); let points = &points[0..n].to_vec();
let control: EdwardsPoint = premultiplied[0..n].iter().sum(); let control: EdwardsPoint = premultiplied[0..n].iter().sum();
let subject = let subject = Pippenger::vartime_multiscalar_mul(scalars.clone(), points.clone());
Pippenger::vartime_multiscalar_mul(scalars.clone(), points.clone());
assert_eq!(subject.compress(), control.compress()); assert_eq!(subject.compress(), control.compress());
@ -180,12 +174,4 @@ macro_rules! implement {
} }
} }
} }
}
};
} }
#[cfg(feature = "simd_avx2")]
implement!(spec_avx2, avx2, "avx2");
#[cfg(all(feature = "simd_avx512", nightly))]
implement!(spec_avx512ifma_avx512vl, ifma, "avx512ifma,avx512vl");