mirror of
https://github.com/saymrwulf/risc0-curve25519-dalek-source.git
synced 2026-09-05 20:10:35 +00:00
Support SIMD on Rust stable (#520)
* Remove dependency on `packed_simd` * Support SIMD on stable Rust * Move `packed_simd.rs` to `vector` module * Add comment header to `packed_simd.rs` * Initialize SIMD registers using intrinsics instead of `transmute` * Use a splat inside of `unpack_pair` * Update README: the AVX2 backend now works on stable Rust * Add a CI job to also build the AVX2 SIMD backend on Rust stable * Added SIMD MSRV test
This commit is contained in:
Koute
GitHub
parent
f460ae149b
commit
4583c472f5
11 changed files with 1192 additions and 860 deletions
16
.github/workflows/rust.yml
vendored
16
.github/workflows/rust.yml
vendored
|
|
@ -55,7 +55,7 @@ jobs:
|
|||
- run: cargo build --target thumbv7em-none-eabi --release
|
||||
- run: cargo build --target thumbv7em-none-eabi --release --features serde
|
||||
|
||||
build-simd:
|
||||
build-simd-nightly:
|
||||
name: Build simd backend (nightly)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
|
@ -69,6 +69,16 @@ jobs:
|
|||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx512ifma'
|
||||
run: cargo build --target x86_64-unknown-linux-gnu
|
||||
|
||||
test-simd-avx2:
|
||||
name: Test simd backend (avx2)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- env:
|
||||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx2'
|
||||
run: cargo test --target x86_64-unknown-linux-gnu
|
||||
|
||||
build-docs:
|
||||
name: Build docs
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -151,6 +161,10 @@ jobs:
|
|||
# deps and the stated MSRV
|
||||
- uses: dtolnay/rust-toolchain@1.60.0
|
||||
- run: cargo build --no-default-features --features serde
|
||||
# Also make sure the AVX2 build works
|
||||
- env:
|
||||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx2'
|
||||
run: cargo build --target x86_64-unknown-linux-gnu
|
||||
|
||||
bench:
|
||||
name: Check that benchmarks compile
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ rand_core = { version = "0.6", default-features = false, features = ["getrandom"
|
|||
|
||||
[build-dependencies]
|
||||
platforms = "3.0.2"
|
||||
rustc_version = "0.4.0"
|
||||
|
||||
[[bench]]
|
||||
name = "dalek_benchmarks"
|
||||
|
|
@ -57,11 +58,6 @@ zeroize = { version = "1", default-features = false, optional = true }
|
|||
[target.'cfg(curve25519_dalek_backend = "fiat")'.dependencies]
|
||||
fiat-crypto = "0.1.19"
|
||||
|
||||
# The original packed_simd package was orphaned, see
|
||||
# https://github.com/rust-lang/packed_simd/issues/303#issuecomment-701361161
|
||||
[target.'cfg(curve25519_dalek_backend = "simd")'.dependencies]
|
||||
packed_simd = { version = "0.3.8", package = "packed_simd_2", features = ["into_bits"] }
|
||||
|
||||
[features]
|
||||
default = ["alloc", "precomputed-tables", "zeroize"]
|
||||
alloc = ["zeroize?/alloc"]
|
||||
|
|
|
|||
12
README.md
12
README.md
|
|
@ -155,15 +155,15 @@ $ cargo build --target i686-unknown-linux-gnu
|
|||
Target backend selection within `simd` must be done manually by setting the
|
||||
`RUSTFLAGS` environment variable to one of the below options:
|
||||
|
||||
| CPU feature | `RUSTFLAGS` |
|
||||
| :--- | :--- |
|
||||
| avx2 | `-C target_feature=+avx2` |
|
||||
| avx512ifma | `-C target_feature=+avx512ifma` |
|
||||
| CPU feature | `RUSTFLAGS` | Requires nightly? |
|
||||
| :--- | :--- | :--- |
|
||||
| avx2 | `-C target_feature=+avx2` | no |
|
||||
| avx512ifma | `-C target_feature=+avx512ifma` | yes |
|
||||
|
||||
Or you can use `-C target_cpu=native` if you don't know what to set.
|
||||
|
||||
The `simd` backend also requires using nightly, e.g. by running `cargo
|
||||
+nightly build`, to build.
|
||||
The AVX512 backend requires Rust nightly. If enabled and when compiled on a non-nightly
|
||||
compiler it will fall back to using the AVX2 backend.
|
||||
|
||||
# Documentation
|
||||
|
||||
|
|
|
|||
8
build.rs
8
build.rs
|
|
@ -19,6 +19,14 @@ fn main() {
|
|||
DalekBits::Dalek64 => println!("cargo:rustc-cfg=curve25519_dalek_bits=\"64\""),
|
||||
DalekBits::Dalek32 => println!("cargo:rustc-cfg=curve25519_dalek_bits=\"32\""),
|
||||
}
|
||||
|
||||
if rustc_version::version_meta()
|
||||
.expect("failed to detect rustc version")
|
||||
.channel
|
||||
== rustc_version::Channel::Nightly
|
||||
{
|
||||
println!("cargo:rustc-cfg=nightly");
|
||||
}
|
||||
}
|
||||
|
||||
// Deterministic cfg(curve25519_dalek_bits) when this is not explicitly set.
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -40,8 +40,8 @@ const C_LANES64: u8 = 0b00_11_00_00;
|
|||
#[allow(unused)]
|
||||
const D_LANES64: u8 = 0b11_00_00_00;
|
||||
|
||||
use crate::backend::vector::packed_simd::{u32x8, u64x4};
|
||||
use core::ops::{Add, Mul, Neg};
|
||||
use packed_simd::{i32x8, u32x8, u64x4, IntoBits};
|
||||
|
||||
use crate::backend::serial::u64::field::FieldElement51;
|
||||
use crate::backend::vector::avx2::constants::{
|
||||
|
|
@ -61,12 +61,12 @@ use crate::backend::vector::avx2::constants::{
|
|||
fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
|
||||
let a: u32x8;
|
||||
let b: u32x8;
|
||||
let zero = i32x8::new(0, 0, 0, 0, 0, 0, 0, 0);
|
||||
let zero = u32x8::splat(0);
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_unpackhi_epi32;
|
||||
use core::arch::x86_64::_mm256_unpacklo_epi32;
|
||||
a = _mm256_unpacklo_epi32(src.into_bits(), zero.into_bits()).into_bits();
|
||||
b = _mm256_unpackhi_epi32(src.into_bits(), zero.into_bits()).into_bits();
|
||||
a = _mm256_unpacklo_epi32(src.into(), zero.into()).into();
|
||||
b = _mm256_unpackhi_epi32(src.into(), zero.into()).into();
|
||||
}
|
||||
(a, b)
|
||||
}
|
||||
|
|
@ -89,13 +89,13 @@ fn repack_pair(x: u32x8, y: u32x8) -> u32x8 {
|
|||
// Input: x = (a0, 0, b0, 0, c0, 0, d0, 0)
|
||||
// Input: y = (a1, 0, b1, 0, c1, 0, d1, 0)
|
||||
|
||||
let x_shuffled = _mm256_shuffle_epi32(x.into_bits(), 0b11_01_10_00);
|
||||
let y_shuffled = _mm256_shuffle_epi32(y.into_bits(), 0b10_00_11_01);
|
||||
let x_shuffled = _mm256_shuffle_epi32(x.into(), 0b11_01_10_00);
|
||||
let y_shuffled = _mm256_shuffle_epi32(y.into(), 0b10_00_11_01);
|
||||
|
||||
// x' = (a0, b0, 0, 0, c0, d0, 0, 0)
|
||||
// y' = ( 0, 0, a1, b1, 0, 0, c1, d1)
|
||||
|
||||
_mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).into_bits()
|
||||
_mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).into()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -180,7 +180,7 @@ impl ConditionallySelectable for FieldElement2625x4 {
|
|||
}
|
||||
|
||||
impl FieldElement2625x4 {
|
||||
pub const ZERO: FieldElement2625x4 = FieldElement2625x4([u32x8::splat(0); 5]);
|
||||
pub const ZERO: FieldElement2625x4 = FieldElement2625x4([u32x8::splat_const::<0>(); 5]);
|
||||
|
||||
/// Split this vector into an array of four (serial) field
|
||||
/// elements.
|
||||
|
|
@ -188,14 +188,14 @@ impl FieldElement2625x4 {
|
|||
pub fn split(&self) -> [FieldElement51; 4] {
|
||||
let mut out = [FieldElement51::ZERO; 4];
|
||||
for i in 0..5 {
|
||||
let a_2i = self.0[i].extract(0) as u64; //
|
||||
let b_2i = self.0[i].extract(1) as u64; //
|
||||
let a_2i_1 = self.0[i].extract(2) as u64; // `.
|
||||
let b_2i_1 = self.0[i].extract(3) as u64; // | pre-swapped to avoid
|
||||
let c_2i = self.0[i].extract(4) as u64; // | a cross lane shuffle
|
||||
let d_2i = self.0[i].extract(5) as u64; // .'
|
||||
let c_2i_1 = self.0[i].extract(6) as u64; //
|
||||
let d_2i_1 = self.0[i].extract(7) as u64; //
|
||||
let a_2i = self.0[i].extract::<0>() as u64; //
|
||||
let b_2i = self.0[i].extract::<1>() as u64; //
|
||||
let a_2i_1 = self.0[i].extract::<2>() as u64; // `.
|
||||
let b_2i_1 = self.0[i].extract::<3>() as u64; // | pre-swapped to avoid
|
||||
let c_2i = self.0[i].extract::<4>() as u64; // | a cross lane shuffle
|
||||
let d_2i = self.0[i].extract::<5>() as u64; // .'
|
||||
let c_2i_1 = self.0[i].extract::<6>() as u64; //
|
||||
let d_2i_1 = self.0[i].extract::<7>() as u64; //
|
||||
|
||||
out[0].0[i] = a_2i + (a_2i_1 << 26);
|
||||
out[1].0[i] = b_2i + (b_2i_1 << 26);
|
||||
|
|
@ -233,7 +233,7 @@ impl FieldElement2625x4 {
|
|||
// Note that this gets turned into a generic LLVM
|
||||
// shuffle-by-constants, which can be lowered to a simpler
|
||||
// instruction than a generic permute.
|
||||
_mm256_permutevar8x32_epi32(x.into_bits(), c.into_bits()).into_bits()
|
||||
_mm256_permutevar8x32_epi32(x.into(), c.into()).into()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -279,38 +279,29 @@ impl FieldElement2625x4 {
|
|||
// which does not require a shuffle immediate but *is* lowered
|
||||
// to immediate shuffles anyways).
|
||||
match control {
|
||||
Lanes::C => {
|
||||
_mm256_blend_epi32(x.into_bits(), y.into_bits(), C_LANES as i32).into_bits()
|
||||
}
|
||||
Lanes::D => {
|
||||
_mm256_blend_epi32(x.into_bits(), y.into_bits(), D_LANES as i32).into_bits()
|
||||
}
|
||||
Lanes::C => _mm256_blend_epi32(x.into(), y.into(), C_LANES as i32).into(),
|
||||
Lanes::D => _mm256_blend_epi32(x.into(), y.into(), D_LANES as i32).into(),
|
||||
Lanes::AD => {
|
||||
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | D_LANES) as i32)
|
||||
.into_bits()
|
||||
_mm256_blend_epi32(x.into(), y.into(), (A_LANES | D_LANES) as i32).into()
|
||||
}
|
||||
Lanes::AB => {
|
||||
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32)
|
||||
.into_bits()
|
||||
_mm256_blend_epi32(x.into(), y.into(), (A_LANES | B_LANES) as i32).into()
|
||||
}
|
||||
Lanes::AC => {
|
||||
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | C_LANES) as i32)
|
||||
.into_bits()
|
||||
_mm256_blend_epi32(x.into(), y.into(), (A_LANES | C_LANES) as i32).into()
|
||||
}
|
||||
Lanes::CD => {
|
||||
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32)
|
||||
.into_bits()
|
||||
_mm256_blend_epi32(x.into(), y.into(), (C_LANES | D_LANES) as i32).into()
|
||||
}
|
||||
Lanes::BC => {
|
||||
_mm256_blend_epi32(x.into_bits(), y.into_bits(), (B_LANES | C_LANES) as i32)
|
||||
.into_bits()
|
||||
_mm256_blend_epi32(x.into(), y.into(), (B_LANES | C_LANES) as i32).into()
|
||||
}
|
||||
Lanes::ABCD => _mm256_blend_epi32(
|
||||
x.into_bits(),
|
||||
y.into_bits(),
|
||||
x.into(),
|
||||
y.into(),
|
||||
(A_LANES | B_LANES | C_LANES | D_LANES) as i32,
|
||||
)
|
||||
.into_bits(),
|
||||
.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -413,7 +404,7 @@ impl FieldElement2625x4 {
|
|||
/// The coefficients of the result are bounded with \\( b < 0.0002 \\).
|
||||
#[inline]
|
||||
pub fn reduce(&self) -> FieldElement2625x4 {
|
||||
let shifts = i32x8::new(26, 26, 25, 25, 26, 26, 25, 25);
|
||||
let shifts = u32x8::new(26, 26, 25, 25, 26, 26, 25, 25);
|
||||
let masks = u32x8::new(
|
||||
(1 << 26) - 1,
|
||||
(1 << 26) - 1,
|
||||
|
|
@ -436,8 +427,8 @@ impl FieldElement2625x4 {
|
|||
use core::arch::x86_64::_mm256_shuffle_epi32;
|
||||
use core::arch::x86_64::_mm256_srlv_epi32;
|
||||
|
||||
let c = _mm256_srlv_epi32(v.into_bits(), shifts.into_bits());
|
||||
_mm256_shuffle_epi32(c, 0b01_00_11_10).into_bits()
|
||||
let c = _mm256_srlv_epi32(v.into(), shifts.into());
|
||||
_mm256_shuffle_epi32(c, 0b01_00_11_10).into()
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -458,7 +449,7 @@ impl FieldElement2625x4 {
|
|||
let combine = |v_lo: u32x8, v_hi: u32x8| -> u32x8 {
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
_mm256_blend_epi32(v_lo.into_bits(), v_hi.into_bits(), 0b11_00_11_00).into_bits()
|
||||
_mm256_blend_epi32(v_lo.into(), v_hi.into(), 0b11_00_11_00).into()
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -488,17 +479,17 @@ impl FieldElement2625x4 {
|
|||
//
|
||||
// c98 = (c(x9), c(y9), c(x8), c(y8), c(z9), c(w9), c(z8), c(w8));
|
||||
// c9_spread = (c(x9), c(x8), c(y9), c(y8), c(z9), c(z8), c(w9), c(w8)).
|
||||
let c9_spread = _mm256_shuffle_epi32(c98.into_bits(), 0b11_01_10_00);
|
||||
let c9_spread = _mm256_shuffle_epi32(c98.into(), 0b11_01_10_00);
|
||||
|
||||
// Since the carryouts are bounded by 2^7, their products with 19
|
||||
// are bounded by 2^11.25. This means that
|
||||
//
|
||||
// c9_19_spread = (19*c(x9), 0, 19*c(y9), 0, 19*c(z9), 0, 19*c(w9), 0).
|
||||
let c9_19_spread = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into_bits());
|
||||
let c9_19_spread = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into());
|
||||
|
||||
// Unshuffle:
|
||||
// c9_19 = (19*c(x9), 19*c(y9), 0, 0, 19*c(z9), 19*c(w9), 0, 0).
|
||||
_mm256_shuffle_epi32(c9_19_spread, 0b11_01_10_00).into_bits()
|
||||
_mm256_shuffle_epi32(c9_19_spread, 0b11_01_10_00).into()
|
||||
};
|
||||
|
||||
// Add the final carryin.
|
||||
|
|
@ -531,11 +522,11 @@ impl FieldElement2625x4 {
|
|||
debug_assert!(i < 9);
|
||||
if i % 2 == 0 {
|
||||
// Even limbs have 26 bits
|
||||
z[i + 1] += z[i] >> 26;
|
||||
z[i + 1] += z[i].shr::<26>();
|
||||
z[i] &= LOW_26_BITS;
|
||||
} else {
|
||||
// Odd limbs have 25 bits
|
||||
z[i + 1] += z[i] >> 25;
|
||||
z[i + 1] += z[i].shr::<25>();
|
||||
z[i] &= LOW_25_BITS;
|
||||
}
|
||||
};
|
||||
|
|
@ -558,17 +549,14 @@ impl FieldElement2625x4 {
|
|||
// big. To ensure c < 2^32, we would need z[9] < 2^57.
|
||||
// Instead, we split the carry in two, with c = c_0 + c_1*2^26.
|
||||
|
||||
let c = z[9] >> 25;
|
||||
let c = z[9].shr::<25>();
|
||||
z[9] &= LOW_25_BITS;
|
||||
let mut c0: u64x4 = c & LOW_26_BITS; // c0 < 2^26;
|
||||
let mut c1: u64x4 = c >> 26; // c1 < 2^(39-26) = 2^13;
|
||||
let mut c1: u64x4 = c.shr::<26>(); // c1 < 2^(39-26) = 2^13;
|
||||
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
let x19 = u64x4::splat(19);
|
||||
c0 = _mm256_mul_epu32(c0.into_bits(), x19.into_bits()).into_bits(); // c0 < 2^30.25
|
||||
c1 = _mm256_mul_epu32(c1.into_bits(), x19.into_bits()).into_bits(); // c1 < 2^17.25
|
||||
}
|
||||
let x19 = u64x4::splat(19);
|
||||
c0 = u32x8::from(c0).mul32(u32x8::from(x19));
|
||||
c1 = u32x8::from(c1).mul32(u32x8::from(x19));
|
||||
|
||||
z[0] += c0; // z0 < 2^26 + 2^30.25 < 2^30.33
|
||||
z[1] += c1; // z1 < 2^25 + 2^17.25 < 2^25.0067
|
||||
|
|
@ -582,11 +570,11 @@ impl FieldElement2625x4 {
|
|||
//
|
||||
// So the packed result is bounded with b = 0.007.
|
||||
FieldElement2625x4([
|
||||
repack_pair(z[0].into_bits(), z[1].into_bits()),
|
||||
repack_pair(z[2].into_bits(), z[3].into_bits()),
|
||||
repack_pair(z[4].into_bits(), z[5].into_bits()),
|
||||
repack_pair(z[6].into_bits(), z[7].into_bits()),
|
||||
repack_pair(z[8].into_bits(), z[9].into_bits()),
|
||||
repack_pair(z[0].into(), z[1].into()),
|
||||
repack_pair(z[2].into(), z[3].into()),
|
||||
repack_pair(z[4].into(), z[5].into()),
|
||||
repack_pair(z[6].into(), z[7].into()),
|
||||
repack_pair(z[8].into(), z[9].into()),
|
||||
])
|
||||
}
|
||||
|
||||
|
|
@ -603,14 +591,12 @@ impl FieldElement2625x4 {
|
|||
pub fn square_and_negate_D(&self) -> FieldElement2625x4 {
|
||||
#[inline(always)]
|
||||
fn m(x: u32x8, y: u32x8) -> u64x4 {
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x.into_bits(), y.into_bits()).into_bits() }
|
||||
x.mul32(y)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn m_lo(x: u32x8, y: u32x8) -> u32x8 {
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x.into_bits(), y.into_bits()).into_bits() }
|
||||
x.mul32(y).into()
|
||||
}
|
||||
|
||||
let v19 = u32x8::new(19, 0, 19, 0, 19, 0, 19, 0);
|
||||
|
|
@ -621,14 +607,14 @@ impl FieldElement2625x4 {
|
|||
let (x6, x7) = unpack_pair(self.0[3]);
|
||||
let (x8, x9) = unpack_pair(self.0[4]);
|
||||
|
||||
let x0_2 = x0 << 1;
|
||||
let x1_2 = x1 << 1;
|
||||
let x2_2 = x2 << 1;
|
||||
let x3_2 = x3 << 1;
|
||||
let x4_2 = x4 << 1;
|
||||
let x5_2 = x5 << 1;
|
||||
let x6_2 = x6 << 1;
|
||||
let x7_2 = x7 << 1;
|
||||
let x0_2 = x0.shl::<1>();
|
||||
let x1_2 = x1.shl::<1>();
|
||||
let x2_2 = x2.shl::<1>();
|
||||
let x3_2 = x3.shl::<1>();
|
||||
let x4_2 = x4.shl::<1>();
|
||||
let x5_2 = x5.shl::<1>();
|
||||
let x6_2 = x6.shl::<1>();
|
||||
let x7_2 = x7.shl::<1>();
|
||||
|
||||
let x5_19 = m_lo(v19, x5);
|
||||
let x6_19 = m_lo(v19, x6);
|
||||
|
|
@ -636,16 +622,16 @@ impl FieldElement2625x4 {
|
|||
let x8_19 = m_lo(v19, x8);
|
||||
let x9_19 = m_lo(v19, x9);
|
||||
|
||||
let mut z0 = m(x0, x0) + m(x2_2, x8_19) + m(x4_2, x6_19) + ((m(x1_2, x9_19) + m(x3_2, x7_19) + m(x5, x5_19)) << 1);
|
||||
let mut z1 = m(x0_2, x1) + m(x3_2, x8_19) + m(x5_2, x6_19) + ((m(x2, x9_19) + m(x4, x7_19)) << 1);
|
||||
let mut z2 = m(x0_2, x2) + m(x1_2, x1) + m(x4_2, x8_19) + m(x6, x6_19) + ((m(x3_2, x9_19) + m(x5_2, x7_19)) << 1);
|
||||
let mut z3 = m(x0_2, x3) + m(x1_2, x2) + m(x5_2, x8_19) + ((m(x4, x9_19) + m(x6, x7_19)) << 1);
|
||||
let mut z4 = m(x0_2, x4) + m(x1_2, x3_2) + m(x2, x2) + m(x6_2, x8_19) + ((m(x5_2, x9_19) + m(x7, x7_19)) << 1);
|
||||
let mut z5 = m(x0_2, x5) + m(x1_2, x4) + m(x2_2, x3) + m(x7_2, x8_19) + ((m(x6, x9_19)) << 1);
|
||||
let mut z6 = m(x0_2, x6) + m(x1_2, x5_2) + m(x2_2, x4) + m(x3_2, x3) + m(x8, x8_19) + ((m(x7_2, x9_19)) << 1);
|
||||
let mut z7 = m(x0_2, x7) + m(x1_2, x6) + m(x2_2, x5) + m(x3_2, x4) + ((m(x8, x9_19)) << 1);
|
||||
let mut z8 = m(x0_2, x8) + m(x1_2, x7_2) + m(x2_2, x6) + m(x3_2, x5_2) + m(x4, x4) + ((m(x9, x9_19)) << 1);
|
||||
let mut z9 = m(x0_2, x9) + m(x1_2, x8) + m(x2_2, x7) + m(x3_2, x6) + m(x4_2, x5) ;
|
||||
let mut z0 = m(x0, x0) + m(x2_2, x8_19) + m(x4_2, x6_19) + ((m(x1_2, x9_19) + m(x3_2, x7_19) + m(x5, x5_19)).shl::<1>());
|
||||
let mut z1 = m(x0_2, x1) + m(x3_2, x8_19) + m(x5_2, x6_19) + ((m(x2, x9_19) + m(x4, x7_19)).shl::<1>());
|
||||
let mut z2 = m(x0_2, x2) + m(x1_2, x1) + m(x4_2, x8_19) + m(x6, x6_19) + ((m(x3_2, x9_19) + m(x5_2, x7_19)).shl::<1>());
|
||||
let mut z3 = m(x0_2, x3) + m(x1_2, x2) + m(x5_2, x8_19) + ((m(x4, x9_19) + m(x6, x7_19)).shl::<1>());
|
||||
let mut z4 = m(x0_2, x4) + m(x1_2, x3_2) + m(x2, x2) + m(x6_2, x8_19) + ((m(x5_2, x9_19) + m(x7, x7_19)).shl::<1>());
|
||||
let mut z5 = m(x0_2, x5) + m(x1_2, x4) + m(x2_2, x3) + m(x7_2, x8_19) + ((m(x6, x9_19)).shl::<1>());
|
||||
let mut z6 = m(x0_2, x6) + m(x1_2, x5_2) + m(x2_2, x4) + m(x3_2, x3) + m(x8, x8_19) + ((m(x7_2, x9_19)).shl::<1>());
|
||||
let mut z7 = m(x0_2, x7) + m(x1_2, x6) + m(x2_2, x5) + m(x3_2, x4) + ((m(x8, x9_19)).shl::<1>());
|
||||
let mut z8 = m(x0_2, x8) + m(x1_2, x7_2) + m(x2_2, x6) + m(x3_2, x5_2) + m(x4, x4) + ((m(x9, x9_19)).shl::<1>());
|
||||
let mut z9 = m(x0_2, x9) + m(x1_2, x8) + m(x2_2, x7) + m(x3_2, x6) + m(x4_2, x5) ;
|
||||
|
||||
// The biggest z_i is bounded as z_i < 249*2^(51 + 2*b);
|
||||
// if b < 1.5 we get z_i < 4485585228861014016.
|
||||
|
|
@ -670,7 +656,7 @@ impl FieldElement2625x4 {
|
|||
let negate_D = |x: u64x4, p: u64x4| -> u64x4 {
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_blend_epi32;
|
||||
_mm256_blend_epi32(x.into_bits(), (p - x).into_bits(), D_LANES64 as i32).into_bits()
|
||||
_mm256_blend_epi32(x.into(), (p - x).into(), D_LANES64 as i32).into()
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -741,30 +727,26 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 {
|
|||
/// The coefficients of the result are bounded with \\( b < 0.007 \\).
|
||||
#[inline]
|
||||
fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement2625x4 {
|
||||
unsafe {
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
let consts = u32x8::new(scalars.0, 0, scalars.1, 0, scalars.2, 0, scalars.3, 0);
|
||||
|
||||
let consts = u32x8::new(scalars.0, 0, scalars.1, 0, scalars.2, 0, scalars.3, 0);
|
||||
let (b0, b1) = unpack_pair(self.0[0]);
|
||||
let (b2, b3) = unpack_pair(self.0[1]);
|
||||
let (b4, b5) = unpack_pair(self.0[2]);
|
||||
let (b6, b7) = unpack_pair(self.0[3]);
|
||||
let (b8, b9) = unpack_pair(self.0[4]);
|
||||
|
||||
let (b0, b1) = unpack_pair(self.0[0]);
|
||||
let (b2, b3) = unpack_pair(self.0[1]);
|
||||
let (b4, b5) = unpack_pair(self.0[2]);
|
||||
let (b6, b7) = unpack_pair(self.0[3]);
|
||||
let (b8, b9) = unpack_pair(self.0[4]);
|
||||
|
||||
FieldElement2625x4::reduce64([
|
||||
_mm256_mul_epu32(b0.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b1.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b2.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b3.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b4.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b5.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b6.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b7.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b8.into_bits(), consts.into_bits()).into_bits(),
|
||||
_mm256_mul_epu32(b9.into_bits(), consts.into_bits()).into_bits(),
|
||||
])
|
||||
}
|
||||
FieldElement2625x4::reduce64([
|
||||
b0.mul32(consts),
|
||||
b1.mul32(consts),
|
||||
b2.mul32(consts),
|
||||
b3.mul32(consts),
|
||||
b4.mul32(consts),
|
||||
b5.mul32(consts),
|
||||
b6.mul32(consts),
|
||||
b7.mul32(consts),
|
||||
b8.mul32(consts),
|
||||
b9.mul32(consts),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -786,14 +768,12 @@ impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 {
|
|||
fn mul(self, rhs: &'b FieldElement2625x4) -> FieldElement2625x4 {
|
||||
#[inline(always)]
|
||||
fn m(x: u32x8, y: u32x8) -> u64x4 {
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x.into_bits(), y.into_bits()).into_bits() }
|
||||
x.mul32(y)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn m_lo(x: u32x8, y: u32x8) -> u32x8 {
|
||||
use core::arch::x86_64::_mm256_mul_epu32;
|
||||
unsafe { _mm256_mul_epu32(x.into_bits(), y.into_bits()).into_bits() }
|
||||
x.mul32(y).into()
|
||||
}
|
||||
|
||||
let (x0, x1) = unpack_pair(self.0[0]);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -11,8 +11,8 @@
|
|||
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
use crate::backend::vector::packed_simd::u64x4;
|
||||
use core::ops::{Add, Mul, Neg};
|
||||
use packed_simd::{u64x4, IntoBits};
|
||||
|
||||
use crate::backend::serial::u64::field::FieldElement51;
|
||||
|
||||
|
|
@ -20,14 +20,14 @@ use crate::backend::serial::u64::field::FieldElement51;
|
|||
#[inline(always)]
|
||||
unsafe fn madd52lo(z: u64x4, x: u64x4, y: u64x4) -> u64x4 {
|
||||
use core::arch::x86_64::_mm256_madd52lo_epu64;
|
||||
_mm256_madd52lo_epu64(z.into_bits(), x.into_bits(), y.into_bits()).into_bits()
|
||||
_mm256_madd52lo_epu64(z.into(), x.into(), y.into()).into()
|
||||
}
|
||||
|
||||
/// A wrapper around `vpmadd52huq` that works on `u64x4`.
|
||||
#[inline(always)]
|
||||
unsafe fn madd52hi(z: u64x4, x: u64x4, y: u64x4) -> u64x4 {
|
||||
use core::arch::x86_64::_mm256_madd52hi_epu64;
|
||||
_mm256_madd52hi_epu64(z.into_bits(), x.into_bits(), y.into_bits()).into_bits()
|
||||
_mm256_madd52hi_epu64(z.into(), x.into(), y.into()).into()
|
||||
}
|
||||
|
||||
/// A vector of four field elements in radix 2^51, with unreduced coefficients.
|
||||
|
|
@ -59,16 +59,16 @@ fn shuffle_lanes(x: u64x4, control: Shuffle) -> u64x4 {
|
|||
use core::arch::x86_64::_mm256_permute4x64_epi64 as perm;
|
||||
|
||||
match control {
|
||||
Shuffle::AAAA => perm(x.into_bits(), 0b00_00_00_00).into_bits(),
|
||||
Shuffle::BBBB => perm(x.into_bits(), 0b01_01_01_01).into_bits(),
|
||||
Shuffle::BADC => perm(x.into_bits(), 0b10_11_00_01).into_bits(),
|
||||
Shuffle::BACD => perm(x.into_bits(), 0b11_10_00_01).into_bits(),
|
||||
Shuffle::ADDA => perm(x.into_bits(), 0b00_11_11_00).into_bits(),
|
||||
Shuffle::CBCB => perm(x.into_bits(), 0b01_10_01_10).into_bits(),
|
||||
Shuffle::ABDC => perm(x.into_bits(), 0b10_11_01_00).into_bits(),
|
||||
Shuffle::ABAB => perm(x.into_bits(), 0b01_00_01_00).into_bits(),
|
||||
Shuffle::DBBD => perm(x.into_bits(), 0b11_01_01_11).into_bits(),
|
||||
Shuffle::CACA => perm(x.into_bits(), 0b00_10_00_10).into_bits(),
|
||||
Shuffle::AAAA => perm(x.into(), 0b00_00_00_00).into(),
|
||||
Shuffle::BBBB => perm(x.into(), 0b01_01_01_01).into(),
|
||||
Shuffle::BADC => perm(x.into(), 0b10_11_00_01).into(),
|
||||
Shuffle::BACD => perm(x.into(), 0b11_10_00_01).into(),
|
||||
Shuffle::ADDA => perm(x.into(), 0b00_11_11_00).into(),
|
||||
Shuffle::CBCB => perm(x.into(), 0b01_10_01_10).into(),
|
||||
Shuffle::ABDC => perm(x.into(), 0b10_11_01_00).into(),
|
||||
Shuffle::ABAB => perm(x.into(), 0b01_00_01_00).into(),
|
||||
Shuffle::DBBD => perm(x.into(), 0b11_01_01_11).into(),
|
||||
Shuffle::CACA => perm(x.into(), 0b00_10_00_10).into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -90,18 +90,18 @@ fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 {
|
|||
use core::arch::x86_64::_mm256_blend_epi32 as blend;
|
||||
|
||||
match control {
|
||||
Lanes::D => blend(x.into_bits(), y.into_bits(), 0b11_00_00_00).into_bits(),
|
||||
Lanes::C => blend(x.into_bits(), y.into_bits(), 0b00_11_00_00).into_bits(),
|
||||
Lanes::AB => blend(x.into_bits(), y.into_bits(), 0b00_00_11_11).into_bits(),
|
||||
Lanes::AC => blend(x.into_bits(), y.into_bits(), 0b00_11_00_11).into_bits(),
|
||||
Lanes::AD => blend(x.into_bits(), y.into_bits(), 0b11_00_00_11).into_bits(),
|
||||
Lanes::BCD => blend(x.into_bits(), y.into_bits(), 0b11_11_11_00).into_bits(),
|
||||
Lanes::D => blend(x.into(), y.into(), 0b11_00_00_00).into(),
|
||||
Lanes::C => blend(x.into(), y.into(), 0b00_11_00_00).into(),
|
||||
Lanes::AB => blend(x.into(), y.into(), 0b00_00_11_11).into(),
|
||||
Lanes::AC => blend(x.into(), y.into(), 0b00_11_00_11).into(),
|
||||
Lanes::AD => blend(x.into(), y.into(), 0b11_00_00_11).into(),
|
||||
Lanes::BCD => blend(x.into(), y.into(), 0b11_11_11_00).into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl F51x4Unreduced {
|
||||
pub const ZERO: F51x4Unreduced = F51x4Unreduced([u64x4::splat(0); 5]);
|
||||
pub const ZERO: F51x4Unreduced = F51x4Unreduced([u64x4::splat_const::<0>(); 5]);
|
||||
|
||||
pub fn new(
|
||||
x0: &FieldElement51,
|
||||
|
|
@ -122,32 +122,32 @@ impl F51x4Unreduced {
|
|||
let x = &self.0;
|
||||
[
|
||||
FieldElement51([
|
||||
x[0].extract(0),
|
||||
x[1].extract(0),
|
||||
x[2].extract(0),
|
||||
x[3].extract(0),
|
||||
x[4].extract(0),
|
||||
x[0].extract::<0>(),
|
||||
x[1].extract::<0>(),
|
||||
x[2].extract::<0>(),
|
||||
x[3].extract::<0>(),
|
||||
x[4].extract::<0>(),
|
||||
]),
|
||||
FieldElement51([
|
||||
x[0].extract(1),
|
||||
x[1].extract(1),
|
||||
x[2].extract(1),
|
||||
x[3].extract(1),
|
||||
x[4].extract(1),
|
||||
x[0].extract::<1>(),
|
||||
x[1].extract::<1>(),
|
||||
x[2].extract::<1>(),
|
||||
x[3].extract::<1>(),
|
||||
x[4].extract::<1>(),
|
||||
]),
|
||||
FieldElement51([
|
||||
x[0].extract(2),
|
||||
x[1].extract(2),
|
||||
x[2].extract(2),
|
||||
x[3].extract(2),
|
||||
x[4].extract(2),
|
||||
x[0].extract::<2>(),
|
||||
x[1].extract::<2>(),
|
||||
x[2].extract::<2>(),
|
||||
x[3].extract::<2>(),
|
||||
x[4].extract::<2>(),
|
||||
]),
|
||||
FieldElement51([
|
||||
x[0].extract(3),
|
||||
x[1].extract(3),
|
||||
x[2].extract(3),
|
||||
x[3].extract(3),
|
||||
x[4].extract(3),
|
||||
x[0].extract::<3>(),
|
||||
x[1].extract::<3>(),
|
||||
x[2].extract::<3>(),
|
||||
x[3].extract::<3>(),
|
||||
x[4].extract::<3>(),
|
||||
]),
|
||||
]
|
||||
}
|
||||
|
|
@ -291,64 +291,64 @@ impl F51x4Reduced {
|
|||
z1_2 = madd52hi(z1_2, x[0], x[0]);
|
||||
|
||||
z2_4 = madd52hi(z2_4, x[0], x[1]);
|
||||
let mut z2_1 = z2_4 << 2;
|
||||
let mut z2_1 = z2_4.shl::<2>();
|
||||
z2_2 = madd52lo(z2_2, x[0], x[2]);
|
||||
z2_1 = madd52lo(z2_1, x[1], x[1]);
|
||||
|
||||
z3_4 = madd52hi(z3_4, x[0], x[2]);
|
||||
let mut z3_1 = z3_4 << 2;
|
||||
let mut z3_1 = z3_4.shl::<2>();
|
||||
z3_2 = madd52lo(z3_2, x[1], x[2]);
|
||||
z3_2 = madd52lo(z3_2, x[0], x[3]);
|
||||
z3_2 = madd52hi(z3_2, x[1], x[1]);
|
||||
|
||||
z4_4 = madd52hi(z4_4, x[1], x[2]);
|
||||
z4_4 = madd52hi(z4_4, x[0], x[3]);
|
||||
let mut z4_1 = z4_4 << 2;
|
||||
let mut z4_1 = z4_4.shl::<2>();
|
||||
z4_2 = madd52lo(z4_2, x[1], x[3]);
|
||||
z4_2 = madd52lo(z4_2, x[0], x[4]);
|
||||
z4_1 = madd52lo(z4_1, x[2], x[2]);
|
||||
|
||||
z5_4 = madd52hi(z5_4, x[1], x[3]);
|
||||
z5_4 = madd52hi(z5_4, x[0], x[4]);
|
||||
let mut z5_1 = z5_4 << 2;
|
||||
let mut z5_1 = z5_4.shl::<2>();
|
||||
z5_2 = madd52lo(z5_2, x[2], x[3]);
|
||||
z5_2 = madd52lo(z5_2, x[1], x[4]);
|
||||
z5_2 = madd52hi(z5_2, x[2], x[2]);
|
||||
|
||||
z6_4 = madd52hi(z6_4, x[2], x[3]);
|
||||
z6_4 = madd52hi(z6_4, x[1], x[4]);
|
||||
let mut z6_1 = z6_4 << 2;
|
||||
let mut z6_1 = z6_4.shl::<2>();
|
||||
z6_2 = madd52lo(z6_2, x[2], x[4]);
|
||||
z6_1 = madd52lo(z6_1, x[3], x[3]);
|
||||
|
||||
z7_4 = madd52hi(z7_4, x[2], x[4]);
|
||||
let mut z7_1 = z7_4 << 2;
|
||||
let mut z7_1 = z7_4.shl::<2>();
|
||||
z7_2 = madd52lo(z7_2, x[3], x[4]);
|
||||
z7_2 = madd52hi(z7_2, x[3], x[3]);
|
||||
|
||||
z8_4 = madd52hi(z8_4, x[3], x[4]);
|
||||
let mut z8_1 = z8_4 << 2;
|
||||
let mut z8_1 = z8_4.shl::<2>();
|
||||
z8_1 = madd52lo(z8_1, x[4], x[4]);
|
||||
|
||||
let mut z9_1 = u64x4::splat(0);
|
||||
z9_2 = madd52hi(z9_2, x[4], x[4]);
|
||||
|
||||
z5_1 += z5_2 << 1;
|
||||
z6_1 += z6_2 << 1;
|
||||
z7_1 += z7_2 << 1;
|
||||
z9_1 += z9_2 << 1;
|
||||
z5_1 += z5_2.shl::<1>();
|
||||
z6_1 += z6_2.shl::<1>();
|
||||
z7_1 += z7_2.shl::<1>();
|
||||
z9_1 += z9_2.shl::<1>();
|
||||
|
||||
let mut t0 = u64x4::splat(0);
|
||||
let mut t1 = u64x4::splat(0);
|
||||
let r19 = u64x4::splat(19);
|
||||
|
||||
t0 = madd52hi(t0, r19, z9_1);
|
||||
t1 = madd52lo(t1, r19, z9_1 >> 52);
|
||||
t1 = madd52lo(t1, r19, z9_1.shr::<52>());
|
||||
|
||||
z4_2 = madd52lo(z4_2, r19, z8_1 >> 52);
|
||||
z3_2 = madd52lo(z3_2, r19, z7_1 >> 52);
|
||||
z2_2 = madd52lo(z2_2, r19, z6_1 >> 52);
|
||||
z1_2 = madd52lo(z1_2, r19, z5_1 >> 52);
|
||||
z4_2 = madd52lo(z4_2, r19, z8_1.shr::<52>());
|
||||
z3_2 = madd52lo(z3_2, r19, z7_1.shr::<52>());
|
||||
z2_2 = madd52lo(z2_2, r19, z6_1.shr::<52>());
|
||||
z1_2 = madd52lo(z1_2, r19, z5_1.shr::<52>());
|
||||
|
||||
z0_2 = madd52lo(z0_2, r19, t0 + t1);
|
||||
z1_2 = madd52hi(z1_2, r19, z5_1);
|
||||
|
|
@ -387,11 +387,11 @@ impl From<F51x4Unreduced> for F51x4Reduced {
|
|||
let r19 = u64x4::splat(19);
|
||||
|
||||
// Compute carryouts in parallel
|
||||
let c0 = x.0[0] >> 51;
|
||||
let c1 = x.0[1] >> 51;
|
||||
let c2 = x.0[2] >> 51;
|
||||
let c3 = x.0[3] >> 51;
|
||||
let c4 = x.0[4] >> 51;
|
||||
let c0 = x.0[0].shr::<51>();
|
||||
let c1 = x.0[1].shr::<51>();
|
||||
let c2 = x.0[2].shr::<51>();
|
||||
let c3 = x.0[3].shr::<51>();
|
||||
let c4 = x.0[4].shr::<51>();
|
||||
|
||||
unsafe {
|
||||
F51x4Reduced([
|
||||
|
|
@ -581,12 +581,12 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced {
|
|||
|
||||
// Wave 6
|
||||
t0 = madd52hi(t0, r19, z9);
|
||||
t1 = madd52lo(t1, r19, z9 >> 52);
|
||||
t1 = madd52lo(t1, r19, z9.shr::<52>());
|
||||
z3_1 = madd52lo(z3_1, x[0], y[3]);
|
||||
z4_2 = madd52hi(z4_2, x[0], y[3]);
|
||||
z1_2 = madd52lo(z1_2, r19, z5 >> 52);
|
||||
z2_2 = madd52lo(z2_2, r19, z6 >> 52);
|
||||
z3_2 = madd52lo(z3_2, r19, z7 >> 52);
|
||||
z1_2 = madd52lo(z1_2, r19, z5.shr::<52>());
|
||||
z2_2 = madd52lo(z2_2, r19, z6.shr::<52>());
|
||||
z3_2 = madd52lo(z3_2, r19, z7.shr::<52>());
|
||||
z0_1 = madd52lo(z0_1, r19, z5);
|
||||
|
||||
// Wave 7
|
||||
|
|
@ -601,7 +601,7 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced {
|
|||
|
||||
// Wave 8
|
||||
z3_1 = madd52lo(z3_1, r19, z8);
|
||||
z4_2 = madd52lo(z4_2, r19, z8 >> 52);
|
||||
z4_2 = madd52lo(z4_2, r19, z8.shr::<52>());
|
||||
|
||||
F51x4Unreduced([
|
||||
z0_1 + z0_2 + z0_2,
|
||||
|
|
|
|||
|
|
@ -11,28 +11,44 @@
|
|||
|
||||
#![doc = include_str!("../../../docs/parallel-formulas.md")]
|
||||
|
||||
#[cfg(not(any(target_feature = "avx2", target_feature = "avx512ifma", docsrs)))]
|
||||
#[cfg(not(any(
|
||||
target_feature = "avx2",
|
||||
all(target_feature = "avx512ifma", nightly),
|
||||
docsrs
|
||||
)))]
|
||||
compile_error!("'simd' backend selected without target_feature=+avx2 or +avx512ifma");
|
||||
|
||||
#[allow(missing_docs)]
|
||||
pub mod packed_simd;
|
||||
|
||||
#[cfg(any(
|
||||
all(target_feature = "avx2", not(target_feature = "avx512ifma")),
|
||||
all(
|
||||
target_feature = "avx2",
|
||||
not(all(target_feature = "avx512ifma", nightly))
|
||||
),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
pub mod avx2;
|
||||
#[cfg(any(
|
||||
all(target_feature = "avx2", not(target_feature = "avx512ifma")),
|
||||
all(
|
||||
target_feature = "avx2",
|
||||
not(all(target_feature = "avx512ifma", nightly))
|
||||
),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
pub(crate) use self::avx2::{edwards::CachedPoint, edwards::ExtendedPoint};
|
||||
|
||||
#[cfg(any(target_feature = "avx512ifma", all(docsrs, target_arch = "x86_64")))]
|
||||
#[cfg(any(
|
||||
all(target_feature = "avx512ifma", nightly),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
pub mod ifma;
|
||||
#[cfg(target_feature = "avx512ifma")]
|
||||
#[cfg(all(target_feature = "avx512ifma", nightly))]
|
||||
pub(crate) use self::ifma::{edwards::CachedPoint, edwards::ExtendedPoint};
|
||||
|
||||
#[cfg(any(
|
||||
target_feature = "avx2",
|
||||
target_feature = "avx512ifma",
|
||||
all(target_feature = "avx512ifma", nightly),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
#[allow(missing_docs)]
|
||||
|
|
@ -43,12 +59,12 @@ pub mod scalar_mul;
|
|||
#[cfg(any(
|
||||
all(
|
||||
target_feature = "avx2",
|
||||
not(target_feature = "avx512ifma"),
|
||||
not(all(target_feature = "avx512ifma", nightly)),
|
||||
feature = "precomputed-tables"
|
||||
),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
pub(crate) use self::avx2::constants::BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
|
||||
#[cfg(all(target_feature = "avx512ifma", feature = "precomputed-tables"))]
|
||||
#[cfg(all(target_feature = "avx512ifma", nightly, feature = "precomputed-tables"))]
|
||||
pub(crate) use self::ifma::constants::BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
|
|
|
|||
311
src/backend/vector/packed_simd.rs
Normal file
311
src/backend/vector/packed_simd.rs
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
// -*- mode: rust; -*-
|
||||
//
|
||||
// This file is part of curve25519-dalek.
|
||||
// See LICENSE for licensing information.
|
||||
|
||||
///! This module defines wrappers over platform-specific SIMD types to make them
|
||||
///! more convenient to use.
|
||||
///!
|
||||
///! UNSAFETY: Everything in this module assumes that we're running on hardware
|
||||
///! which supports at least AVX2. This invariant *must* be enforced
|
||||
///! by the callers of this code.
|
||||
use core::ops::{Add, AddAssign, BitAnd, BitAndAssign, BitXor, BitXorAssign, Sub};
|
||||
|
||||
macro_rules! impl_shared {
|
||||
(
|
||||
$ty:ident,
|
||||
$lane_ty:ident,
|
||||
$add_intrinsic:ident,
|
||||
$sub_intrinsic:ident,
|
||||
$shl_intrinsic:ident,
|
||||
$shr_intrinsic:ident,
|
||||
$extract_intrinsic:ident
|
||||
) => {
|
||||
#[allow(non_camel_case_types)]
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
#[repr(transparent)]
|
||||
pub struct $ty(core::arch::x86_64::__m256i);
|
||||
|
||||
impl From<$ty> for core::arch::x86_64::__m256i {
|
||||
#[inline]
|
||||
fn from(value: $ty) -> core::arch::x86_64::__m256i {
|
||||
value.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<core::arch::x86_64::__m256i> for $ty {
|
||||
#[inline]
|
||||
fn from(value: core::arch::x86_64::__m256i) -> $ty {
|
||||
$ty(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for $ty {
|
||||
#[inline]
|
||||
fn eq(&self, rhs: &$ty) -> bool {
|
||||
unsafe {
|
||||
// This compares each pair of 8-bit packed integers and returns either 0xFF or
|
||||
// 0x00 depending on whether they're equal.
|
||||
//
|
||||
// So the values are equal if (and only if) this returns a value that's filled
|
||||
// with only 0xFF.
|
||||
//
|
||||
// Pseudocode of what this does:
|
||||
// self.0
|
||||
// .bytes()
|
||||
// .zip(rhs.0.bytes())
|
||||
// .map(|a, b| if a == b { 0xFF } else { 0x00 })
|
||||
// .join();
|
||||
let m = core::arch::x86_64::_mm256_cmpeq_epi8(self.0, rhs.0);
|
||||
|
||||
// Now we need to reduce the 256-bit value to something on which we can branch.
|
||||
//
|
||||
// This will just take the most significant bit of every 8-bit packed integer
|
||||
// and build an `i32` out of it. If the values we previously compared were
|
||||
// equal then all off the most significant bits will be equal to 1, which means
|
||||
// that this will return 0xFFFFFFFF, which is equal to -1 when represented as
|
||||
// an `i32`.
|
||||
core::arch::x86_64::_mm256_movemask_epi8(m) == -1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for $ty {}
|
||||
|
||||
impl Add for $ty {
|
||||
type Output = Self;
|
||||
|
||||
#[inline]
|
||||
fn add(self, rhs: $ty) -> Self {
|
||||
unsafe { core::arch::x86_64::$add_intrinsic(self.0, rhs.0).into() }
|
||||
}
|
||||
}
|
||||
|
||||
impl AddAssign for $ty {
|
||||
#[inline]
|
||||
fn add_assign(&mut self, rhs: $ty) {
|
||||
*self = *self + rhs
|
||||
}
|
||||
}
|
||||
|
||||
impl Sub for $ty {
|
||||
type Output = Self;
|
||||
|
||||
#[inline]
|
||||
fn sub(self, rhs: $ty) -> Self {
|
||||
unsafe { core::arch::x86_64::$sub_intrinsic(self.0, rhs.0).into() }
|
||||
}
|
||||
}
|
||||
|
||||
impl BitAnd for $ty {
|
||||
type Output = Self;
|
||||
|
||||
#[inline]
|
||||
fn bitand(self, rhs: $ty) -> Self {
|
||||
unsafe { core::arch::x86_64::_mm256_and_si256(self.0, rhs.0).into() }
|
||||
}
|
||||
}
|
||||
|
||||
impl BitXor for $ty {
|
||||
type Output = Self;
|
||||
|
||||
#[inline]
|
||||
fn bitxor(self, rhs: $ty) -> Self {
|
||||
unsafe { core::arch::x86_64::_mm256_xor_si256(self.0, rhs.0).into() }
|
||||
}
|
||||
}
|
||||
|
||||
impl BitAndAssign for $ty {
|
||||
#[inline]
|
||||
fn bitand_assign(&mut self, rhs: $ty) {
|
||||
*self = *self & rhs;
|
||||
}
|
||||
}
|
||||
|
||||
impl BitXorAssign for $ty {
|
||||
#[inline]
|
||||
fn bitxor_assign(&mut self, rhs: $ty) {
|
||||
*self = *self ^ rhs;
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl $ty {
|
||||
#[inline]
|
||||
pub fn shl<const N: i32>(self) -> Self {
|
||||
unsafe { core::arch::x86_64::$shl_intrinsic(self.0, N).into() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn shr<const N: i32>(self) -> Self {
|
||||
unsafe { core::arch::x86_64::$shr_intrinsic(self.0, N).into() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn extract<const N: i32>(self) -> $lane_ty {
|
||||
unsafe { core::arch::x86_64::$extract_intrinsic(self.0, N) as $lane_ty }
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! impl_conv {
|
||||
($src:ident => $($dst:ident),+) => {
|
||||
$(
|
||||
impl From<$src> for $dst {
|
||||
#[inline]
|
||||
fn from(value: $src) -> $dst {
|
||||
$dst(value.0)
|
||||
}
|
||||
}
|
||||
)+
|
||||
}
|
||||
}
|
||||
|
||||
// We define SIMD functionality over packed unsigned integer types. However, all the integer
|
||||
// intrinsics deal with signed integers. So we cast unsigned to signed, pack it into SIMD, do
|
||||
// add/sub/shl/shr arithmetic, and finally cast back to unsigned at the end. Why is this equivalent
|
||||
// to doing the same thing on unsigned integers? Shl/shr is clear, because casting does not change
|
||||
// the bits of the integer. But what about add/sub? This is due to the following:
|
||||
//
|
||||
// 1) Rust uses two's complement to represent signed integers. So we're assured that the values
|
||||
// we cast into SIMD and extract out at the end are two's complement.
|
||||
//
|
||||
// https://doc.rust-lang.org/reference/types/numeric.html
|
||||
//
|
||||
// 2) Wrapping add/sub is compatible between two's complement signed and unsigned integers.
|
||||
// That is, for all x,y: u64 (or any unsigned integer type),
|
||||
//
|
||||
// x.wrapping_add(y) == (x as i64).wrapping_add(y as i64) as u64, and
|
||||
// x.wrapping_sub(y) == (x as i64).wrapping_sub(y as i64) as u64
|
||||
//
|
||||
// https://julesjacobs.com/2019/03/20/why-twos-complement-works.html
|
||||
//
|
||||
// 3) The add/sub functions we use for SIMD are indeed wrapping. The docs indicate that
|
||||
// __mm256_add/sub compile to vpaddX/vpsubX instructions where X = w, d, or q depending on
|
||||
// the bitwidth. From x86 docs:
|
||||
//
|
||||
// When an individual result is too large to be represented in X bits (overflow), the
|
||||
// result is wrapped around and the low X bits are written to the destination operand
|
||||
// (that is, the carry is ignored).
|
||||
//
|
||||
// https://www.felixcloutier.com/x86/paddb:paddw:paddd:paddq
|
||||
// https://www.felixcloutier.com/x86/psubb:psubw:psubd
|
||||
// https://www.felixcloutier.com/x86/psubq
|
||||
|
||||
impl_shared!(
|
||||
u64x4,
|
||||
u64,
|
||||
_mm256_add_epi64,
|
||||
_mm256_sub_epi64,
|
||||
_mm256_slli_epi64,
|
||||
_mm256_srli_epi64,
|
||||
_mm256_extract_epi64
|
||||
);
|
||||
impl_shared!(
|
||||
u32x8,
|
||||
u32,
|
||||
_mm256_add_epi32,
|
||||
_mm256_sub_epi32,
|
||||
_mm256_slli_epi32,
|
||||
_mm256_srli_epi32,
|
||||
_mm256_extract_epi32
|
||||
);
|
||||
|
||||
impl_conv!(u64x4 => u32x8);
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl u64x4 {
|
||||
/// A constified variant of `new`.
|
||||
///
|
||||
/// Should only be called from `const` contexts. At runtime `new` is going to be faster.
|
||||
#[inline]
|
||||
pub const fn new_const(x0: u64, x1: u64, x2: u64, x3: u64) -> Self {
|
||||
// SAFETY: Transmuting between an array and a SIMD type is safe
|
||||
// https://rust-lang.github.io/unsafe-code-guidelines/layout/packed-simd-vectors.html
|
||||
unsafe { Self(core::mem::transmute([x0, x1, x2, x3])) }
|
||||
}
|
||||
|
||||
/// A constified variant of `splat`.
|
||||
///
|
||||
/// Should only be called from `const` contexts. At runtime `splat` is going to be faster.
|
||||
#[inline]
|
||||
pub const fn splat_const<const N: u64>() -> Self {
|
||||
Self::new_const(N, N, N, N)
|
||||
}
|
||||
|
||||
/// Constructs a new instance.
|
||||
#[inline]
|
||||
pub fn new(x0: u64, x1: u64, x2: u64, x3: u64) -> Self {
|
||||
unsafe {
|
||||
// _mm256_set_epi64 sets the underlying vector in reverse order of the args
|
||||
Self(core::arch::x86_64::_mm256_set_epi64x(
|
||||
x3 as i64, x2 as i64, x1 as i64, x0 as i64,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructs a new instance with all of the elements initialized to the given value.
|
||||
#[inline]
|
||||
pub fn splat(x: u64) -> Self {
|
||||
unsafe { Self(core::arch::x86_64::_mm256_set1_epi64x(x as i64)) }
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl u32x8 {
|
||||
/// A constified variant of `new`.
|
||||
///
|
||||
/// Should only be called from `const` contexts. At runtime `new` is going to be faster.
|
||||
#[inline]
|
||||
pub const fn new_const(
|
||||
x0: u32,
|
||||
x1: u32,
|
||||
x2: u32,
|
||||
x3: u32,
|
||||
x4: u32,
|
||||
x5: u32,
|
||||
x6: u32,
|
||||
x7: u32,
|
||||
) -> Self {
|
||||
// SAFETY: Transmuting between an array and a SIMD type is safe
|
||||
// https://rust-lang.github.io/unsafe-code-guidelines/layout/packed-simd-vectors.html
|
||||
unsafe { Self(core::mem::transmute([x0, x1, x2, x3, x4, x5, x6, x7])) }
|
||||
}
|
||||
|
||||
/// A constified variant of `splat`.
|
||||
///
|
||||
/// Should only be called from `const` contexts. At runtime `splat` is going to be faster.
|
||||
#[inline]
|
||||
pub const fn splat_const<const N: u32>() -> Self {
|
||||
Self::new_const(N, N, N, N, N, N, N, N)
|
||||
}
|
||||
|
||||
/// Constructs a new instance.
|
||||
#[inline]
|
||||
pub fn new(x0: u32, x1: u32, x2: u32, x3: u32, x4: u32, x5: u32, x6: u32, x7: u32) -> Self {
|
||||
unsafe {
|
||||
// _mm256_set_epi32 sets the underlying vector in reverse order of the args
|
||||
Self(core::arch::x86_64::_mm256_set_epi32(
|
||||
x7 as i32, x6 as i32, x5 as i32, x4 as i32, x3 as i32, x2 as i32, x1 as i32,
|
||||
x0 as i32,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructs a new instance with all of the elements initialized to the given value.
|
||||
#[inline]
|
||||
pub fn splat(x: u32) -> Self {
|
||||
unsafe { Self(core::arch::x86_64::_mm256_set1_epi32(x as i32)) }
|
||||
}
|
||||
|
||||
/// Multiplies the low unsigned 32-bits from each packed 64-bit element
|
||||
/// and returns the unsigned 64-bit results.
|
||||
///
|
||||
/// (This ignores the upper 32-bits from each packed 64-bits!)
|
||||
#[inline]
|
||||
pub fn mul32(self, rhs: u32x8) -> u64x4 {
|
||||
// NOTE: This ignores the upper 32-bits from each packed 64-bits.
|
||||
unsafe { core::arch::x86_64::_mm256_mul_epu32(self.0, rhs.0).into() }
|
||||
}
|
||||
}
|
||||
|
|
@ -10,7 +10,14 @@
|
|||
// - Henry de Valence <hdevalence@hdevalence.ca>
|
||||
|
||||
#![no_std]
|
||||
#![cfg_attr(curve25519_dalek_backend = "simd", feature(stdsimd))]
|
||||
#![cfg_attr(
|
||||
all(
|
||||
curve25519_dalek_backend = "simd",
|
||||
target_feature = "avx512ifma",
|
||||
nightly
|
||||
),
|
||||
feature(stdsimd)
|
||||
)]
|
||||
#![cfg_attr(docsrs, feature(doc_auto_cfg, doc_cfg, doc_cfg_hide))]
|
||||
#![cfg_attr(docsrs, doc(cfg_hide(docsrs)))]
|
||||
//------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Reference in a new issue