saymrwulf/curve25519-dalek-source
1
0
Fork 0
mirror of https://github.com/saymrwulf/curve25519-dalek-source.git synced 2026-09-05 20:30:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support AVX512 on stable Rust (#913)

Browse source 
Fixes #758
This commit is contained in:
Nazar Mokrynskyi 2026-06-29 17:44:40 +03:00 committed by GitHub
parent d995caf38f
commit 4cf8db2369
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 133 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
.github/workflows/curve25519-dalek.yml vendored
View file

@ -65,7 +65,7 @@ jobs:
run: cargo build -p ${{ matrix.crate }} --target thumbv7em-none-eabi --release --no-default-features
- name: no_std fiat / cargo hack ${{ matrix.crate }}
env:
RUSTFLAGS: '--cfg curve25519_dalek_backend="fiat"'
RUSTFLAGS: '--cfg curve25519_dalek_backend="fiat"'
run: cargo hack build -p ${{ matrix.crate }} --target thumbv7em-none-eabi --release --each-feature --exclude-features default,std,getrandom
test-serial:
@ -129,6 +129,24 @@ jobs:
RUSTFLAGS: '-C target_feature=+avx2'
run: cargo test --no-default-features --features alloc,precomputed-tables,zeroize,group --target x86_64-unknown-linux-gnu
test-avx512-sde:
name: Test avx512 backend under Intel SDE
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@stable
- uses: RustCrypto/actions/intel-sde-install@master
- name: Route the target through sde64
run: |
mkdir -p .cargo
echo '[target.x86_64-unknown-linux-gnu]' > .cargo/config.toml
echo 'runner = "sde64 -future --"' >> .cargo/config.toml
- env:
# avx512ifma + avx512vl are the two extensions curve25519-dalek's avx512 backend needs
RUSTFLAGS: '-C target_feature=+avx512ifma,+avx512vl'
RUSTDOCFLAGS: '-C target_feature=+avx512ifma,+avx512vl'
run: cargo test --target x86_64-unknown-linux-gnu
msrv:
name: Current MSRV is 1.85.0
runs-on: ubuntu-latest

6
Cargo.lock generated
View file

@ -305,7 +305,7 @@ dependencies = [
[[package]]
name = "curve25519-dalek"
version = "5.0.0-rc.0"
version = "5.0.0-rc.1"
dependencies = [
"cfg-if",
"cpufeatures",
@ -372,7 +372,7 @@ dependencies = [
[[package]]
name = "ed25519-dalek"
version = "3.0.0-rc.0"
version = "3.0.0-rc.1"
dependencies = [
"blake2",
"chacha20",
@ -1489,7 +1489,7 @@ dependencies = [
[[package]]
name = "x25519-dalek"
version = "3.0.0-rc.0"
version = "3.0.0-rc.1"
dependencies = [
"criterion",
"curve25519-dalek",

32
curve25519-dalek-derive/src/lib.rs
View file

@ -117,6 +117,15 @@ pub fn unsafe_target_feature_specialize(
let ident = syn::Ident::new(&name, item_mod.ident.span());
let mut attrs = item_mod.attrs.clone();
if let Some(condition) = attributes.condition() {
// Build target_feature conditions for each feature
let target_features = features.iter().map(|feature| {
let feature_str = syn::LitStr::new(feature, attributes.lit().span());
quote::quote! { target_feature = #feature_str }
});
// Combine all conditions with 'all'
let cfg_tokens = quote::quote! { all(#(#target_features),*, #condition) };
attrs.push(syn::Attribute {
pound_token: Default::default(),
style: syn::AttrStyle::Outer,
@ -124,7 +133,7 @@ pub fn unsafe_target_feature_specialize(
meta: syn::Meta::List(syn::MetaList {
path: syn::Ident::new("cfg", attributes.lit().span()).into(),
delimiter: syn::MacroDelimiter::Paren(Default::default()),
tokens: condition.clone(),
tokens: cfg_tokens,
}),
});
}
@ -289,9 +298,22 @@ fn process_function(
function: syn::ItemFn,
outer: Option<(syn::Generics, Box<syn::Type>)>,
) -> TokenStream {
// Split comma-separated features and create enable tokens for each
let (enable_features, target_features) = attributes
.value()
.split(',')
.map(|f| {
let feature_lit = syn::LitStr::new(f, attributes.span());
(
quote::quote! { enable = #feature_lit },
quote::quote! { target_feature = #feature_lit },
)
})
.unzip::<_, _, Vec<_>, Vec<_>>();
if function.sig.unsafety.is_some() {
return quote::quote! {
#[target_feature(enable = #attributes)]
#[target_feature(#(#enable_features),*)]
#function
}
.into();
@ -386,7 +408,7 @@ fn process_function(
}
syn::Meta::Path(path) if is_path_eq(path, "test") => {
maybe_outer_attributes.push(attribute);
maybe_cfg = quote::quote! { #[cfg(target_feature = #attributes)] };
maybe_cfg = quote::quote! { #[cfg(all(#(#target_features),*))] };
}
syn::Meta::List(syn::MetaList { path, tokens, .. })
if is_path_eq(path, "inline") && tokens.to_string() == "always" =>
@ -428,7 +450,7 @@ fn process_function(
let item_trait_impl = quote::quote! {
impl #outer_impl_generics #trait_ident #outer_ty_generics for #self_ty #outer_where_clause {
#[target_feature(enable = #attributes)]
#[target_feature(#(#enable_features),*)]
#maybe_inline
unsafe fn #function_inner_name #fn_impl_generics (#(#function_args_inner),*) #function_return #fn_where_clause #function_body
}
@ -451,7 +473,7 @@ fn process_function(
#maybe_cfg
#(#maybe_outer_attributes)*
#function_visibility fn #function_name #fn_impl_generics (#(#function_args_outer),*) #function_return #fn_where_clause {
#[target_feature(enable = #attributes)]
#[target_feature(#(#enable_features),*)]
#maybe_inline
unsafe fn #function_inner_name #fn_impl_generics (#(#function_args_inner),*) #function_return #fn_where_clause #function_body
unsafe {

2
curve25519-dalek/Cargo.toml
View file

@ -81,7 +81,7 @@ curve25519-dalek-derive = "0.1"
level = "warn"
check-cfg = [
'cfg(allow_unused_unsafe)',
'cfg(curve25519_dalek_backend, values("fiat", "serial", "simd", "unstable_avx512"))',
'cfg(curve25519_dalek_backend, values("fiat", "serial", "simd", "avx512"))',
'cfg(curve25519_dalek_diagnostics, values("build"))',
'cfg(curve25519_dalek_bits, values("32", "64"))',
'cfg(nightly)',

8
curve25519-dalek/README.md
View file

@ -84,7 +84,7 @@ Curve arithmetic is implemented and used by one of the following backends:
| `serial` | Automatic | An optimized, non-parllel implementation | `32` and `64` |
| `fiat` | Manual | Formally verified field arithmetic from [fiat-crypto] | `32` and `64` |
| `simd` | Automatic | Intel AVX2 accelerated backend | `64` only |
| `unstable_avx512` | Manual | Intel AVX512 IFMA accelerated backend (requires nightly) | `64` only |
| `avx512` | Automatic | Intel AVX512 IFMA accelerated backend | `64` only |
At runtime, `curve25519-dalek` selects an arithmetic backend from the set of backends it was compiled to support. For Intel x86-64 targets, unless otherwise specified, it will build itself with `simd` support, and default to `serial` at runtime if the appropriate CPU features aren't detected. See [SIMD backend] for more details.
@ -138,16 +138,14 @@ $ cargo build --target i686-unknown-linux-gnu
## SIMD backend
When the `simd` backend is selected, the AVX2 or `serial` implementation is selected automatically at runtime, depending on the currently available CPU features. Similarly, when the `unstable_avx512` backend is selected, the AVX512 implementation is selected automatically at runtime if available, or else selection falls through to the aforementioned `simd` backend logic.
When the `simd` backend is selected, the AVX2 or `serial` implementation is selected automatically at runtime, depending on the currently available CPU features. Similarly, when the `avx512` backend is selected, the AVX512 implementation is selected automatically at runtime if available, or else selection falls through to the aforementioned `simd` backend logic.
For a given CPU feature, you can also specify an appropriate `-C target_feature` to build a binary which assumes the required SIMD instructions are always available. Don't do this if you don't have a good reason.
| Backend | `RUSTFLAGS` | Requires nightly? |
| :--- | :--- | :--- |
| AVX2 | `-C target_feature=+avx2` | no |
| AVX512 | `-C target_feature=+avx512ifma,+avx512vl` | yes |
To reiterate, the `simd` backend will NOT use AVX512 code under any circumstance. The only way to enable AVX512 currently is to select the `unstable_avx512` backend and use a nightly compiler.
| AVX512 | `-C target_feature=+avx512ifma,+avx512vl` | yes if `<= 1.89` |
# Documentation

112
curve25519-dalek/build.rs
View file

@ -3,7 +3,7 @@
#![deny(clippy::unwrap_used, dead_code)]
#[allow(non_camel_case_types)]
#[derive(PartialEq, Debug)]
#[derive(PartialEq, Debug, Clone, Copy)]
enum DalekBits {
Dalek32,
Dalek64,
@ -21,6 +21,12 @@ impl std::fmt::Display for DalekBits {
}
}
fn target_has_feature(feature: &str) -> bool {
std::env::var("CARGO_CFG_TARGET_FEATURE")
.map(|features| features.split(',').any(|f| f == feature))
.unwrap_or(false)
}
fn main() {
let target_arch = match std::env::var("CARGO_CFG_TARGET_ARCH") {
Ok(arch) => arch,
@ -35,17 +41,6 @@ fn main() {
println!("cargo:rustc-cfg=curve25519_dalek_bits=\"{curve25519_dalek_bits}\"");
let nightly = if rustc_version::version_meta()
.expect("failed to detect rustc version")
.channel
== rustc_version::Channel::Nightly
{
println!("cargo:rustc-cfg=nightly");
true
} else {
false
};
let rustc_version = rustc_version::version().expect("failed to detect rustc version");
if rustc_version.major == 1 && rustc_version.minor <= 64 {
// Old versions of Rust complain when you have an `unsafe fn` and you use `unsafe {}` inside,
@ -54,46 +49,45 @@ fn main() {
}
// Backend overrides / defaults
let curve25519_dalek_backend = match std::env::var("CARGO_CFG_CURVE25519_DALEK_BACKEND")
.as_deref()
{
Ok("fiat") => "fiat",
Ok("serial") => "serial",
Ok("simd") => {
// simd can only be enabled on x86_64 & 64bit target_pointer_width
match is_capable_simd(&target_arch, curve25519_dalek_bits) {
true => "simd",
// If override is not possible this must result to compile error
// See: issues/532
false => panic!("Could not override curve25519_dalek_backend to simd"),
}
}
Ok("unstable_avx512") if nightly => {
// simd can only be enabled on x86_64 & 64bit target_pointer_width
match is_capable_simd(&target_arch, curve25519_dalek_bits) {
true => {
// In addition enable Avx2 fallback through simd stable backend
// NOTE: Compiler permits duplicate / multi value on the same key
println!("cargo:rustc-cfg=curve25519_dalek_backend=\"simd\"");
"unstable_avx512"
let curve25519_dalek_backend =
match std::env::var("CARGO_CFG_CURVE25519_DALEK_BACKEND").as_deref() {
Ok("fiat") => "fiat",
Ok("serial") => "serial",
Ok("simd") => {
// simd can only be enabled on x86_64 & 64bit target_pointer_width
match is_capable_simd(&target_arch, curve25519_dalek_bits) {
true => "simd",
// If override is not possible this must result to compile error
// See: issues/532
false => panic!("Could not override curve25519_dalek_backend to simd"),
}
// If override is not possible this must result to compile error
// See: issues/532
false => panic!("Could not override curve25519_dalek_backend to unstable_avx512"),
}
}
Ok("unstable_avx512") if !nightly => {
panic!(
"Could not override curve25519_dalek_backend to unstable_avx512, as this is nightly only"
);
}
// default between serial / simd (if potentially capable)
_ => match is_capable_simd(&target_arch, curve25519_dalek_bits) {
true => "simd",
false => "serial",
},
};
Ok("avx512") => {
// AVX-512 can only be enabled on x86_64 & 64bit target_pointer_width
match is_capable_simd(&target_arch, curve25519_dalek_bits) {
true => {
// Enable SIMD as fallback through stable backend
// NOTE: Compiler permits duplicate / multi value on the same key
println!("cargo:rustc-cfg=curve25519_dalek_backend=\"simd\"");
"avx512"
}
// If override is not possible this must result to compile error
// See: issues/532
false => panic!("Could not override curve25519_dalek_backend to avx512"),
}
}
// default between serial / simd / avx512 (if potentially capable)
_ => match is_capable_avx512(&target_arch, curve25519_dalek_bits, rustc_version) {
true => {
println!("cargo:rustc-cfg=curve25519_dalek_backend=\"simd\"");
"avx512"
}
false => match is_capable_simd(&target_arch, curve25519_dalek_bits) {
true => "simd",
false => "serial",
},
},
};
println!("cargo:rustc-cfg=curve25519_dalek_backend=\"{curve25519_dalek_backend}\"");
}
@ -102,6 +96,24 @@ fn is_capable_simd(arch: &str, bits: DalekBits) -> bool {
arch == "x86_64" && bits == DalekBits::Dalek64
}
// Is the target arch & curve25519_dalek_bits potentially AVX-512 capable ?
fn is_capable_avx512(arch: &str, bits: DalekBits, rustc_version: rustc_version::Version) -> bool {
// AVX-512 requires rustc >=1.89 or nightly
if rustc_version.major == 1 && rustc_version.minor < 89 {
let channel = rustc_version::version_meta()
.expect("failed to detect rustc version")
.channel;
if channel != rustc_version::Channel::Nightly {
return false;
}
}
is_capable_simd(arch, bits)
&& target_has_feature("avx512ifma")
&& target_has_feature("avx512vl")
}
// Deterministic cfg(curve25519_dalek_bits) when this is not explicitly set.
mod deterministic {

24
curve25519-dalek/src/backend.rs
View file

@ -46,14 +46,14 @@ pub mod vector;
enum BackendKind {
#[cfg(curve25519_dalek_backend = "simd")]
Avx2,
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
Avx512,
Serial,
}
#[inline]
fn get_selected_backend() -> BackendKind {
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
{
cpufeatures::new!(cpuid_avx512, "avx512ifma", "avx512vl");
let token_avx512: cpuid_avx512::InitToken = cpuid_avx512::init();
@ -88,7 +88,7 @@ where
#[cfg(curve25519_dalek_backend = "simd")]
BackendKind::Avx2 =>
vector::scalar_mul::pippenger::spec_avx2::Pippenger::optional_multiscalar_mul::<I, J>(scalars, points),
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
BackendKind::Avx512 =>
vector::scalar_mul::pippenger::spec_avx512ifma_avx512vl::Pippenger::optional_multiscalar_mul::<I, J>(scalars, points),
BackendKind::Serial =>
@ -100,7 +100,7 @@ where
pub(crate) enum VartimePrecomputedStraus {
#[cfg(curve25519_dalek_backend = "simd")]
Avx2(vector::scalar_mul::precomputed_straus::spec_avx2::VartimePrecomputedStraus),
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
Avx512ifma(
vector::scalar_mul::precomputed_straus::spec_avx512ifma_avx512vl::VartimePrecomputedStraus,
),
@ -120,7 +120,7 @@ impl VartimePrecomputedStraus {
#[cfg(curve25519_dalek_backend = "simd")]
BackendKind::Avx2 =>
VartimePrecomputedStraus::Avx2(vector::scalar_mul::precomputed_straus::spec_avx2::VartimePrecomputedStraus::new(static_points)),
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
BackendKind::Avx512 =>
VartimePrecomputedStraus::Avx512ifma(vector::scalar_mul::precomputed_straus::spec_avx512ifma_avx512vl::VartimePrecomputedStraus::new(static_points)),
BackendKind::Serial =>
@ -135,7 +135,7 @@ impl VartimePrecomputedStraus {
match self {
#[cfg(curve25519_dalek_backend = "simd")]
VartimePrecomputedStraus::Avx2(inner) => inner.len(),
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
VartimePrecomputedStraus::Avx512ifma(inner) => inner.len(),
VartimePrecomputedStraus::Scalar(inner) => inner.len(),
}
@ -148,7 +148,7 @@ impl VartimePrecomputedStraus {
match self {
#[cfg(curve25519_dalek_backend = "simd")]
VartimePrecomputedStraus::Avx2(inner) => inner.is_empty(),
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
VartimePrecomputedStraus::Avx512ifma(inner) => inner.is_empty(),
VartimePrecomputedStraus::Scalar(inner) => inner.is_empty(),
}
@ -176,7 +176,7 @@ impl VartimePrecomputedStraus {
dynamic_scalars,
dynamic_points,
),
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
VartimePrecomputedStraus::Avx512ifma(inner) => inner.optional_mixed_multiscalar_mul(
static_scalars,
dynamic_scalars,
@ -207,7 +207,7 @@ where
BackendKind::Avx2 => {
vector::scalar_mul::straus::spec_avx2::Straus::multiscalar_mul::<I, J>(scalars, points)
}
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
BackendKind::Avx512 => {
vector::scalar_mul::straus::spec_avx512ifma_avx512vl::Straus::multiscalar_mul::<I, J>(
scalars, points,
@ -236,7 +236,7 @@ where
scalars, points,
)
}
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
BackendKind::Avx512 => {
vector::scalar_mul::straus::spec_avx512ifma_avx512vl::Straus::optional_multiscalar_mul::<
I,
@ -254,7 +254,7 @@ pub fn variable_base_mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint
match get_selected_backend() {
#[cfg(curve25519_dalek_backend = "simd")]
BackendKind::Avx2 => vector::scalar_mul::variable_base::spec_avx2::mul(point, scalar),
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
BackendKind::Avx512 => {
vector::scalar_mul::variable_base::spec_avx512ifma_avx512vl::mul(point, scalar)
}
@ -268,7 +268,7 @@ pub fn vartime_double_base_mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> Edwa
match get_selected_backend() {
#[cfg(curve25519_dalek_backend = "simd")]
BackendKind::Avx2 => vector::scalar_mul::vartime_double_base::spec_avx2::mul(a, A, b),
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
BackendKind::Avx512 => {
vector::scalar_mul::vartime_double_base::spec_avx512ifma_avx512vl::mul(a, A, b)
}

2
curve25519-dalek/src/backend/vector.rs
View file

@ -16,7 +16,7 @@ pub mod packed_simd;
pub mod avx2;
#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
#[cfg(curve25519_dalek_backend = "avx512")]
pub mod ifma;
pub mod scalar_mul;

5
curve25519-dalek/src/backend/vector/scalar_mul/pippenger.rs
View file

@ -11,10 +11,7 @@
#[curve25519_dalek_derive::unsafe_target_feature_specialize(
"avx2",
conditional(
"avx512ifma,avx512vl",
all(curve25519_dalek_backend = "unstable_avx512", nightly)
)
conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
)]
pub mod spec {

5
curve25519-dalek/src/backend/vector/scalar_mul/precomputed_straus.rs
View file

@ -13,10 +13,7 @@
#[curve25519_dalek_derive::unsafe_target_feature_specialize(
"avx2",
conditional(
"avx512ifma,avx512vl",
all(curve25519_dalek_backend = "unstable_avx512", nightly)
)
conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
)]
pub mod spec {

5
curve25519-dalek/src/backend/vector/scalar_mul/straus.rs
View file

@ -13,10 +13,7 @@
#[curve25519_dalek_derive::unsafe_target_feature_specialize(
"avx2",
conditional(
"avx512ifma,avx512vl",
all(curve25519_dalek_backend = "unstable_avx512", nightly)
)
conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
)]
pub mod spec {

5
curve25519-dalek/src/backend/vector/scalar_mul/variable_base.rs
View file

@ -2,10 +2,7 @@
#[curve25519_dalek_derive::unsafe_target_feature_specialize(
"avx2",
conditional(
"avx512ifma,avx512vl",
all(curve25519_dalek_backend = "unstable_avx512", nightly)
)
conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
)]
pub mod spec {

5
curve25519-dalek/src/backend/vector/scalar_mul/vartime_double_base.rs
View file

@ -13,10 +13,7 @@
#[curve25519_dalek_derive::unsafe_target_feature_specialize(
"avx2",
conditional(
"avx512ifma,avx512vl",
all(curve25519_dalek_backend = "unstable_avx512", nightly)
)
conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
)]
pub mod spec {