Support AVX512 on stable Rust (#913)

Fixes #758

.github/workflows/curve25519-dalek.yml

@@ -65,7 +65,7 @@ jobs:
         run: cargo build -p ${{ matrix.crate }} --target thumbv7em-none-eabi --release --no-default-features
       - name: no_std fiat / cargo hack ${{ matrix.crate }}
         env:
-          RUSTFLAGS: '--cfg curve25519_dalek_backend="fiat"'      
+          RUSTFLAGS: '--cfg curve25519_dalek_backend="fiat"'
         run: cargo hack build -p ${{ matrix.crate }} --target thumbv7em-none-eabi --release --each-feature --exclude-features default,std,getrandom
 
   test-serial:
@@ -129,6 +129,24 @@ jobs:
         RUSTFLAGS: '-C target_feature=+avx2'
       run: cargo test --no-default-features --features alloc,precomputed-tables,zeroize,group --target x86_64-unknown-linux-gnu
 
+  test-avx512-sde:
+    name: Test avx512 backend under Intel SDE
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: dtolnay/rust-toolchain@stable
+    - uses: RustCrypto/actions/intel-sde-install@master
+    - name: Route the target through sde64
+      run: |
+        mkdir -p .cargo
+        echo '[target.x86_64-unknown-linux-gnu]' > .cargo/config.toml
+        echo 'runner = "sde64 -future --"' >> .cargo/config.toml
+    - env:
+        # avx512ifma + avx512vl are the two extensions curve25519-dalek's avx512 backend needs
+        RUSTFLAGS: '-C target_feature=+avx512ifma,+avx512vl'
+        RUSTDOCFLAGS: '-C target_feature=+avx512ifma,+avx512vl'
+      run: cargo test --target x86_64-unknown-linux-gnu
+
   msrv:
     name: Current MSRV is 1.85.0
     runs-on: ubuntu-latest

Cargo.lock

@@ -305,7 +305,7 @@ dependencies = [
 
 [[package]]
 name = "curve25519-dalek"
-version = "5.0.0-rc.0"
+version = "5.0.0-rc.1"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -372,7 +372,7 @@ dependencies = [
 
 [[package]]
 name = "ed25519-dalek"
-version = "3.0.0-rc.0"
+version = "3.0.0-rc.1"
 dependencies = [
  "blake2",
  "chacha20",
@@ -1489,7 +1489,7 @@ dependencies = [
 
 [[package]]
 name = "x25519-dalek"
-version = "3.0.0-rc.0"
+version = "3.0.0-rc.1"
 dependencies = [
  "criterion",
  "curve25519-dalek",

curve25519-dalek-derive/src/lib.rs

@@ -117,6 +117,15 @@ pub fn unsafe_target_feature_specialize(
         let ident = syn::Ident::new(&name, item_mod.ident.span());
         let mut attrs = item_mod.attrs.clone();
         if let Some(condition) = attributes.condition() {
+            // Build target_feature conditions for each feature
+            let target_features = features.iter().map(|feature| {
+                let feature_str = syn::LitStr::new(feature, attributes.lit().span());
+                quote::quote! { target_feature = #feature_str }
+            });
+
+            // Combine all conditions with 'all'
+            let cfg_tokens = quote::quote! { all(#(#target_features),*, #condition) };
+
             attrs.push(syn::Attribute {
                 pound_token: Default::default(),
                 style: syn::AttrStyle::Outer,
@@ -124,7 +133,7 @@ pub fn unsafe_target_feature_specialize(
                 meta: syn::Meta::List(syn::MetaList {
                     path: syn::Ident::new("cfg", attributes.lit().span()).into(),
                     delimiter: syn::MacroDelimiter::Paren(Default::default()),
-                    tokens: condition.clone(),
+                    tokens: cfg_tokens,
                 }),
             });
         }
@@ -289,9 +298,22 @@ fn process_function(
     function: syn::ItemFn,
     outer: Option<(syn::Generics, Box<syn::Type>)>,
 ) -> TokenStream {
+    // Split comma-separated features and create enable tokens for each
+    let (enable_features, target_features) = attributes
+        .value()
+        .split(',')
+        .map(|f| {
+            let feature_lit = syn::LitStr::new(f, attributes.span());
+            (
+                quote::quote! { enable = #feature_lit },
+                quote::quote! { target_feature = #feature_lit },
+            )
+        })
+        .unzip::<_, _, Vec<_>, Vec<_>>();
+
     if function.sig.unsafety.is_some() {
         return quote::quote! {
-            #[target_feature(enable = #attributes)]
+            #[target_feature(#(#enable_features),*)]
             #function
         }
         .into();
@@ -386,7 +408,7 @@ fn process_function(
             }
             syn::Meta::Path(path) if is_path_eq(path, "test") => {
                 maybe_outer_attributes.push(attribute);
-                maybe_cfg = quote::quote! { #[cfg(target_feature = #attributes)] };
+                maybe_cfg = quote::quote! { #[cfg(all(#(#target_features),*))] };
             }
             syn::Meta::List(syn::MetaList { path, tokens, .. })
                 if is_path_eq(path, "inline") && tokens.to_string() == "always" =>
@@ -428,7 +450,7 @@ fn process_function(
 
         let item_trait_impl = quote::quote! {
             impl #outer_impl_generics #trait_ident #outer_ty_generics for #self_ty #outer_where_clause {
-                #[target_feature(enable = #attributes)]
+                #[target_feature(#(#enable_features),*)]
                 #maybe_inline
                 unsafe fn #function_inner_name #fn_impl_generics (#(#function_args_inner),*) #function_return #fn_where_clause #function_body
             }
@@ -451,7 +473,7 @@ fn process_function(
             #maybe_cfg
             #(#maybe_outer_attributes)*
             #function_visibility fn #function_name #fn_impl_generics (#(#function_args_outer),*) #function_return #fn_where_clause {
-                #[target_feature(enable = #attributes)]
+                #[target_feature(#(#enable_features),*)]
                 #maybe_inline
                 unsafe fn #function_inner_name #fn_impl_generics (#(#function_args_inner),*) #function_return #fn_where_clause #function_body
                 unsafe {

curve25519-dalek/Cargo.toml

@@ -81,7 +81,7 @@ curve25519-dalek-derive = "0.1"
 level = "warn"
 check-cfg = [
     'cfg(allow_unused_unsafe)',
-    'cfg(curve25519_dalek_backend, values("fiat", "serial", "simd", "unstable_avx512"))',
+    'cfg(curve25519_dalek_backend, values("fiat", "serial", "simd", "avx512"))',
     'cfg(curve25519_dalek_diagnostics, values("build"))',
     'cfg(curve25519_dalek_bits, values("32", "64"))',
     'cfg(nightly)',

curve25519-dalek/README.md

@@ -84,7 +84,7 @@ Curve arithmetic is implemented and used by one of the following backends:
 | `serial`          | Automatic | An optimized, non-parllel implementation                 | `32` and `64`     |
 | `fiat`            | Manual    | Formally verified field arithmetic from [fiat-crypto]    | `32` and `64`     |
 | `simd`            | Automatic | Intel AVX2 accelerated backend                           | `64` only         |
-| `unstable_avx512` | Manual    | Intel AVX512 IFMA accelerated backend (requires nightly) | `64` only         |
+| `avx512`          | Automatic | Intel AVX512 IFMA accelerated backend                    | `64` only         |
 
 At runtime, `curve25519-dalek` selects an arithmetic backend from the set of backends it was compiled to support. For Intel x86-64 targets, unless otherwise specified, it will build itself with `simd` support, and default to `serial` at runtime if the appropriate CPU features aren't detected. See [SIMD backend] for more details.
 
@@ -138,16 +138,14 @@ $ cargo build --target i686-unknown-linux-gnu
 
 ## SIMD backend
 
-When the `simd` backend is selected, the AVX2 or `serial` implementation is selected automatically at runtime, depending on the currently available CPU features. Similarly, when the `unstable_avx512` backend is selected, the AVX512 implementation is selected automatically at runtime if available, or else selection falls through to the aforementioned `simd` backend logic.
+When the `simd` backend is selected, the AVX2 or `serial` implementation is selected automatically at runtime, depending on the currently available CPU features. Similarly, when the `avx512` backend is selected, the AVX512 implementation is selected automatically at runtime if available, or else selection falls through to the aforementioned `simd` backend logic.
 
 For a given CPU feature, you can also specify an appropriate `-C target_feature` to build a binary which assumes the required SIMD instructions are always available. Don't do this if you don't have a good reason.
 
 | Backend | `RUSTFLAGS`                               | Requires nightly? |
 | :---    | :---                                      | :---              |
 | AVX2    | `-C target_feature=+avx2`                 | no                |
-| AVX512  | `-C target_feature=+avx512ifma,+avx512vl` | yes               |
+| AVX512  | `-C target_feature=+avx512ifma,+avx512vl` | yes if `<= 1.89`  |
-
-To reiterate, the `simd` backend will NOT use AVX512 code under any circumstance. The only way to enable AVX512 currently is to select the `unstable_avx512` backend and use a nightly compiler.
 
 # Documentation
 

curve25519-dalek/build.rs

@@ -3,7 +3,7 @@
 #![deny(clippy::unwrap_used, dead_code)]
 
 #[allow(non_camel_case_types)]
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Debug, Clone, Copy)]
 enum DalekBits {
     Dalek32,
     Dalek64,
@@ -21,6 +21,12 @@ impl std::fmt::Display for DalekBits {
     }
 }
 
+fn target_has_feature(feature: &str) -> bool {
+    std::env::var("CARGO_CFG_TARGET_FEATURE")
+        .map(|features| features.split(',').any(|f| f == feature))
+        .unwrap_or(false)
+}
+
 fn main() {
     let target_arch = match std::env::var("CARGO_CFG_TARGET_ARCH") {
         Ok(arch) => arch,
@@ -35,17 +41,6 @@ fn main() {
 
     println!("cargo:rustc-cfg=curve25519_dalek_bits=\"{curve25519_dalek_bits}\"");
 
-    let nightly = if rustc_version::version_meta()
-        .expect("failed to detect rustc version")
-        .channel
-        == rustc_version::Channel::Nightly
-    {
-        println!("cargo:rustc-cfg=nightly");
-        true
-    } else {
-        false
-    };
-
     let rustc_version = rustc_version::version().expect("failed to detect rustc version");
     if rustc_version.major == 1 && rustc_version.minor <= 64 {
         // Old versions of Rust complain when you have an `unsafe fn` and you use `unsafe {}` inside,
@@ -54,46 +49,45 @@ fn main() {
     }
 
     // Backend overrides / defaults
-    let curve25519_dalek_backend = match std::env::var("CARGO_CFG_CURVE25519_DALEK_BACKEND")
+    let curve25519_dalek_backend =
-        .as_deref()
+        match std::env::var("CARGO_CFG_CURVE25519_DALEK_BACKEND").as_deref() {
-    {
+            Ok("fiat") => "fiat",
-        Ok("fiat") => "fiat",
+            Ok("serial") => "serial",
-        Ok("serial") => "serial",
+            Ok("simd") => {
-        Ok("simd") => {
+                // simd can only be enabled on x86_64 & 64bit target_pointer_width
-            // simd can only be enabled on x86_64 & 64bit target_pointer_width
+                match is_capable_simd(&target_arch, curve25519_dalek_bits) {
-            match is_capable_simd(&target_arch, curve25519_dalek_bits) {
+                    true => "simd",
-                true => "simd",
+                    // If override is not possible this must result to compile error
-                // If override is not possible this must result to compile error
+                    // See: issues/532
-                // See: issues/532
+                    false => panic!("Could not override curve25519_dalek_backend to simd"),
-                false => panic!("Could not override curve25519_dalek_backend to simd"),
-            }
-        }
-        Ok("unstable_avx512") if nightly => {
-            // simd can only be enabled on x86_64 & 64bit target_pointer_width
-            match is_capable_simd(&target_arch, curve25519_dalek_bits) {
-                true => {
-                    // In addition enable Avx2 fallback through simd stable backend
-                    // NOTE: Compiler permits duplicate / multi value on the same key
-                    println!("cargo:rustc-cfg=curve25519_dalek_backend=\"simd\"");
-
-                    "unstable_avx512"
                 }
-                // If override is not possible this must result to compile error
-                // See: issues/532
-                false => panic!("Could not override curve25519_dalek_backend to unstable_avx512"),
             }
-        }
+            Ok("avx512") => {
-        Ok("unstable_avx512") if !nightly => {
+                // AVX-512 can only be enabled on x86_64 & 64bit target_pointer_width
-            panic!(
+                match is_capable_simd(&target_arch, curve25519_dalek_bits) {
-                "Could not override curve25519_dalek_backend to unstable_avx512, as this is nightly only"
+                    true => {
-            );
+                        // Enable SIMD as fallback through stable backend
-        }
+                        // NOTE: Compiler permits duplicate / multi value on the same key
-        // default between serial / simd (if potentially capable)
+                        println!("cargo:rustc-cfg=curve25519_dalek_backend=\"simd\"");
-        _ => match is_capable_simd(&target_arch, curve25519_dalek_bits) {
+                        "avx512"
-            true => "simd",
+                    }
-            false => "serial",
+                    // If override is not possible this must result to compile error
-        },
+                    // See: issues/532
-    };
+                    false => panic!("Could not override curve25519_dalek_backend to avx512"),
+                }
+            }
+            // default between serial / simd / avx512 (if potentially capable)
+            _ => match is_capable_avx512(&target_arch, curve25519_dalek_bits, rustc_version) {
+                true => {
+                    println!("cargo:rustc-cfg=curve25519_dalek_backend=\"simd\"");
+                    "avx512"
+                }
+                false => match is_capable_simd(&target_arch, curve25519_dalek_bits) {
+                    true => "simd",
+                    false => "serial",
+                },
+            },
+        };
     println!("cargo:rustc-cfg=curve25519_dalek_backend=\"{curve25519_dalek_backend}\"");
 }
 
@@ -102,6 +96,24 @@ fn is_capable_simd(arch: &str, bits: DalekBits) -> bool {
     arch == "x86_64" && bits == DalekBits::Dalek64
 }
 
+// Is the target arch & curve25519_dalek_bits potentially AVX-512 capable ?
+fn is_capable_avx512(arch: &str, bits: DalekBits, rustc_version: rustc_version::Version) -> bool {
+    // AVX-512 requires rustc >=1.89 or nightly
+    if rustc_version.major == 1 && rustc_version.minor < 89 {
+        let channel = rustc_version::version_meta()
+            .expect("failed to detect rustc version")
+            .channel;
+
+        if channel != rustc_version::Channel::Nightly {
+            return false;
+        }
+    }
+
+    is_capable_simd(arch, bits)
+        && target_has_feature("avx512ifma")
+        && target_has_feature("avx512vl")
+}
+
 // Deterministic cfg(curve25519_dalek_bits) when this is not explicitly set.
 mod deterministic {
 

curve25519-dalek/src/backend.rs

@@ -46,14 +46,14 @@ pub mod vector;
 enum BackendKind {
     #[cfg(curve25519_dalek_backend = "simd")]
     Avx2,
-    #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+    #[cfg(curve25519_dalek_backend = "avx512")]
     Avx512,
     Serial,
 }
 
 #[inline]
 fn get_selected_backend() -> BackendKind {
-    #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+    #[cfg(curve25519_dalek_backend = "avx512")]
     {
         cpufeatures::new!(cpuid_avx512, "avx512ifma", "avx512vl");
         let token_avx512: cpuid_avx512::InitToken = cpuid_avx512::init();
@@ -88,7 +88,7 @@ where
         #[cfg(curve25519_dalek_backend = "simd")]
         BackendKind::Avx2 =>
             vector::scalar_mul::pippenger::spec_avx2::Pippenger::optional_multiscalar_mul::<I, J>(scalars, points),
-        #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+        #[cfg(curve25519_dalek_backend = "avx512")]
         BackendKind::Avx512 =>
             vector::scalar_mul::pippenger::spec_avx512ifma_avx512vl::Pippenger::optional_multiscalar_mul::<I, J>(scalars, points),
         BackendKind::Serial =>
@@ -100,7 +100,7 @@ where
 pub(crate) enum VartimePrecomputedStraus {
     #[cfg(curve25519_dalek_backend = "simd")]
     Avx2(vector::scalar_mul::precomputed_straus::spec_avx2::VartimePrecomputedStraus),
-    #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+    #[cfg(curve25519_dalek_backend = "avx512")]
     Avx512ifma(
         vector::scalar_mul::precomputed_straus::spec_avx512ifma_avx512vl::VartimePrecomputedStraus,
     ),
@@ -120,7 +120,7 @@ impl VartimePrecomputedStraus {
             #[cfg(curve25519_dalek_backend = "simd")]
             BackendKind::Avx2 =>
                 VartimePrecomputedStraus::Avx2(vector::scalar_mul::precomputed_straus::spec_avx2::VartimePrecomputedStraus::new(static_points)),
-            #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+            #[cfg(curve25519_dalek_backend = "avx512")]
             BackendKind::Avx512 =>
                 VartimePrecomputedStraus::Avx512ifma(vector::scalar_mul::precomputed_straus::spec_avx512ifma_avx512vl::VartimePrecomputedStraus::new(static_points)),
             BackendKind::Serial =>
@@ -135,7 +135,7 @@ impl VartimePrecomputedStraus {
         match self {
             #[cfg(curve25519_dalek_backend = "simd")]
             VartimePrecomputedStraus::Avx2(inner) => inner.len(),
-            #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+            #[cfg(curve25519_dalek_backend = "avx512")]
             VartimePrecomputedStraus::Avx512ifma(inner) => inner.len(),
             VartimePrecomputedStraus::Scalar(inner) => inner.len(),
         }
@@ -148,7 +148,7 @@ impl VartimePrecomputedStraus {
         match self {
             #[cfg(curve25519_dalek_backend = "simd")]
             VartimePrecomputedStraus::Avx2(inner) => inner.is_empty(),
-            #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+            #[cfg(curve25519_dalek_backend = "avx512")]
             VartimePrecomputedStraus::Avx512ifma(inner) => inner.is_empty(),
             VartimePrecomputedStraus::Scalar(inner) => inner.is_empty(),
         }
@@ -176,7 +176,7 @@ impl VartimePrecomputedStraus {
                 dynamic_scalars,
                 dynamic_points,
             ),
-            #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+            #[cfg(curve25519_dalek_backend = "avx512")]
             VartimePrecomputedStraus::Avx512ifma(inner) => inner.optional_mixed_multiscalar_mul(
                 static_scalars,
                 dynamic_scalars,
@@ -207,7 +207,7 @@ where
         BackendKind::Avx2 => {
             vector::scalar_mul::straus::spec_avx2::Straus::multiscalar_mul::<I, J>(scalars, points)
         }
-        #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+        #[cfg(curve25519_dalek_backend = "avx512")]
         BackendKind::Avx512 => {
             vector::scalar_mul::straus::spec_avx512ifma_avx512vl::Straus::multiscalar_mul::<I, J>(
                 scalars, points,
@@ -236,7 +236,7 @@ where
                 scalars, points,
             )
         }
-        #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+        #[cfg(curve25519_dalek_backend = "avx512")]
         BackendKind::Avx512 => {
             vector::scalar_mul::straus::spec_avx512ifma_avx512vl::Straus::optional_multiscalar_mul::<
                 I,
@@ -254,7 +254,7 @@ pub fn variable_base_mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint
     match get_selected_backend() {
         #[cfg(curve25519_dalek_backend = "simd")]
         BackendKind::Avx2 => vector::scalar_mul::variable_base::spec_avx2::mul(point, scalar),
-        #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+        #[cfg(curve25519_dalek_backend = "avx512")]
         BackendKind::Avx512 => {
             vector::scalar_mul::variable_base::spec_avx512ifma_avx512vl::mul(point, scalar)
         }
@@ -268,7 +268,7 @@ pub fn vartime_double_base_mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> Edwa
     match get_selected_backend() {
         #[cfg(curve25519_dalek_backend = "simd")]
         BackendKind::Avx2 => vector::scalar_mul::vartime_double_base::spec_avx2::mul(a, A, b),
-        #[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+        #[cfg(curve25519_dalek_backend = "avx512")]
         BackendKind::Avx512 => {
             vector::scalar_mul::vartime_double_base::spec_avx512ifma_avx512vl::mul(a, A, b)
         }

curve25519-dalek/src/backend/vector.rs

@@ -16,7 +16,7 @@ pub mod packed_simd;
 
 pub mod avx2;
 
-#[cfg(all(curve25519_dalek_backend = "unstable_avx512", nightly))]
+#[cfg(curve25519_dalek_backend = "avx512")]
 pub mod ifma;
 
 pub mod scalar_mul;

curve25519-dalek/src/backend/vector/scalar_mul/pippenger.rs

@@ -11,10 +11,7 @@
 
 #[curve25519_dalek_derive::unsafe_target_feature_specialize(
     "avx2",
-    conditional(
+    conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
-        "avx512ifma,avx512vl",
-        all(curve25519_dalek_backend = "unstable_avx512", nightly)
-    )
 )]
 pub mod spec {
 

curve25519-dalek/src/backend/vector/scalar_mul/precomputed_straus.rs

@@ -13,10 +13,7 @@
 
 #[curve25519_dalek_derive::unsafe_target_feature_specialize(
     "avx2",
-    conditional(
+    conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
-        "avx512ifma,avx512vl",
-        all(curve25519_dalek_backend = "unstable_avx512", nightly)
-    )
 )]
 pub mod spec {
 

curve25519-dalek/src/backend/vector/scalar_mul/straus.rs

@@ -13,10 +13,7 @@
 
 #[curve25519_dalek_derive::unsafe_target_feature_specialize(
     "avx2",
-    conditional(
+    conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
-        "avx512ifma,avx512vl",
-        all(curve25519_dalek_backend = "unstable_avx512", nightly)
-    )
 )]
 pub mod spec {
 

curve25519-dalek/src/backend/vector/scalar_mul/variable_base.rs

@@ -2,10 +2,7 @@
 
 #[curve25519_dalek_derive::unsafe_target_feature_specialize(
     "avx2",
-    conditional(
+    conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
-        "avx512ifma,avx512vl",
-        all(curve25519_dalek_backend = "unstable_avx512", nightly)
-    )
 )]
 pub mod spec {
 

curve25519-dalek/src/backend/vector/scalar_mul/vartime_double_base.rs

@@ -13,10 +13,7 @@
 
 #[curve25519_dalek_derive::unsafe_target_feature_specialize(
     "avx2",
-    conditional(
+    conditional("avx512ifma,avx512vl", curve25519_dalek_backend = "avx512")
-        "avx512ifma,avx512vl",
-        all(curve25519_dalek_backend = "unstable_avx512", nightly)
-    )
 )]
 pub mod spec {