mirror of
https://github.com/saymrwulf/curve25519-dalek-source.git
synced 2026-09-05 20:30:57 +00:00
Runtime backend autodetection
This commit is contained in:
Jan Bujak
parent
91e839aae5
commit
0db8783be8
22 changed files with 570 additions and 167 deletions
33
.github/workflows/rust.yml
vendored
33
.github/workflows/rust.yml
vendored
|
|
@ -55,19 +55,19 @@ jobs:
|
|||
- run: cargo build --target thumbv7em-none-eabi --release
|
||||
- run: cargo build --target thumbv7em-none-eabi --release --features serde
|
||||
|
||||
build-simd-nightly:
|
||||
name: Build simd backend (nightly)
|
||||
test-simd-native:
|
||||
name: Test simd backend (native)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
# Build with AVX2 features, then with AVX512 features
|
||||
- env:
|
||||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx2'
|
||||
run: cargo build --target x86_64-unknown-linux-gnu
|
||||
- env:
|
||||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx512ifma'
|
||||
run: cargo build --target x86_64-unknown-linux-gnu
|
||||
# This will:
|
||||
# 1) build all of the x86_64 SIMD code,
|
||||
# 2) run all of the SIMD-specific tests that the test runner supports,
|
||||
# 3) run all of the normal tests using the best available SIMD backend.
|
||||
RUSTFLAGS: '-C target_cpu=native'
|
||||
run: cargo test --features simd --target x86_64-unknown-linux-gnu
|
||||
|
||||
test-simd-avx2:
|
||||
name: Test simd backend (avx2)
|
||||
|
|
@ -76,8 +76,10 @@ jobs:
|
|||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- env:
|
||||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx2'
|
||||
run: cargo test --target x86_64-unknown-linux-gnu
|
||||
# This will run AVX2-specific tests and run all of the normal tests
|
||||
# with the AVX2 backend, even if the runner supports AVX512.
|
||||
RUSTFLAGS: '-C target_feature=+avx2'
|
||||
run: cargo test --no-default-features --features alloc,precomputed-tables,zeroize,simd_avx2 --target x86_64-unknown-linux-gnu
|
||||
|
||||
build-docs:
|
||||
name: Build docs
|
||||
|
|
@ -131,12 +133,7 @@ jobs:
|
|||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: clippy
|
||||
- env:
|
||||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx2'
|
||||
run: cargo clippy --target x86_64-unknown-linux-gnu
|
||||
- env:
|
||||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx512ifma'
|
||||
run: cargo clippy --target x86_64-unknown-linux-gnu
|
||||
- run: cargo clippy --target x86_64-unknown-linux-gnu
|
||||
|
||||
rustfmt:
|
||||
name: Check formatting
|
||||
|
|
@ -162,9 +159,7 @@ jobs:
|
|||
- uses: dtolnay/rust-toolchain@1.60.0
|
||||
- run: cargo build --no-default-features --features serde
|
||||
# Also make sure the AVX2 build works
|
||||
- env:
|
||||
RUSTFLAGS: '--cfg curve25519_dalek_backend="simd" -C target_feature=+avx2'
|
||||
run: cargo build --target x86_64-unknown-linux-gnu
|
||||
- run: cargo build --target x86_64-unknown-linux-gnu
|
||||
|
||||
bench:
|
||||
name: Check that benchmarks compile
|
||||
|
|
|
|||
17
Cargo.toml
17
Cargo.toml
|
|
@ -27,7 +27,6 @@ rustdoc-args = [
|
|||
"--html-in-header", "docs/assets/rustdoc-include-katex-header.html",
|
||||
"--cfg", "docsrs",
|
||||
]
|
||||
rustc-args = ["--cfg", "curve25519_dalek_backend=\"simd\""]
|
||||
features = ["serde", "rand_core", "digest", "legacy_compatibility"]
|
||||
|
||||
[dev-dependencies]
|
||||
|
|
@ -54,15 +53,29 @@ digest = { version = "0.10", default-features = false, optional = true }
|
|||
subtle = { version = "2.3.0", default-features = false }
|
||||
serde = { version = "1.0", default-features = false, optional = true, features = ["derive"] }
|
||||
zeroize = { version = "1", default-features = false, optional = true }
|
||||
unsafe_target_feature = { version = "0.1.1", optional = true }
|
||||
|
||||
[target.'cfg(target_arch = "x86_64")'.dependencies]
|
||||
cpufeatures = "0.2.6"
|
||||
|
||||
[target.'cfg(curve25519_dalek_backend = "fiat")'.dependencies]
|
||||
fiat-crypto = "0.1.19"
|
||||
|
||||
[features]
|
||||
default = ["alloc", "precomputed-tables", "zeroize"]
|
||||
default = ["alloc", "precomputed-tables", "zeroize", "simd"]
|
||||
alloc = ["zeroize?/alloc"]
|
||||
precomputed-tables = []
|
||||
legacy_compatibility = []
|
||||
|
||||
# Whether to allow the use of the AVX2 SIMD backend.
|
||||
simd_avx2 = ["unsafe_target_feature"]
|
||||
|
||||
# Whether to allow the use of the AVX512 SIMD backend.
|
||||
# (Note: This requires Rust nightly; on Rust stable this feature will be ignored.)
|
||||
simd_avx512 = ["unsafe_target_feature"]
|
||||
|
||||
# A meta-feature to allow all SIMD backends to be used.
|
||||
simd = ["simd_avx2", "simd_avx512"]
|
||||
|
||||
[profile.dev]
|
||||
opt-level = 2
|
||||
|
|
|
|||
1
Makefile
1
Makefile
|
|
@ -1,6 +1,5 @@
|
|||
FEATURES := serde rand_core digest legacy_compatibility
|
||||
|
||||
export RUSTFLAGS := --cfg=curve25519_dalek_backend="simd"
|
||||
export RUSTDOCFLAGS := \
|
||||
--cfg docsrs \
|
||||
--html-in-header docs/assets/rustdoc-include-katex-header.html
|
||||
|
|
|
|||
50
README.md
50
README.md
|
|
@ -53,6 +53,9 @@ curve25519-dalek = "4.0.0-rc.2"
|
|||
| `alloc` | ✓ | Enables Edwards and Ristretto multiscalar multiplication, batch scalar inversion, and batch Ristretto double-and-compress. Also enables `zeroize`. |
|
||||
| `zeroize` | ✓ | Enables [`Zeroize`][zeroize-trait] for all scalar and curve point types. |
|
||||
| `precomputed-tables` | ✓ | Includes precomputed basepoint multiplication tables. This speeds up `EdwardsPoint::mul_base` and `RistrettoPoint::mul_base` by ~4x, at the cost of ~30KB added to the code size. |
|
||||
| `simd_avx2` | ✓ | Allows the AVX2 SIMD backend to be used, if available. |
|
||||
| `simd_avx512` | ✓ | Allows the AVX512 SIMD backend to be used, if available. |
|
||||
| `simd` | ✓ | Allows every SIMD backend to be used, if available. |
|
||||
| `rand_core` | | Enables `Scalar::random` and `RistrettoPoint::random`. This is an optional dependency whose version is not subject to SemVer. See [below](#public-api-semver-exemptions) for more details. |
|
||||
| `digest` | | Enables `RistrettoPoint::{from_hash, hash_from_bytes}` and `Scalar::{from_hash, hash_from_bytes}`. This is an optional dependency whose version is not subject to SemVer. See [below](#public-api-semver-exemptions) for more details. |
|
||||
| `serde` | | Enables `serde` serialization/deserialization for all the point and scalar types. |
|
||||
|
|
@ -95,18 +98,17 @@ See tracking issue: [curve25519-dalek/issues/521](https://github.com/dalek-crypt
|
|||
|
||||
Curve arithmetic is implemented and used by selecting one of the following backends:
|
||||
|
||||
| Backend | Implementation | Target backends |
|
||||
| :--- | :--- | :--- |
|
||||
| `[default]` | Serial formulas | `u32` <br/> `u64` |
|
||||
| `simd` | [Parallel][parallel_doc], using Advanced Vector Extensions | `avx2` <br/> `avx512ifma` |
|
||||
| `fiat` | Formally verified field arithmetic from [fiat-crypto] | `fiat_u32` <br/> `fiat_u64` |
|
||||
| Backend | Implementation | Target backends |
|
||||
| :--- | :--- | :--- |
|
||||
| `[default]` | Automatic runtime backend selection (either serial or SIMD) | `u32` <br/> `u64` <br/> `avx2` <br/> `avx512` |
|
||||
| `fiat` | Formally verified field arithmetic from [fiat-crypto] | `fiat_u32` <br/> `fiat_u64` |
|
||||
|
||||
To choose a backend other than the `[default]` serial backend, set the
|
||||
To choose a backend other than the `[default]` backend, set the
|
||||
environment variable:
|
||||
```sh
|
||||
RUSTFLAGS='--cfg curve25519_dalek_backend="BACKEND"'
|
||||
```
|
||||
where `BACKEND` is `simd` or `fiat`. Equivalently, you can write to
|
||||
where `BACKEND` is `fiat`. Equivalently, you can write to
|
||||
`~/.cargo/config`:
|
||||
```toml
|
||||
[build]
|
||||
|
|
@ -114,11 +116,8 @@ rustflags = ['--cfg=curve25519_dalek_backend="BACKEND"']
|
|||
```
|
||||
More info [here](https://doc.rust-lang.org/cargo/reference/config.html#buildrustflags).
|
||||
|
||||
The `simd` backend requires extra configuration. See [the SIMD
|
||||
section](#simd-target-backends).
|
||||
|
||||
Note for contributors: The target backends are not entirely independent of each
|
||||
other. The `simd` backend directly depends on parts of the the `u64` backend to
|
||||
other. The SIMD backend directly depends on parts of the the `u64` backend to
|
||||
function.
|
||||
|
||||
## Word size for serial backends
|
||||
|
|
@ -137,7 +136,7 @@ RUSTFLAGS='--cfg curve25519_dalek_bits="SIZE"'
|
|||
where `SIZE` is `32` or `64`. As in the above section, this can also be placed
|
||||
in `~/.cargo/config`.
|
||||
|
||||
**NOTE:** The `simd` backend CANNOT be used with word size 32.
|
||||
**NOTE:** Using a word size of 32 will automatically disable SIMD support.
|
||||
|
||||
### Cross-compilation
|
||||
|
||||
|
|
@ -152,18 +151,19 @@ $ cargo build --target i686-unknown-linux-gnu
|
|||
|
||||
## SIMD target backends
|
||||
|
||||
Target backend selection within `simd` must be done manually by setting the
|
||||
`RUSTFLAGS` environment variable to one of the below options:
|
||||
The SIMD target backend selection is done automatically at runtime depending
|
||||
on the available CPU features, provided the appropriate feature flag is enabled.
|
||||
|
||||
| CPU feature | `RUSTFLAGS` | Requires nightly? |
|
||||
| :--- | :--- | :--- |
|
||||
| avx2 | `-C target_feature=+avx2` | no |
|
||||
| avx512ifma | `-C target_feature=+avx512ifma` | yes |
|
||||
You can also specify an appropriate `-C target_feature` to build a binary
|
||||
which assumes the required SIMD instructions are always available.
|
||||
|
||||
Or you can use `-C target_cpu=native` if you don't know what to set.
|
||||
| Backend | Feature flag | `RUSTFLAGS` | Requires nightly? |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| avx2 | `simd_avx2` | `-C target_feature=+avx2` | no |
|
||||
| avx512 | `simd_avx512` | `-C target_feature=+avx512ifma,+avx512vl` | yes |
|
||||
|
||||
The AVX512 backend requires Rust nightly. If enabled and when compiled on a non-nightly
|
||||
compiler it will fall back to using the AVX2 backend.
|
||||
The AVX512 backend requires Rust nightly. When compiled on a non-nightly
|
||||
compiler it will always be disabled.
|
||||
|
||||
# Documentation
|
||||
|
||||
|
|
@ -243,7 +243,8 @@ The implementation is memory-safe, and contains no significant
|
|||
`unsafe` code. The SIMD backend uses `unsafe` internally to call SIMD
|
||||
intrinsics. These are marked `unsafe` only because invoking them on an
|
||||
inappropriate CPU would cause `SIGILL`, but the entire backend is only
|
||||
compiled with appropriate `target_feature`s, so this cannot occur.
|
||||
invoked when the appropriate CPU features are detected at runtime, or
|
||||
when the whole program is compiled with the appropriate `target_feature`s.
|
||||
|
||||
# Performance
|
||||
|
||||
|
|
@ -251,8 +252,7 @@ Benchmarks are run using [`criterion.rs`][criterion]:
|
|||
|
||||
```sh
|
||||
cargo bench --features "rand_core"
|
||||
# Uses avx2 or ifma only if compiled for an appropriate target.
|
||||
export RUSTFLAGS='--cfg curve25519_dalek_backend="simd" -C target_cpu=native'
|
||||
export RUSTFLAGS='-C target_cpu=native'
|
||||
cargo +nightly bench --features "rand_core"
|
||||
```
|
||||
|
||||
|
|
@ -294,7 +294,7 @@ universe's beauty, but also his deep hatred of the Daleks. Rusty destroys the
|
|||
other Daleks and departs the ship, determined to track down and bring an end
|
||||
to the Dalek race.*
|
||||
|
||||
`curve25519-dalek` is authored by Isis Agora Lovecruft and Henry de Valence.
|
||||
`curve25519-dalek` is authored by Isis Agora Lovecruft and Henry de Valence.
|
||||
|
||||
Portions of this library were originally a port of [Adam Langley's
|
||||
Golang ed25519 library](https://github.com/agl/ed25519), which was in
|
||||
|
|
|
|||
|
|
@ -34,7 +34,305 @@
|
|||
//! The [`vector`] backend is selected by the `simd_backend` cargo
|
||||
//! feature; it uses the [`serial`] backend for non-vectorized operations.
|
||||
|
||||
use crate::EdwardsPoint;
|
||||
use crate::Scalar;
|
||||
|
||||
pub mod serial;
|
||||
|
||||
#[cfg(any(curve25519_dalek_backend = "simd", docsrs))]
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
any(feature = "simd_avx2", all(feature = "simd_avx512", nightly)),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
pub mod vector;
|
||||
|
||||
#[derive(Copy, Clone)]
|
||||
enum BackendKind {
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
feature = "simd_avx2",
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
Avx2,
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
all(feature = "simd_avx512", nightly),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
Avx512,
|
||||
Serial,
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_selected_backend() -> BackendKind {
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
all(feature = "simd_avx512", nightly),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
{
|
||||
cpufeatures::new!(cpuid_avx512, "avx512ifma", "avx512vl");
|
||||
let token_avx512: cpuid_avx512::InitToken = cpuid_avx512::init();
|
||||
if token_avx512.get() {
|
||||
return BackendKind::Avx512;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
feature = "simd_avx2",
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
{
|
||||
cpufeatures::new!(cpuid_avx2, "avx2");
|
||||
let token_avx2: cpuid_avx2::InitToken = cpuid_avx2::init();
|
||||
if token_avx2.get() {
|
||||
return BackendKind::Avx2;
|
||||
}
|
||||
}
|
||||
|
||||
BackendKind::Serial
|
||||
}
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
pub fn pippenger_optional_multiscalar_mul<I, J>(scalars: I, points: J) -> Option<EdwardsPoint>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: core::borrow::Borrow<Scalar>,
|
||||
J: IntoIterator<Item = Option<EdwardsPoint>>,
|
||||
{
|
||||
use crate::traits::VartimeMultiscalarMul;
|
||||
|
||||
match get_selected_backend() {
|
||||
#[cfg(all(target_arch = "x86_64", feature = "simd_avx2", curve25519_dalek_bits = "64", not(curve25519_dalek_backend = "fiat")))]
|
||||
BackendKind::Avx2 =>
|
||||
self::vector::scalar_mul::pippenger::spec_avx2::Pippenger::optional_multiscalar_mul::<I, J>(scalars, points),
|
||||
#[cfg(all(target_arch = "x86_64", all(feature = "simd_avx512", nightly), curve25519_dalek_bits = "64", not(curve25519_dalek_backend = "fiat")))]
|
||||
BackendKind::Avx512 =>
|
||||
self::vector::scalar_mul::pippenger::spec_avx512ifma_avx512vl::Pippenger::optional_multiscalar_mul::<I, J>(scalars, points),
|
||||
BackendKind::Serial =>
|
||||
self::serial::scalar_mul::pippenger::Pippenger::optional_multiscalar_mul::<I, J>(scalars, points),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
pub(crate) enum VartimePrecomputedStraus {
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
feature = "simd_avx2",
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
Avx2(self::vector::scalar_mul::precomputed_straus::spec_avx2::VartimePrecomputedStraus),
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
all(feature = "simd_avx512", nightly),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
Avx512ifma(
|
||||
self::vector::scalar_mul::precomputed_straus::spec_avx512ifma_avx512vl::VartimePrecomputedStraus,
|
||||
),
|
||||
Scalar(self::serial::scalar_mul::precomputed_straus::VartimePrecomputedStraus),
|
||||
}
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
impl VartimePrecomputedStraus {
|
||||
pub fn new<I>(static_points: I) -> Self
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: core::borrow::Borrow<EdwardsPoint>,
|
||||
{
|
||||
use crate::traits::VartimePrecomputedMultiscalarMul;
|
||||
|
||||
match get_selected_backend() {
|
||||
#[cfg(all(target_arch = "x86_64", feature = "simd_avx2", curve25519_dalek_bits = "64", not(curve25519_dalek_backend = "fiat")))]
|
||||
BackendKind::Avx2 =>
|
||||
VartimePrecomputedStraus::Avx2(self::vector::scalar_mul::precomputed_straus::spec_avx2::VartimePrecomputedStraus::new(static_points)),
|
||||
#[cfg(all(target_arch = "x86_64", all(feature = "simd_avx512", nightly), curve25519_dalek_bits = "64", not(curve25519_dalek_backend = "fiat")))]
|
||||
BackendKind::Avx512 =>
|
||||
VartimePrecomputedStraus::Avx512ifma(self::vector::scalar_mul::precomputed_straus::spec_avx512ifma_avx512vl::VartimePrecomputedStraus::new(static_points)),
|
||||
BackendKind::Serial =>
|
||||
VartimePrecomputedStraus::Scalar(self::serial::scalar_mul::precomputed_straus::VartimePrecomputedStraus::new(static_points))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn optional_mixed_multiscalar_mul<I, J, K>(
|
||||
&self,
|
||||
static_scalars: I,
|
||||
dynamic_scalars: J,
|
||||
dynamic_points: K,
|
||||
) -> Option<EdwardsPoint>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: core::borrow::Borrow<Scalar>,
|
||||
J: IntoIterator,
|
||||
J::Item: core::borrow::Borrow<Scalar>,
|
||||
K: IntoIterator<Item = Option<EdwardsPoint>>,
|
||||
{
|
||||
use crate::traits::VartimePrecomputedMultiscalarMul;
|
||||
|
||||
match self {
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
feature = "simd_avx2",
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
VartimePrecomputedStraus::Avx2(inner) => inner.optional_mixed_multiscalar_mul(
|
||||
static_scalars,
|
||||
dynamic_scalars,
|
||||
dynamic_points,
|
||||
),
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
all(feature = "simd_avx512", nightly),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
VartimePrecomputedStraus::Avx512ifma(inner) => inner.optional_mixed_multiscalar_mul(
|
||||
static_scalars,
|
||||
dynamic_scalars,
|
||||
dynamic_points,
|
||||
),
|
||||
VartimePrecomputedStraus::Scalar(inner) => inner.optional_mixed_multiscalar_mul(
|
||||
static_scalars,
|
||||
dynamic_scalars,
|
||||
dynamic_points,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
pub fn straus_multiscalar_mul<I, J>(scalars: I, points: J) -> EdwardsPoint
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: core::borrow::Borrow<Scalar>,
|
||||
J: IntoIterator,
|
||||
J::Item: core::borrow::Borrow<EdwardsPoint>,
|
||||
{
|
||||
use crate::traits::MultiscalarMul;
|
||||
|
||||
match get_selected_backend() {
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
feature = "simd_avx2",
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
BackendKind::Avx2 => {
|
||||
self::vector::scalar_mul::straus::spec_avx2::Straus::multiscalar_mul::<I, J>(
|
||||
scalars, points,
|
||||
)
|
||||
}
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
all(feature = "simd_avx512", nightly),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
BackendKind::Avx512 => {
|
||||
self::vector::scalar_mul::straus::spec_avx512ifma_avx512vl::Straus::multiscalar_mul::<
|
||||
I,
|
||||
J,
|
||||
>(scalars, points)
|
||||
}
|
||||
BackendKind::Serial => {
|
||||
self::serial::scalar_mul::straus::Straus::multiscalar_mul::<I, J>(scalars, points)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
pub fn straus_optional_multiscalar_mul<I, J>(scalars: I, points: J) -> Option<EdwardsPoint>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: core::borrow::Borrow<Scalar>,
|
||||
J: IntoIterator<Item = Option<EdwardsPoint>>,
|
||||
{
|
||||
use crate::traits::VartimeMultiscalarMul;
|
||||
|
||||
match get_selected_backend() {
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
feature = "simd_avx2",
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
BackendKind::Avx2 => {
|
||||
self::vector::scalar_mul::straus::spec_avx2::Straus::optional_multiscalar_mul::<I, J>(
|
||||
scalars, points,
|
||||
)
|
||||
}
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
all(feature = "simd_avx512", nightly),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
BackendKind::Avx512 => {
|
||||
self::vector::scalar_mul::straus::spec_avx512ifma_avx512vl::Straus::optional_multiscalar_mul::<
|
||||
I,
|
||||
J,
|
||||
>(scalars, points)
|
||||
}
|
||||
BackendKind::Serial => {
|
||||
self::serial::scalar_mul::straus::Straus::optional_multiscalar_mul::<I, J>(
|
||||
scalars, points,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Perform constant-time, variable-base scalar multiplication.
|
||||
pub fn variable_base_mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint {
|
||||
match get_selected_backend() {
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
feature = "simd_avx2",
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
BackendKind::Avx2 => self::vector::scalar_mul::variable_base::spec_avx2::mul(point, scalar),
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
all(feature = "simd_avx512", nightly),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
BackendKind::Avx512 => {
|
||||
self::vector::scalar_mul::variable_base::spec_avx512ifma_avx512vl::mul(point, scalar)
|
||||
}
|
||||
BackendKind::Serial => self::serial::scalar_mul::variable_base::mul(point, scalar),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint.
|
||||
#[allow(non_snake_case)]
|
||||
pub fn vartime_double_base_mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint {
|
||||
match get_selected_backend() {
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
feature = "simd_avx2",
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
BackendKind::Avx2 => self::vector::scalar_mul::vartime_double_base::spec_avx2::mul(a, A, b),
|
||||
#[cfg(all(
|
||||
target_arch = "x86_64",
|
||||
all(feature = "simd_avx512", nightly),
|
||||
curve25519_dalek_bits = "64",
|
||||
not(curve25519_dalek_backend = "fiat")
|
||||
))]
|
||||
BackendKind::Avx512 => {
|
||||
self::vector::scalar_mul::vartime_double_base::spec_avx512ifma_avx512vl::mul(a, A, b)
|
||||
}
|
||||
BackendKind::Serial => self::serial::scalar_mul::vartime_double_base::mul(a, A, b),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,8 +42,4 @@ cfg_if! {
|
|||
|
||||
pub mod curve_models;
|
||||
|
||||
#[cfg(not(all(
|
||||
curve25519_dalek_backend = "simd",
|
||||
any(target_feature = "avx2", target_feature = "avx512ifma")
|
||||
)))]
|
||||
pub mod scalar_mul;
|
||||
|
|
|
|||
|
|
@ -41,8 +41,13 @@ use core::ops::{Add, Neg, Sub};
|
|||
use subtle::Choice;
|
||||
use subtle::ConditionallySelectable;
|
||||
|
||||
use unsafe_target_feature::unsafe_target_feature;
|
||||
|
||||
use crate::edwards;
|
||||
use crate::window::{LookupTable, NafLookupTable5, NafLookupTable8};
|
||||
use crate::window::{LookupTable, NafLookupTable5};
|
||||
|
||||
#[cfg(any(feature = "precomputed-tables", feature = "alloc"))]
|
||||
use crate::window::NafLookupTable8;
|
||||
|
||||
use crate::traits::Identity;
|
||||
|
||||
|
|
@ -59,12 +64,14 @@ use super::field::{FieldElement2625x4, Lanes, Shuffle};
|
|||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct ExtendedPoint(pub(super) FieldElement2625x4);
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl From<edwards::EdwardsPoint> for ExtendedPoint {
|
||||
fn from(P: edwards::EdwardsPoint) -> ExtendedPoint {
|
||||
ExtendedPoint(FieldElement2625x4::new(&P.X, &P.Y, &P.Z, &P.T))
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl From<ExtendedPoint> for edwards::EdwardsPoint {
|
||||
fn from(P: ExtendedPoint) -> edwards::EdwardsPoint {
|
||||
let tmp = P.0.split();
|
||||
|
|
@ -77,6 +84,7 @@ impl From<ExtendedPoint> for edwards::EdwardsPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl ConditionallySelectable for ExtendedPoint {
|
||||
fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self {
|
||||
ExtendedPoint(FieldElement2625x4::conditional_select(&a.0, &b.0, choice))
|
||||
|
|
@ -87,18 +95,21 @@ impl ConditionallySelectable for ExtendedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Default for ExtendedPoint {
|
||||
fn default() -> ExtendedPoint {
|
||||
ExtendedPoint::identity()
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Identity for ExtendedPoint {
|
||||
fn identity() -> ExtendedPoint {
|
||||
constants::EXTENDEDPOINT_IDENTITY
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl ExtendedPoint {
|
||||
/// Compute the double of this point.
|
||||
pub fn double(&self) -> ExtendedPoint {
|
||||
|
|
@ -184,6 +195,7 @@ impl ExtendedPoint {
|
|||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct CachedPoint(pub(super) FieldElement2625x4);
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl From<ExtendedPoint> for CachedPoint {
|
||||
fn from(P: ExtendedPoint) -> CachedPoint {
|
||||
let mut x = P.0;
|
||||
|
|
@ -202,18 +214,21 @@ impl From<ExtendedPoint> for CachedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Default for CachedPoint {
|
||||
fn default() -> CachedPoint {
|
||||
CachedPoint::identity()
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Identity for CachedPoint {
|
||||
fn identity() -> CachedPoint {
|
||||
constants::CACHEDPOINT_IDENTITY
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl ConditionallySelectable for CachedPoint {
|
||||
fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self {
|
||||
CachedPoint(FieldElement2625x4::conditional_select(&a.0, &b.0, choice))
|
||||
|
|
@ -224,6 +239,7 @@ impl ConditionallySelectable for CachedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl<'a> Neg for &'a CachedPoint {
|
||||
type Output = CachedPoint;
|
||||
/// Lazily negate the point.
|
||||
|
|
@ -238,6 +254,7 @@ impl<'a> Neg for &'a CachedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
|
||||
type Output = ExtendedPoint;
|
||||
|
||||
|
|
@ -275,6 +292,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint {
|
||||
type Output = ExtendedPoint;
|
||||
|
||||
|
|
@ -288,6 +306,7 @@ impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable<CachedPoint> {
|
||||
fn from(point: &'a edwards::EdwardsPoint) -> Self {
|
||||
let P = ExtendedPoint::from(*point);
|
||||
|
|
@ -299,6 +318,7 @@ impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable<CachedPoint> {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable5<CachedPoint> {
|
||||
fn from(point: &'a edwards::EdwardsPoint) -> Self {
|
||||
let A = ExtendedPoint::from(*point);
|
||||
|
|
@ -312,6 +332,8 @@ impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable5<CachedPoint> {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "precomputed-tables", feature = "alloc"))]
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable8<CachedPoint> {
|
||||
fn from(point: &'a edwards::EdwardsPoint) -> Self {
|
||||
let A = ExtendedPoint::from(*point);
|
||||
|
|
@ -325,6 +347,7 @@ impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable8<CachedPoint> {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(target_feature = "avx2")]
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
|
@ -524,6 +547,7 @@ mod test {
|
|||
doubling_test_helper(P);
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "precomputed-tables", feature = "alloc"))]
|
||||
#[test]
|
||||
fn basepoint_odd_lookup_table_verify() {
|
||||
use crate::backend::vector::avx2::constants::BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ use crate::backend::vector::avx2::constants::{
|
|||
P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO,
|
||||
};
|
||||
|
||||
use unsafe_target_feature::unsafe_target_feature;
|
||||
|
||||
/// Unpack 32-bit lanes into 64-bit lanes:
|
||||
/// ```ascii,no_run
|
||||
/// (a0, b0, a1, b1, c0, d0, c1, d1)
|
||||
|
|
@ -57,6 +59,7 @@ use crate::backend::vector::avx2::constants::{
|
|||
/// (a0, 0, b0, 0, c0, 0, d0, 0)
|
||||
/// (a1, 0, b1, 0, c1, 0, d1, 0)
|
||||
/// ```
|
||||
#[unsafe_target_feature("avx2")]
|
||||
#[inline(always)]
|
||||
fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
|
||||
let a: u32x8;
|
||||
|
|
@ -80,6 +83,7 @@ fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
|
|||
/// ```ascii,no_run
|
||||
/// (a0, b0, a1, b1, c0, d0, c1, d1)
|
||||
/// ```
|
||||
#[unsafe_target_feature("avx2")]
|
||||
#[inline(always)]
|
||||
fn repack_pair(x: u32x8, y: u32x8) -> u32x8 {
|
||||
unsafe {
|
||||
|
|
@ -151,6 +155,7 @@ pub struct FieldElement2625x4(pub(crate) [u32x8; 5]);
|
|||
use subtle::Choice;
|
||||
use subtle::ConditionallySelectable;
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl ConditionallySelectable for FieldElement2625x4 {
|
||||
fn conditional_select(
|
||||
a: &FieldElement2625x4,
|
||||
|
|
@ -179,6 +184,7 @@ impl ConditionallySelectable for FieldElement2625x4 {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl FieldElement2625x4 {
|
||||
pub const ZERO: FieldElement2625x4 = FieldElement2625x4([u32x8::splat_const::<0>(); 5]);
|
||||
|
||||
|
|
@ -675,6 +681,7 @@ impl FieldElement2625x4 {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Neg for FieldElement2625x4 {
|
||||
type Output = FieldElement2625x4;
|
||||
|
||||
|
|
@ -703,6 +710,7 @@ impl Neg for FieldElement2625x4 {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Add<FieldElement2625x4> for FieldElement2625x4 {
|
||||
type Output = FieldElement2625x4;
|
||||
/// Add two `FieldElement2625x4`s, without performing a reduction.
|
||||
|
|
@ -718,6 +726,7 @@ impl Add<FieldElement2625x4> for FieldElement2625x4 {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 {
|
||||
type Output = FieldElement2625x4;
|
||||
/// Perform a multiplication by a vector of small constants.
|
||||
|
|
@ -750,6 +759,7 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 {
|
||||
type Output = FieldElement2625x4;
|
||||
/// Multiply `self` by `rhs`.
|
||||
|
|
@ -860,6 +870,7 @@ impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(target_feature = "avx2")]
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -16,3 +16,5 @@ pub(crate) mod field;
|
|||
pub(crate) mod edwards;
|
||||
|
||||
pub(crate) mod constants;
|
||||
|
||||
pub(crate) use self::edwards::{CachedPoint, ExtendedPoint};
|
||||
|
|
|
|||
|
|
@ -16,8 +16,13 @@ use core::ops::{Add, Neg, Sub};
|
|||
use subtle::Choice;
|
||||
use subtle::ConditionallySelectable;
|
||||
|
||||
use unsafe_target_feature::unsafe_target_feature;
|
||||
|
||||
use crate::edwards;
|
||||
use crate::window::{LookupTable, NafLookupTable5, NafLookupTable8};
|
||||
use crate::window::{LookupTable, NafLookupTable5};
|
||||
|
||||
#[cfg(any(feature = "precomputed-tables", feature = "alloc"))]
|
||||
use crate::window::NafLookupTable8;
|
||||
|
||||
use super::constants;
|
||||
use super::field::{F51x4Reduced, F51x4Unreduced, Lanes, Shuffle};
|
||||
|
|
@ -28,12 +33,14 @@ pub struct ExtendedPoint(pub(super) F51x4Unreduced);
|
|||
#[derive(Copy, Clone, Debug)]
|
||||
pub struct CachedPoint(pub(super) F51x4Reduced);
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl From<edwards::EdwardsPoint> for ExtendedPoint {
|
||||
fn from(P: edwards::EdwardsPoint) -> ExtendedPoint {
|
||||
ExtendedPoint(F51x4Unreduced::new(&P.X, &P.Y, &P.Z, &P.T))
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl From<ExtendedPoint> for edwards::EdwardsPoint {
|
||||
fn from(P: ExtendedPoint) -> edwards::EdwardsPoint {
|
||||
let reduced = F51x4Reduced::from(P.0);
|
||||
|
|
@ -47,6 +54,7 @@ impl From<ExtendedPoint> for edwards::EdwardsPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl From<ExtendedPoint> for CachedPoint {
|
||||
fn from(P: ExtendedPoint) -> CachedPoint {
|
||||
let mut x = P.0;
|
||||
|
|
@ -59,18 +67,21 @@ impl From<ExtendedPoint> for CachedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl Default for ExtendedPoint {
|
||||
fn default() -> ExtendedPoint {
|
||||
ExtendedPoint::identity()
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl Identity for ExtendedPoint {
|
||||
fn identity() -> ExtendedPoint {
|
||||
constants::EXTENDEDPOINT_IDENTITY
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl ExtendedPoint {
|
||||
pub fn double(&self) -> ExtendedPoint {
|
||||
// (Y1 X1 T1 Z1) -- uses vpshufd (1c latency @ 1/c)
|
||||
|
|
@ -122,6 +133,7 @@ impl ExtendedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
|
||||
type Output = ExtendedPoint;
|
||||
|
||||
|
|
@ -151,18 +163,21 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl Default for CachedPoint {
|
||||
fn default() -> CachedPoint {
|
||||
CachedPoint::identity()
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl Identity for CachedPoint {
|
||||
fn identity() -> CachedPoint {
|
||||
constants::CACHEDPOINT_IDENTITY
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl ConditionallySelectable for CachedPoint {
|
||||
fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self {
|
||||
CachedPoint(F51x4Reduced::conditional_select(&a.0, &b.0, choice))
|
||||
|
|
@ -173,6 +188,7 @@ impl ConditionallySelectable for CachedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl<'a> Neg for &'a CachedPoint {
|
||||
type Output = CachedPoint;
|
||||
|
||||
|
|
@ -182,6 +198,7 @@ impl<'a> Neg for &'a CachedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint {
|
||||
type Output = ExtendedPoint;
|
||||
|
||||
|
|
@ -191,6 +208,7 @@ impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable<CachedPoint> {
|
||||
fn from(point: &'a edwards::EdwardsPoint) -> Self {
|
||||
let P = ExtendedPoint::from(*point);
|
||||
|
|
@ -202,6 +220,7 @@ impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable<CachedPoint> {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable5<CachedPoint> {
|
||||
fn from(point: &'a edwards::EdwardsPoint) -> Self {
|
||||
let A = ExtendedPoint::from(*point);
|
||||
|
|
@ -215,6 +234,8 @@ impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable5<CachedPoint> {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "precomputed-tables", feature = "alloc"))]
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable8<CachedPoint> {
|
||||
fn from(point: &'a edwards::EdwardsPoint) -> Self {
|
||||
let A = ExtendedPoint::from(*point);
|
||||
|
|
@ -228,6 +249,7 @@ impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable8<CachedPoint> {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(target_feature = "avx512ifma,avx512vl")]
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -16,15 +16,19 @@ use core::ops::{Add, Mul, Neg};
|
|||
|
||||
use crate::backend::serial::u64::field::FieldElement51;
|
||||
|
||||
use unsafe_target_feature::unsafe_target_feature;
|
||||
|
||||
/// A wrapper around `vpmadd52luq` that works on `u64x4`.
|
||||
#[inline(always)]
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
#[inline]
|
||||
unsafe fn madd52lo(z: u64x4, x: u64x4, y: u64x4) -> u64x4 {
|
||||
use core::arch::x86_64::_mm256_madd52lo_epu64;
|
||||
_mm256_madd52lo_epu64(z.into(), x.into(), y.into()).into()
|
||||
}
|
||||
|
||||
/// A wrapper around `vpmadd52huq` that works on `u64x4`.
|
||||
#[inline(always)]
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
#[inline]
|
||||
unsafe fn madd52hi(z: u64x4, x: u64x4, y: u64x4) -> u64x4 {
|
||||
use core::arch::x86_64::_mm256_madd52hi_epu64;
|
||||
_mm256_madd52hi_epu64(z.into(), x.into(), y.into()).into()
|
||||
|
|
@ -53,6 +57,7 @@ pub enum Shuffle {
|
|||
CACA,
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
#[inline(always)]
|
||||
fn shuffle_lanes(x: u64x4, control: Shuffle) -> u64x4 {
|
||||
unsafe {
|
||||
|
|
@ -84,6 +89,7 @@ pub enum Lanes {
|
|||
BCD,
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
#[inline]
|
||||
fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 {
|
||||
unsafe {
|
||||
|
|
@ -100,6 +106,7 @@ fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl F51x4Unreduced {
|
||||
pub const ZERO: F51x4Unreduced = F51x4Unreduced([u64x4::splat_const::<0>(); 5]);
|
||||
|
||||
|
|
@ -198,6 +205,7 @@ impl F51x4Unreduced {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl Neg for F51x4Reduced {
|
||||
type Output = F51x4Reduced;
|
||||
|
||||
|
|
@ -209,6 +217,7 @@ impl Neg for F51x4Reduced {
|
|||
use subtle::Choice;
|
||||
use subtle::ConditionallySelectable;
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl ConditionallySelectable for F51x4Reduced {
|
||||
#[inline]
|
||||
fn conditional_select(a: &F51x4Reduced, b: &F51x4Reduced, choice: Choice) -> F51x4Reduced {
|
||||
|
|
@ -235,6 +244,7 @@ impl ConditionallySelectable for F51x4Reduced {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl F51x4Reduced {
|
||||
#[inline]
|
||||
pub fn shuffle(&self, control: Shuffle) -> F51x4Reduced {
|
||||
|
|
@ -373,6 +383,7 @@ impl F51x4Reduced {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl From<F51x4Reduced> for F51x4Unreduced {
|
||||
#[inline]
|
||||
fn from(x: F51x4Reduced) -> F51x4Unreduced {
|
||||
|
|
@ -380,6 +391,7 @@ impl From<F51x4Reduced> for F51x4Unreduced {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl From<F51x4Unreduced> for F51x4Reduced {
|
||||
#[inline]
|
||||
fn from(x: F51x4Unreduced) -> F51x4Reduced {
|
||||
|
|
@ -405,6 +417,7 @@ impl From<F51x4Unreduced> for F51x4Reduced {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl Add<F51x4Unreduced> for F51x4Unreduced {
|
||||
type Output = F51x4Unreduced;
|
||||
#[inline]
|
||||
|
|
@ -419,6 +432,7 @@ impl Add<F51x4Unreduced> for F51x4Unreduced {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl<'a> Mul<(u32, u32, u32, u32)> for &'a F51x4Reduced {
|
||||
type Output = F51x4Unreduced;
|
||||
#[inline]
|
||||
|
|
@ -470,6 +484,7 @@ impl<'a> Mul<(u32, u32, u32, u32)> for &'a F51x4Reduced {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx512ifma,avx512vl")]
|
||||
impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced {
|
||||
type Output = F51x4Unreduced;
|
||||
#[inline]
|
||||
|
|
@ -614,6 +629,7 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(target_feature = "avx512ifma,avx512vl")]
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -16,3 +16,5 @@ pub mod field;
|
|||
pub mod edwards;
|
||||
|
||||
pub mod constants;
|
||||
|
||||
pub(crate) use self::edwards::{CachedPoint, ExtendedPoint};
|
||||
|
|
|
|||
|
|
@ -11,60 +11,13 @@
|
|||
|
||||
#![doc = include_str!("../../../docs/parallel-formulas.md")]
|
||||
|
||||
#[cfg(not(any(
|
||||
target_feature = "avx2",
|
||||
all(target_feature = "avx512ifma", nightly),
|
||||
docsrs
|
||||
)))]
|
||||
compile_error!("'simd' backend selected without target_feature=+avx2 or +avx512ifma");
|
||||
|
||||
#[allow(missing_docs)]
|
||||
pub mod packed_simd;
|
||||
|
||||
#[cfg(any(
|
||||
all(
|
||||
target_feature = "avx2",
|
||||
not(all(target_feature = "avx512ifma", nightly))
|
||||
),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
#[cfg(feature = "simd_avx2")]
|
||||
pub mod avx2;
|
||||
#[cfg(any(
|
||||
all(
|
||||
target_feature = "avx2",
|
||||
not(all(target_feature = "avx512ifma", nightly))
|
||||
),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
pub(crate) use self::avx2::{edwards::CachedPoint, edwards::ExtendedPoint};
|
||||
|
||||
#[cfg(any(
|
||||
all(target_feature = "avx512ifma", nightly),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
#[cfg(all(feature = "simd_avx512", nightly))]
|
||||
pub mod ifma;
|
||||
#[cfg(all(target_feature = "avx512ifma", nightly))]
|
||||
pub(crate) use self::ifma::{edwards::CachedPoint, edwards::ExtendedPoint};
|
||||
|
||||
#[cfg(any(
|
||||
target_feature = "avx2",
|
||||
all(target_feature = "avx512ifma", nightly),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
#[allow(missing_docs)]
|
||||
pub mod scalar_mul;
|
||||
|
||||
// Precomputed table re-exports
|
||||
|
||||
#[cfg(any(
|
||||
all(
|
||||
target_feature = "avx2",
|
||||
not(all(target_feature = "avx512ifma", nightly)),
|
||||
feature = "precomputed-tables"
|
||||
),
|
||||
all(docsrs, target_arch = "x86_64")
|
||||
))]
|
||||
pub(crate) use self::avx2::constants::BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
|
||||
#[cfg(all(target_feature = "avx512ifma", nightly, feature = "precomputed-tables"))]
|
||||
pub(crate) use self::ifma::constants::BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@
|
|||
///! by the callers of this code.
|
||||
use core::ops::{Add, AddAssign, BitAnd, BitAndAssign, BitXor, BitXorAssign, Sub};
|
||||
|
||||
use unsafe_target_feature::unsafe_target_feature;
|
||||
|
||||
macro_rules! impl_shared {
|
||||
(
|
||||
$ty:ident,
|
||||
|
|
@ -26,6 +28,7 @@ macro_rules! impl_shared {
|
|||
#[repr(transparent)]
|
||||
pub struct $ty(core::arch::x86_64::__m256i);
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl From<$ty> for core::arch::x86_64::__m256i {
|
||||
#[inline]
|
||||
fn from(value: $ty) -> core::arch::x86_64::__m256i {
|
||||
|
|
@ -33,6 +36,7 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl From<core::arch::x86_64::__m256i> for $ty {
|
||||
#[inline]
|
||||
fn from(value: core::arch::x86_64::__m256i) -> $ty {
|
||||
|
|
@ -40,6 +44,7 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl PartialEq for $ty {
|
||||
#[inline]
|
||||
fn eq(&self, rhs: &$ty) -> bool {
|
||||
|
|
@ -72,6 +77,7 @@ macro_rules! impl_shared {
|
|||
|
||||
impl Eq for $ty {}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Add for $ty {
|
||||
type Output = Self;
|
||||
|
||||
|
|
@ -81,6 +87,8 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::assign_op_pattern)]
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl AddAssign for $ty {
|
||||
#[inline]
|
||||
fn add_assign(&mut self, rhs: $ty) {
|
||||
|
|
@ -88,6 +96,7 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl Sub for $ty {
|
||||
type Output = Self;
|
||||
|
||||
|
|
@ -97,6 +106,7 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl BitAnd for $ty {
|
||||
type Output = Self;
|
||||
|
||||
|
|
@ -106,6 +116,7 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl BitXor for $ty {
|
||||
type Output = Self;
|
||||
|
||||
|
|
@ -115,6 +126,8 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::assign_op_pattern)]
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl BitAndAssign for $ty {
|
||||
#[inline]
|
||||
fn bitand_assign(&mut self, rhs: $ty) {
|
||||
|
|
@ -122,6 +135,8 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::assign_op_pattern)]
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl BitXorAssign for $ty {
|
||||
#[inline]
|
||||
fn bitxor_assign(&mut self, rhs: $ty) {
|
||||
|
|
@ -129,6 +144,7 @@ macro_rules! impl_shared {
|
|||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
#[allow(dead_code)]
|
||||
impl $ty {
|
||||
#[inline]
|
||||
|
|
@ -152,6 +168,7 @@ macro_rules! impl_shared {
|
|||
macro_rules! impl_conv {
|
||||
($src:ident => $($dst:ident),+) => {
|
||||
$(
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl From<$src> for $dst {
|
||||
#[inline]
|
||||
fn from(value: $src) -> $dst {
|
||||
|
|
@ -235,8 +252,9 @@ impl u64x4 {
|
|||
}
|
||||
|
||||
/// Constructs a new instance.
|
||||
#[unsafe_target_feature("avx2")]
|
||||
#[inline]
|
||||
pub fn new(x0: u64, x1: u64, x2: u64, x3: u64) -> Self {
|
||||
pub fn new(x0: u64, x1: u64, x2: u64, x3: u64) -> u64x4 {
|
||||
unsafe {
|
||||
// _mm256_set_epi64 sets the underlying vector in reverse order of the args
|
||||
Self(core::arch::x86_64::_mm256_set_epi64x(
|
||||
|
|
@ -246,8 +264,9 @@ impl u64x4 {
|
|||
}
|
||||
|
||||
/// Constructs a new instance with all of the elements initialized to the given value.
|
||||
#[unsafe_target_feature("avx2")]
|
||||
#[inline]
|
||||
pub fn splat(x: u64) -> Self {
|
||||
pub fn splat(x: u64) -> u64x4 {
|
||||
unsafe { Self(core::arch::x86_64::_mm256_set1_epi64x(x as i64)) }
|
||||
}
|
||||
}
|
||||
|
|
@ -257,6 +276,7 @@ impl u32x8 {
|
|||
/// A constified variant of `new`.
|
||||
///
|
||||
/// Should only be called from `const` contexts. At runtime `new` is going to be faster.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[inline]
|
||||
pub const fn new_const(
|
||||
x0: u32,
|
||||
|
|
@ -282,8 +302,10 @@ impl u32x8 {
|
|||
}
|
||||
|
||||
/// Constructs a new instance.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[unsafe_target_feature("avx2")]
|
||||
#[inline]
|
||||
pub fn new(x0: u32, x1: u32, x2: u32, x3: u32, x4: u32, x5: u32, x6: u32, x7: u32) -> Self {
|
||||
pub fn new(x0: u32, x1: u32, x2: u32, x3: u32, x4: u32, x5: u32, x6: u32, x7: u32) -> u32x8 {
|
||||
unsafe {
|
||||
// _mm256_set_epi32 sets the underlying vector in reverse order of the args
|
||||
Self(core::arch::x86_64::_mm256_set_epi32(
|
||||
|
|
@ -294,11 +316,15 @@ impl u32x8 {
|
|||
}
|
||||
|
||||
/// Constructs a new instance with all of the elements initialized to the given value.
|
||||
#[unsafe_target_feature("avx2")]
|
||||
#[inline]
|
||||
pub fn splat(x: u32) -> Self {
|
||||
pub fn splat(x: u32) -> u32x8 {
|
||||
unsafe { Self(core::arch::x86_64::_mm256_set1_epi32(x as i32)) }
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe_target_feature("avx2")]
|
||||
impl u32x8 {
|
||||
/// Multiplies the low unsigned 32-bits from each packed 64-bit element
|
||||
/// and returns the unsigned 64-bit results.
|
||||
///
|
||||
|
|
|
|||
|
|
@ -9,12 +9,23 @@
|
|||
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
#[unsafe_target_feature::unsafe_target_feature_specialize(
|
||||
conditional("avx2", feature = "simd_avx2"),
|
||||
conditional("avx512ifma,avx512vl", all(feature = "simd_avx512", nightly))
|
||||
)]
|
||||
pub mod spec {
|
||||
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use core::borrow::Borrow;
|
||||
use core::cmp::Ordering;
|
||||
|
||||
use crate::backend::vector::{CachedPoint, ExtendedPoint};
|
||||
#[for_target_feature("avx2")]
|
||||
use crate::backend::vector::avx2::{CachedPoint, ExtendedPoint};
|
||||
|
||||
#[for_target_feature("avx512ifma")]
|
||||
use crate::backend::vector::ifma::{CachedPoint, ExtendedPoint};
|
||||
|
||||
use crate::edwards::EdwardsPoint;
|
||||
use crate::scalar::Scalar;
|
||||
use crate::traits::{Identity, VartimeMultiscalarMul};
|
||||
|
|
@ -49,7 +60,7 @@ impl VartimeMultiscalarMul for Pippenger {
|
|||
|
||||
// Collect optimized scalars and points in a buffer for repeated access
|
||||
// (scanning the whole collection per each digit position).
|
||||
let scalars = scalars.into_iter().map(|s| s.borrow().as_radix_2w(w));
|
||||
let scalars = scalars.map(|s| s.borrow().as_radix_2w(w));
|
||||
|
||||
let points = points
|
||||
.into_iter()
|
||||
|
|
@ -127,12 +138,12 @@ impl VartimeMultiscalarMul for Pippenger {
|
|||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use crate::constants;
|
||||
use crate::scalar::Scalar;
|
||||
|
||||
#[test]
|
||||
fn test_vartime_pippenger() {
|
||||
use super::*;
|
||||
use crate::constants;
|
||||
use crate::scalar::Scalar;
|
||||
|
||||
// Reuse points across different tests
|
||||
let mut n = 512;
|
||||
let x = Scalar::from(2128506u64).invert();
|
||||
|
|
@ -163,3 +174,5 @@ mod test {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,12 +11,23 @@
|
|||
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
#[unsafe_target_feature::unsafe_target_feature_specialize(
|
||||
conditional("avx2", feature = "simd_avx2"),
|
||||
conditional("avx512ifma,avx512vl", all(feature = "simd_avx512", nightly))
|
||||
)]
|
||||
pub mod spec {
|
||||
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use core::borrow::Borrow;
|
||||
use core::cmp::Ordering;
|
||||
|
||||
use crate::backend::vector::{CachedPoint, ExtendedPoint};
|
||||
#[for_target_feature("avx2")]
|
||||
use crate::backend::vector::avx2::{CachedPoint, ExtendedPoint};
|
||||
|
||||
#[for_target_feature("avx512ifma")]
|
||||
use crate::backend::vector::ifma::{CachedPoint, ExtendedPoint};
|
||||
|
||||
use crate::edwards::EdwardsPoint;
|
||||
use crate::scalar::Scalar;
|
||||
use crate::traits::Identity;
|
||||
|
|
@ -33,7 +44,7 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus {
|
|||
fn new<I>(static_points: I) -> Self
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: Borrow<Self::Point>,
|
||||
I::Item: Borrow<EdwardsPoint>,
|
||||
{
|
||||
Self {
|
||||
static_lookup_tables: static_points
|
||||
|
|
@ -48,13 +59,13 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus {
|
|||
static_scalars: I,
|
||||
dynamic_scalars: J,
|
||||
dynamic_points: K,
|
||||
) -> Option<Self::Point>
|
||||
) -> Option<EdwardsPoint>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: Borrow<Scalar>,
|
||||
J: IntoIterator,
|
||||
J::Item: Borrow<Scalar>,
|
||||
K: IntoIterator<Item = Option<Self::Point>>,
|
||||
K: IntoIterator<Item = Option<EdwardsPoint>>,
|
||||
{
|
||||
let static_nafs = static_scalars
|
||||
.into_iter()
|
||||
|
|
@ -113,3 +124,5 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus {
|
|||
Some(R.into())
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,12 @@
|
|||
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
#[unsafe_target_feature::unsafe_target_feature_specialize(
|
||||
conditional("avx2", feature = "simd_avx2"),
|
||||
conditional("avx512ifma,avx512vl", all(feature = "simd_avx512", nightly))
|
||||
)]
|
||||
pub mod spec {
|
||||
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use core::borrow::Borrow;
|
||||
|
|
@ -18,7 +24,12 @@ use core::cmp::Ordering;
|
|||
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use crate::backend::vector::{CachedPoint, ExtendedPoint};
|
||||
#[for_target_feature("avx2")]
|
||||
use crate::backend::vector::avx2::{CachedPoint, ExtendedPoint};
|
||||
|
||||
#[for_target_feature("avx512ifma")]
|
||||
use crate::backend::vector::ifma::{CachedPoint, ExtendedPoint};
|
||||
|
||||
use crate::edwards::EdwardsPoint;
|
||||
use crate::scalar::Scalar;
|
||||
use crate::traits::{Identity, MultiscalarMul, VartimeMultiscalarMul};
|
||||
|
|
@ -110,3 +121,5 @@ impl VartimeMultiscalarMul for Straus {
|
|||
Some(Q.into())
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,17 @@
|
|||
#![allow(non_snake_case)]
|
||||
|
||||
use crate::backend::vector::{CachedPoint, ExtendedPoint};
|
||||
#[unsafe_target_feature::unsafe_target_feature_specialize(
|
||||
conditional("avx2", feature = "simd_avx2"),
|
||||
conditional("avx512ifma,avx512vl", all(feature = "simd_avx512", nightly))
|
||||
)]
|
||||
pub mod spec {
|
||||
|
||||
#[for_target_feature("avx2")]
|
||||
use crate::backend::vector::avx2::{CachedPoint, ExtendedPoint};
|
||||
|
||||
#[for_target_feature("avx512ifma")]
|
||||
use crate::backend::vector::ifma::{CachedPoint, ExtendedPoint};
|
||||
|
||||
use crate::edwards::EdwardsPoint;
|
||||
use crate::scalar::Scalar;
|
||||
use crate::traits::Identity;
|
||||
|
|
@ -30,3 +41,5 @@ pub fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint {
|
|||
}
|
||||
Q.into()
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,28 @@
|
|||
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
#[unsafe_target_feature::unsafe_target_feature_specialize(
|
||||
conditional("avx2", feature = "simd_avx2"),
|
||||
conditional("avx512ifma,avx512vl", all(feature = "simd_avx512", nightly))
|
||||
)]
|
||||
pub mod spec {
|
||||
|
||||
use core::cmp::Ordering;
|
||||
|
||||
use crate::backend::vector::{CachedPoint, ExtendedPoint};
|
||||
#[for_target_feature("avx2")]
|
||||
use crate::backend::vector::avx2::{CachedPoint, ExtendedPoint};
|
||||
|
||||
#[for_target_feature("avx512ifma")]
|
||||
use crate::backend::vector::ifma::{CachedPoint, ExtendedPoint};
|
||||
|
||||
#[cfg(feature = "precomputed-tables")]
|
||||
#[for_target_feature("avx2")]
|
||||
use crate::backend::vector::avx2::constants::BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
|
||||
#[cfg(feature = "precomputed-tables")]
|
||||
#[for_target_feature("avx512ifma")]
|
||||
use crate::backend::vector::ifma::constants::BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
|
||||
use crate::edwards::EdwardsPoint;
|
||||
use crate::scalar::Scalar;
|
||||
use crate::traits::Identity;
|
||||
|
|
@ -40,7 +59,8 @@ pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint {
|
|||
let table_A = NafLookupTable5::<CachedPoint>::from(A);
|
||||
|
||||
#[cfg(feature = "precomputed-tables")]
|
||||
let table_B = &crate::backend::vector::BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
let table_B = &BASEPOINT_ODD_LOOKUP_TABLE;
|
||||
|
||||
#[cfg(not(feature = "precomputed-tables"))]
|
||||
let table_B = &NafLookupTable5::<CachedPoint>::from(&crate::constants::ED25519_BASEPOINT_POINT);
|
||||
|
||||
|
|
@ -77,3 +97,5 @@ pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint {
|
|||
|
||||
Q.into()
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -144,17 +144,6 @@ use crate::traits::MultiscalarMul;
|
|||
#[cfg(feature = "alloc")]
|
||||
use crate::traits::{VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul};
|
||||
|
||||
#[cfg(not(all(
|
||||
curve25519_dalek_backend = "simd",
|
||||
any(target_feature = "avx2", target_feature = "avx512ifma")
|
||||
)))]
|
||||
use crate::backend::serial::scalar_mul;
|
||||
#[cfg(all(
|
||||
curve25519_dalek_backend = "simd",
|
||||
any(target_feature = "avx2", target_feature = "avx512ifma")
|
||||
))]
|
||||
use crate::backend::vector::scalar_mul;
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// Compressed points
|
||||
// ------------------------------------------------------------------------
|
||||
|
|
@ -696,7 +685,7 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsPoint {
|
|||
/// For scalar multiplication of a basepoint,
|
||||
/// `EdwardsBasepointTable` is approximately 4x faster.
|
||||
fn mul(self, scalar: &'b Scalar) -> EdwardsPoint {
|
||||
scalar_mul::variable_base::mul(self, scalar)
|
||||
crate::backend::variable_base_mul(self, scalar)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -793,7 +782,7 @@ impl MultiscalarMul for EdwardsPoint {
|
|||
// size-dependent algorithm dispatch, use this as the hint.
|
||||
let _size = s_lo;
|
||||
|
||||
scalar_mul::straus::Straus::multiscalar_mul(scalars, points)
|
||||
crate::backend::straus_multiscalar_mul(scalars, points)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -825,9 +814,9 @@ impl VartimeMultiscalarMul for EdwardsPoint {
|
|||
let size = s_lo;
|
||||
|
||||
if size < 190 {
|
||||
scalar_mul::straus::Straus::optional_multiscalar_mul(scalars, points)
|
||||
crate::backend::straus_optional_multiscalar_mul(scalars, points)
|
||||
} else {
|
||||
scalar_mul::pippenger::Pippenger::optional_multiscalar_mul(scalars, points)
|
||||
crate::backend::pippenger_optional_multiscalar_mul(scalars, points)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -837,7 +826,7 @@ impl VartimeMultiscalarMul for EdwardsPoint {
|
|||
// decouple stability of the inner type from the stability of the
|
||||
// outer type.
|
||||
#[cfg(feature = "alloc")]
|
||||
pub struct VartimeEdwardsPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus);
|
||||
pub struct VartimeEdwardsPrecomputation(crate::backend::VartimePrecomputedStraus);
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
impl VartimePrecomputedMultiscalarMul for VartimeEdwardsPrecomputation {
|
||||
|
|
@ -848,7 +837,7 @@ impl VartimePrecomputedMultiscalarMul for VartimeEdwardsPrecomputation {
|
|||
I: IntoIterator,
|
||||
I::Item: Borrow<Self::Point>,
|
||||
{
|
||||
Self(scalar_mul::precomputed_straus::VartimePrecomputedStraus::new(static_points))
|
||||
Self(crate::backend::VartimePrecomputedStraus::new(static_points))
|
||||
}
|
||||
|
||||
fn optional_mixed_multiscalar_mul<I, J, K>(
|
||||
|
|
@ -876,7 +865,7 @@ impl EdwardsPoint {
|
|||
A: &EdwardsPoint,
|
||||
b: &Scalar,
|
||||
) -> EdwardsPoint {
|
||||
scalar_mul::vartime_double_base::mul(a, A, b)
|
||||
crate::backend::vartime_double_base_mul(a, A, b)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
10
src/lib.rs
10
src/lib.rs
|
|
@ -11,13 +11,13 @@
|
|||
|
||||
#![no_std]
|
||||
#![cfg_attr(
|
||||
all(
|
||||
curve25519_dalek_backend = "simd",
|
||||
target_feature = "avx512ifma",
|
||||
nightly
|
||||
),
|
||||
all(target_arch = "x86_64", feature = "simd_avx512", nightly),
|
||||
feature(stdsimd)
|
||||
)]
|
||||
#![cfg_attr(
|
||||
all(target_arch = "x86_64", feature = "simd_avx512", nightly),
|
||||
feature(avx512_target_feature)
|
||||
)]
|
||||
#![cfg_attr(docsrs, feature(doc_auto_cfg, doc_cfg, doc_cfg_hide))]
|
||||
#![cfg_attr(docsrs, doc(cfg_hide(docsrs)))]
|
||||
//------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -180,9 +180,6 @@ use digest::Digest;
|
|||
use crate::constants;
|
||||
use crate::field::FieldElement;
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
use cfg_if::cfg_if;
|
||||
|
||||
use subtle::Choice;
|
||||
use subtle::ConditionallyNegatable;
|
||||
use subtle::ConditionallySelectable;
|
||||
|
|
@ -203,18 +200,6 @@ use crate::traits::Identity;
|
|||
#[cfg(feature = "alloc")]
|
||||
use crate::traits::{MultiscalarMul, VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul};
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
cfg_if! {
|
||||
if #[cfg(all(
|
||||
curve25519_dalek_backend = "simd",
|
||||
any(target_feature = "avx2", target_feature = "avx512ifma")
|
||||
))] {
|
||||
use crate::backend::vector::scalar_mul;
|
||||
} else {
|
||||
use crate::backend::serial::scalar_mul;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
// Compressed points
|
||||
// ------------------------------------------------------------------------
|
||||
|
|
@ -999,7 +984,7 @@ impl VartimeMultiscalarMul for RistrettoPoint {
|
|||
// decouple stability of the inner type from the stability of the
|
||||
// outer type.
|
||||
#[cfg(feature = "alloc")]
|
||||
pub struct VartimeRistrettoPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus);
|
||||
pub struct VartimeRistrettoPrecomputation(crate::backend::VartimePrecomputedStraus);
|
||||
|
||||
#[cfg(feature = "alloc")]
|
||||
impl VartimePrecomputedMultiscalarMul for VartimeRistrettoPrecomputation {
|
||||
|
|
@ -1010,11 +995,9 @@ impl VartimePrecomputedMultiscalarMul for VartimeRistrettoPrecomputation {
|
|||
I: IntoIterator,
|
||||
I::Item: Borrow<Self::Point>,
|
||||
{
|
||||
Self(
|
||||
scalar_mul::precomputed_straus::VartimePrecomputedStraus::new(
|
||||
static_points.into_iter().map(|P| P.borrow().0),
|
||||
),
|
||||
)
|
||||
Self(crate::backend::VartimePrecomputedStraus::new(
|
||||
static_points.into_iter().map(|P| P.borrow().0),
|
||||
))
|
||||
}
|
||||
|
||||
fn optional_mixed_multiscalar_mul<I, J, K>(
|
||||
|
|
|
|||
Loading…
Reference in a new issue