saymrwulf/betrusted-curve25519-dalek-source
1
0
Fork 0
mirror of https://github.com/saymrwulf/betrusted-curve25519-dalek-source.git synced 2026-09-13 21:50:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'release/1.1.0-pre.0'

Browse source
This commit is contained in:
Henry de Valence 2019-02-14 14:51:03 -08:00
commit 8b2742cb9d
47 changed files with 5790 additions and 1169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
CHANGELOG.md Normal file
View file

@ -0,0 +1,32 @@
# Changelog
Entries are listed in reverse chronological order.
## 1.1.0-pre.0
* Restructures the source tree into `serial` and `vector` backends.
* Adds a new IFMA backend which sets speed records.
* Adds support for precomputation for multiscalar multiplication.
* Replaces the `rand` dependency with `rand_core`.
* Generalizes trait bounds on
- `RistrettoPoint::random()`
- `Scalar::random()`
to allow owned and borrowed RNGs and to allow `RngCore` instead of
`Rng`.
## 1.0.3
* Adds `ConstantTimeEq` implementation for compressed points.
## 1.0.2
* Fixes a typo in the naming of variables in Ristretto formulas (no change to functionality).
## 1.0.1
* Depends on the stable `2.0` version of `subtle` instead of `2.0.0-pre.0`.
## 1.0.0
Initial stable release. Yanked due to a dependency mistake (see above).

17
Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "curve25519-dalek"
version = "1.0.3"
version = "1.1.0-pre.0"
authors = ["Isis Lovecruft <isis@patternsinthevoid.net>",
"Henry de Valence <hdevalence@hdevalence.ca>"]
readme = "README.md"
@ -26,9 +26,11 @@ features = ["nightly"]
travis-ci = { repository = "dalek-cryptography/curve25519-dalek", branch = "master"}
[dev-dependencies]
rand_os = "0.1.0"
sha2 = { version = "0.8", default-features = false }
bincode = "1"
criterion = "0.2"
rand = "0.6"
[[bench]]
name = "dalek_benchmarks"
@ -41,7 +43,7 @@ harness = false
# match exactly, since the build.rs uses the crate itself as a library.
[dependencies]
rand = { version = "0.6.0", default-features = false }
rand_core = { version = "0.3.0", default-features = false }
byteorder = { version = "^1.2.3", default-features = false, features = ["i128"] }
digest = { version = "0.8", default-features = false }
clear_on_drop = "=0.2.3"
@ -50,7 +52,7 @@ serde = { version = "1.0", optional = true }
packed_simd = { version = "0.3.0", features = ["into_bits"], optional = true }
[build-dependencies]
rand = { version = "0.6.0", default-features = false }
rand_core = { version = "0.3.0", default-features = false }
byteorder = { version = "^1.2.3", default-features = false, features = ["i128"] }
digest = { version = "0.8", default-features = false }
clear_on_drop = "=0.2.3"
@ -61,7 +63,7 @@ packed_simd = { version = "0.3.0", features = ["into_bits"], optional = true }
[features]
nightly = ["subtle/nightly", "clear_on_drop/nightly"]
default = ["std", "u64_backend"]
std = ["alloc", "subtle/std", "rand/std"]
std = ["alloc", "subtle/std", "rand_core/std"]
alloc = []
yolocrypto = []
@ -69,9 +71,10 @@ yolocrypto = []
u32_backend = []
# The u64 backend uses u64s with u128 products.
u64_backend = []
# The AVX2 backend uses u32x8s with u64x4 products.
# It uses the u64 code for serial operations.
avx2_backend = ["nightly", "u64_backend", "packed_simd"]
# The SIMD backend uses parallel formulas, using either AVX2 or AVX512-IFMA.
simd_backend = ["nightly", "u64_backend", "packed_simd"]
# Old name for the SIMD backend, preserved for compatibility
avx2_backend = ["simd_backend"]
# Signals that we're in the main build stage. This is off by default,
# to signal stage 1 of the build, where build.rs loads the library

187
benches/dalek_benchmarks.rs
View file

@ -2,10 +2,12 @@
extern crate rand;
use rand::rngs::OsRng;
use rand::thread_rng;
#[macro_use]
extern crate criterion;
use criterion::BatchSize;
use criterion::Criterion;
extern crate curve25519_dalek;
@ -20,14 +22,10 @@ mod edwards_benches {
use super::*;
use curve25519_dalek::edwards;
use curve25519_dalek::edwards::EdwardsPoint;
use curve25519_dalek::traits::MultiscalarMul;
use curve25519_dalek::traits::VartimeMultiscalarMul;
fn compress(c: &mut Criterion) {
let B = &constants::ED25519_BASEPOINT_POINT;
c.bench_function("EdwardsPoint compression", move |b| {
b.iter(|| B.compress())
});
c.bench_function("EdwardsPoint compression", move |b| b.iter(|| B.compress()));
}
fn decompress(c: &mut Criterion) {
@ -55,25 +53,65 @@ mod edwards_benches {
fn vartime_double_base_scalar_mul(c: &mut Criterion) {
c.bench_function("Variable-time aA+bB, A variable, B fixed", |bench| {
let B = &constants::ED25519_BASEPOINT_POINT;
let a = Scalar::from(298374928u64).invert();
let b = Scalar::from(897987897u64).invert();
let A = B * (b * a);
bench.iter(|| EdwardsPoint::vartime_double_scalar_mul_basepoint(&a, &A, &b));
let mut rng = thread_rng();
let A = &Scalar::random(&mut rng) * &constants::ED25519_BASEPOINT_TABLE;
bench.iter_batched(
|| (Scalar::random(&mut rng), Scalar::random(&mut rng)),
|(a, b)| EdwardsPoint::vartime_double_scalar_mul_basepoint(&a, &A, &b),
BatchSize::SmallInput,
);
});
}
criterion_group! {
name = edwards_benches;
config = Criterion::default();
targets =
compress,
decompress,
consttime_fixed_base_scalar_mul,
consttime_variable_base_scalar_mul,
vartime_double_base_scalar_mul,
}
}
mod multiscalar_benches {
use super::*;
use curve25519_dalek::edwards;
use curve25519_dalek::edwards::EdwardsPoint;
use curve25519_dalek::edwards::VartimeEdwardsPrecomputation;
use curve25519_dalek::traits::MultiscalarMul;
use curve25519_dalek::traits::VartimeMultiscalarMul;
use curve25519_dalek::traits::VartimePrecomputedMultiscalarMul;
fn construct_scalars(n: usize) -> Vec<Scalar> {
let mut rng = thread_rng();
(0..n).map(|_| Scalar::random(&mut rng)).collect()
}
fn construct_points(n: usize) -> Vec<EdwardsPoint> {
let mut rng = thread_rng();
(0..n)
.map(|_| &Scalar::random(&mut rng) * &constants::ED25519_BASEPOINT_TABLE)
.collect()
}
fn construct(n: usize) -> (Vec<Scalar>, Vec<EdwardsPoint>) {
(construct_scalars(n), construct_points(n))
}
fn consttime_multiscalar_mul(c: &mut Criterion) {
c.bench_function_over_inputs(
"Constant-time variable-base multiscalar multiplication",
|b, &&size| {
let mut rng = OsRng::new().unwrap();
let scalars: Vec<Scalar> = (0..size).map(|_| Scalar::random(&mut rng)).collect();
let points: Vec<EdwardsPoint> = scalars
.iter()
.map(|s| s * &constants::ED25519_BASEPOINT_TABLE)
.collect();
b.iter(|| EdwardsPoint::multiscalar_mul(&scalars, &points));
let points = construct_points(size);
// This is supposed to be constant-time, but we might as well
// rerandomize the scalars for every call just in case.
b.iter_batched(
|| construct_scalars(size),
|scalars| EdwardsPoint::multiscalar_mul(&scalars, &points),
BatchSize::SmallInput,
);
},
&MULTISCALAR_SIZES,
);
@ -83,29 +121,107 @@ mod edwards_benches {
c.bench_function_over_inputs(
"Variable-time variable-base multiscalar multiplication",
|b, &&size| {
let mut rng = OsRng::new().unwrap();
let scalars: Vec<Scalar> = (0..size).map(|_| Scalar::random(&mut rng)).collect();
let points: Vec<EdwardsPoint> = scalars
.iter()
.map(|s| s * &constants::ED25519_BASEPOINT_TABLE)
.collect();
b.iter(|| EdwardsPoint::vartime_multiscalar_mul(&scalars, &points));
let points = construct_points(size);
// Rerandomize the scalars for every call to prevent
// false timings from better caching (e.g., the CPU
// cache lifts exactly the right table entries for the
// benchmark into the highest cache levels).
b.iter_batched(
|| construct_scalars(size),
|scalars| EdwardsPoint::vartime_multiscalar_mul(&scalars, &points),
BatchSize::SmallInput,
);
},
&MULTISCALAR_SIZES,
);
}
criterion_group!{
name = edwards_benches;
config = Criterion::default();
fn vartime_precomputed_pure_static(c: &mut Criterion) {
c.bench_function_over_inputs(
"Variable-time fixed-base multiscalar multiplication",
move |b, &&total_size| {
let static_size = total_size;
let static_points = construct_points(static_size);
let precomp = VartimeEdwardsPrecomputation::new(&static_points);
// Rerandomize the scalars for every call to prevent
// false timings from better caching (e.g., the CPU
// cache lifts exactly the right table entries for the
// benchmark into the highest cache levels).
b.iter_batched(
|| construct_scalars(static_size),
|scalars| precomp.vartime_multiscalar_mul(&scalars),
BatchSize::SmallInput,
);
},
&MULTISCALAR_SIZES,
);
}
fn vartime_precomputed_helper(c: &mut Criterion, dynamic_fraction: f64) {
let label = format!(
"Variable-time mixed-base multiscalar multiplication ({:.0}pct dyn)",
100.0 * dynamic_fraction,
);
c.bench_function_over_inputs(
&label,
move |b, &&total_size| {
let dynamic_size = ((total_size as f64) * dynamic_fraction) as usize;
let static_size = total_size - dynamic_size;
let static_points = construct_points(static_size);
let dynamic_points = construct_points(dynamic_size);
let precomp = VartimeEdwardsPrecomputation::new(&static_points);
// Rerandomize the scalars for every call to prevent
// false timings from better caching (e.g., the CPU
// cache lifts exactly the right table entries for the
// benchmark into the highest cache levels). Timings
// should be independent of points so we don't
// randomize them.
b.iter_batched(
|| {
(
construct_scalars(static_size),
construct_scalars(dynamic_size),
)
},
|(static_scalars, dynamic_scalars)| {
precomp.vartime_mixed_multiscalar_mul(
&static_scalars,
&dynamic_scalars,
&dynamic_points,
)
},
BatchSize::SmallInput,
);
},
&MULTISCALAR_SIZES,
);
}
fn vartime_precomputed_00_pct_dynamic(c: &mut Criterion) {
vartime_precomputed_helper(c, 0.0);
}
fn vartime_precomputed_20_pct_dynamic(c: &mut Criterion) {
vartime_precomputed_helper(c, 0.2);
}
fn vartime_precomputed_50_pct_dynamic(c: &mut Criterion) {
vartime_precomputed_helper(c, 0.5);
}
criterion_group! {
name = multiscalar_benches;
// Lower the sample size to run the benchmarks faster
config = Criterion::default().sample_size(15);
targets =
compress,
decompress,
consttime_fixed_base_scalar_mul,
consttime_variable_base_scalar_mul,
vartime_double_base_scalar_mul,
consttime_multiscalar_mul,
vartime_multiscalar_mul,
vartime_precomputed_pure_static,
vartime_precomputed_00_pct_dynamic,
vartime_precomputed_20_pct_dynamic,
vartime_precomputed_50_pct_dynamic,
}
}
@ -141,7 +257,7 @@ mod ristretto_benches {
);
}
criterion_group!{
criterion_group! {
name = ristretto_benches;
config = Criterion::default();
targets =
@ -162,7 +278,7 @@ mod montgomery_benches {
});
}
criterion_group!{
criterion_group! {
name = montgomery_benches;
config = Criterion::default();
targets = montgomery_ladder,
@ -194,7 +310,7 @@ mod scalar_benches {
);
}
criterion_group!{
criterion_group! {
name = scalar_benches;
config = Criterion::default();
targets =
@ -208,4 +324,5 @@ criterion_main!(
montgomery_benches::montgomery_benches,
ristretto_benches::ristretto_benches,
edwards_benches::edwards_benches,
multiscalar_benches::multiscalar_benches,
);

86
build.rs
View file

@ -1,5 +1,13 @@
#![cfg_attr(
all(feature = "simd_backend", target_feature = "avx512ifma"),
feature(simd_ffi)
)]
#![cfg_attr(
all(feature = "simd_backend", target_feature = "avx512ifma"),
feature(link_llvm_intrinsics)
)]
#![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))]
#![cfg_attr(feature = "nightly", feature(cfg_target_feature))]
#![cfg_attr(feature = "nightly", feature(doc_cfg))]
#![allow(unused_variables)]
#![allow(non_snake_case)]
#![allow(dead_code)]
@ -10,10 +18,10 @@ extern crate byteorder;
extern crate clear_on_drop;
extern crate core;
extern crate digest;
extern crate rand;
extern crate rand_core;
extern crate subtle;
#[cfg(all(feature = "nightly", feature = "avx2_backend"))]
#[cfg(all(feature = "nightly", feature = "packed_simd"))]
extern crate packed_simd;
use std::env;
@ -36,35 +44,31 @@ mod macros;
// Public modules
#[path = "src/scalar.rs"]
mod scalar;
#[path = "src/montgomery.rs"]
mod montgomery;
#[path = "src/edwards.rs"]
mod edwards;
#[path = "src/ristretto.rs"]
mod ristretto;
#[path = "src/constants.rs"]
mod constants;
#[path = "src/edwards.rs"]
mod edwards;
#[path = "src/montgomery.rs"]
mod montgomery;
#[path = "src/ristretto.rs"]
mod ristretto;
#[path = "src/scalar.rs"]
mod scalar;
#[path = "src/traits.rs"]
mod traits;
// Internal modules
#[path = "src/field.rs"]
mod field;
#[path = "src/curve_models/mod.rs"]
mod curve_models;
#[path = "src/backend/mod.rs"]
mod backend;
#[path = "src/field.rs"]
mod field;
#[path = "src/prelude.rs"]
mod prelude;
#[path = "src/scalar_mul/mod.rs"]
mod scalar_mul;
#[path = "src/window.rs"]
mod window;
use edwards::EdwardsBasepointTable;
use curve_models::AffineNielsPoint;
use scalar_mul::window::NafLookupTable8;
fn main() {
// Enable the "stage2_build" feature in the main build stage
@ -81,17 +85,16 @@ fn main() {
format!(
"\n
#[cfg(feature = \"u32_backend\")]
use backend::u32::field::FieldElement32;
use backend::serial::u32::field::FieldElement2625;
#[cfg(feature = \"u64_backend\")]
use backend::u64::field::FieldElement64;
use backend::serial::u64::field::FieldElement51;
use edwards::EdwardsBasepointTable;
use curve_models::AffineNielsPoint;
use backend::serial::curve_models::AffineNielsPoint;
use scalar_mul::window::LookupTable;
use scalar_mul::window::NafLookupTable8;
use window::LookupTable;
/// Table containing precomputed multiples of the Ed25519 basepoint \\\\(B = (x, 4/5)\\\\).
pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN;
@ -101,20 +104,35 @@ pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TAB
pub const ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN: EdwardsBasepointTable = {:?};
\n\n",
&table
).as_bytes(),
).unwrap();
)
.as_bytes(),
)
.unwrap();
// Now generate AFFINE_ODD_MULTIPLES_OF_BASEPOINT
let B = &constants::ED25519_BASEPOINT_POINT;
let odd_multiples = NafLookupTable8::<AffineNielsPoint>::from(B);
// if we are going to build the serial scalar_mul backend
#[cfg(not(all(
feature = "simd_backend",
any(target_feature = "avx2", target_feature = "avx512ifma")
)))]
{
use backend::serial::curve_models::AffineNielsPoint;
use window::NafLookupTable8;
f.write_all(
format!(
"\n
let B = &constants::ED25519_BASEPOINT_POINT;
let odd_multiples = NafLookupTable8::<AffineNielsPoint>::from(B);
f.write_all(
format!(
"\n
use window::NafLookupTable8;
/// Odd multiples of the basepoint `[B, 3B, 5B, 7B, 9B, 11B, 13B, 15B, ..., 127B]`.
pub(crate) const AFFINE_ODD_MULTIPLES_OF_BASEPOINT: NafLookupTable8<AffineNielsPoint> = {:?};
\n\n",
&odd_multiples
).as_bytes(),
).unwrap();
&odd_multiples
)
.as_bytes(),
)
.unwrap();
}
}

323
docs/avx2-notes.md
View file

@ -1,218 +1,4 @@
A vectorized implementation of group operations on the twisted Edwards
form of Curve25519, using a modification of the 4-way parallel
formulas of Hisil, Wong, Carter, and Dawson.
# Overview
The 2008 paper [_Twisted Edwards Curves Revisited_][hwcd08] by Hisil,
Wong, Carter, and Dawson (HWCD) introduced the “extended coordinates”
and mixed-model representations which are used by most Edwards curve
implementations.
However, they also describe 4-way parallel formulas for point addition
and doubling: a unified addition algorithm taking an effective
\\(2\mathbf M + 1\mathbf D\\), a doubling algorithm taking an
effective \\(1\mathbf M + 1\mathbf S\\), and a dedicated (i.e., for
distinct points) addition algorithm taking an effective \\(2 \mathbf M
\\). They compare these formulas with a 2-way parallel variant of the
Montgomery ladder.
Unlike their serial formulas, which are used widely, their parallel
formulas do not seem to have been implemented in software before. The
2-way parallel Montgomery ladder was used in 2015 by Tung Chou's
`sandy2x` implementation. Curiously, however, although the [`sandy2x`
paper][sandy2x] also implements Edwards arithmetic, and cites HWCD08,
it doesn't mention their parallel Edwards formulas.
A 2015 paper by Hernández and López describes an AVX2 implementation
of X25519. Neither the paper nor the code are publicly available, but
it apparently gives only a [slight speedup][avx2trac], suggesting that
it uses a 4-way parallel Montgomery ladder rather than parallel
Edwards formulas.
The reason may be that HWCD08 describe their formulas as operating on
four independent processors, which would make a software
implementation impractical: all of the operations are too low-latency
to effectively synchronize. But a closer inspection reveals that the
(more expensive) multiplication and squaring steps are uniform, while
the instruction divergence occurs in the (much cheaper) addition and
subtraction steps. This means that a SIMD implementation can perform
the expensive steps uniformly, and handle divergence in the
inexpensive steps using masking.
These notes describe modifications to the original parallel formulas
to allow a SIMD implementation, and this module contains an
implementation of the modified formulas using 256-bit AVX2 vector
operations.
# Parallel formulas in HWCD'08
The doubling formula is presented in the HWCD paper as follows:
| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 |
|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|
| | idle | idle | idle | \\( R\_1 \gets X\_1 + Y\_1 \\) |
| \\(1\mathbf S\\) | \\( R\_2 \gets X\_1\^2 \\) | \\( R\_3 \gets Y\_1\^2 \\) | \\( R\_4 \gets Z\_1\^2 \\) | \\( R\_5 \gets R\_1\^2 \\) |
| | \\( R\_6 \gets R\_2 + R\_3 \\) | \\( R\_7 \gets R\_2 - R\_3 \\) | \\( R\_4 \gets 2 R\_4 \\) | idle |
| | idle | \\( R\_1 \gets R\_4 + R\_7 \\) | idle | \\( R\_2 \gets R\_6 - R\_5 \\) |
| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_6 R\_7 \\) | \\( T\_3 \gets R\_2 R\_6 \\) | \\( Z\_3 \gets R\_1 R\_7 \\) |
and the unified addition algorithm is presented as follows:
| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 |
|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|
| | \\( R\_1 \gets Y\_1 - X\_1 \\) | \\( R\_2 \gets Y\_2 - X\_2 \\) | \\( R\_3 \gets Y\_1 + X\_1 \\) | \\( R\_4 \gets Y\_2 + X\_2 \\) |
| \\(1\mathbf M\\) | \\( R\_5 \gets R\_1 R\_2 \\) | \\( R\_6 \gets R\_3 R\_4 \\) | \\( R\_7 \gets T\_1 T\_2 \\) | \\( R\_8 \gets Z\_1 Z\_2 \\) |
| \\(1\mathbf D\\) | idle | idle | \\( R\_7 \gets k R\_7 \\) | \\( R\_8 \gets 2 R\_8 \\) |
| | \\( R\_1 \gets R\_6 - R\_5 \\) | \\( R\_2 \gets R\_8 - R\_7 \\) | \\( R\_3 \gets R\_8 + R\_7 \\) | \\( R\_4 \gets R\_6 + R\_5 \\) |
| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_3 R\_4 \\) | \\( T\_3 \gets R\_1 R\_4 \\) | \\( Z\_3 \gets R\_2 R\_3 \\) |
Here \\(\mathbf M\\) and \\(\mathbf S\\) represent the cost of
multiplication and squaring of generic field elements, \\(\mathbf D\\)
represents the cost of multiplication by a curve constant (in this
case \\( k = 2d \\)).
Notice that the \\(1\mathbf M\\) and \\(1\mathbf S\\) steps are
uniform. The non-uniform steps are all inexpensive additions or
subtractions, with the exception of the multiplication by the curve
constant \\(k = 2d\\):
$$
R\_7 \gets 2 d R\_7.
$$
HWCD suggest parallelising this step by breaking \\(k = 2d\\) into four
parts as \\(k = k_0 + 2\^n k_1 + 2\^{2n} k_2 + 2\^{3n} k_3 \\) and
computing \\(k_i R_7 \\) in parallel. This is quite awkward, but if
the curve constant is a ratio \\( d = d\_1/d\_2 \\), then projective
coordinates allow us to instead compute
$$
(R\_5, R\_6, R\_7, R\_8) \gets (d\_2 R\_5, d\_2 R\_6, 2d\_1 R\_7, d\_2 R\_8).
$$
This can be performed as a uniform multiplication by a vector of
constants, and if \\(d\_1, d\_2\\) are small, it is relatively
inexpensive. (This trick was suggested by Mike Hamburg).
In the Curve25519 case, we have
$$
d = \frac{d\_1}{d\_2} = \frac{-121665}{121666};
$$
Since \\(2 \cdot 121666 < 2\^{18}\\), all the constants above fit (up
to sign) in 32 bits, so this can be done in parallel as four
multiplications by small constants \\( (121666, 121666, 2\cdot 121665,
2\cdot 121666) \\), followed by a negation to compute \\( - 2\cdot 121665\\).
# Modified parallel formulas
Using the modifications sketched above, we can write SIMD-friendly
versions of the parallel formulas as follows. To avoid confusion with
the original formulas, temporary variables are named \\(S\\) instead
of \\(R\\) and are in static single-assignment form.
## Addition
To add points
\\(P_1 = (X_1 : Y_1 : Z_1 : T_1) \\)
and
\\(P_2 = (X_2 : Y_2 : Z_2 : T_2 ) \\),
we compute
$$
\begin{aligned}
(S\_0 &&,&& S\_1 &&,&& S\_2 &&,&& S\_3 )
&\gets
(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Y\_2 - X\_2&&,&& Y\_2 + X\_2)
\\\\
(S\_4 &&,&& S\_5 &&,&& S\_6 &&,&& S\_7 )
&\gets
(S\_0 \cdot S\_2&&,&& S\_1 \cdot S\_3&&,&& Z\_1 \cdot Z\_2&&,&& T\_1 \cdot T\_2)
\\\\
(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} )
&\gets
(d\_2 \cdot S\_4 &&,&& d\_2 \cdot S\_5 &&,&& 2 d\_2 \cdot S\_6 &&,&& 2 d\_1 \cdot S\_7 )
\\\\
(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15})
&\gets
(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11})
\\\\
(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3)
&\gets
(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13})
\end{aligned}
$$
to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\).
This costs \\( 2\mathbf M + 1 \mathbf D\\).
## Readdition
If the point \\( P_2 = (X\_2 : Y\_2 : Z\_2 : T\_2) \\) is fixed, we
can cache the multiplication of the curve constants by computing
$$
\begin{aligned}
(S\_2' &&,&& S\_3' &&,&& Z\_2' &&,&& T\_2' )
&\gets
(d\_2 \cdot (Y\_2 - X\_2)&&,&& d\_2 \cdot (Y\_1 + X\_1)&&,&& 2d\_2 \cdot Z\_2 &&,&& 2d\_1 \cdot T\_2).
\end{aligned}
$$
This costs \\( 1\mathbf D\\); with \\( (S\_2', S\_3', Z\_2', T\_2')\\)
in hand, the addition formulas above become
$$
\begin{aligned}
(S\_0 &&,&& S\_1 &&,&& Z\_1 &&,&& T\_1 )
&\gets
(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Z\_1 &&,&& T\_1)
\\\\
(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} )
&\gets
(S\_0 \cdot S\_2' &&,&& S\_1 \cdot S\_3'&&,&& Z\_1 \cdot Z\_2' &&,&& T\_1 \cdot T\_2')
\\\\
(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15})
&\gets
(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11})
\\\\
(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3)
&\gets
(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13})
\end{aligned}
$$
which costs only \\( 2\mathbf M \\). This precomputation is
essentially similar to the precomputation that HWCD suggest for their
serial formulas. Because the cost of precomputation and then
readdition is the same as addition, it's sufficient to only
implement caching and readdition.
## Doubling
The non-uniform portions of the (re)addition formulas have a fairly
regular structure. Unfortunately, this is not the case for the
doubling formulas, which are much less nice.
To double a point \\( P = (X\_1 : Y\_1 : Z\_1 : T\_1) \\), we compute
$$
\begin{aligned}
(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& S\_0)
&\gets
(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& X\_1 + Y\_1)
\\\\
(S\_1 &&,&& S\_2 &&,&& S\_3 &&,&& S\_4 )
&\gets
(X\_1\^2 &&,&& Y\_1\^2&&,&& Z\_1\^2 &&,&& S\_0\^2)
\\\\
(S\_5 &&,&& S\_6 &&,&& S\_8 &&,&& S\_9 )
&\gets
(S\_1 + S\_2 &&,&& S\_1 - S\_2 &&,&& S\_1 + 2S\_3 - S\_2 &&,&& S\_1 + S\_2 - S\_4)
\\\\
(X\_3 &&,&& Y\_3 &&,&& Z\_3 &&,&& T\_3 )
&\gets
(S\_8 \cdot S\_9 &&,&& S\_5 \cdot S\_6 &&,&& S\_8 \cdot S\_6 &&,&& S\_5 \cdot S\_9)
\end{aligned}
$$
to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = [2]P\_1 \\).
The intermediate step between the squaring and multiplication requires
a long chain of additions, but with some care and finesse,
described below, it is possible (in our case) to arrange this
computation without requiring an intermediate reduction.
However, it does mean that the doubling formulas have proportionately
more vectorization overhead than the (re)addition formulas. The
effects of this are discussed in the comparison section below.
An AVX2 implementation of the vectorized point operation strategy.
# Field element representation
@ -266,9 +52,7 @@ the form
Since this breaks cleanly into two 128-bit lanes, it may be possible
to adapt it to 128-bit vector instructions such as NEON without too
much difficulty. Going the other direction, to extend this to AVX512,
we could either run two point operations in parallel in lower and upper
halves of the registers, or use 2-way parallelism within a field operation.
much difficulty.
# Avoiding Overflow in Doubling
@ -354,106 +138,3 @@ $$
whose right-hand sides are all bounded with \\( b < 1.75 \\) and
whose left-hand sides are all bounded with \\( b < 2.5 \\),
so that we can avoid any intermediate reductions.
# Comparison to non-vectorized formulas
In theory, the parallel Edwards formulas seem to allow a \\(4\\)-way
speedup from parallelism. However, an actual vectorized
implementation has several slowdowns that cut into this speedup.
First, the parallel formulas can only use a \\( 32 \times 32
\rightarrow 64 \\)-bit integer multiplier, so the speedup from
vectorization must overcome the disadvantage of losing the \\( 64
\times 64 \rightarrow 128\\)-bit (serial) integer multiplier. The
effect of this slowdown is microarchitecture-dependent, since it
requires accounting for the total number of multiplications and
additions and their relative costs. In the future, it will probably
be possible to avoid this slowdown by using the `IFMA52` instructions,
whose parallelism is perfectly suited to these formulas.
Second, the parallel doubling formulas incur both a theoretical and
practical slowdown. The parallel formulas described above work on the
\\( \mathbb P\^3 \\) “extended” coordinates. The \\( \mathbb P\^2 \\)
model introduced earlier by [Bernstein, Birkner, Joye, Lange, and
Peters][bbjlp08] allows slightly faster doublings, so HWCD suggest
mixing coordinate systems while performing scalar multiplication
(attributing the idea to [a 1998 paper][cmo98] by Cohen, Miyagi, and
Ono). The \\( T \\) coordinate is not required for doublings, so when
doublings are followed by doublings, its computation can be skipped.
More details on this approach and the different coordinate systems can
be found in the [`curve_models` module documentation][curve_models].
Unfortunately, this optimization is not compatible with the parallel
formulas, which cannot save time by skipping a single variable, so the
parallel doubling formulas do slightly more work when counting the
total number of field multiplications and squarings.
In addition, the parallel doubling formulas have a less regular
pattern of additions and subtractions than the parallel addition
formulas, so the vectorization overhead is proportionately greater.
Both the parallel addition and parallel doubling formulas also require
some shuffling to rearrange data within the vectors, which places more
pressure on the shuffle unit than is desirable.
This means that the speedup from using a vectorized implementation of
parallel Edwards formulas is likely to be greatest in applications
that do fewer doublings and more additions (like a large multiscalar
multiplication) rather than applications that do fewer additions and
more doublings (like a double-base scalar multiplication).
Third, current Intel CPUs perform thermal throttling when using wide
vector instructions. A detailed description can be found in §15.26 of
[the Intel Optimization Manual][intel], but using wide vector
instructions prevents the core from operating at higher frequencies.
The core can return to the higher-frequency state after 2
milliseconds, but this timer is reset every time high-power
instructions are used.
Any speedup from vectorization therefore has to be weighed against a
slowdown for the next few million instructions. For a mixed workload,
where point operations are interspersed with other tasks, this can
reduce overall performance. This implementation is therefore probably
not suitable for basic applications, like signatures, but is
worthwhile for complex applications, like zero-knowledge proofs, which
do sustained work.
For this reason, the AVX2 backend is not enabled by default, but can
be selected using the `avx2_backend` feature.
# Future work
There are several directions for future improvement:
* Using the vectorized field arithmetic code to parallelize across
point operations rather than within a single point operation. This
is less flexible, but would give a speedup both from allowing use of
the faster mixed-model arithmetic and from reducing shuffle
pressure. One approach in this direction would be to implement
batched scalar-point operations using vectors of points (AoSoA
layout). This less generally useful but would give a speedup for
Bulletproofs.
* Extending the implementation to use the full width of AVX512, either
handling the extra parallelism internally to a single point
operation (by using a 2-way parallel implementation of field
arithmetic instead of a wordsliced one), or externally,
parallelizing across point operations. Internal parallelism would
be preferable but might require too much shuffle pressure.
* Generalizing the implementation to non-AVX2 instructions,
particularly NEON. The current point arithmetic code is written in
terms of field element vectors, which are in turn implemented using
platform SIMD vectors. It should be possible to write an alternate
implementation of the `FieldElement32x4` using NEON without changing
the point arithmetic. NEON has 128-bit vectors rather than 256-bit
vectors, but this may still be worthwhile compared to a serial
implementation.
[sandy2x]: https://eprint.iacr.org/2015/943.pdf
[avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28
[hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf
[curve_models]: https://doc-internal.dalek.rs/curve25519_dalek/curve_models/index.html
[bbjlp08]: https://eprint.iacr.org/2008/013
[cmo98]: https://link.springer.com/content/pdf/10.1007%2F3-540-49649-1_6.pdf
[intel]: https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf

580
docs/ifma-notes.md Normal file
View file

@ -0,0 +1,580 @@
An AVX512-IFMA implementation of the vectorized point operation
strategy.
# IFMA instructions
AVX512-IFMA is an extension to AVX-512 consisting of two instructions:
* `vpmadd52luq`: packed multiply of unsigned 52-bit integers and add
the low 52 product bits to 64-bit accumulators;
* `vpmadd52huq`: packed multiply of unsigned 52-bit integers and add
the high 52 product bits to 64-bit accumulators;
These operate on 64-bit lanes of their source vectors, taking the low
52 bits of each lane of each source vector, computing the 104-bit
products of each pair, and then adding either the high or low 52 bits
of the 104-bit products to the 64-bit lanes of the destination vector.
The multiplication is performed internally by reusing circuitry for
floating-point arithmetic. Although these instructions are part of
AVX512, the AVX512VL (vector length) extension (present whenever IFMA
is) allows using them with 512, 256, or 128-bit operands.
This provides a major advantage to vectorized integer operations:
previously, vector operations could only use a \\(32 \times 32
\rightarrow 64\\)-bit multiplier, while serial code could use a
\\(64\times 64 \rightarrow 128\\)-bit multiplier.
## IFMA for big-integer multiplications
A detailed example of the intended use of the IFMA instructions can be
found in a 2016 paper by Gueron and Krasnov, [_Accelerating Big
Integer Arithmetic Using Intel IFMA Extensions_][2016_gueron_krasnov].
The basic idea is that multiplication of large integers (such as 1024,
2048, or more bits) can be performed as follows.
First, convert a “packed” 64-bit representation
\\[
\begin{aligned}
x &= x'_0 + x'_1 2^{64} + x'_2 2^{128} + \cdots \\\\
y &= y'_0 + y'_1 2^{64} + y'_2 2^{128} + \cdots
\end{aligned}
\\]
into a “redundant” 52-bit representation
\\[
\begin{aligned}
x &= x_0 + x_1 2^{52} + x_2 2^{104} + \cdots \\\\
y &= y_0 + y_1 2^{52} + y_2 2^{104} + \cdots
\end{aligned}
\\]
with each \\(x_i, y_j\\) in a 64-bit lane.
Writing the product as \\(z = z_0 + z_1 2^{52} + z_2 2^{104} + \cdots\\),
the “schoolbook” multiplication strategy gives
\\[
\begin{aligned}
&z_0 &&=& x_0 & y_0 & & & & & & & & \\\\
&z_1 &&=& x_1 & y_0 &+ x_0 & y_1 & & & & & & \\\\
&z_2 &&=& x_2 & y_0 &+ x_1 & y_1 &+ x_0 & y_2 & & & & \\\\
&z_3 &&=& x_3 & y_0 &+ x_2 & y_1 &+ x_1 & y_2 &+ x_0 & y_3 & & \\\\
&z_4 &&=& \vdots\\;&\\;\vdots &+ x_3 & y_1 &+ x_2 & y_2 &+ x_1 & y_3 &+ \cdots& \\\\
&z_5 &&=& & & \vdots\\;&\\;\vdots &+ x_3 & y_2 &+ x_2 & y_3 &+ \cdots& \\\\
&z_6 &&=& & & & & \vdots\\;&\\;\vdots &+ x_3 & y_3 &+ \cdots& \\\\
&z_7 &&=& & & & & & & \vdots\\;&\\;\vdots &+ \cdots& \\\\
&\vdots&&=& & & & & & & & & \ddots& \\\\
\end{aligned}
\\]
Notice that the product coefficient \\(z_k\\), representing the value
\\(z_k 2^{52k}\\), is the sum of all product terms
\\(
(x_i 2^{52 i}) (y_j 2^{52 j})
\\)
with \\(k = i + j\\).
Write the IFMA operators \\(\mathrm{lo}(a,b)\\), denoting the low
\\(52\\) bits of \\(ab\\), and
\\(\mathrm{hi}(a,b)\\), denoting the high \\(52\\) bits of
\\(ab\\).
Now we can rewrite the product terms as
\\[
\begin{aligned}
(x_i 2^{52 i}) (y_j 2^{52 j})
&=
2^{52 (i+j)}(
\mathrm{lo}(x_i, y_j) +
\mathrm{hi}(x_i, y_j) 2^{52}
)
\\\\
&=
\mathrm{lo}(x_i, y_j) 2^{52 (i+j)} +
\mathrm{hi}(x_i, y_j) 2^{52 (i+j+1)}.
\end{aligned}
\\]
This means that the low half of \\(x_i y_j\\) can be accumulated onto
the product limb \\(z_{i+j}\\) and the high half can be directly
accumulated onto the next-higher product limb \\(z_{i+j+1}\\) with no
additional operations. This allows rewriting the schoolbook
multiplication into the form
\\[
\begin{aligned}
&z_0 &&=& \mathrm{lo}(x_0,&y_0) & & & & & & & & & & \\\\
&z_1 &&=& \mathrm{lo}(x_1,&y_0) &+\mathrm{hi}(x_0,&y_0) &+\mathrm{lo}(x_0,&y_1) & & & & & & \\\\
&z_2 &&=& \mathrm{lo}(x_2,&y_0) &+\mathrm{hi}(x_1,&y_0) &+\mathrm{lo}(x_1,&y_1) &+\mathrm{hi}(x_0,&y_1) &+\mathrm{lo}(x_0,&y_2) & & \\\\
&z_3 &&=& \mathrm{lo}(x_3,&y_0) &+\mathrm{hi}(x_2,&y_0) &+\mathrm{lo}(x_2,&y_1) &+\mathrm{hi}(x_1,&y_1) &+\mathrm{lo}(x_1,&y_2) &+ \cdots& \\\\
&z_4 &&=& \vdots\\;&\\;\vdots &+\mathrm{hi}(x_3,&y_0) &+\mathrm{lo}(x_3,&y_1) &+\mathrm{hi}(x_2,&y_1) &+\mathrm{lo}(x_2,&y_2) &+ \cdots& \\\\
&z_5 &&=& & & \vdots\\;&\\;\vdots & \vdots\\;&\\;\vdots &+\mathrm{hi}(x_3,&y_1) &+\mathrm{lo}(x_3,&y_2) &+ \cdots& \\\\
&z_6 &&=& & & & & & & \vdots\\;&\\;\vdots & \vdots\\;&\\;\vdots &+ \cdots& \\\\
&\vdots&&=& & & & & & & & & & & \ddots& \\\\
\end{aligned}
\\]
Gueron and Krasnov implement multiplication by constructing vectors
out of the columns of this diagram, so that the source operands for
the IFMA instructions are of the form \\((x_0, x_1, x_2, \ldots)\\)
and \\((y_i, y_i, y_i, \ldots)\\).
After performing the multiplication,
the product terms \\(z_i\\) are then repacked into a 64-bit representation.
## An alternative strategy
The strategy described above is aimed at big-integer multiplications,
such as 1024, 2048, or 4096 bits, which would be used for applications
like RSA. However, elliptic curve cryptography uses much smaller field
sizes, such as 256 or 384 bits, so a different strategy is needed.
The parallel Edwards formulas provide parallelism at the level of the
formulas for curve operations. This means that instead of scanning
through the terms of the source operands and parallelizing *within* a
field element (as described above), we can arrange the computation in
product-scanning form and parallelize *across* field elements (as
described below).
The parallel Edwards
formulas provide 4-way parallelism, so they can be implemented using
256-bit vectors using a single 64-bit lane for each element, or using
512-bit vectors using two 64-bit lanes.
The only available CPU supporting IFMA (the
i3-8121U) executes 512-bit IFMA instructions at half rate compared to
256-bit instructions, so for now there's no throughput advantage to
using 512-bit IFMA instructions, and this implementation uses 256-bit
vectors.
To extend this to 512-bit vectors, it's only only necessary to achieve
2-way parallelism, and it's possible (with a small amount of overhead)
to create a hybrid strategy that operates entirely within 128-bit
lanes. This means that cross-lane operations can use the faster
`vpshufd` (1c latency) instead of a general shuffle instruction (3c
latency).
# Choice of radix
The inputs to IFMA instructions are 52 bits wide, so the radix \\(r\\)
used to represent a multiprecision integer must be \\( r \leq 52 \\).
The obvious choice is the "native" radix \\(r = 52\\).
As described above, this choice
has the advantage that for \\(x_i, y_j \in [0,2^{52})\\), the product term
\\[
\begin{aligned}
(x_i 2^{52 i}) (y_j 2^{52 j})
&=
2^{52 (i+j)}(
\mathrm{lo}(x_i, y_j) +
\mathrm{hi}(x_i, y_j) 2^{52}
)
\\\\
&=
\mathrm{lo}(x_i, y_j) 2^{52 (i+j)} +
\mathrm{hi}(x_i, y_j) 2^{52 (i+j+1)},
\end{aligned}
\\]
so that the low and high halves of the product can be directly accumulated
onto the product limbs.
In contrast, when using a smaller radix \\(r = 52 - k\\),
the product term has the form
\\[
\begin{aligned}
(x_i 2^{r i}) (y_j 2^{r j})
&=
2^{r (i+j)}(
\mathrm{lo}(x_i, y_j) +
\mathrm{hi}(x_i, y_j) 2^{52}
)
\\\\
&=
\mathrm{lo}(x_i, y_j) 2^{r (i+j)} +
(
\mathrm{hi}(x_i, y_j) 2^k
)
2^{r (i+j+1)}.
\end{aligned}
\\]
What's happening is that the product \\(x_i y_j\\) of size \\(2r\\)
bits is split not at \\(r\\) but at \\(52\\), so \\(k\\) product bits
are placed into the low half instead of the high half. This means
that the high half of the product cannot be directly accumulated onto
\\(z_{i+j+1}\\), but must first be multiplied by \\(2^k\\) (i.e., left
shifted by \\(k\\)). In addition, the low half of the product is
\\(52\\) bits large instead of \\(r\\) bits.
## Handling offset product terms
[Drucker and Gueron][2018_drucker_gueron] analyze the choice of radix
in the context of big-integer squaring, outlining three ways to handle
the offset product terms, before concluding that all of them are
suboptimal:
1. Shift the results after accumulation;
2. Shift the input operands before multiplication;
3. Split the MAC operation, accumulating into a zeroed register,
shifting the result, and then adding.
The first option is rejected because it could double-shift some
previously accumulated terms, the second doesn't work because the
inputs could become larger than \\(52\\) bits, and the third requires
additional instructions to handle the shifting and adding.
Based on an analysis of total number of instructions, they suggest an
addition to the instruction set, which they call `FMSA` (fused
multiply-shift-add). This would shift the result according to an 8-bit
immediate value before accumulating it into the destination register.
However, this change to the instruction set doesn't seem to be
necessary. Instead, the product terms can be grouped according to
their coefficients, accumulated together, then shifted once before
adding them to the final sum. This uses an extra register, shift, and
add, but only once per product term (accumulation target), not once
per source term (as in the Drucker-Gueron paper).
Moreover, because IFMA instructions execute only on two ports
(presumably 0 and 1), while adds and shifts can execute on three ports
(0, 1, and 5), the adds and shifts can execute independently of the
IFMA operations, as long as there is not too much pressure on port 5.
This means that, although the total number of instructions increases,
the shifts and adds do not necessarily increase the execution time, as
long as throughput is limited by IFMA operations.
Finally, because IFMA instructions have 4 cycle latency and 0.5/1
cycle throughput (for 256/512 bit vectors), maximizing IFMA throughput
requires either 8 (for 256) or 4 (for 512) independent operations. So
accumulating groups of terms independently before adding them at the
end may be necessary anyways, in order to prevent long chains of
dependent instructions.
## Advantages of a smaller radix
Using a smaller radix has other advantages. Although radix \\(52\\)
is an unsaturated representation from the point of view of the
\\(64\\)-bit accumulators (because up to 4096 product terms can be
accumulated without carries), it's a saturated representation from the
point of view of the multiplier (since \\(52\\)-bit values are the
maximum input size).
Because the inputs to a multiplication must have all of their limbs
bounded by \\(2^{52}\\), limbs in excess of \\(2^{52}\\) must be
reduced before they can be used as an input. The
[Gueron-Krasnov][2016_gueron_krasnov] paper suggests normalizing
values using a standard, sequential carry chain: for each limb, add
the carryin from reducing the previous limb, compute the carryout and
reduce the current limb, then move to the next limb.
However, when using a smaller radix, such as \\(51\\), each limb can
store a carry bit and still be used as the input to a multiplication.
This means that the inputs do not need to be normalized, and instead
of using a sequential carry chain, we can compute all carryouts in
parallel, reduce all limbs in parallel, and then add the carryins in
parallel (possibly growing the limb values by one bit).
Because the output of this partial reduction is an acceptable
multiplication input, we can "close the loop" using partial reductions
and never have to normalize to a canonical representation through the
entire computation, in contrast to the Gueron-Krasnov approach, which
converts back to a packed representation after every operation. (This
idea seems to trace back to at least as early as [this 1999
paper][1999_walter]).
Using \\(r = 51\\) is enough to keep a carry bit in each limb and
avoid normalizations. What about an even smaller radix? One reason
to choose a smaller radix would be to align the limb boundaries with
an inline reduction (for instance, choosing \\(r = 43\\) for the
Mersenne field \\(p = 2^{127} - 1\\)), but for \\(p = 2^{255 - 19}\\),
\\(r = 51 = 255/5\\) is the natural choice.
# Multiplication
The inputs to a multiplication are two field elements
\\[
\begin{aligned}
x &= x_0 + x_1 2^{51} + x_2 2^{102} + x_3 2^{153} + x_4 2^{204} \\\\
y &= y_0 + y_1 2^{51} + y_2 2^{102} + y_3 2^{153} + y_4 2^{204},
\end{aligned}
\\]
with limbs in range \\([0,2^{52})\\).
Writing the product terms as
\\[
\begin{aligned}
z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\
&+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459},
\end{aligned}
\\]
a schoolbook multiplication in product scanning form takes the form
\\[
\begin{aligned}
z_0 &= x_0 y_0 \\\\
z_1 &= x_1 y_0 + x_0 y_1 \\\\
z_2 &= x_2 y_0 + x_1 y_1 + x_0 y_2 \\\\
z_3 &= x_3 y_0 + x_2 y_1 + x_1 y_2 + x_0 y_3 \\\\
z_4 &= x_4 y_0 + x_3 y_1 + x_2 y_2 + x_1 y_3 + x_0 y_4 \\\\
z_5 &= x_4 y_1 + x_3 y_2 + x_2 y_3 + x_1 y_4 \\\\
z_6 &= x_4 y_2 + x_3 y_3 + x_2 y_4 \\\\
z_7 &= x_4 y_3 + x_3 y_4 \\\\
z_8 &= x_4 y_4 \\\\
z_9 &= 0 \\\\
\end{aligned}
\\]
Each term \\(x_i y_j\\) can be written in terms of IFMA operations as
\\[
x_i y_j = \mathrm{lo}(x_i,y_j) + 2\mathrm{hi}(x_i,y_j)2^{51}.
\\]
Substituting this equation into the schoolbook multiplication, then
moving terms to eliminate the \\(2^{51}\\) factors gives
\\[
\begin{aligned}
z_0 &= \mathrm{lo}(x_0, y_0) \\\\
&+ \qquad 0 \\\\
z_1 &= \mathrm{lo}(x_1, y_0) + \mathrm{lo}(x_0, y_1) \\\\
&+ \qquad 2( \mathrm{hi}(x_0, y_0) )\\\\
z_2 &= \mathrm{lo}(x_2, y_0) + \mathrm{lo}(x_1, y_1) + \mathrm{lo}(x_0, y_2) \\\\
&+ \qquad 2( \mathrm{hi}(x_1, y_0) + \mathrm{hi}(x_0, y_1) )\\\\
z_3 &= \mathrm{lo}(x_3, y_0) + \mathrm{lo}(x_2, y_1) + \mathrm{lo}(x_1, y_2) + \mathrm{lo}(x_0, y_3) \\\\
&+ \qquad 2( \mathrm{hi}(x_2, y_0) + \mathrm{hi}(x_1, y_1) + \mathrm{hi}(x_0, y_2) )\\\\
z_4 &= \mathrm{lo}(x_4, y_0) + \mathrm{lo}(x_3, y_1) + \mathrm{lo}(x_2, y_2) + \mathrm{lo}(x_1, y_3) + \mathrm{lo}(x_0, y_4) \\\\
&+ \qquad 2( \mathrm{hi}(x_3, y_0) + \mathrm{hi}(x_2, y_1) + \mathrm{hi}(x_1, y_2) + \mathrm{hi}(x_0, y_3) )\\\\
z_5 &= \mathrm{lo}(x_4, y_1) + \mathrm{lo}(x_3, y_2) + \mathrm{lo}(x_2, y_3) + \mathrm{lo}(x_1, y_4) \\\\
&+ \qquad 2( \mathrm{hi}(x_4, y_0) + \mathrm{hi}(x_3, y_1) + \mathrm{hi}(x_2, y_2) + \mathrm{hi}(x_1, y_3) + \mathrm{hi}(x_0, y_4) )\\\\
z_6 &= \mathrm{lo}(x_4, y_2) + \mathrm{lo}(x_3, y_3) + \mathrm{lo}(x_2, y_4) \\\\
&+ \qquad 2( \mathrm{hi}(x_4, y_1) + \mathrm{hi}(x_3, y_2) + \mathrm{hi}(x_2, y_3) + \mathrm{hi}(x_1, y_4) )\\\\
z_7 &= \mathrm{lo}(x_4, y_3) + \mathrm{lo}(x_3, y_4) \\\\
&+ \qquad 2( \mathrm{hi}(x_4, y_2) + \mathrm{hi}(x_3, y_3) + \mathrm{hi}(x_2, y_4) )\\\\
z_8 &= \mathrm{lo}(x_4, y_4) \\\\
&+ \qquad 2( \mathrm{hi}(x_4, y_3) + \mathrm{hi}(x_3, y_4) )\\\\
z_9 &= 0 \\\\
&+ \qquad 2( \mathrm{hi}(x_4, y_4) )\\\\
\end{aligned}
\\]
As noted above, our strategy will be to multiply and accumulate the
terms with coefficient \\(2\\) separately from those with coefficient
\\(1\\), before combining them at the end. This can alternately be
thought of as accumulating product terms into a *doubly-redundant*
representation, with two limbs for each digit, before collapsing
the doubly-redundant representation by shifts and adds.
This computation requires 25 `vpmadd52luq` and 25 `vpmadd52huq`
operations. For 256-bit vectors, IFMA operations execute on an
i3-8121U with latency 4 cycles, throughput 0.5 cycles, so executing 50
instructions requires 25 cycles' worth of throughput. Accumulating
terms with coefficient \\(1\\) and \\(2\\) seperately means that the
longest dependency chain has length 5, so the critical path has length
20 cycles and the bottleneck is throughput.
# Reduction modulo \\(p\\)
The next question is how to handle the reduction modulo \\(p\\).
Because \\(p = 2^{255} - 19\\), \\(2^{255} = 19 \pmod p\\), so we can
alternately write
\\[
\begin{aligned}
z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\
&+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459}
\end{aligned}
\\]
as
\\[
\begin{aligned}
z &= (z_0 + 19z_5) + (z_1 + 19z_6) 2^{51} + (z_2 + 19z_7) 2^{102} + (z_3 + 19z_8) 2^{153} + (z_4 + 19z_9) 2^{204}.
\end{aligned}
\\]
When using a \\(64 \times 64 \rightarrow 128\\)-bit multiplier, this
can be handled (as in [Ed25519][ed25519_paper]) by premultiplying
source terms by \\(19\\). Since \\(\lg(19) < 4.25\\), this increases
their size by less than \\(4.25\\) bits, and the rest of the
multiplication can be shown to work out.
Here, we have at most \\(1\\) bit of headroom. In order to allow
premultiplication, we would need to use radix \\(2^{47}\\), which
would require six limbs instead of five. Instead, we compute the high
terms \\(z_5, \ldots, z_9\\), each using two chains of IFMA
operations, then multiply by \\(19\\) and combine with the lower terms
\\(z_0, \ldots, z_4\\). There are two ways to perform the
multiplication by \\(19\\): using more IFMA operations, or using the
`vpmullq` instruction, which computes the low \\(64\\) bits of a \\(64
\times 64\\)-bit product. However, `vpmullq` has 15c/1.5c
latency/throughput, in contrast to the 4c/0.5c latency/throughput of
IFMA operations, so it seems like a worse choice.
The high terms \\(z_5, \ldots, z_9\\) are sums of \\(52\\)-bit terms,
so they are larger than \\(52\\) bits. Write these terms in radix \\(52\\) as
\\[
z_{5+i} = z_{5+i}' + z_{5+i}'' 2^{52}, \qquad z_{5+i}' < 2^{52}.
\\]
Then the contribution of \\(z_{5+i}\\), taken modulo \\(p\\), is
\\[
\begin{aligned}
z_{5+i} 2^{255} 2^{51 i}
&=
19 (z_{5+i}' + z_{5+i}'' 2^{52}) 2^{51 i}
\\\\
&=
19 z_{5+i}' 2^{51 i} + 2 \cdot 19 z_{5+i}'' 2^{51 (i+1)}
\\\\
\end{aligned}
\\]
The products \\(19 z_{5+i}', 19 z_{5+i}''\\) can be written in terms of IFMA operations as
\\[
\begin{aligned}
19 z_{5+i}' &= \mathrm{lo}(19, z_{5+i}') + 2 \mathrm{hi}(19, z_{5+i}') 2^{51}, \\\\
19 z_{5+i}'' &= \mathrm{lo}(19, z_{5+i}'') + 2 \mathrm{hi}(19, z_{5+i}'') 2^{51}. \\\\
\end{aligned}
\\]
Because \\(z_{5+i} < 2^{64}\\), \\(z_{5+i}'' < 2^{12} \\), so \\(19
z_{5+i}'' < 2^{17} < 2^{52} \\) and \\(\mathrm{hi}(19, z_{5+i}'') = 0\\).
Because IFMA operations ignore the high bits of their source
operands, we do not need to compute \\(z\_{5+i}'\\) explicitly:
the high bits will be ignored.
Combining these observations, we can write
\\[
\begin{aligned}
z_{5+i} 2^{255} 2^{51 i}
&=
19 z_{5+i}' 2^{51 i} + 2 \cdot 19 z_{5+i}'' 2^{51 (i+1)}
\\\\
&=
\mathrm{lo}(19, z_{5+i}) 2^{51 i}
\+ 2 \mathrm{hi}(19, z_{5+i}) 2^{51 (i+1)}
\+ 2 \mathrm{lo}(19, z_{5+i}/2^{52}) 2^{51 (i+1)}.
\end{aligned}
\\]
For \\(i = 0,1,2,3\\), this allows reducing \\(z_{5+i}\\) onto
\\(z_{i}, z_{i+1}\\), and if the low terms are computed using a
doubly-redundant representation, no additional shifts are needed to
handle the \\(2\\) coefficients. For \\(i = 4\\), there's a
complication: the contribution becomes
\\[
\begin{aligned}
z_{9} 2^{255} 2^{204}
&=
\mathrm{lo}(19, z_{9}) 2^{204}
\+ 2 \mathrm{hi}(19, z_{9}) 2^{255}
\+ 2 \mathrm{lo}(19, z_{9}/2^{52}) 2^{255}
\\\\
&=
\mathrm{lo}(19, z_{9}) 2^{204}
\+ 2 \mathrm{hi}(19, z_{9}) 19
\+ 2 \mathrm{lo}(19, z_{9}/2^{52}) 19
\\\\
&=
\mathrm{lo}(19, z_{9}) 2^{204}
\+ 2
\mathrm{lo}(19, \mathrm{hi}(19, z_{9}) + \mathrm{lo}(19, z_{9}/2^{52})).
\\\\
\end{aligned}
\\]
It would be possible to cut the number of multiplications from 3 to 2
by carrying the high part of each \\(z_i\\) onto \\(z_{i+1}\\). This
would eliminate 5 multiplications, clearing 2.5 cycles of port
pressure, at the cost of 5 additions, adding 1.66 cycles of port
pressure. But doing this would create a dependency between terms
(e.g., \\(z_{5}\\) must be computed before the reduction of
\\(z_{6}\\) can begin), whereas with the approach above, all
contributions to all terms are computed independently, to maximize ILP
and flexibility for the processor to schedule instructions.
This strategy performs 16 IFMA operations, adding two IFMA operations
to each of the \\(2\\)-coefficient terms and one to each of the
\\(1\\)-coefficient terms. Considering the multiplication and
reduction together, we use 66 IFMA operations, requiring 33 cycles'
throughput, while the longest chain of IFMA operations is in the
reduction of \\(z_5\\) onto \\(z_1\\), of length 7 (so 28 cycles, plus
2 cycles to combine the two parts of \\(z_5\\), and the bottleneck is
again throughput.
Once this is done, we have computed the product terms
\\[
z = z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204},
\\]
without reducing the \\(z_i\\) to fit in \\(52\\) bits. Because the
overall flow of operations alternates multiplications and additions or
subtractions, we would have to perform a reduction after an addition
but before the next multiplication anyways, so there's no benefit to
fully reducing the limbs at the end of a multiplication. Instead, we
leave them unreduced, and track the reduction state using the type
system to ensure that unreduced limbs are not accidentally used as an
input to a multiplication.
# Squaring
Squaring operates similarly to multiplication, but with the
possibility to combine identical terms.
As before, we write the input as
\\[
\begin{aligned}
x &= x_0 + x_1 2^{51} + x_2 2^{102} + x_3 2^{153} + x_4 2^{204}
\end{aligned}
\\]
with limbs in range \\([0,2^{52})\\).
Writing the product terms as
\\[
\begin{aligned}
z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\
&+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459},
\end{aligned}
\\]
a schoolbook squaring in product scanning form takes the form
\\[
\begin{aligned}
z_0 &= x_0 x_0 \\\\
z_1 &= 2 x_1 x_0 \\\\
z_2 &= 2 x_2 x_0 + x_1 x_1 \\\\
z_3 &= 2 x_3 x_0 + 2 x_2 x_1 \\\\
z_4 &= 2 x_4 x_0 + 2 x_3 x_1 + x_2 x_2 \\\\
z_5 &= 2 x_4 x_1 + 2 x_3 x_2 \\\\
z_6 &= 2 x_4 x_2 + x_3 x_3 \\\\
z_7 &= 2 x_4 x_3 \\\\
z_8 &= x_4 x_4 \\\\
z_9 &= 0 \\\\
\end{aligned}
\\]
As before, we write \\(x_i x_j\\) as
\\[
x_i x_j = \mathrm{lo}(x_i,x_j) + 2\mathrm{hi}(x_i,x_j)2^{51},
\\]
and substitute to obtain
\\[
\begin{aligned}
z_0 &= \mathrm{lo}(x_0, x_0) + 0 \\\\
z_1 &= 2 \mathrm{lo}(x_1, x_0) + 2 \mathrm{hi}(x_0, x_0) \\\\
z_2 &= 2 \mathrm{lo}(x_2, x_0) + \mathrm{lo}(x_1, x_1) + 4 \mathrm{hi}(x_1, x_0) \\\\
z_3 &= 2 \mathrm{lo}(x_3, x_0) + 2 \mathrm{lo}(x_2, x_1) + 4 \mathrm{hi}(x_2, x_0) + 2 \mathrm{hi}(x_1, x_1) \\\\
z_4 &= 2 \mathrm{lo}(x_4, x_0) + 2 \mathrm{lo}(x_3, x_1) + \mathrm{lo}(x_2, x_2) + 4 \mathrm{hi}(x_3, x_0) + 4 \mathrm{hi}(x_2, x_1) \\\\
z_5 &= 2 \mathrm{lo}(x_4, x_1) + 2 \mathrm{lo}(x_3, x_2) + 4 \mathrm{hi}(x_4, x_0) + 4 \mathrm{hi}(x_3, x_1) + 2 \mathrm{hi}(x_2, x_2) \\\\
z_6 &= 2 \mathrm{lo}(x_4, x_2) + \mathrm{lo}(x_3, x_3) + 4 \mathrm{hi}(x_4, x_1) + 4 \mathrm{hi}(x_3, x_2) \\\\
z_7 &= 2 \mathrm{lo}(x_4, x_3) + 4 \mathrm{hi}(x_4, x_2) + 2 \mathrm{hi}(x_3, x_3) \\\\
z_8 &= \mathrm{lo}(x_4, x_4) + 4 \mathrm{hi}(x_4, x_3) \\\\
z_9 &= 0 + 2 \mathrm{hi}(x_4, x_4) \\\\
\end{aligned}
\\]
To implement these, we group terms by their coefficient, computing
those with coefficient \\(2\\) on set of IFMA chains, and on another
set of chains, we begin with coefficient-\\(4\\) terms, then shift
left before continuing with the coefficient-\\(1\\) terms.
The reduction strategy is the same as for multiplication.
# Future improvements
LLVM won't use blend operations on [256-bit vectors yet][llvm_blend],
so there's a bunch of blend instructions that could be omitted.
Although the multiplications and squarings are much faster, there's no
speedup to the additions and subtractions, so there are diminishing
returns. In fact, the complications in the doubling formulas mean
that doubling is actually slower than readdition. This also suggests
that moving to 512-bit vectors won't be much help for a strategy aimed
at parallelism within a group operation, so to extract performance
gains from 512-bit vectors it will probably be necessary to create a
parallel-friendly multiscalar multiplication algorithm. This could
also help with reducing shuffle pressure.
The squaring implementation could probably be optimized, but without
`perf` support on Cannonlake it's difficult to make actual
measurements.
Another improvement would be to implement vectorized square root
computations, which would allow creating an iterator adaptor for point
decompression that bunched decompression operations and executed them
in parallel. This would accelerate batch verification.
[2016_gueron_krasnov]: https://ieeexplore.ieee.org/document/7563269
[2018_drucker_gueron]: https://eprint.iacr.org/2018/335
[1999_walter]: https://pdfs.semanticscholar.org/0e6a/3e8f30b63b556679f5dff2cbfdfe9523f4fa.pdf
[ed25519_paper]: https://ed25519.cr.yp.to/ed25519-20110926.pdf
[llvm_blend]: https://bugs.llvm.org/show_bug.cgi?id=38343

333
docs/parallel-formulas.md Normal file
View file

@ -0,0 +1,333 @@
Vectorized implementations of field and point operations, using a
modification of the 4-way parallel formulas of Hisil, Wong, Carter,
and Dawson.
These notes explain the parallel formulas and our strategy for using
them with SIMD operations. There are two backend implementations: one
using AVX2, and the other using AVX512-IFMA.
# Overview
The 2008 paper [_Twisted Edwards Curves Revisited_][hwcd08] by Hisil,
Wong, Carter, and Dawson (HWCD) introduced the “extended coordinates”
and mixed-model representations which are used by most Edwards curve
implementations.
However, they also describe 4-way parallel formulas for point addition
and doubling: a unified addition algorithm taking an effective
\\(2\mathbf M + 1\mathbf D\\), a doubling algorithm taking an
effective \\(1\mathbf M + 1\mathbf S\\), and a dedicated (i.e., for
distinct points) addition algorithm taking an effective \\(2 \mathbf M
\\). They compare these formulas with a 2-way parallel variant of the
Montgomery ladder.
Unlike their serial formulas, which are used widely, their parallel
formulas do not seem to have been implemented in software before. The
2-way parallel Montgomery ladder was used in 2015 by Tung Chou's
`sandy2x` implementation. Curiously, however, although the [`sandy2x`
paper][sandy2x] also implements Edwards arithmetic, and cites HWCD08,
it doesn't mention their parallel Edwards formulas.
A 2015 paper by Hernández and López describes an AVX2 implementation
of X25519. Neither the paper nor the code are publicly available, but
it apparently gives only a [slight speedup][avx2trac], suggesting that
it uses a 4-way parallel Montgomery ladder rather than parallel
Edwards formulas.
The reason may be that HWCD08 describe their formulas as operating on
four independent processors, which would make a software
implementation impractical: all of the operations are too low-latency
to effectively synchronize. But a closer inspection reveals that the
(more expensive) multiplication and squaring steps are uniform, while
the instruction divergence occurs in the (much cheaper) addition and
subtraction steps. This means that a SIMD implementation can perform
the expensive steps uniformly, and handle divergence in the
inexpensive steps using masking.
These notes describe modifications to the original parallel formulas
to allow a SIMD implementation, and this module contains
implementations of the modified formulas targeting either AVX2 or
AVX512-IFMA.
# Parallel formulas in HWCD'08
The doubling formula is presented in the HWCD paper as follows:
| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 |
|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|
| | idle | idle | idle | \\( R\_1 \gets X\_1 + Y\_1 \\) |
| \\(1\mathbf S\\) | \\( R\_2 \gets X\_1\^2 \\) | \\( R\_3 \gets Y\_1\^2 \\) | \\( R\_4 \gets Z\_1\^2 \\) | \\( R\_5 \gets R\_1\^2 \\) |
| | \\( R\_6 \gets R\_2 + R\_3 \\) | \\( R\_7 \gets R\_2 - R\_3 \\) | \\( R\_4 \gets 2 R\_4 \\) | idle |
| | idle | \\( R\_1 \gets R\_4 + R\_7 \\) | idle | \\( R\_2 \gets R\_6 - R\_5 \\) |
| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_6 R\_7 \\) | \\( T\_3 \gets R\_2 R\_6 \\) | \\( Z\_3 \gets R\_1 R\_7 \\) |
and the unified addition algorithm is presented as follows:
| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 |
|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|
| | \\( R\_1 \gets Y\_1 - X\_1 \\) | \\( R\_2 \gets Y\_2 - X\_2 \\) | \\( R\_3 \gets Y\_1 + X\_1 \\) | \\( R\_4 \gets Y\_2 + X\_2 \\) |
| \\(1\mathbf M\\) | \\( R\_5 \gets R\_1 R\_2 \\) | \\( R\_6 \gets R\_3 R\_4 \\) | \\( R\_7 \gets T\_1 T\_2 \\) | \\( R\_8 \gets Z\_1 Z\_2 \\) |
| \\(1\mathbf D\\) | idle | idle | \\( R\_7 \gets k R\_7 \\) | \\( R\_8 \gets 2 R\_8 \\) |
| | \\( R\_1 \gets R\_6 - R\_5 \\) | \\( R\_2 \gets R\_8 - R\_7 \\) | \\( R\_3 \gets R\_8 + R\_7 \\) | \\( R\_4 \gets R\_6 + R\_5 \\) |
| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_3 R\_4 \\) | \\( T\_3 \gets R\_1 R\_4 \\) | \\( Z\_3 \gets R\_2 R\_3 \\) |
Here \\(\mathbf M\\) and \\(\mathbf S\\) represent the cost of
multiplication and squaring of generic field elements, \\(\mathbf D\\)
represents the cost of multiplication by a curve constant (in this
case \\( k = 2d \\)).
Notice that the \\(1\mathbf M\\) and \\(1\mathbf S\\) steps are
uniform. The non-uniform steps are all inexpensive additions or
subtractions, with the exception of the multiplication by the curve
constant \\(k = 2d\\):
$$
R\_7 \gets 2 d R\_7.
$$
HWCD suggest parallelising this step by breaking \\(k = 2d\\) into four
parts as \\(k = k_0 + 2\^n k_1 + 2\^{2n} k_2 + 2\^{3n} k_3 \\) and
computing \\(k_i R_7 \\) in parallel. This is quite awkward, but if
the curve constant is a ratio \\( d = d\_1/d\_2 \\), then projective
coordinates allow us to instead compute
$$
(R\_5, R\_6, R\_7, R\_8) \gets (d\_2 R\_5, d\_2 R\_6, 2d\_1 R\_7, d\_2 R\_8).
$$
This can be performed as a uniform multiplication by a vector of
constants, and if \\(d\_1, d\_2\\) are small, it is relatively
inexpensive. (This trick was suggested by Mike Hamburg).
In the Curve25519 case, we have
$$
d = \frac{d\_1}{d\_2} = \frac{-121665}{121666};
$$
Since \\(2 \cdot 121666 < 2\^{18}\\), all the constants above fit (up
to sign) in 32 bits, so this can be done in parallel as four
multiplications by small constants \\( (121666, 121666, 2\cdot 121665,
2\cdot 121666) \\), followed by a negation to compute \\( - 2\cdot 121665\\).
# Modified parallel formulas
Using the modifications sketched above, we can write SIMD-friendly
versions of the parallel formulas as follows. To avoid confusion with
the original formulas, temporary variables are named \\(S\\) instead
of \\(R\\) and are in static single-assignment form.
## Addition
To add points
\\(P_1 = (X_1 : Y_1 : Z_1 : T_1) \\)
and
\\(P_2 = (X_2 : Y_2 : Z_2 : T_2 ) \\),
we compute
$$
\begin{aligned}
(S\_0 &&,&& S\_1 &&,&& S\_2 &&,&& S\_3 )
&\gets
(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Y\_2 - X\_2&&,&& Y\_2 + X\_2)
\\\\
(S\_4 &&,&& S\_5 &&,&& S\_6 &&,&& S\_7 )
&\gets
(S\_0 \cdot S\_2&&,&& S\_1 \cdot S\_3&&,&& Z\_1 \cdot Z\_2&&,&& T\_1 \cdot T\_2)
\\\\
(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} )
&\gets
(d\_2 \cdot S\_4 &&,&& d\_2 \cdot S\_5 &&,&& 2 d\_2 \cdot S\_6 &&,&& 2 d\_1 \cdot S\_7 )
\\\\
(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15})
&\gets
(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11})
\\\\
(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3)
&\gets
(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13})
\end{aligned}
$$
to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\).
This costs \\( 2\mathbf M + 1 \mathbf D\\).
## Readdition
If the point \\( P_2 = (X\_2 : Y\_2 : Z\_2 : T\_2) \\) is fixed, we
can cache the multiplication of the curve constants by computing
$$
\begin{aligned}
(S\_2' &&,&& S\_3' &&,&& Z\_2' &&,&& T\_2' )
&\gets
(d\_2 \cdot (Y\_2 - X\_2)&&,&& d\_2 \cdot (Y\_1 + X\_1)&&,&& 2d\_2 \cdot Z\_2 &&,&& 2d\_1 \cdot T\_2).
\end{aligned}
$$
This costs \\( 1\mathbf D\\); with \\( (S\_2', S\_3', Z\_2', T\_2')\\)
in hand, the addition formulas above become
$$
\begin{aligned}
(S\_0 &&,&& S\_1 &&,&& Z\_1 &&,&& T\_1 )
&\gets
(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Z\_1 &&,&& T\_1)
\\\\
(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} )
&\gets
(S\_0 \cdot S\_2' &&,&& S\_1 \cdot S\_3'&&,&& Z\_1 \cdot Z\_2' &&,&& T\_1 \cdot T\_2')
\\\\
(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15})
&\gets
(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11})
\\\\
(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3)
&\gets
(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13})
\end{aligned}
$$
which costs only \\( 2\mathbf M \\). This precomputation is
essentially similar to the precomputation that HWCD suggest for their
serial formulas. Because the cost of precomputation and then
readdition is the same as addition, it's sufficient to only
implement caching and readdition.
## Doubling
The non-uniform portions of the (re)addition formulas have a fairly
regular structure. Unfortunately, this is not the case for the
doubling formulas, which are much less nice.
To double a point \\( P = (X\_1 : Y\_1 : Z\_1 : T\_1) \\), we compute
$$
\begin{aligned}
(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& S\_0)
&\gets
(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& X\_1 + Y\_1)
\\\\
(S\_1 &&,&& S\_2 &&,&& S\_3 &&,&& S\_4 )
&\gets
(X\_1\^2 &&,&& Y\_1\^2&&,&& Z\_1\^2 &&,&& S\_0\^2)
\\\\
(S\_5 &&,&& S\_6 &&,&& S\_8 &&,&& S\_9 )
&\gets
(S\_1 + S\_2 &&,&& S\_1 - S\_2 &&,&& S\_1 + 2S\_3 - S\_2 &&,&& S\_1 + S\_2 - S\_4)
\\\\
(X\_3 &&,&& Y\_3 &&,&& Z\_3 &&,&& T\_3 )
&\gets
(S\_8 \cdot S\_9 &&,&& S\_5 \cdot S\_6 &&,&& S\_8 \cdot S\_6 &&,&& S\_5 \cdot S\_9)
\end{aligned}
$$
to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = [2]P\_1 \\).
The intermediate step between the squaring and multiplication requires
a long chain of additions. For the IFMA-based implementation, this is not a problem; for the AVX2-based implementation, it is, but with some care and finesse, it's possible to arrange the computation without requiring an intermediate reduction.
# Implementation
These formulas aren't specific to a particular representation of field
element vectors, whose optimum choice is determined by the details of
the instruction set. However, it's not possible to perfectly separate
the implementation of the field element vectors from the
implementation of the point operations. Instead, the [`avx2`] and
[`ifma`] backends provide `ExtendedPoint` and `CachedPoint` types, and
the [`scalar_mul`] code uses one of the backend types by a type alias.
# Comparison to non-vectorized formulas
In theory, the parallel Edwards formulas seem to allow a \\(4\\)-way
speedup from parallelism. However, an actual vectorized
implementation has several slowdowns that cut into this speedup.
First, the parallel formulas can only use the available vector
multiplier. For AVX2, this is a \\( 32 \times 32 \rightarrow 64
\\)-bit integer multiplier, so the speedup from vectorization must
overcome the disadvantage of losing the \\( 64 \times 64 \rightarrow
128\\)-bit (serial) integer multiplier. The effect of this slowdown
is microarchitecture-dependent, since it requires accounting for the
total number of multiplications and additions and their relative
costs. IFMA allows using a \\( 52 \times 52 \rightarrow 104 \\)-bit
multiplier, but the high and low halves need to be computed
separately, and the reduction requires extra work because it's not
possible to pre-multiply by \\(19\\).
Second, the parallel doubling formulas incur both a theoretical and
practical slowdown. The parallel formulas described above work on the
\\( \mathbb P\^3 \\) “extended” coordinates. The \\( \mathbb P\^2 \\)
model introduced earlier by [Bernstein, Birkner, Joye, Lange, and
Peters][bbjlp08] allows slightly faster doublings, so HWCD suggest
mixing coordinate systems while performing scalar multiplication
(attributing the idea to [a 1998 paper][cmo98] by Cohen, Miyagi, and
Ono). The \\( T \\) coordinate is not required for doublings, so when
doublings are followed by doublings, its computation can be skipped.
More details on this approach and the different coordinate systems can
be found in the [`curve_models` module documentation][curve_models].
Unfortunately, this optimization is not compatible with the parallel
formulas, which cannot save time by skipping a single variable, so the
parallel doubling formulas do slightly more work when counting the
total number of field multiplications and squarings.
In addition, the parallel doubling formulas have a less regular
pattern of additions and subtractions than the parallel addition
formulas, so the vectorization overhead is proportionately greater.
Both the parallel addition and parallel doubling formulas also require
some shuffling to rearrange data within the vectors, which places more
pressure on the shuffle unit than is desirable.
This means that the speedup from using a vectorized implementation of
parallel Edwards formulas is likely to be greatest in applications
that do fewer doublings and more additions (like a large multiscalar
multiplication) rather than applications that do fewer additions and
more doublings (like a double-base scalar multiplication).
Third, Amdahl's law says that the speedup is limited to the portion
which can be parallelized. Normally, the field multiplications
dominate the cost of point operations, but with the IFMA backend, the
multiplications are so fast that the non-parallel additions end up as
a significant portion of the total time.
Fourth, current Intel CPUs perform thermal throttling when using wide
vector instructions. A detailed description can be found in §15.26 of
[the Intel Optimization Manual][intel], but using wide vector
instructions prevents the core from operating at higher frequencies.
The core can return to the higher-frequency state after 2
milliseconds, but this timer is reset every time high-power
instructions are used.
Any speedup from vectorization therefore has to be weighed against a
slowdown for the next few million instructions. For a mixed workload,
where point operations are interspersed with other tasks, this can
reduce overall performance. This implementation is therefore probably
not suitable for basic applications, like signatures, but is
worthwhile for complex applications, like zero-knowledge proofs, which
do sustained work.
# Future work
There are several directions for future improvement:
* Using the vectorized field arithmetic code to parallelize across
point operations rather than within a single point operation. This
is less flexible, but would give a speedup both from allowing use of
the faster mixed-model arithmetic and from reducing shuffle
pressure. One approach in this direction would be to implement
batched scalar-point operations using vectors of points (AoSoA
layout). This less generally useful but would give a speedup for
Bulletproofs.
* Extending the IFMA implementation to use the full width of AVX512,
either handling the extra parallelism internally to a single point
operation (by using a 2-way parallel implementation of field
arithmetic instead of a wordsliced one), or externally,
parallelizing across point operations. Internal parallelism would
be preferable but might require too much shuffle pressure. For now,
the only available CPU which runs IFMA operations executes them at
256-bits wide anyways, so this isn't yet important.
* Generalizing the implementation to NEON instructions. The current
point arithmetic code is written in terms of field element vectors,
which are in turn implemented using platform SIMD vectors. It
should be possible to write an alternate implementation of the
`FieldElement2625x4` using NEON without changing the point
arithmetic. NEON has 128-bit vectors rather than 256-bit vectors,
but this may still be worthwhile compared to a serial
implementation.
[sandy2x]: https://eprint.iacr.org/2015/943.pdf
[avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28
[hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf
[curve_models]: https://doc-internal.dalek.rs/curve25519_dalek/curve_models/index.html
[bbjlp08]: https://eprint.iacr.org/2008/013
[cmo98]: https://link.springer.com/content/pdf/10.1007%2F3-540-49649-1_6.pdf
[intel]: https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf

62
src/backend/mod.rs
View file

@ -10,29 +10,53 @@
//! Pluggable implementations for different architectures.
//!
//! The naming of the `u32` and `u64` modules is somewhat unfortunate,
//! since these are also the names of primitive types. Since types have
//! a different namespace than modules, this isn't a problem to the
//! compiler, but it could cause confusion.
//! The backend code is split into two parts: a serial backend,
//! and a vector backend.
//!
//! However, it's unlikely that the names of those modules would be
//! brought into scope directly, instead of used as
//! `backend::u32::field` or similar. Unfortunately we can't use
//! `32bit` since identifiers can't start with letters, and the backends
//! do use `u32`/`u64`, so this seems like a least-bad option.
//! The [`serial`] backend contains 32- and 64-bit implementations of
//! field arithmetic and scalar arithmetic, as well as implementations
//! of point operations using the mixed-model strategy (passing
//! between different curve models depending on the operation).
//!
//! The [`vector`] backend contains implementations of vectorized
//! field arithmetic, used to implement point operations using a novel
//! implementation strategy derived from parallel formulas of Hisil,
//! Wong, Carter, and Dawson.
//!
//! Because the two strategies give rise to different curve models,
//! it's not possible to reuse exactly the same scalar multiplication
//! code (or to write it generically), so both serial and vector
//! backends contain matching implementations of scalar multiplication
//! algorithms. These are intended to be selected by a `#[cfg]`-based
//! type alias.
//!
//! The [`vector`] backend is selected by the `simd_backend` cargo
//! feature; it uses the [`serial`] backend for non-vectorized operations.
#[cfg(not(any(feature = "u32_backend", feature = "u64_backend", feature = "avx2_backend")))]
#[cfg(not(any(
feature = "u32_backend",
feature = "u64_backend",
feature = "simd_backend",
)))]
compile_error!(
"no curve25519-dalek backend cargo feature enabled! \
please enable one of: u32_backend, u64_backend, avx2_backend"
please enable one of: u32_backend, u64_backend, simd_backend"
);
#[cfg(feature = "u32_backend")]
pub mod u32;
#[cfg(feature = "u64_backend")]
pub mod u64;
#[cfg(all(feature = "avx2_backend", target_feature = "avx2"))]
pub mod avx2;
pub mod serial;
#[cfg(any(
all(
feature = "simd_backend",
any(target_feature = "avx2", target_feature = "avx512ifma")
),
all(feature = "nightly", rustdoc)
))]
#[cfg_attr(
feature = "nightly",
doc(cfg(any(all(
feature = "simd_backend",
any(target_feature = "avx2", target_feature = "avx512ifma")
))))
)]
pub mod vector;

0
src/curve_models/mod.rs → src/backend/serial/curve_models/mod.rs
View file

43
src/backend/serial/mod.rs Normal file
View file

@ -0,0 +1,43 @@
// -*- mode: rust; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence
// See LICENSE for licensing information.
//
// Authors:
// - Isis Agora Lovecruft <isis@patternsinthevoid.net>
// - Henry de Valence <hdevalence@hdevalence.ca>
//! Serial implementations of field, scalar, point arithmetic.
//!
//! When the vector backend is disabled, the crate uses the
//! mixed-model strategy for implementing point operations and scalar
//! multiplication; see the [`curve_models`](self::curve_models) and
//! [`scalar_mul`](self::scalar_mul) documentation for more
//! information.
//!
//! When the vector backend is enabled, the field and scalar
//! implementations are still used for non-vectorized operations.
//!
//! Note: at this time the `u32` and `u64` backends cannot be built
//! together.
#[cfg(not(any(feature = "u32_backend", feature = "u64_backend")))]
compile_error!(
"no curve25519-dalek backend cargo feature enabled! \
please enable one of: u32_backend, u64_backend"
);
#[cfg(feature = "u32_backend")]
pub mod u32;
#[cfg(feature = "u64_backend")]
pub mod u64;
pub mod curve_models;
#[cfg(not(all(
feature = "simd_backend",
any(target_feature = "avx2", target_feature = "avx512ifma")
)))]
pub mod scalar_mul;

6
src/scalar_mul/mod.rs → src/backend/serial/scalar_mul/mod.rs
View file

@ -16,11 +16,13 @@
//! scalar multiplication implementations, since it only uses one
//! curve model.
pub mod window;
pub mod variable_base;
#[cfg(feature = "stage2_build")]
pub mod vartime_double_base;
#[cfg(feature = "alloc")]
pub mod straus;
#[cfg(feature = "alloc")]
pub mod precomputed_straus;

114
src/backend/serial/scalar_mul/precomputed_straus.rs Normal file
View file

@ -0,0 +1,114 @@
// -*- mode: rust; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2019 Henry de Valence.
// See LICENSE for licensing information.
//
// Authors:
// - Henry de Valence <hdevalence@hdevalence.ca>
//! Precomputation for Straus's method.
#![allow(non_snake_case)]
use core::borrow::Borrow;
use backend::serial::curve_models::{
AffineNielsPoint, CompletedPoint, ProjectiveNielsPoint, ProjectivePoint,
};
use edwards::EdwardsPoint;
use scalar::Scalar;
use traits::Identity;
use traits::VartimePrecomputedMultiscalarMul;
use window::{NafLookupTable5, NafLookupTable8};
#[allow(unused_imports)]
use prelude::*;
pub struct VartimePrecomputedStraus {
static_lookup_tables: Vec<NafLookupTable8<AffineNielsPoint>>,
}
impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus {
type Point = EdwardsPoint;
fn new<I>(static_points: I) -> Self
where
I: IntoIterator,
I::Item: Borrow<Self::Point>,
{
Self {
static_lookup_tables: static_points
.into_iter()
.map(|P| NafLookupTable8::<AffineNielsPoint>::from(P.borrow()))
.collect(),
}
}
fn optional_mixed_multiscalar_mul<I, J, K>(
&self,
static_scalars: I,
dynamic_scalars: J,
dynamic_points: K,
) -> Option<Self::Point>
where
I: IntoIterator,
I::Item: Borrow<Scalar>,
J: IntoIterator,
J::Item: Borrow<Scalar>,
K: IntoIterator<Item = Option<Self::Point>>,
{
let static_nafs = static_scalars
.into_iter()
.map(|c| c.borrow().non_adjacent_form(5))
.collect::<Vec<_>>();
let dynamic_nafs: Vec<_> = dynamic_scalars
.into_iter()
.map(|c| c.borrow().non_adjacent_form(5))
.collect::<Vec<_>>();
let dynamic_lookup_tables = match dynamic_points
.into_iter()
.map(|P_opt| P_opt.map(|P| NafLookupTable5::<ProjectiveNielsPoint>::from(&P)))
.collect::<Option<Vec<_>>>()
{
Some(x) => x,
None => return None,
};
let sp = self.static_lookup_tables.len();
let dp = dynamic_lookup_tables.len();
assert_eq!(sp, static_nafs.len());
assert_eq!(dp, dynamic_nafs.len());
// We could save some doublings by looking for the highest
// nonzero NAF coefficient, but since we might have a lot of
// them to search, it's not clear it's worthwhile to check.
let mut S = ProjectivePoint::identity();
for j in (0..255).rev() {
let mut R: CompletedPoint = S.double();
for i in 0..dp {
let t_ij = dynamic_nafs[i][j];
if t_ij > 0 {
R = &R.to_extended() + &dynamic_lookup_tables[i].select(t_ij as usize);
} else if t_ij < 0 {
R = &R.to_extended() - &dynamic_lookup_tables[i].select(-t_ij as usize);
}
}
for i in 0..sp {
let t_ij = static_nafs[i][j];
if t_ij > 0 {
R = &R.to_extended() + &self.static_lookup_tables[i].select(t_ij as usize);
} else if t_ij < 0 {
R = &R.to_extended() - &self.static_lookup_tables[i].select(-t_ij as usize);
}
}
S = R.to_projective();
}
Some(S.to_extended())
}
}

16
src/scalar_mul/straus.rs → src/backend/serial/scalar_mul/straus.rs
View file

@ -12,16 +12,11 @@
#![allow(non_snake_case)]
#[cfg(any(feature = "alloc", feature = "std"))]
use core::borrow::Borrow;
#[cfg(any(feature = "alloc", feature = "std"))]
use edwards::EdwardsPoint;
#[cfg(any(feature = "alloc", feature = "std"))]
use scalar::Scalar;
#[cfg(any(feature = "alloc", feature = "std"))]
use traits::MultiscalarMul;
#[cfg(any(feature = "alloc", feature = "std"))]
use traits::VartimeMultiscalarMul;
#[allow(unused_imports)]
@ -48,10 +43,8 @@ use prelude::*;
///
/// [solution]: https://www.jstor.org/stable/2310929
/// [problem]: https://www.jstor.org/stable/2312273
#[cfg(any(feature = "alloc", feature = "std"))]
pub struct Straus {}
#[cfg(feature = "alloc")]
impl MultiscalarMul for Straus {
type Point = EdwardsPoint;
@ -115,8 +108,8 @@ impl MultiscalarMul for Straus {
{
use clear_on_drop::ClearOnDrop;
use curve_models::ProjectiveNielsPoint;
use scalar_mul::window::LookupTable;
use backend::serial::curve_models::ProjectiveNielsPoint;
use window::LookupTable;
use traits::Identity;
let lookup_tables: Vec<_> = points
@ -148,7 +141,6 @@ impl MultiscalarMul for Straus {
}
}
#[cfg(feature = "alloc")]
impl VartimeMultiscalarMul for Straus {
type Point = EdwardsPoint;
@ -167,8 +159,8 @@ impl VartimeMultiscalarMul for Straus {
I::Item: Borrow<Scalar>,
J: IntoIterator<Item = Option<EdwardsPoint>>,
{
use curve_models::{CompletedPoint, ProjectiveNielsPoint, ProjectivePoint};
use scalar_mul::window::NafLookupTable5;
use backend::serial::curve_models::{CompletedPoint, ProjectiveNielsPoint, ProjectivePoint};
use window::NafLookupTable5;
use traits::Identity;
let nafs: Vec<_> = scalars

4
src/scalar_mul/variable_base.rs → src/backend/serial/scalar_mul/variable_base.rs
View file

@ -3,8 +3,8 @@
use traits::Identity;
use scalar::Scalar;
use edwards::EdwardsPoint;
use curve_models::ProjectiveNielsPoint;
use scalar_mul::window::LookupTable;
use backend::serial::curve_models::ProjectiveNielsPoint;
use window::LookupTable;
/// Perform constant-time, variable-base scalar multiplication.
pub(crate) fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint {

4
src/scalar_mul/vartime_double_base.rs → src/backend/serial/scalar_mul/vartime_double_base.rs
View file

@ -13,8 +13,8 @@ use constants;
use traits::Identity;
use scalar::Scalar;
use edwards::EdwardsPoint;
use curve_models::{ProjectiveNielsPoint, ProjectivePoint};
use scalar_mul::window::NafLookupTable5;
use backend::serial::curve_models::{ProjectiveNielsPoint, ProjectivePoint};
use window::NafLookupTable5;
/// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint.
pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint {

149
src/backend/serial/u32/constants.rs Normal file
View file

@ -0,0 +1,149 @@
// -*- mode: rust; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence
// See LICENSE for licensing information.
//
// Authors:
// - Isis Agora Lovecruft <isis@patternsinthevoid.net>
// - Henry de Valence <hdevalence@hdevalence.ca>
//! This module contains various constants (such as curve parameters
//! and useful field elements like `sqrt(-1)`), as well as
//! lookup tables of pre-computed points.
use backend::serial::u32::field::FieldElement2625;
use backend::serial::u32::scalar::Scalar29;
use edwards::EdwardsPoint;
/// Edwards `d` value, equal to `-121665/121666 mod p`.
pub(crate) const EDWARDS_D: FieldElement2625 = FieldElement2625([
56195235, 13857412, 51736253, 6949390, 114729,
24766616, 60832955, 30306712, 48412415, 21499315,
]);
/// Edwards `2*d` value, equal to `2*(-121665/121666) mod p`.
pub(crate) const EDWARDS_D2: FieldElement2625 = FieldElement2625([
45281625, 27714825, 36363642, 13898781, 229458,
15978800, 54557047, 27058993, 29715967, 9444199,
]);
/// `= sqrt(a*d - 1)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters.
pub(crate) const SQRT_AD_MINUS_ONE: FieldElement2625 = FieldElement2625([
24849947, 33400850, 43495378, 6347714, 46036536,
32887293, 41837720, 18186727, 66238516, 14525638,
]);
/// `= 1/sqrt(a-d)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters.
pub(crate) const INVSQRT_A_MINUS_D: FieldElement2625 = FieldElement2625([
6111466, 4156064, 39310137, 12243467, 41204824,
120896, 20826367, 26493656, 6093567, 31568420,
]);
/// Precomputed value of one of the square roots of -1 (mod p)
pub(crate) const SQRT_M1: FieldElement2625 = FieldElement2625([
34513072, 25610706, 9377949, 3500415, 12389472,
33281959, 41962654, 31548777, 326685, 11406482,
]);
/// `APLUS2_OVER_FOUR` is (A+2)/4. (This is used internally within the Montgomery ladder.)
pub(crate) const APLUS2_OVER_FOUR: FieldElement2625 = FieldElement2625([
121666, 0, 0, 0, 0, 0, 0, 0, 0, 0
]);
/// `L` is the order of base point, i.e. 2^252 +
/// 27742317777372353535851937790883648493
pub(crate) const L: Scalar29 = Scalar29([ 0x1cf5d3ed, 0x009318d2, 0x1de73596, 0x1df3bd45,
0x0000014d, 0x00000000, 0x00000000, 0x00000000,
0x00100000 ]);
/// `L` * `LFACTOR` = -1 (mod 2^29)
pub(crate) const LFACTOR: u32 = 0x12547e1b;
/// `R` = R % L where R = 2^261
pub(crate) const R: Scalar29 = Scalar29([ 0x114df9ed, 0x1a617303, 0x0f7c098c, 0x16793167,
0x1ffd656e, 0x1fffffff, 0x1fffffff, 0x1fffffff,
0x000fffff ]);
/// `RR` = (R^2) % L where R = 2^261
pub(crate) const RR: Scalar29 = Scalar29([ 0x0b5f9d12, 0x1e141b17, 0x158d7f3d, 0x143f3757,
0x1972d781, 0x042feb7c, 0x1ceec73d, 0x1e184d1e,
0x0005046d ]);
/// The Ed25519 basepoint, as an `EdwardsPoint`.
///
/// This is called `_POINT` to distinguish it from
/// `ED25519_BASEPOINT_TABLE`, which should be used for scalar
/// multiplication (it's much faster).
pub const ED25519_BASEPOINT_POINT: EdwardsPoint = EdwardsPoint{
X: FieldElement2625([52811034, 25909283, 16144682, 17082669, 27570973, 30858332, 40966398, 8378388, 20764389, 8758491]),
Y: FieldElement2625([40265304, 26843545, 13421772, 20132659, 26843545, 6710886, 53687091, 13421772, 40265318, 26843545]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([28827043, 27438313, 39759291, 244362, 8635006, 11264893, 19351346, 13413597, 16611511, 27139452]),
};
/// The 8-torsion subgroup \\(\mathcal E [8]\\).
///
/// In the case of Curve25519, it is cyclic; the \\(i\\)-th element of
/// the array is \\([i]P\\), where \\(P\\) is a point of order \\(8\\)
/// generating \\(\mathcal E[8]\\).
///
/// Thus \\(\mathcal E[8]\\) is the points indexed by `0,2,4,6`, and
/// \\(\mathcal E[2]\\) is the points indexed by `0,4`.
/// The Ed25519 basepoint has y = 4/5. This is called `_POINT` to
/// distinguish it from `_TABLE`, which should be used for scalar
/// multiplication (it's much faster).
pub const EIGHT_TORSION: [EdwardsPoint; 8] = EIGHT_TORSION_INNER_DOC_HIDDEN;
/// Inner item used to hide limb constants from cargo doc output.
#[doc(hidden)]
pub const EIGHT_TORSION_INNER_DOC_HIDDEN: [EdwardsPoint; 8] = [
EdwardsPoint{
X: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Y: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
},
EdwardsPoint{
X: FieldElement2625([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]),
Y: FieldElement2625([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251])
},
EdwardsPoint{
X: FieldElement2625([32595773, 7943725, 57730914, 30054016, 54719391, 272472, 25146209, 2005654, 66782178, 22147949]),
Y: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
},
EdwardsPoint{
X: FieldElement2625([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]),
Y: FieldElement2625([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180])
},
EdwardsPoint{
X: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Y: FieldElement2625([67108844, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
},
EdwardsPoint{
X: FieldElement2625([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]),
Y: FieldElement2625([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251])
},
EdwardsPoint{
X: FieldElement2625([34513072, 25610706, 9377949, 3500415, 12389472, 33281959, 41962654, 31548777, 326685, 11406482]),
Y: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
},
EdwardsPoint{
X: FieldElement2625([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]),
Y: FieldElement2625([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]),
Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement2625([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180])
},
];

114
src/backend/u32/field.rs → src/backend/serial/u32/field.rs
View file

@ -24,7 +24,7 @@ use core::ops::{Sub, SubAssign};
use subtle::Choice;
use subtle::ConditionallySelectable;
/// A `FieldElement32` represents an element of the field
/// A `FieldElement2625` represents an element of the field
/// \\( \mathbb Z / (2\^{255} - 19)\\).
///
/// In the 32-bit implementation, a `FieldElement` is represented in
@ -41,44 +41,44 @@ use subtle::ConditionallySelectable;
/// # Note
///
/// The `curve25519_dalek::field` module provides a type alias
/// `curve25519_dalek::field::FieldElement` to either `FieldElement64`
/// or `FieldElement32`.
/// `curve25519_dalek::field::FieldElement` to either `FieldElement51`
/// or `FieldElement2625`.
///
/// The backend-specific type `FieldElement32` should not be used
/// The backend-specific type `FieldElement2625` should not be used
/// outside of the `curve25519_dalek::field` module.
#[derive(Copy, Clone)]
pub struct FieldElement32(pub (crate) [u32; 10]);
pub struct FieldElement2625(pub (crate) [u32; 10]);
impl Debug for FieldElement32 {
impl Debug for FieldElement2625 {
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
write!(f, "FieldElement32({:?})", &self.0[..])
write!(f, "FieldElement2625({:?})", &self.0[..])
}
}
impl<'b> AddAssign<&'b FieldElement32> for FieldElement32 {
fn add_assign(&mut self, _rhs: &'b FieldElement32) {
impl<'b> AddAssign<&'b FieldElement2625> for FieldElement2625 {
fn add_assign(&mut self, _rhs: &'b FieldElement2625) {
for i in 0..10 {
self.0[i] += _rhs.0[i];
}
}
}
impl<'a, 'b> Add<&'b FieldElement32> for &'a FieldElement32 {
type Output = FieldElement32;
fn add(self, _rhs: &'b FieldElement32) -> FieldElement32 {
impl<'a, 'b> Add<&'b FieldElement2625> for &'a FieldElement2625 {
type Output = FieldElement2625;
fn add(self, _rhs: &'b FieldElement2625) -> FieldElement2625 {
let mut output = *self;
output += _rhs;
output
}
}
impl<'b> SubAssign<&'b FieldElement32> for FieldElement32 {
fn sub_assign(&mut self, _rhs: &'b FieldElement32) {
// See comment in FieldElement64::Sub
impl<'b> SubAssign<&'b FieldElement2625> for FieldElement2625 {
fn sub_assign(&mut self, _rhs: &'b FieldElement2625) {
// See comment in FieldElement51::Sub
//
// Compute a - b as ((a + 2^4 * p) - b) to avoid underflow.
let b = &_rhs.0;
self.0 = FieldElement32::reduce([
self.0 = FieldElement2625::reduce([
((self.0[0] + (0x3ffffed << 4)) - b[0]) as u64,
((self.0[1] + (0x1ffffff << 4)) - b[1]) as u64,
((self.0[2] + (0x3ffffff << 4)) - b[2]) as u64,
@ -93,25 +93,25 @@ impl<'b> SubAssign<&'b FieldElement32> for FieldElement32 {
}
}
impl<'a, 'b> Sub<&'b FieldElement32> for &'a FieldElement32 {
type Output = FieldElement32;
fn sub(self, _rhs: &'b FieldElement32) -> FieldElement32 {
impl<'a, 'b> Sub<&'b FieldElement2625> for &'a FieldElement2625 {
type Output = FieldElement2625;
fn sub(self, _rhs: &'b FieldElement2625) -> FieldElement2625 {
let mut output = *self;
output -= _rhs;
output
}
}
impl<'b> MulAssign<&'b FieldElement32> for FieldElement32 {
fn mul_assign(&mut self, _rhs: &'b FieldElement32) {
let result = (self as &FieldElement32) * _rhs;
impl<'b> MulAssign<&'b FieldElement2625> for FieldElement2625 {
fn mul_assign(&mut self, _rhs: &'b FieldElement2625) {
let result = (self as &FieldElement2625) * _rhs;
self.0 = result.0;
}
}
impl<'a, 'b> Mul<&'b FieldElement32> for &'a FieldElement32 {
type Output = FieldElement32;
fn mul(self, _rhs: &'b FieldElement32) -> FieldElement32 {
impl<'a, 'b> Mul<&'b FieldElement2625> for &'a FieldElement2625 {
type Output = FieldElement2625;
fn mul(self, _rhs: &'b FieldElement2625) -> FieldElement2625 {
/// Helper function to multiply two 32-bit integers with 64 bits
/// of output.
#[inline(always)]
@ -206,26 +206,26 @@ impl<'a, 'b> Mul<&'b FieldElement32> for &'a FieldElement32 {
//
// So z[0] fits into a u64 if 51 + 2*b + lg(249) < 64
// if b < 2.5.
FieldElement32::reduce([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
FieldElement2625::reduce([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
}
}
impl<'a> Neg for &'a FieldElement32 {
type Output = FieldElement32;
fn neg(self) -> FieldElement32 {
impl<'a> Neg for &'a FieldElement2625 {
type Output = FieldElement2625;
fn neg(self) -> FieldElement2625 {
let mut output = *self;
output.negate();
output
}
}
impl ConditionallySelectable for FieldElement32 {
impl ConditionallySelectable for FieldElement2625 {
fn conditional_select(
a: &FieldElement32,
b: &FieldElement32,
a: &FieldElement2625,
b: &FieldElement2625,
choice: Choice,
) -> FieldElement32 {
FieldElement32([
) -> FieldElement2625 {
FieldElement2625([
u32::conditional_select(&a.0[0], &b.0[0], choice),
u32::conditional_select(&a.0[1], &b.0[1], choice),
u32::conditional_select(&a.0[2], &b.0[2], choice),
@ -239,7 +239,7 @@ impl ConditionallySelectable for FieldElement32 {
])
}
fn conditional_assign(&mut self, other: &FieldElement32, choice: Choice) {
fn conditional_assign(&mut self, other: &FieldElement2625, choice: Choice) {
self.0[0].conditional_assign(&other.0[0], choice);
self.0[1].conditional_assign(&other.0[1], choice);
self.0[2].conditional_assign(&other.0[2], choice);
@ -252,7 +252,7 @@ impl ConditionallySelectable for FieldElement32 {
self.0[9].conditional_assign(&other.0[9], choice);
}
fn conditional_swap(a: &mut FieldElement32, b: &mut FieldElement32, choice: Choice) {
fn conditional_swap(a: &mut FieldElement2625, b: &mut FieldElement2625, choice: Choice) {
u32::conditional_swap(&mut a.0[0], &mut b.0[0], choice);
u32::conditional_swap(&mut a.0[1], &mut b.0[1], choice);
u32::conditional_swap(&mut a.0[2], &mut b.0[2], choice);
@ -266,11 +266,11 @@ impl ConditionallySelectable for FieldElement32 {
}
}
impl FieldElement32 {
impl FieldElement2625 {
/// Invert the sign of this field element
pub fn negate(&mut self) {
// Compute -b as ((2^4 * p) - b) to avoid underflow.
let neg = FieldElement32::reduce([
let neg = FieldElement2625::reduce([
((0x3ffffed << 4) - self.0[0]) as u64,
((0x1ffffff << 4) - self.0[1]) as u64,
((0x3ffffff << 4) - self.0[2]) as u64,
@ -286,25 +286,25 @@ impl FieldElement32 {
}
/// Construct zero.
pub fn zero() -> FieldElement32 {
FieldElement32([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
pub fn zero() -> FieldElement2625 {
FieldElement2625([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
}
/// Construct one.
pub fn one() -> FieldElement32 {
FieldElement32([ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
pub fn one() -> FieldElement2625 {
FieldElement2625([ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
}
/// Construct -1.
pub fn minus_one() -> FieldElement32 {
FieldElement32([
pub fn minus_one() -> FieldElement2625 {
FieldElement2625([
0x3ffffec, 0x1ffffff, 0x3ffffff, 0x1ffffff, 0x3ffffff,
0x1ffffff, 0x3ffffff, 0x1ffffff, 0x3ffffff, 0x1ffffff,
])
}
/// Given `k > 0`, return `self^(2^k)`.
pub fn pow2k(&self, k: u32) -> FieldElement32 {
pub fn pow2k(&self, k: u32) -> FieldElement2625 {
debug_assert!( k > 0 );
let mut z = self.square();
for _ in 1..k {
@ -314,12 +314,12 @@ impl FieldElement32 {
}
/// Given unreduced coefficients `z[0], ..., z[9]` of any size,
/// carry and reduce them mod p to obtain a `FieldElement32`
/// carry and reduce them mod p to obtain a `FieldElement2625`
/// whose coefficients have excess `b < 0.007`.
///
/// In other words, each coefficient of the result is bounded by
/// either `2^(25 + 0.007)` or `2^(26 + 0.007)`, as appropriate.
fn reduce(mut z: [u64; 10]) -> FieldElement32 {
fn reduce(mut z: [u64; 10]) -> FieldElement2625 {
const LOW_25_BITS: u64 = (1 << 25) - 1;
const LOW_26_BITS: u64 = (1 << 26) - 1;
@ -361,13 +361,13 @@ impl FieldElement32 {
// < 2^25.007 (good enough)
// and we're done.
FieldElement32([
FieldElement2625([
z[0] as u32, z[1] as u32, z[2] as u32, z[3] as u32, z[4] as u32,
z[5] as u32, z[6] as u32, z[7] as u32, z[8] as u32, z[9] as u32,
])
}
/// Load a `FieldElement64` from the low 255 bits of a 256-bit
/// Load a `FieldElement51` from the low 255 bits of a 256-bit
/// input.
///
/// # Warning
@ -378,7 +378,7 @@ impl FieldElement32 {
/// encoding of every field element should decode, re-encode to
/// the canonical encoding, and check that the input was
/// canonical.
pub fn from_bytes(data: &[u8; 32]) -> FieldElement32 { //FeFromBytes
pub fn from_bytes(data: &[u8; 32]) -> FieldElement2625 { //FeFromBytes
#[inline]
fn load3(b: &[u8]) -> u64 {
(b[0] as u64) | ((b[1] as u64) << 8) | ((b[2] as u64) << 16)
@ -402,16 +402,16 @@ impl FieldElement32 {
h[8] = load3(&data[26..]) << 4;
h[9] = (load3(&data[29..]) & LOW_23_BITS) << 2;
FieldElement32::reduce(h)
FieldElement2625::reduce(h)
}
/// Serialize this `FieldElement64` to a 32-byte array. The
/// Serialize this `FieldElement51` to a 32-byte array. The
/// encoding is canonical.
pub fn to_bytes(&self) -> [u8; 32] {
let inp = &self.0;
// Reduce the value represented by `in` to the range [0,2*p)
let mut h: [u32; 10] = FieldElement32::reduce([
let mut h: [u32; 10] = FieldElement2625::reduce([
// XXX this cast is annoying
inp[0] as u64, inp[1] as u64, inp[2] as u64, inp[3] as u64, inp[4] as u64,
inp[5] as u64, inp[6] as u64, inp[7] as u64, inp[8] as u64, inp[9] as u64,
@ -554,16 +554,16 @@ impl FieldElement32 {
}
/// Compute `self^2`.
pub fn square(&self) -> FieldElement32 {
FieldElement32::reduce(self.square_inner())
pub fn square(&self) -> FieldElement2625 {
FieldElement2625::reduce(self.square_inner())
}
/// Compute `2*self^2`.
pub fn square2(&self) -> FieldElement32 {
pub fn square2(&self) -> FieldElement2625 {
let mut coeffs = self.square_inner();
for i in 0..self.0.len() {
coeffs[i] += coeffs[i];
}
FieldElement32::reduce(coeffs)
FieldElement2625::reduce(coeffs)
}
}

0
src/backend/u32/mod.rs → src/backend/serial/u32/mod.rs
View file

122
src/backend/u32/scalar.rs → src/backend/serial/u32/scalar.rs
View file

@ -15,24 +15,24 @@ use core::ops::{Index, IndexMut};
use constants;
/// The `Scalar32` struct represents an element in ℤ/lℤ as 9 29-bit limbs
/// The `Scalar29` struct represents an element in ℤ/lℤ as 9 29-bit limbs
#[derive(Copy,Clone)]
pub struct Scalar32(pub [u32; 9]);
pub struct Scalar29(pub [u32; 9]);
impl Debug for Scalar32 {
impl Debug for Scalar29 {
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
write!(f, "Scalar32: {:?}", &self.0[..])
write!(f, "Scalar29: {:?}", &self.0[..])
}
}
impl Index<usize> for Scalar32 {
impl Index<usize> for Scalar29 {
type Output = u32;
fn index(&self, _index: usize) -> &u32 {
&(self.0[_index])
}
}
impl IndexMut<usize> for Scalar32 {
impl IndexMut<usize> for Scalar29 {
fn index_mut(&mut self, _index: usize) -> &mut u32 {
&mut (self.0[_index])
}
@ -44,14 +44,14 @@ fn m(x: u32, y: u32) -> u64 {
(x as u64) * (y as u64)
}
impl Scalar32 {
impl Scalar29 {
/// Return the zero scalar.
pub fn zero() -> Scalar32 {
Scalar32([0,0,0,0,0,0,0,0,0])
pub fn zero() -> Scalar29 {
Scalar29([0,0,0,0,0,0,0,0,0])
}
/// Unpack a 32 byte / 256 bit scalar into 9 29-bit limbs.
pub fn from_bytes(bytes: &[u8; 32]) -> Scalar32 {
pub fn from_bytes(bytes: &[u8; 32]) -> Scalar29 {
let mut words = [0u32; 8];
for i in 0..8 {
for j in 0..4 {
@ -61,7 +61,7 @@ impl Scalar32 {
let mask = (1u32 << 29) - 1;
let top_mask = (1u32 << 24) - 1;
let mut s = Scalar32::zero();
let mut s = Scalar29::zero();
s[ 0] = words[0] & mask;
s[ 1] = ((words[0] >> 29) | (words[1] << 3)) & mask;
@ -77,7 +77,7 @@ impl Scalar32 {
}
/// Reduce a 64 byte / 512 bit scalar mod l.
pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar32 {
pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar29 {
let mut words = [0u32; 16];
for i in 0..16 {
for j in 0..4 {
@ -86,8 +86,8 @@ impl Scalar32 {
}
let mask = (1u32 << 29) - 1;
let mut lo = Scalar32::zero();
let mut hi = Scalar32::zero();
let mut lo = Scalar29::zero();
let mut hi = Scalar29::zero();
lo[0] = words[ 0] & mask;
lo[1] = ((words[ 0] >> 29) | (words[ 1] << 3)) & mask;
@ -108,13 +108,13 @@ impl Scalar32 {
hi[7] = ((words[14] >> 16) | (words[15] << 16)) & mask;
hi[8] = words[15] >> 13 ;
lo = Scalar32::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo
hi = Scalar32::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R
lo = Scalar29::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo
hi = Scalar29::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R
Scalar32::add(&hi, &lo) // (hi * R) + lo
Scalar29::add(&hi, &lo) // (hi * R) + lo
}
/// Pack the limbs of this `Scalar32` into 32 bytes.
/// Pack the limbs of this `Scalar29` into 32 bytes.
pub fn to_bytes(&self) -> [u8; 32] {
let mut s = [0u8; 32];
@ -155,8 +155,8 @@ impl Scalar32 {
}
/// Compute `a + b` (mod l).
pub fn add(a: &Scalar32, b: &Scalar32) -> Scalar32 {
let mut sum = Scalar32::zero();
pub fn add(a: &Scalar29, b: &Scalar29) -> Scalar29 {
let mut sum = Scalar29::zero();
let mask = (1u32 << 29) - 1;
// a + b
@ -167,12 +167,12 @@ impl Scalar32 {
}
// subtract l if the sum is >= l
Scalar32::sub(&sum, &constants::L)
Scalar29::sub(&sum, &constants::L)
}
/// Compute `a - b` (mod l).
pub fn sub(a: &Scalar32, b: &Scalar32) -> Scalar32 {
let mut difference = Scalar32::zero();
pub fn sub(a: &Scalar29, b: &Scalar29) -> Scalar29 {
let mut difference = Scalar29::zero();
let mask = (1u32 << 29) - 1;
// a - b
@ -197,7 +197,7 @@ impl Scalar32 {
///
/// This is implemented with a one-level refined Karatsuba decomposition
#[inline(always)]
pub (crate) fn mul_internal(a: &Scalar32, b: &Scalar32) -> [u64; 17] {
pub (crate) fn mul_internal(a: &Scalar29, b: &Scalar29) -> [u64; 17] {
let mut z = [0u64; 17];
z[0] = m(a[0],b[0]); // c00
@ -254,7 +254,7 @@ impl Scalar32 {
/// Compute `a^2`.
#[inline(always)]
fn square_internal(a: &Scalar32) -> [u64; 17] {
fn square_internal(a: &Scalar29) -> [u64; 17] {
let aa = [
a[0]*2,
a[1]*2,
@ -289,7 +289,7 @@ impl Scalar32 {
/// Compute `limbs/R` (mod l), where R is the Montgomery modulus 2^261
#[inline(always)]
pub (crate) fn montgomery_reduce(limbs: &[u64; 17]) -> Scalar32 {
pub (crate) fn montgomery_reduce(limbs: &[u64; 17]) -> Scalar29 {
#[inline(always)]
fn part1(sum: u64) -> (u64, u32) {
@ -329,49 +329,49 @@ impl Scalar32 {
let r8 = carry as u32;
// result may be >= l, so attempt to subtract l
Scalar32::sub(&Scalar32([r0,r1,r2,r3,r4,r5,r6,r7,r8]), l)
Scalar29::sub(&Scalar29([r0,r1,r2,r3,r4,r5,r6,r7,r8]), l)
}
/// Compute `a * b` (mod l).
#[inline(never)]
pub fn mul(a: &Scalar32, b: &Scalar32) -> Scalar32 {
let ab = Scalar32::montgomery_reduce(&Scalar32::mul_internal(a, b));
Scalar32::montgomery_reduce(&Scalar32::mul_internal(&ab, &constants::RR))
pub fn mul(a: &Scalar29, b: &Scalar29) -> Scalar29 {
let ab = Scalar29::montgomery_reduce(&Scalar29::mul_internal(a, b));
Scalar29::montgomery_reduce(&Scalar29::mul_internal(&ab, &constants::RR))
}
/// Compute `a^2` (mod l).
#[inline(never)]
#[allow(dead_code)] // XXX we don't expose square() via the Scalar API
pub fn square(&self) -> Scalar32 {
let aa = Scalar32::montgomery_reduce(&Scalar32::square_internal(self));
Scalar32::montgomery_reduce(&Scalar32::mul_internal(&aa, &constants::RR))
pub fn square(&self) -> Scalar29 {
let aa = Scalar29::montgomery_reduce(&Scalar29::square_internal(self));
Scalar29::montgomery_reduce(&Scalar29::mul_internal(&aa, &constants::RR))
}
/// Compute `(a * b) / R` (mod l), where R is the Montgomery modulus 2^261
#[inline(never)]
pub fn montgomery_mul(a: &Scalar32, b: &Scalar32) -> Scalar32 {
Scalar32::montgomery_reduce(&Scalar32::mul_internal(a, b))
pub fn montgomery_mul(a: &Scalar29, b: &Scalar29) -> Scalar29 {
Scalar29::montgomery_reduce(&Scalar29::mul_internal(a, b))
}
/// Compute `(a^2) / R` (mod l) in Montgomery form, where R is the Montgomery modulus 2^261
#[inline(never)]
pub fn montgomery_square(&self) -> Scalar32 {
Scalar32::montgomery_reduce(&Scalar32::square_internal(self))
pub fn montgomery_square(&self) -> Scalar29 {
Scalar29::montgomery_reduce(&Scalar29::square_internal(self))
}
/// Puts a Scalar32 in to Montgomery form, i.e. computes `a*R (mod l)`
/// Puts a Scalar29 in to Montgomery form, i.e. computes `a*R (mod l)`
#[inline(never)]
pub fn to_montgomery(&self) -> Scalar32 {
Scalar32::montgomery_mul(self, &constants::RR)
pub fn to_montgomery(&self) -> Scalar29 {
Scalar29::montgomery_mul(self, &constants::RR)
}
/// Takes a Scalar32 out of Montgomery form, i.e. computes `a/R (mod l)`
pub fn from_montgomery(&self) -> Scalar32 {
/// Takes a Scalar29 out of Montgomery form, i.e. computes `a/R (mod l)`
pub fn from_montgomery(&self) -> Scalar29 {
let mut limbs = [0u64; 17];
for i in 0..9 {
limbs[i] = self[i] as u64;
}
Scalar32::montgomery_reduce(&limbs)
Scalar29::montgomery_reduce(&limbs)
}
}
@ -385,69 +385,69 @@ mod test {
/// x = 2^253-1 = 14474011154664524427946373126085988481658748083205070504932198000989141204991
/// x = 7237005577332262213973186563042994240801631723825162898930247062703686954002 mod l
/// x = 5147078182513738803124273553712992179887200054963030844803268920753008712037*R mod l in Montgomery form
pub static X: Scalar32 = Scalar32(
pub static X: Scalar29 = Scalar29(
[0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff,
0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff,
0x001fffff]);
/// x^2 = 3078544782642840487852506753550082162405942681916160040940637093560259278169 mod l
pub static XX: Scalar32 = Scalar32(
pub static XX: Scalar29 = Scalar29(
[0x00217559, 0x000b3401, 0x103ff43b, 0x1462a62c,
0x1d6f9f38, 0x18e7a42f, 0x09a3dcee, 0x008dbe18,
0x0006ce65]);
/// x^2 = 2912514428060642753613814151688322857484807845836623976981729207238463947987*R mod l in Montgomery form
pub static XX_MONT: Scalar32 = Scalar32(
pub static XX_MONT: Scalar29 = Scalar29(
[0x152b4d2e, 0x0571d53b, 0x1da6d964, 0x188663b6,
0x1d1b5f92, 0x19d50e3f, 0x12306c29, 0x0c6f26fe,
0x00030edb]);
/// y = 6145104759870991071742105800796537629880401874866217824609283457819451087098
pub static Y: Scalar32 = Scalar32(
pub static Y: Scalar29 = Scalar29(
[0x1e1458fa, 0x165ba838, 0x1d787b36, 0x0e577f3a,
0x1d2baf06, 0x1d689a19, 0x1fff3047, 0x117704ab,
0x000d9601]);
/// x*y = 36752150652102274958925982391442301741
pub static XY: Scalar32 = Scalar32(
pub static XY: Scalar29 = Scalar29(
[0x0ba7632d, 0x017736bb, 0x15c76138, 0x0c69daa1,
0x000001ba, 0x00000000, 0x00000000, 0x00000000,
0x00000000]);
/// x*y = 3783114862749659543382438697751927473898937741870308063443170013240655651591*R mod l in Montgomery form
pub static XY_MONT: Scalar32 = Scalar32(
pub static XY_MONT: Scalar29 = Scalar29(
[0x077b51e1, 0x1c64e119, 0x02a19ef5, 0x18d2129e,
0x00de0430, 0x045a7bc8, 0x04cfc7c9, 0x1c002681,
0x000bdc1c]);
/// a = 2351415481556538453565687241199399922945659411799870114962672658845158063753
pub static A: Scalar32 = Scalar32(
pub static A: Scalar29 = Scalar29(
[0x07b3be89, 0x02291b60, 0x14a99f03, 0x07dc3787,
0x0a782aae, 0x16262525, 0x0cfdb93f, 0x13f5718d,
0x000532da]);
/// b = 4885590095775723760407499321843594317911456947580037491039278279440296187236
pub static B: Scalar32 = Scalar32(
pub static B: Scalar29 = Scalar29(
[0x15421564, 0x1e69fd72, 0x093d9692, 0x161785be,
0x1587d69f, 0x09d9dada, 0x130246c0, 0x0c0a8e72,
0x000acd25]);
/// a+b = 0
/// a-b = 4702830963113076907131374482398799845891318823599740229925345317690316127506
pub static AB: Scalar32 = Scalar32(
pub static AB: Scalar29 = Scalar29(
[0x0f677d12, 0x045236c0, 0x09533e06, 0x0fb86f0f,
0x14f0555c, 0x0c4c4a4a, 0x19fb727f, 0x07eae31a,
0x000a65b5]);
// c = (2^512 - 1) % l = 1627715501170711445284395025044413883736156588369414752970002579683115011840
pub static C: Scalar32 = Scalar32(
pub static C: Scalar29 = Scalar29(
[0x049c0f00, 0x00308f1a, 0x0164d1e9, 0x1c374ed1,
0x1be65d00, 0x19e90bfa, 0x08f73bb1, 0x036f8613,
0x00039941]);
#[test]
fn mul_max() {
let res = Scalar32::mul(&X, &X);
let res = Scalar29::mul(&X, &X);
for i in 0..9 {
assert!(res[i] == XX[i]);
}
@ -463,7 +463,7 @@ mod test {
#[test]
fn montgomery_mul_max() {
let res = Scalar32::montgomery_mul(&X, &X);
let res = Scalar29::montgomery_mul(&X, &X);
for i in 0..9 {
assert!(res[i] == XX_MONT[i]);
}
@ -479,7 +479,7 @@ mod test {
#[test]
fn mul() {
let res = Scalar32::mul(&X, &Y);
let res = Scalar29::mul(&X, &Y);
for i in 0..9 {
assert!(res[i] == XY[i]);
}
@ -487,7 +487,7 @@ mod test {
#[test]
fn montgomery_mul() {
let res = Scalar32::montgomery_mul(&X, &Y);
let res = Scalar29::montgomery_mul(&X, &Y);
for i in 0..9 {
assert!(res[i] == XY_MONT[i]);
}
@ -495,8 +495,8 @@ mod test {
#[test]
fn add() {
let res = Scalar32::add(&A, &B);
let zero = Scalar32::zero();
let res = Scalar29::add(&A, &B);
let zero = Scalar29::zero();
for i in 0..9 {
assert!(res[i] == zero[i]);
}
@ -504,7 +504,7 @@ mod test {
#[test]
fn sub() {
let res = Scalar32::sub(&A, &B);
let res = Scalar29::sub(&A, &B);
for i in 0..9 {
assert!(res[i] == AB[i]);
}
@ -513,7 +513,7 @@ mod test {
#[test]
fn from_bytes_wide() {
let bignum = [255u8; 64]; // 2^512 - 1
let reduced = Scalar32::from_bytes_wide(&bignum);
let reduced = Scalar29::from_bytes_wide(&bignum);
for i in 0..9 {
assert!(reduced[i] == C[i]);
}

94
src/backend/u64/constants.rs → src/backend/serial/u64/constants.rs
View file

@ -10,43 +10,43 @@
//! This module contains backend-specific constant values, such as the 64-bit limbs of curve constants.
use backend::u64::field::FieldElement64;
use backend::u64::scalar::Scalar64;
use backend::serial::u64::field::FieldElement51;
use backend::serial::u64::scalar::Scalar52;
use edwards::EdwardsPoint;
/// Edwards `d` value, equal to `-121665/121666 mod p`.
pub(crate) const EDWARDS_D: FieldElement64 = FieldElement64([929955233495203, 466365720129213, 1662059464998953, 2033849074728123, 1442794654840575]);
pub(crate) const EDWARDS_D: FieldElement51 = FieldElement51([929955233495203, 466365720129213, 1662059464998953, 2033849074728123, 1442794654840575]);
/// Edwards `2*d` value, equal to `2*(-121665/121666) mod p`.
pub(crate) const EDWARDS_D2: FieldElement64 = FieldElement64([1859910466990425, 932731440258426, 1072319116312658, 1815898335770999, 633789495995903]);
pub(crate) const EDWARDS_D2: FieldElement51 = FieldElement51([1859910466990425, 932731440258426, 1072319116312658, 1815898335770999, 633789495995903]);
/// `= sqrt(a*d - 1)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters.
pub(crate) const SQRT_AD_MINUS_ONE: FieldElement64 = FieldElement64([
pub(crate) const SQRT_AD_MINUS_ONE: FieldElement51 = FieldElement51([
2241493124984347, 425987919032274, 2207028919301688, 1220490630685848, 974799131293748
]);
/// `= 1/sqrt(a-d)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters.
pub(crate) const INVSQRT_A_MINUS_D: FieldElement64 = FieldElement64([
pub(crate) const INVSQRT_A_MINUS_D: FieldElement51 = FieldElement51([
278908739862762, 821645201101625, 8113234426968, 1777959178193151, 2118520810568447
]);
/// Precomputed value of one of the square roots of -1 (mod p)
pub(crate) const SQRT_M1: FieldElement64 = FieldElement64([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]);
pub(crate) const SQRT_M1: FieldElement51 = FieldElement51([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]);
/// `APLUS2_OVER_FOUR` is (A+2)/4. (This is used internally within the Montgomery ladder.)
pub(crate) const APLUS2_OVER_FOUR: FieldElement64 = FieldElement64([121666, 0, 0, 0, 0]);
pub(crate) const APLUS2_OVER_FOUR: FieldElement51 = FieldElement51([121666, 0, 0, 0, 0]);
/// `L` is the order of base point, i.e. 2^252 + 27742317777372353535851937790883648493
pub(crate) const L: Scalar64 = Scalar64([ 0x0002631a5cf5d3ed, 0x000dea2f79cd6581, 0x000000000014def9, 0x0000000000000000, 0x0000100000000000 ]);
pub(crate) const L: Scalar52 = Scalar52([ 0x0002631a5cf5d3ed, 0x000dea2f79cd6581, 0x000000000014def9, 0x0000000000000000, 0x0000100000000000 ]);
/// `L` * `LFACTOR` = -1 (mod 2^52)
pub(crate) const LFACTOR: u64 = 0x51da312547e1b;
/// `R` = R % L where R = 2^260
pub(crate) const R: Scalar64 = Scalar64([ 0x000f48bd6721e6ed, 0x0003bab5ac67e45a, 0x000fffffeb35e51b, 0x000fffffffffffff, 0x00000fffffffffff ]);
pub(crate) const R: Scalar52 = Scalar52([ 0x000f48bd6721e6ed, 0x0003bab5ac67e45a, 0x000fffffeb35e51b, 0x000fffffffffffff, 0x00000fffffffffff ]);
/// `RR` = (R^2) % L where R = 2^260
pub(crate) const RR: Scalar64 = Scalar64([ 0x0009d265e952d13b, 0x000d63c715bea69f, 0x0005be65cb687604, 0x0003dceec73d217f, 0x000009411b7c309a ]);
pub(crate) const RR: Scalar52 = Scalar52([ 0x0009d265e952d13b, 0x000d63c715bea69f, 0x0005be65cb687604, 0x0003dceec73d217f, 0x000009411b7c309a ]);
/// The Ed25519 basepoint, as an `EdwardsPoint`.
///
@ -54,10 +54,10 @@ pub(crate) const RR: Scalar64 = Scalar64([ 0x0009d265e952d13b, 0x000d63c715bea69
/// `ED25519_BASEPOINT_TABLE`, which should be used for scalar
/// multiplication (it's much faster).
pub const ED25519_BASEPOINT_POINT: EdwardsPoint = EdwardsPoint{
X: FieldElement64([1738742601995546, 1146398526822698, 2070867633025821, 562264141797630, 587772402128613]),
Y: FieldElement64([1801439850948184, 1351079888211148, 450359962737049, 900719925474099, 1801439850948198]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([1841354044333475, 16398895984059, 755974180946558, 900171276175154, 1821297809914039]),
X: FieldElement51([1738742601995546, 1146398526822698, 2070867633025821, 562264141797630, 587772402128613]),
Y: FieldElement51([1801439850948184, 1351079888211148, 450359962737049, 900719925474099, 1801439850948198]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([1841354044333475, 16398895984059, 755974180946558, 900171276175154, 1821297809914039]),
};
/// The 8-torsion subgroup \\(\mathcal E [8]\\).
@ -74,58 +74,58 @@ pub const EIGHT_TORSION: [EdwardsPoint; 8] = EIGHT_TORSION_INNER_DOC_HIDDEN;
#[doc(hidden)]
pub const EIGHT_TORSION_INNER_DOC_HIDDEN: [EdwardsPoint; 8] = [
EdwardsPoint {
X: FieldElement64([0, 0, 0, 0, 0]),
Y: FieldElement64([1, 0, 0, 0, 0]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([0, 0, 0, 0, 0]),
X: FieldElement51([0, 0, 0, 0, 0]),
Y: FieldElement51([1, 0, 0, 0, 0]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([0, 0, 0, 0, 0]),
}
,
EdwardsPoint {
X: FieldElement64([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]),
Y: FieldElement64([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]),
X: FieldElement51([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]),
Y: FieldElement51([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]),
}
,
EdwardsPoint {
X: FieldElement64([533094393274173, 2016890930128738, 18285341111199, 134597186663265, 1486323764102114]),
Y: FieldElement64([0, 0, 0, 0, 0]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([0, 0, 0, 0, 0]),
X: FieldElement51([533094393274173, 2016890930128738, 18285341111199, 134597186663265, 1486323764102114]),
Y: FieldElement51([0, 0, 0, 0, 0]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([0, 0, 0, 0, 0]),
}
,
EdwardsPoint {
X: FieldElement64([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]),
Y: FieldElement64([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]),
X: FieldElement51([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]),
Y: FieldElement51([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]),
}
,
EdwardsPoint {
X: FieldElement64([0, 0, 0, 0, 0]),
Y: FieldElement64([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([0, 0, 0, 0, 0]),
X: FieldElement51([0, 0, 0, 0, 0]),
Y: FieldElement51([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([0, 0, 0, 0, 0]),
}
,
EdwardsPoint {
X: FieldElement64([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]),
Y: FieldElement64([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]),
X: FieldElement51([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]),
Y: FieldElement51([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]),
}
,
EdwardsPoint {
X: FieldElement64([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]),
Y: FieldElement64([0, 0, 0, 0, 0]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([0, 0, 0, 0, 0]),
X: FieldElement51([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]),
Y: FieldElement51([0, 0, 0, 0, 0]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([0, 0, 0, 0, 0]),
}
,
EdwardsPoint {
X: FieldElement64([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]),
Y: FieldElement64([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]),
Z: FieldElement64([1, 0, 0, 0, 0]),
T: FieldElement64([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]),
X: FieldElement51([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]),
Y: FieldElement51([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]),
Z: FieldElement51([1, 0, 0, 0, 0]),
T: FieldElement51([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]),
}
];

112
src/backend/u64/field.rs → src/backend/serial/u64/field.rs
View file

@ -20,7 +20,7 @@ use core::ops::{Sub, SubAssign};
use subtle::Choice;
use subtle::ConditionallySelectable;
/// A `FieldElement64` represents an element of the field
/// A `FieldElement51` represents an element of the field
/// \\( \mathbb Z / (2\^{255} - 19)\\).
///
/// In the 64-bit implementation, a `FieldElement` is represented in
@ -30,57 +30,57 @@ use subtle::ConditionallySelectable;
/// # Note
///
/// The `curve25519_dalek::field` module provides a type alias
/// `curve25519_dalek::field::FieldElement` to either `FieldElement64`
/// or `FieldElement32`.
/// `curve25519_dalek::field::FieldElement` to either `FieldElement51`
/// or `FieldElement2625`.
///
/// The backend-specific type `FieldElement64` should not be used
/// The backend-specific type `FieldElement51` should not be used
/// outside of the `curve25519_dalek::field` module.
#[derive(Copy, Clone)]
pub struct FieldElement64(pub (crate) [u64; 5]);
pub struct FieldElement51(pub (crate) [u64; 5]);
impl Debug for FieldElement64 {
impl Debug for FieldElement51 {
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
write!(f, "FieldElement64({:?})", &self.0[..])
write!(f, "FieldElement51({:?})", &self.0[..])
}
}
impl<'b> AddAssign<&'b FieldElement64> for FieldElement64 {
fn add_assign(&mut self, _rhs: &'b FieldElement64) {
impl<'b> AddAssign<&'b FieldElement51> for FieldElement51 {
fn add_assign(&mut self, _rhs: &'b FieldElement51) {
for i in 0..5 {
self.0[i] += _rhs.0[i];
}
}
}
impl<'a, 'b> Add<&'b FieldElement64> for &'a FieldElement64 {
type Output = FieldElement64;
fn add(self, _rhs: &'b FieldElement64) -> FieldElement64 {
impl<'a, 'b> Add<&'b FieldElement51> for &'a FieldElement51 {
type Output = FieldElement51;
fn add(self, _rhs: &'b FieldElement51) -> FieldElement51 {
let mut output = *self;
output += _rhs;
output
}
}
impl<'b> SubAssign<&'b FieldElement64> for FieldElement64 {
fn sub_assign(&mut self, _rhs: &'b FieldElement64) {
let result = (self as &FieldElement64) - _rhs;
impl<'b> SubAssign<&'b FieldElement51> for FieldElement51 {
fn sub_assign(&mut self, _rhs: &'b FieldElement51) {
let result = (self as &FieldElement51) - _rhs;
self.0 = result.0;
}
}
impl<'a, 'b> Sub<&'b FieldElement64> for &'a FieldElement64 {
type Output = FieldElement64;
fn sub(self, _rhs: &'b FieldElement64) -> FieldElement64 {
impl<'a, 'b> Sub<&'b FieldElement51> for &'a FieldElement51 {
type Output = FieldElement51;
fn sub(self, _rhs: &'b FieldElement51) -> FieldElement51 {
// To avoid underflow, first add a multiple of p.
// Choose 16*p = p << 4 to be larger than 54-bit _rhs.
//
// If we could statically track the bitlengths of the limbs
// of every FieldElement64, we could choose a multiple of p
// of every FieldElement51, we could choose a multiple of p
// just bigger than _rhs and avoid having to do a reduction.
//
// Since we don't yet have type-level integers to do this, we
// have to add an explicit reduction call here.
FieldElement64::reduce([
FieldElement51::reduce([
(self.0[0] + 36028797018963664u64) - _rhs.0[0],
(self.0[1] + 36028797018963952u64) - _rhs.0[1],
(self.0[2] + 36028797018963952u64) - _rhs.0[2],
@ -90,16 +90,16 @@ impl<'a, 'b> Sub<&'b FieldElement64> for &'a FieldElement64 {
}
}
impl<'b> MulAssign<&'b FieldElement64> for FieldElement64 {
fn mul_assign(&mut self, _rhs: &'b FieldElement64) {
let result = (self as &FieldElement64) * _rhs;
impl<'b> MulAssign<&'b FieldElement51> for FieldElement51 {
fn mul_assign(&mut self, _rhs: &'b FieldElement51) {
let result = (self as &FieldElement51) * _rhs;
self.0 = result.0;
}
}
impl<'a, 'b> Mul<&'b FieldElement64> for &'a FieldElement64 {
type Output = FieldElement64;
fn mul(self, _rhs: &'b FieldElement64) -> FieldElement64 {
impl<'a, 'b> Mul<&'b FieldElement51> for &'a FieldElement51 {
type Output = FieldElement51;
fn mul(self, _rhs: &'b FieldElement51) -> FieldElement51 {
/// Helper function to multiply two 64-bit integers with 128
/// bits of output.
#[inline(always)]
@ -196,26 +196,26 @@ impl<'a, 'b> Mul<&'b FieldElement64> for &'a FieldElement64 {
out[0] &= LOW_51_BIT_MASK;
// Now out[i] < 2^(51 + epsilon) for all i.
FieldElement64(out)
FieldElement51(out)
}
}
impl<'a> Neg for &'a FieldElement64 {
type Output = FieldElement64;
fn neg(self) -> FieldElement64 {
impl<'a> Neg for &'a FieldElement51 {
type Output = FieldElement51;
fn neg(self) -> FieldElement51 {
let mut output = *self;
output.negate();
output
}
}
impl ConditionallySelectable for FieldElement64 {
impl ConditionallySelectable for FieldElement51 {
fn conditional_select(
a: &FieldElement64,
b: &FieldElement64,
a: &FieldElement51,
b: &FieldElement51,
choice: Choice,
) -> FieldElement64 {
FieldElement64([
) -> FieldElement51 {
FieldElement51([
u64::conditional_select(&a.0[0], &b.0[0], choice),
u64::conditional_select(&a.0[1], &b.0[1], choice),
u64::conditional_select(&a.0[2], &b.0[2], choice),
@ -224,7 +224,7 @@ impl ConditionallySelectable for FieldElement64 {
])
}
fn conditional_swap(a: &mut FieldElement64, b: &mut FieldElement64, choice: Choice) {
fn conditional_swap(a: &mut FieldElement51, b: &mut FieldElement51, choice: Choice) {
u64::conditional_swap(&mut a.0[0], &mut b.0[0], choice);
u64::conditional_swap(&mut a.0[1], &mut b.0[1], choice);
u64::conditional_swap(&mut a.0[2], &mut b.0[2], choice);
@ -232,7 +232,7 @@ impl ConditionallySelectable for FieldElement64 {
u64::conditional_swap(&mut a.0[4], &mut b.0[4], choice);
}
fn conditional_assign(&mut self, other: &FieldElement64, choice: Choice) {
fn conditional_assign(&mut self, other: &FieldElement51, choice: Choice) {
self.0[0].conditional_assign(&other.0[0], choice);
self.0[1].conditional_assign(&other.0[1], choice);
self.0[2].conditional_assign(&other.0[2], choice);
@ -241,11 +241,11 @@ impl ConditionallySelectable for FieldElement64 {
}
}
impl FieldElement64 {
impl FieldElement51 {
/// Invert the sign of this field element
pub fn negate(&mut self) {
// See commentary in the Sub impl
let neg = FieldElement64::reduce([
let neg = FieldElement51::reduce([
36028797018963664u64 - self.0[0],
36028797018963952u64 - self.0[1],
36028797018963952u64 - self.0[2],
@ -256,23 +256,23 @@ impl FieldElement64 {
}
/// Construct zero.
pub fn zero() -> FieldElement64 {
FieldElement64([ 0, 0, 0, 0, 0 ])
pub fn zero() -> FieldElement51 {
FieldElement51([ 0, 0, 0, 0, 0 ])
}
/// Construct one.
pub fn one() -> FieldElement64 {
FieldElement64([ 1, 0, 0, 0, 0 ])
pub fn one() -> FieldElement51 {
FieldElement51([ 1, 0, 0, 0, 0 ])
}
/// Construct -1.
pub fn minus_one() -> FieldElement64 {
FieldElement64([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247])
pub fn minus_one() -> FieldElement51 {
FieldElement51([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247])
}
/// Given 64-bit input limbs, reduce to enforce the bound 2^(51 + epsilon).
#[inline(always)]
fn reduce(mut limbs: [u64; 5]) -> FieldElement64 {
fn reduce(mut limbs: [u64; 5]) -> FieldElement51 {
const LOW_51_BIT_MASK: u64 = (1u64 << 51) - 1;
// Since the input limbs are bounded by 2^64, the biggest
@ -304,10 +304,10 @@ impl FieldElement64 {
limbs[3] += c2;
limbs[4] += c3;
FieldElement64(limbs)
FieldElement51(limbs)
}
/// Load a `FieldElement64` from the low 255 bits of a 256-bit
/// Load a `FieldElement51` from the low 255 bits of a 256-bit
/// input.
///
/// # Warning
@ -319,7 +319,7 @@ impl FieldElement64 {
/// the canonical encoding, and check that the input was
/// canonical.
///
pub fn from_bytes(bytes: &[u8; 32]) -> FieldElement64 {
pub fn from_bytes(bytes: &[u8; 32]) -> FieldElement51 {
let load8 = |input: &[u8]| -> u64 {
(input[0] as u64)
| ((input[1] as u64) << 8)
@ -332,7 +332,7 @@ impl FieldElement64 {
};
let low_51_bit_mask = (1u64 << 51) - 1;
FieldElement64(
FieldElement51(
// load bits [ 0, 64), no shift
[ load8(&bytes[ 0..]) & low_51_bit_mask
// load bits [ 48,112), shift to [ 51,112)
@ -346,7 +346,7 @@ impl FieldElement64 {
])
}
/// Serialize this `FieldElement64` to a 32-byte array. The
/// Serialize this `FieldElement51` to a 32-byte array. The
/// encoding is canonical.
pub fn to_bytes(&self) -> [u8; 32] {
// Let h = limbs[0] + limbs[1]*2^51 + ... + limbs[4]*2^204.
@ -365,7 +365,7 @@ impl FieldElement64 {
// Therefore q can be computed as the carry bit of h + 19.
// First, reduce the limbs to ensure h < 2*p.
let mut limbs = FieldElement64::reduce(self.0).0;
let mut limbs = FieldElement51::reduce(self.0).0;
let mut q = (limbs[0] + 19) >> 51;
q = (limbs[1] + q) >> 51;
@ -433,7 +433,7 @@ impl FieldElement64 {
}
/// Given `k > 0`, return `self^(2^k)`.
pub fn pow2k(&self, mut k: u32) -> FieldElement64 {
pub fn pow2k(&self, mut k: u32) -> FieldElement51 {
debug_assert!( k > 0 );
@ -535,16 +535,16 @@ impl FieldElement64 {
}
}
FieldElement64(a)
FieldElement51(a)
}
/// Returns the square of this field element.
pub fn square(&self) -> FieldElement64 {
pub fn square(&self) -> FieldElement51 {
self.pow2k(1)
}
/// Returns 2 times the square of this field element.
pub fn square2(&self) -> FieldElement64 {
pub fn square2(&self) -> FieldElement51 {
let mut square = self.pow2k(1);
for i in 0..5 {
square.0[i] *= 2;

0
src/backend/u64/mod.rs → src/backend/serial/u64/mod.rs
View file

122
src/backend/u64/scalar.rs → src/backend/serial/u64/scalar.rs
View file

@ -16,25 +16,25 @@ use core::ops::{Index, IndexMut};
use constants;
/// The `Scalar64` struct represents an element in
/// The `Scalar52` struct represents an element in
/// \\(\mathbb Z / \ell \mathbb Z\\) as 5 \\(52\\)-bit limbs.
#[derive(Copy,Clone)]
pub struct Scalar64(pub [u64; 5]);
pub struct Scalar52(pub [u64; 5]);
impl Debug for Scalar64 {
impl Debug for Scalar52 {
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
write!(f, "Scalar64: {:?}", &self.0[..])
write!(f, "Scalar52: {:?}", &self.0[..])
}
}
impl Index<usize> for Scalar64 {
impl Index<usize> for Scalar52 {
type Output = u64;
fn index(&self, _index: usize) -> &u64 {
&(self.0[_index])
}
}
impl IndexMut<usize> for Scalar64 {
impl IndexMut<usize> for Scalar52 {
fn index_mut(&mut self, _index: usize) -> &mut u64 {
&mut (self.0[_index])
}
@ -46,14 +46,14 @@ fn m(x: u64, y: u64) -> u128 {
(x as u128) * (y as u128)
}
impl Scalar64 {
impl Scalar52 {
/// Return the zero scalar
pub fn zero() -> Scalar64 {
Scalar64([0,0,0,0,0])
pub fn zero() -> Scalar52 {
Scalar52([0,0,0,0,0])
}
/// Unpack a 32 byte / 256 bit scalar into 5 52-bit limbs.
pub fn from_bytes(bytes: &[u8; 32]) -> Scalar64 {
pub fn from_bytes(bytes: &[u8; 32]) -> Scalar52 {
let mut words = [0u64; 4];
for i in 0..4 {
for j in 0..8 {
@ -63,7 +63,7 @@ impl Scalar64 {
let mask = (1u64 << 52) - 1;
let top_mask = (1u64 << 48) - 1;
let mut s = Scalar64::zero();
let mut s = Scalar52::zero();
s[ 0] = words[0] & mask;
s[ 1] = ((words[0] >> 52) | (words[1] << 12)) & mask;
@ -75,7 +75,7 @@ impl Scalar64 {
}
/// Reduce a 64 byte / 512 bit scalar mod l
pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar64 {
pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar52 {
let mut words = [0u64; 8];
for i in 0..8 {
for j in 0..8 {
@ -84,8 +84,8 @@ impl Scalar64 {
}
let mask = (1u64 << 52) - 1;
let mut lo = Scalar64::zero();
let mut hi = Scalar64::zero();
let mut lo = Scalar52::zero();
let mut hi = Scalar52::zero();
lo[0] = words[ 0] & mask;
lo[1] = ((words[ 0] >> 52) | (words[ 1] << 12)) & mask;
@ -98,13 +98,13 @@ impl Scalar64 {
hi[3] = ((words[ 6] >> 32) | (words[ 7] << 32)) & mask;
hi[4] = words[ 7] >> 20 ;
lo = Scalar64::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo
hi = Scalar64::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R
lo = Scalar52::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo
hi = Scalar52::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R
Scalar64::add(&hi, &lo)
Scalar52::add(&hi, &lo)
}
/// Pack the limbs of this `Scalar64` into 32 bytes
/// Pack the limbs of this `Scalar52` into 32 bytes
pub fn to_bytes(&self) -> [u8; 32] {
let mut s = [0u8; 32];
@ -145,8 +145,8 @@ impl Scalar64 {
}
/// Compute `a + b` (mod l)
pub fn add(a: &Scalar64, b: &Scalar64) -> Scalar64 {
let mut sum = Scalar64::zero();
pub fn add(a: &Scalar52, b: &Scalar52) -> Scalar52 {
let mut sum = Scalar52::zero();
let mask = (1u64 << 52) - 1;
// a + b
@ -157,12 +157,12 @@ impl Scalar64 {
}
// subtract l if the sum is >= l
Scalar64::sub(&sum, &constants::L)
Scalar52::sub(&sum, &constants::L)
}
/// Compute `a - b` (mod l)
pub fn sub(a: &Scalar64, b: &Scalar64) -> Scalar64 {
let mut difference = Scalar64::zero();
pub fn sub(a: &Scalar52, b: &Scalar52) -> Scalar52 {
let mut difference = Scalar52::zero();
let mask = (1u64 << 52) - 1;
// a - b
@ -185,7 +185,7 @@ impl Scalar64 {
/// Compute `a * b`
#[inline(always)]
pub (crate) fn mul_internal(a: &Scalar64, b: &Scalar64) -> [u128; 9] {
pub (crate) fn mul_internal(a: &Scalar52, b: &Scalar52) -> [u128; 9] {
let mut z = [0u128; 9];
z[0] = m(a[0],b[0]);
@ -203,7 +203,7 @@ impl Scalar64 {
/// Compute `a^2`
#[inline(always)]
fn square_internal(a: &Scalar64) -> [u128; 9] {
fn square_internal(a: &Scalar52) -> [u128; 9] {
let aa = [
a[0]*2,
a[1]*2,
@ -226,7 +226,7 @@ impl Scalar64 {
/// Compute `limbs/R` (mod l), where R is the Montgomery modulus 2^260
#[inline(always)]
pub (crate) fn montgomery_reduce(limbs: &[u128; 9]) -> Scalar64 {
pub (crate) fn montgomery_reduce(limbs: &[u128; 9]) -> Scalar52 {
#[inline(always)]
fn part1(sum: u128) -> (u128, u64) {
@ -258,50 +258,50 @@ impl Scalar64 {
let r4 = carry as u64;
// result may be >= l, so attempt to subtract l
Scalar64::sub(&Scalar64([r0,r1,r2,r3,r4]), l)
Scalar52::sub(&Scalar52([r0,r1,r2,r3,r4]), l)
}
/// Compute `a * b` (mod l)
#[inline(never)]
pub fn mul(a: &Scalar64, b: &Scalar64) -> Scalar64 {
let ab = Scalar64::montgomery_reduce(&Scalar64::mul_internal(a, b));
Scalar64::montgomery_reduce(&Scalar64::mul_internal(&ab, &constants::RR))
pub fn mul(a: &Scalar52, b: &Scalar52) -> Scalar52 {
let ab = Scalar52::montgomery_reduce(&Scalar52::mul_internal(a, b));
Scalar52::montgomery_reduce(&Scalar52::mul_internal(&ab, &constants::RR))
}
/// Compute `a^2` (mod l)
#[inline(never)]
#[allow(dead_code)] // XXX we don't expose square() via the Scalar API
pub fn square(&self) -> Scalar64 {
let aa = Scalar64::montgomery_reduce(&Scalar64::square_internal(self));
Scalar64::montgomery_reduce(&Scalar64::mul_internal(&aa, &constants::RR))
pub fn square(&self) -> Scalar52 {
let aa = Scalar52::montgomery_reduce(&Scalar52::square_internal(self));
Scalar52::montgomery_reduce(&Scalar52::mul_internal(&aa, &constants::RR))
}
/// Compute `(a * b) / R` (mod l), where R is the Montgomery modulus 2^260
#[inline(never)]
pub fn montgomery_mul(a: &Scalar64, b: &Scalar64) -> Scalar64 {
Scalar64::montgomery_reduce(&Scalar64::mul_internal(a, b))
pub fn montgomery_mul(a: &Scalar52, b: &Scalar52) -> Scalar52 {
Scalar52::montgomery_reduce(&Scalar52::mul_internal(a, b))
}
/// Compute `(a^2) / R` (mod l) in Montgomery form, where R is the Montgomery modulus 2^260
#[inline(never)]
pub fn montgomery_square(&self) -> Scalar64 {
Scalar64::montgomery_reduce(&Scalar64::square_internal(self))
pub fn montgomery_square(&self) -> Scalar52 {
Scalar52::montgomery_reduce(&Scalar52::square_internal(self))
}
/// Puts a Scalar64 in to Montgomery form, i.e. computes `a*R (mod l)`
/// Puts a Scalar52 in to Montgomery form, i.e. computes `a*R (mod l)`
#[inline(never)]
pub fn to_montgomery(&self) -> Scalar64 {
Scalar64::montgomery_mul(self, &constants::RR)
pub fn to_montgomery(&self) -> Scalar52 {
Scalar52::montgomery_mul(self, &constants::RR)
}
/// Takes a Scalar64 out of Montgomery form, i.e. computes `a/R (mod l)`
/// Takes a Scalar52 out of Montgomery form, i.e. computes `a/R (mod l)`
#[inline(never)]
pub fn from_montgomery(&self) -> Scalar64 {
pub fn from_montgomery(&self) -> Scalar52 {
let mut limbs = [0u128; 9];
for i in 0..5 {
limbs[i] = self[i] as u128;
}
Scalar64::montgomery_reduce(&limbs)
Scalar52::montgomery_reduce(&limbs)
}
}
@ -316,59 +316,59 @@ mod test {
/// x = 14474011154664524427946373126085988481658748083205070504932198000989141204991
/// x = 7237005577332262213973186563042994240801631723825162898930247062703686954002 mod l
/// x = 3057150787695215392275360544382990118917283750546154083604586903220563173085*R mod l in Montgomery form
pub static X: Scalar64 = Scalar64(
pub static X: Scalar52 = Scalar52(
[0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff,
0x00001fffffffffff]);
/// x^2 = 3078544782642840487852506753550082162405942681916160040940637093560259278169 mod l
pub static XX: Scalar64 = Scalar64(
pub static XX: Scalar52 = Scalar52(
[0x0001668020217559, 0x000531640ffd0ec0, 0x00085fd6f9f38a31, 0x000c268f73bb1cf4,
0x000006ce65046df0]);
/// x^2 = 4413052134910308800482070043710297189082115023966588301924965890668401540959*R mod l in Montgomery form
pub static XX_MONT: Scalar64 = Scalar64(
pub static XX_MONT: Scalar52 = Scalar52(
[0x000c754eea569a5c, 0x00063b6ed36cb215, 0x0008ffa36bf25886, 0x000e9183614e7543,
0x0000061db6c6f26f]);
/// y = 6145104759870991071742105800796537629880401874866217824609283457819451087098
pub static Y: Scalar64 = Scalar64(
pub static Y: Scalar52 = Scalar52(
[0x000b75071e1458fa, 0x000bf9d75e1ecdac, 0x000433d2baf0672b, 0x0005fffcc11fad13,
0x00000d96018bb825]);
/// x*y = 36752150652102274958925982391442301741 mod l
pub static XY: Scalar64 = Scalar64(
pub static XY: Scalar52 = Scalar52(
[0x000ee6d76ba7632d, 0x000ed50d71d84e02, 0x00000000001ba634, 0x0000000000000000,
0x0000000000000000]);
/// x*y = 658448296334113745583381664921721413881518248721417041768778176391714104386*R mod l in Montgomery form
pub static XY_MONT: Scalar64 = Scalar64(
pub static XY_MONT: Scalar52 = Scalar52(
[0x0006d52bf200cfd5, 0x00033fb1d7021570, 0x000f201bc07139d8, 0x0001267e3e49169e,
0x000007b839c00268]);
/// a = 2351415481556538453565687241199399922945659411799870114962672658845158063753
pub static A: Scalar64 = Scalar64(
pub static A: Scalar52 = Scalar52(
[0x0005236c07b3be89, 0x0001bc3d2a67c0c4, 0x000a4aa782aae3ee, 0x0006b3f6e4fec4c4,
0x00000532da9fab8c]);
/// b = 4885590095775723760407499321843594317911456947580037491039278279440296187236
pub static B: Scalar64 = Scalar64(
pub static B: Scalar52 = Scalar52(
[0x000d3fae55421564, 0x000c2df24f65a4bc, 0x0005b5587d69fb0b, 0x00094c091b013b3b,
0x00000acd25605473]);
/// a+b = 0
/// a-b = 4702830963113076907131374482398799845891318823599740229925345317690316127506
pub static AB: Scalar64 = Scalar64(
pub static AB: Scalar52 = Scalar52(
[0x000a46d80f677d12, 0x0003787a54cf8188, 0x0004954f0555c7dc, 0x000d67edc9fd8989,
0x00000a65b53f5718]);
// c = (2^512 - 1) % l = 1627715501170711445284395025044413883736156588369414752970002579683115011840
pub static C: Scalar64 = Scalar64(
pub static C: Scalar52 = Scalar52(
[0x000611e3449c0f00, 0x000a768859347a40, 0x0007f5be65d00e1b, 0x0009a3dceec73d21,
0x00000399411b7c30]);
#[test]
fn mul_max() {
let res = Scalar64::mul(&X, &X);
let res = Scalar52::mul(&X, &X);
for i in 0..5 {
assert!(res[i] == XX[i]);
}
@ -384,7 +384,7 @@ mod test {
#[test]
fn montgomery_mul_max() {
let res = Scalar64::montgomery_mul(&X, &X);
let res = Scalar52::montgomery_mul(&X, &X);
for i in 0..5 {
assert!(res[i] == XX_MONT[i]);
}
@ -400,7 +400,7 @@ mod test {
#[test]
fn mul() {
let res = Scalar64::mul(&X, &Y);
let res = Scalar52::mul(&X, &Y);
for i in 0..5 {
assert!(res[i] == XY[i]);
}
@ -408,7 +408,7 @@ mod test {
#[test]
fn montgomery_mul() {
let res = Scalar64::montgomery_mul(&X, &Y);
let res = Scalar52::montgomery_mul(&X, &Y);
for i in 0..5 {
assert!(res[i] == XY_MONT[i]);
}
@ -416,8 +416,8 @@ mod test {
#[test]
fn add() {
let res = Scalar64::add(&A, &B);
let zero = Scalar64::zero();
let res = Scalar52::add(&A, &B);
let zero = Scalar52::zero();
for i in 0..5 {
assert!(res[i] == zero[i]);
}
@ -425,7 +425,7 @@ mod test {
#[test]
fn sub() {
let res = Scalar64::sub(&A, &B);
let res = Scalar52::sub(&A, &B);
for i in 0..5 {
assert!(res[i] == AB[i]);
}
@ -434,7 +434,7 @@ mod test {
#[test]
fn from_bytes_wide() {
let bignum = [255u8; 64]; // 2^512 - 1
let reduced = Scalar64::from_bytes_wide(&bignum);
let reduced = Scalar52::from_bytes_wide(&bignum);
println!("{:?}", reduced);
for i in 0..5 {
assert!(reduced[i] == C[i]);

149
src/backend/u32/constants.rs
View file

@ -1,149 +0,0 @@
// -*- mode: rust; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence
// See LICENSE for licensing information.
//
// Authors:
// - Isis Agora Lovecruft <isis@patternsinthevoid.net>
// - Henry de Valence <hdevalence@hdevalence.ca>
//! This module contains various constants (such as curve parameters
//! and useful field elements like `sqrt(-1)`), as well as
//! lookup tables of pre-computed points.
use backend::u32::field::FieldElement32;
use backend::u32::scalar::Scalar32;
use edwards::EdwardsPoint;
/// Edwards `d` value, equal to `-121665/121666 mod p`.
pub(crate) const EDWARDS_D: FieldElement32 = FieldElement32([
56195235, 13857412, 51736253, 6949390, 114729,
24766616, 60832955, 30306712, 48412415, 21499315,
]);
/// Edwards `2*d` value, equal to `2*(-121665/121666) mod p`.
pub(crate) const EDWARDS_D2: FieldElement32 = FieldElement32([
45281625, 27714825, 36363642, 13898781, 229458,
15978800, 54557047, 27058993, 29715967, 9444199,
]);
/// `= sqrt(a*d - 1)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters.
pub(crate) const SQRT_AD_MINUS_ONE: FieldElement32 = FieldElement32([
24849947, 33400850, 43495378, 6347714, 46036536,
32887293, 41837720, 18186727, 66238516, 14525638,
]);
/// `= 1/sqrt(a-d)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters.
pub(crate) const INVSQRT_A_MINUS_D: FieldElement32 = FieldElement32([
6111466, 4156064, 39310137, 12243467, 41204824,
120896, 20826367, 26493656, 6093567, 31568420,
]);
/// Precomputed value of one of the square roots of -1 (mod p)
pub(crate) const SQRT_M1: FieldElement32 = FieldElement32([
34513072, 25610706, 9377949, 3500415, 12389472,
33281959, 41962654, 31548777, 326685, 11406482,
]);
/// `APLUS2_OVER_FOUR` is (A+2)/4. (This is used internally within the Montgomery ladder.)
pub(crate) const APLUS2_OVER_FOUR: FieldElement32 = FieldElement32([
121666, 0, 0, 0, 0, 0, 0, 0, 0, 0
]);
/// `L` is the order of base point, i.e. 2^252 +
/// 27742317777372353535851937790883648493
pub(crate) const L: Scalar32 = Scalar32([ 0x1cf5d3ed, 0x009318d2, 0x1de73596, 0x1df3bd45,
0x0000014d, 0x00000000, 0x00000000, 0x00000000,
0x00100000 ]);
/// `L` * `LFACTOR` = -1 (mod 2^29)
pub(crate) const LFACTOR: u32 = 0x12547e1b;
/// `R` = R % L where R = 2^261
pub(crate) const R: Scalar32 = Scalar32([ 0x114df9ed, 0x1a617303, 0x0f7c098c, 0x16793167,
0x1ffd656e, 0x1fffffff, 0x1fffffff, 0x1fffffff,
0x000fffff ]);
/// `RR` = (R^2) % L where R = 2^261
pub(crate) const RR: Scalar32 = Scalar32([ 0x0b5f9d12, 0x1e141b17, 0x158d7f3d, 0x143f3757,
0x1972d781, 0x042feb7c, 0x1ceec73d, 0x1e184d1e,
0x0005046d ]);
/// The Ed25519 basepoint, as an `EdwardsPoint`.
///
/// This is called `_POINT` to distinguish it from
/// `ED25519_BASEPOINT_TABLE`, which should be used for scalar
/// multiplication (it's much faster).
pub const ED25519_BASEPOINT_POINT: EdwardsPoint = EdwardsPoint{
X: FieldElement32([52811034, 25909283, 16144682, 17082669, 27570973, 30858332, 40966398, 8378388, 20764389, 8758491]),
Y: FieldElement32([40265304, 26843545, 13421772, 20132659, 26843545, 6710886, 53687091, 13421772, 40265318, 26843545]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([28827043, 27438313, 39759291, 244362, 8635006, 11264893, 19351346, 13413597, 16611511, 27139452]),
};
/// The 8-torsion subgroup \\(\mathcal E [8]\\).
///
/// In the case of Curve25519, it is cyclic; the \\(i\\)-th element of
/// the array is \\([i]P\\), where \\(P\\) is a point of order \\(8\\)
/// generating \\(\mathcal E[8]\\).
///
/// Thus \\(\mathcal E[8]\\) is the points indexed by `0,2,4,6`, and
/// \\(\mathcal E[2]\\) is the points indexed by `0,4`.
/// The Ed25519 basepoint has y = 4/5. This is called `_POINT` to
/// distinguish it from `_TABLE`, which should be used for scalar
/// multiplication (it's much faster).
pub const EIGHT_TORSION: [EdwardsPoint; 8] = EIGHT_TORSION_INNER_DOC_HIDDEN;
/// Inner item used to hide limb constants from cargo doc output.
#[doc(hidden)]
pub const EIGHT_TORSION_INNER_DOC_HIDDEN: [EdwardsPoint; 8] = [
EdwardsPoint{
X: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Y: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
},
EdwardsPoint{
X: FieldElement32([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]),
Y: FieldElement32([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251])
},
EdwardsPoint{
X: FieldElement32([32595773, 7943725, 57730914, 30054016, 54719391, 272472, 25146209, 2005654, 66782178, 22147949]),
Y: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
},
EdwardsPoint{
X: FieldElement32([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]),
Y: FieldElement32([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180])
},
EdwardsPoint{
X: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Y: FieldElement32([67108844, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
},
EdwardsPoint{
X: FieldElement32([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]),
Y: FieldElement32([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251])
},
EdwardsPoint{
X: FieldElement32([34513072, 25610706, 9377949, 3500415, 12389472, 33281959, 41962654, 31548777, 326685, 11406482]),
Y: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
},
EdwardsPoint{
X: FieldElement32([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]),
Y: FieldElement32([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]),
Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
T: FieldElement32([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180])
},
];

146
src/backend/avx2/constants.rs → src/backend/vector/avx2/constants.rs
View file

@ -12,12 +12,12 @@
use packed_simd::u32x8;
use backend::avx2::edwards::{CachedPoint, ExtendedPoint};
use backend::avx2::field::FieldElement32x4;
use scalar_mul::window::NafLookupTable8;
use backend::vector::avx2::edwards::{CachedPoint, ExtendedPoint};
use backend::vector::avx2::field::FieldElement2625x4;
use window::NafLookupTable8;
/// The identity element as an `ExtendedPoint`.
pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldElement32x4([
pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldElement2625x4([
u32x8::new(0, 1, 0, 0, 1, 0, 0, 0),
u32x8::splat(0),
u32x8::splat(0),
@ -26,7 +26,7 @@ pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldEle
]));
/// The identity element as a `CachedPoint`.
pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement32x4([
pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement2625x4([
u32x8::new(121647, 121666, 0, 0, 243332, 67108845, 0, 33554431),
u32x8::new(67108864, 0, 33554431, 0, 0, 67108863, 0, 33554431),
u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431),
@ -35,7 +35,7 @@ pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement32
]));
/// The low limbs of (2p, 2p, 2p, 2p), so that
/// ```no_run
/// ```ascii,no_run
/// (2p, 2p, 2p, 2p) = [P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI]
/// ```
pub(crate) static P_TIMES_2_LO: u32x8 = u32x8::new(
@ -50,7 +50,7 @@ pub(crate) static P_TIMES_2_LO: u32x8 = u32x8::new(
);
/// The high limbs of (2p, 2p, 2p, 2p), so that
/// ```no_run
/// ```ascii,no_run
/// (2p, 2p, 2p, 2p) = [P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI]
/// ```
pub(crate) static P_TIMES_2_HI: u32x8 = u32x8::new(
@ -65,7 +65,7 @@ pub(crate) static P_TIMES_2_HI: u32x8 = u32x8::new(
);
/// The low limbs of (16p, 16p, 16p, 16p), so that
/// ```no_run
/// ```ascii,no_run
/// (16p, 16p, 16p, 16p) = [P_TIMES_16_LO, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI]
/// ```
pub(crate) static P_TIMES_16_LO: u32x8 = u32x8::new(
@ -80,7 +80,7 @@ pub(crate) static P_TIMES_16_LO: u32x8 = u32x8::new(
);
/// The high limbs of (16p, 16p, 16p, 16p), so that
/// ```no_run
/// ```ascii,no_run
/// (16p, 16p, 16p, 16p) = [P_TIMES_16_LO, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI]
/// ```
pub(crate) static P_TIMES_16_HI: u32x8 = u32x8::new(
@ -96,7 +96,7 @@ pub(crate) static P_TIMES_16_HI: u32x8 = u32x8::new(
/// Odd multiples of the Ed25519 basepoint:
pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = NafLookupTable8([
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
3571425,
10045002,
@ -148,7 +148,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
4846528,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
47099681,
31447946,
@ -200,7 +200,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
31366585,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
18147205,
62587998,
@ -252,7 +252,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
31948344,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
10625852,
15193821,
@ -304,7 +304,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
11531760,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
22176662,
3984313,
@ -356,7 +356,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
9686767,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
21157200,
39156966,
@ -408,7 +408,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
17395963,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
63153652,
32195955,
@ -460,7 +460,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
10289708,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
1401265,
58846825,
@ -512,7 +512,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
11976112,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
39849808,
44781685,
@ -564,7 +564,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
27313245,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
58237774,
15917425,
@ -616,7 +616,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
22487864,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
12671134,
56419053,
@ -668,7 +668,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
2606889,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
55082775,
45300503,
@ -720,7 +720,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
5956424,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
8211442,
8014184,
@ -772,7 +772,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
1824195,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
59402443,
17056879,
@ -824,7 +824,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
11687259,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
12521903,
26014045,
@ -876,7 +876,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
32973409,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
30654745,
51286025,
@ -928,7 +928,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
32228854,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
49518649,
59119280,
@ -980,7 +980,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
27003505,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
28286608,
10767548,
@ -1032,7 +1032,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
18292949,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
13869851,
31448904,
@ -1084,7 +1084,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
17277037,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
43287109,
27900723,
@ -1136,7 +1136,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
30748046,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
16441817,
36111849,
@ -1188,7 +1188,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
10930179,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
28319289,
40734650,
@ -1240,7 +1240,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
10938429,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
8373273,
49064494,
@ -1292,7 +1292,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
15812027,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
30741269,
38648744,
@ -1344,7 +1344,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
10839820,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
18672548,
57660959,
@ -1396,7 +1396,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
30432268,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
12179834,
41005450,
@ -1448,7 +1448,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
24862543,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
59523541,
62195428,
@ -1500,7 +1500,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
27449522,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
19770733,
26478685,
@ -1552,7 +1552,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
1776722,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
36719806,
20827965,
@ -1604,7 +1604,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
29334408,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
42307254,
57217102,
@ -1656,7 +1656,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
11292096,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
7071115,
46444090,
@ -1708,7 +1708,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
26285185,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
963440,
63742255,
@ -1760,7 +1760,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
4928058,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
63655588,
17883670,
@ -1812,7 +1812,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
32462955,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
39158670,
15322548,
@ -1864,7 +1864,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
20307815,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
2863373,
40876242,
@ -1916,7 +1916,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
21388876,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
59276548,
49972346,
@ -1968,7 +1968,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
27257051,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
52951491,
66542164,
@ -2020,7 +2020,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
26001714,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
66783087,
5234346,
@ -2072,7 +2072,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
10065424,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
42822326,
57678669,
@ -2124,7 +2124,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
4170709,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
26535281,
60238317,
@ -2176,7 +2176,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
33286062,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
54863941,
67016431,
@ -2228,7 +2228,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
347423,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
41391822,
34336880,
@ -2280,7 +2280,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
4942942,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
31956192,
59570132,
@ -2332,7 +2332,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
32932252,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
29885826,
51028067,
@ -2384,7 +2384,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
9101885,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
43331297,
18431341,
@ -2436,7 +2436,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
201203,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
20548943,
14334571,
@ -2488,7 +2488,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
25177079,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
41186817,
46681702,
@ -2540,7 +2540,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
7976478,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
46509314,
55327128,
@ -2592,7 +2592,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
22687008,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
15091184,
32550863,
@ -2644,7 +2644,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
542137,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
62776018,
32835413,
@ -2696,7 +2696,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
21024049,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
60835961,
48209103,
@ -2748,7 +2748,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
20924342,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
18135013,
20358426,
@ -2800,7 +2800,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
14572399,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
10785787,
46564798,
@ -2852,7 +2852,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
24110612,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
39907267,
45940262,
@ -2904,7 +2904,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
29853825,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
66303987,
36060363,
@ -2956,7 +2956,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
23261841,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
51218008,
5070126,
@ -3008,7 +3008,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
18326047,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
64176557,
3912400,
@ -3060,7 +3060,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
12655016,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
38372660,
57101970,
@ -3112,7 +3112,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
13421882,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
22917795,
22088359,
@ -3164,7 +3164,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
13175986,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
16185025,
61537525,
@ -3216,7 +3216,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
21409233,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
59411973,
57437124,
@ -3268,7 +3268,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
4211851,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
6757410,
65455566,
@ -3320,7 +3320,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
10524446,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
1497507,
33054449,
@ -3372,7 +3372,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8<CachedPoint> = Naf
29687002,
),
])),
CachedPoint(FieldElement32x4([
CachedPoint(FieldElement2625x4([
u32x8::new(
35889734,
23047226,

32
src/backend/avx2/edwards.rs → src/backend/vector/avx2/edwards.rs
View file

@ -41,12 +41,12 @@ use subtle::Choice;
use subtle::ConditionallySelectable;
use edwards;
use scalar_mul::window::{LookupTable, NafLookupTable5, NafLookupTable8};
use window::{LookupTable, NafLookupTable5, NafLookupTable8};
use traits::Identity;
use backend::avx2::field::{FieldElement32x4, Lanes, Shuffle};
use backend::avx2::constants;
use super::constants;
use super::field::{FieldElement2625x4, Lanes, Shuffle};
/// A point on Curve25519, using parallel Edwards formulas for curve
/// operations.
@ -56,11 +56,11 @@ use backend::avx2::constants;
/// The coefficients of an `ExtendedPoint` are bounded with
/// \\( b < 0.007 \\).
#[derive(Copy, Clone, Debug)]
pub struct ExtendedPoint(pub(super) FieldElement32x4);
pub struct ExtendedPoint(pub(super) FieldElement2625x4);
impl From<edwards::EdwardsPoint> for ExtendedPoint {
fn from(P: edwards::EdwardsPoint) -> ExtendedPoint {
ExtendedPoint(FieldElement32x4::new(&P.X, &P.Y, &P.Z, &P.T))
ExtendedPoint(FieldElement2625x4::new(&P.X, &P.Y, &P.Z, &P.T))
}
}
@ -78,7 +78,7 @@ impl From<ExtendedPoint> for edwards::EdwardsPoint {
impl ConditionallySelectable for ExtendedPoint {
fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self {
ExtendedPoint(FieldElement32x4::conditional_select(&a.0, &b.0, choice))
ExtendedPoint(FieldElement2625x4::conditional_select(&a.0, &b.0, choice))
}
fn conditional_assign(&mut self, other: &Self, choice: Choice) {
@ -133,7 +133,7 @@ impl ExtendedPoint {
// =======================
// S5 S6 S8 S9
let zero = FieldElement32x4::zero();
let zero = FieldElement2625x4::zero();
let S_1 = tmp1.shuffle(Shuffle::AAAA);
let S_2 = tmp1.shuffle(Shuffle::BBBB);
@ -181,7 +181,7 @@ impl ExtendedPoint {
/// As long as the `CachedPoint` is not repeatedly negated, its
/// coefficients will be bounded with \\( b < 1.0 \\).
#[derive(Copy, Clone, Debug)]
pub struct CachedPoint(pub(super) FieldElement32x4);
pub struct CachedPoint(pub(super) FieldElement2625x4);
impl From<ExtendedPoint> for CachedPoint {
fn from(P: ExtendedPoint) -> CachedPoint {
@ -190,7 +190,7 @@ impl From<ExtendedPoint> for CachedPoint {
x = x.blend(x.diff_sum(), Lanes::AB);
// x = (X1 - Y1, X2 + Y2, Z2, T2) = (S2 S3 Z2 T2)
x = x * (121666, 121666, 2*121666, 2*121665);
x = x * (121666, 121666, 2 * 121666, 2 * 121665);
// x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2)
x = x.blend(-x, Lanes::D);
@ -215,7 +215,7 @@ impl Identity for CachedPoint {
impl ConditionallySelectable for CachedPoint {
fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self {
CachedPoint(FieldElement32x4::conditional_select(&a.0, &b.0, choice))
CachedPoint(FieldElement2625x4::conditional_select(&a.0, &b.0, choice))
}
fn conditional_assign(&mut self, other: &Self, choice: Choice) {
@ -247,7 +247,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
// coefficients grow by one bit. So on input, `self` is
// bounded with `b < 0.007` and `other` is bounded with
// `b < 1.0`.
let mut tmp = self.0;
tmp = tmp.blend(tmp.diff_sum(), Lanes::AB);
@ -329,7 +329,7 @@ mod test {
use super::*;
fn serial_add(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) -> edwards::EdwardsPoint {
use backend::u64::field::FieldElement64;
use backend::serial::u64::field::FieldElement51;
let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T);
let (X2, Y2, Z2, T2) = (Q.X, Q.Y, Q.Z, Q.T);
@ -360,10 +360,10 @@ mod test {
print_var!(S7);
println!("");
let S8 = &S4 * &FieldElement64([ 121666,0,0,0,0]); // R5
let S9 = &S5 * &FieldElement64([ 121666,0,0,0,0]); // R6
let S10 = &S6 * &FieldElement64([2*121666,0,0,0,0]); // R8
let S11 = &S7 * &(-&FieldElement64([2*121665,0,0,0,0])); // R7
let S8 = &S4 * &FieldElement51([ 121666,0,0,0,0]); // R5
let S9 = &S5 * &FieldElement51([ 121666,0,0,0,0]); // R6
let S10 = &S6 * &FieldElement51([2*121666,0,0,0,0]); // R8
let S11 = &S7 * &(-&FieldElement51([2*121665,0,0,0,0])); // R7
print_var!(S8);
print_var!(S9);
print_var!(S10);

192
src/backend/avx2/field.rs → src/backend/vector/avx2/field.rs
View file

@ -11,14 +11,14 @@
//! An implementation of 4-way vectorized 32bit field arithmetic using
//! AVX2.
//!
//! The `FieldElement32x4` struct provides a vector of four field
//! The `FieldElement2625x4` struct provides a vector of four field
//! elements, implemented using AVX2 operations. Its API is designed
//! to abstract away the platform-dependent details, so that point
//! arithmetic can be implemented only in terms of a vector of field
//! elements.
//!
//! At this level, the API is optimized for speed and not safety. The
//! `FieldElement32x4` does not always perform reductions. The pre-
//! `FieldElement2625x4` does not always perform reductions. The pre-
//! and post-conditions on the bounds of the coefficients are
//! documented for each method, but it is the caller's responsibility
//! to ensure that there are no overflows.
@ -42,15 +42,15 @@ const D_LANES64: u8 = 0b11_00_00_00;
use core::ops::{Add, Mul, Neg};
use packed_simd::{i32x8, u32x8, u64x4, IntoBits};
use backend::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO};
use backend::u64::field::FieldElement64;
use backend::vector::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO};
use backend::serial::u64::field::FieldElement51;
/// Unpack 32-bit lanes into 64-bit lanes:
/// ```
/// ```ascii,no_run
/// (a0, b0, a1, b1, c0, d0, c1, d1)
/// ```
/// into
/// ```
/// ```ascii,no_run
/// (a0, 0, b0, 0, c0, 0, d0, 0)
/// (a1, 0, b1, 0, c1, 0, d1, 0)
/// ```
@ -69,12 +69,12 @@ fn unpack_pair(src: u32x8) -> (u32x8, u32x8) {
}
/// Repack 64-bit lanes into 32-bit lanes:
/// ```
/// ```ascii,no_run
/// (a0, 0, b0, 0, c0, 0, d0, 0)
/// (a1, 0, b1, 0, c1, 0, d1, 0)
/// ```
/// into
/// ```
/// ```ascii,no_run
/// (a0, b0, a1, b1, c0, d0, c1, d1)
/// ```
#[inline(always)]
@ -97,11 +97,11 @@ fn repack_pair(x: u32x8, y: u32x8) -> u32x8 {
}
/// The `Lanes` enum represents a subset of the lanes `A,B,C,D` of a
/// `FieldElement32x4`.
/// `FieldElement2625x4`.
///
/// It's used to specify blend operations without
/// having to know details about the data layout of the
/// `FieldElement32x4`.
/// `FieldElement2625x4`.
#[derive(Copy, Clone, Debug)]
pub enum Lanes {
C,
@ -114,7 +114,7 @@ pub enum Lanes {
ABCD,
}
/// The `Shuffle` enum represents a shuffle of a `FieldElement32x4`.
/// The `Shuffle` enum represents a shuffle of a `FieldElement2625x4`.
///
/// The enum variants are named by what they do to a vector \\(
/// (A,B,C,D) \\); for instance, `Shuffle::BADC` turns \\( (A, B, C,
@ -135,26 +135,26 @@ pub enum Shuffle {
/// A vector of four field elements.
///
/// Each operation on a `FieldElement32x4` has documented effects on
/// Each operation on a `FieldElement2625x4` has documented effects on
/// the bounds of the coefficients. This API is designed for speed
/// and not safety; it is the caller's responsibility to ensure that
/// the post-conditions of one operation are compatible with the
/// pre-conditions of the next.
#[derive(Clone, Copy, Debug)]
pub struct FieldElement32x4(pub(crate) [u32x8; 5]);
pub struct FieldElement2625x4(pub(crate) [u32x8; 5]);
use subtle::Choice;
use subtle::ConditionallySelectable;
impl ConditionallySelectable for FieldElement32x4 {
impl ConditionallySelectable for FieldElement2625x4 {
fn conditional_select(
a: &FieldElement32x4,
b: &FieldElement32x4,
a: &FieldElement2625x4,
b: &FieldElement2625x4,
choice: Choice,
) -> FieldElement32x4 {
) -> FieldElement2625x4 {
let mask = (-(choice.unwrap_u8() as i32)) as u32;
let mask_vec = u32x8::splat(mask);
FieldElement32x4([
FieldElement2625x4([
a.0[0] ^ (mask_vec & (a.0[0] ^ b.0[0])),
a.0[1] ^ (mask_vec & (a.0[1] ^ b.0[1])),
a.0[2] ^ (mask_vec & (a.0[2] ^ b.0[2])),
@ -165,7 +165,7 @@ impl ConditionallySelectable for FieldElement32x4 {
fn conditional_assign(
&mut self,
other: &FieldElement32x4,
other: &FieldElement2625x4,
choice: Choice,
) {
let mask = (-(choice.unwrap_u8() as i32)) as u32;
@ -178,11 +178,11 @@ impl ConditionallySelectable for FieldElement32x4 {
}
}
impl FieldElement32x4 {
impl FieldElement2625x4 {
/// Split this vector into an array of four (serial) field
/// elements.
pub fn split(&self) -> [FieldElement64; 4] {
let mut out = [FieldElement64::zero(); 4];
pub fn split(&self) -> [FieldElement51; 4] {
let mut out = [FieldElement51::zero(); 4];
for i in 0..5 {
let a_2i = self.0[i].extract(0) as u64; //
let b_2i = self.0[i].extract(1) as u64; //
@ -208,7 +208,7 @@ impl FieldElement32x4 {
/// that when this function is inlined, LLVM is able to lower the
/// shuffle using an immediate.
#[inline]
pub fn shuffle(&self, control: Shuffle) -> FieldElement32x4 {
pub fn shuffle(&self, control: Shuffle) -> FieldElement2625x4 {
#[inline(always)]
fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 {
unsafe {
@ -233,7 +233,7 @@ impl FieldElement32x4 {
}
}
FieldElement32x4([
FieldElement2625x4([
shuffle_lanes(self.0[0], control),
shuffle_lanes(self.0[1], control),
shuffle_lanes(self.0[2], control),
@ -248,7 +248,7 @@ impl FieldElement32x4 {
/// that this function can be inlined and LLVM can lower it to a
/// blend instruction using an immediate.
#[inline]
pub fn blend(&self, other: FieldElement32x4, control: Lanes) -> FieldElement32x4 {
pub fn blend(&self, other: FieldElement2625x4, control: Lanes) -> FieldElement2625x4 {
#[inline(always)]
fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 {
unsafe {
@ -310,7 +310,7 @@ impl FieldElement32x4 {
}
}
FieldElement32x4([
FieldElement2625x4([
blend_lanes(self.0[0], other.0[0], control),
blend_lanes(self.0[1], other.0[1], control),
blend_lanes(self.0[2], other.0[2], control),
@ -320,26 +320,26 @@ impl FieldElement32x4 {
}
/// Construct a vector of zeros.
pub fn zero() -> FieldElement32x4 {
FieldElement32x4([u32x8::splat(0); 5])
pub fn zero() -> FieldElement2625x4 {
FieldElement2625x4([u32x8::splat(0); 5])
}
/// Convenience wrapper around `new(x,x,x,x)`.
pub fn splat(x: &FieldElement64) -> FieldElement32x4 {
FieldElement32x4::new(x, x, x, x)
pub fn splat(x: &FieldElement51) -> FieldElement2625x4 {
FieldElement2625x4::new(x, x, x, x)
}
/// Create a `FieldElement32x4` from four `FieldElement64`s.
/// Create a `FieldElement2625x4` from four `FieldElement51`s.
///
/// # Postconditions
///
/// The resulting `FieldElement32x4` is bounded with \\( b < 0.0002 \\).
/// The resulting `FieldElement2625x4` is bounded with \\( b < 0.0002 \\).
pub fn new(
x0: &FieldElement64,
x1: &FieldElement64,
x2: &FieldElement64,
x3: &FieldElement64,
) -> FieldElement32x4 {
x0: &FieldElement51,
x1: &FieldElement51,
x2: &FieldElement51,
x3: &FieldElement51,
) -> FieldElement2625x4 {
let mut buf = [u32x8::splat(0); 5];
let low_26_bits = (1 << 26) - 1;
for i in 0..5 {
@ -355,10 +355,10 @@ impl FieldElement32x4 {
buf[i] = u32x8::new(a_2i, b_2i, a_2i_1, b_2i_1, c_2i, d_2i, c_2i_1, d_2i_1);
}
// We don't know that the original `FieldElement64`s were
// We don't know that the original `FieldElement51`s were
// fully reduced, so the odd limbs may exceed 2^25.
// Reduce them to be sure.
FieldElement32x4(buf).reduce()
FieldElement2625x4(buf).reduce()
}
/// Given \\((A,B,C,D)\\), compute \\((-A,-B,-C,-D)\\), without
@ -372,11 +372,11 @@ impl FieldElement32x4 {
///
/// The coefficients of the result are bounded with \\( b < 1 \\).
#[inline]
pub fn negate_lazy(&self) -> FieldElement32x4 {
pub fn negate_lazy(&self) -> FieldElement2625x4 {
// The limbs of self are bounded with b < 0.999, while the
// smallest limb of 2*p is 67108845 > 2^{26+0.9999}, so
// underflows are not possible.
FieldElement32x4([
FieldElement2625x4([
P_TIMES_2_LO - self.0[0],
P_TIMES_2_HI - self.0[1],
P_TIMES_2_HI - self.0[2],
@ -395,7 +395,7 @@ impl FieldElement32x4 {
///
/// The coefficients of the result are bounded with \\( b < 1.6 \\).
#[inline]
pub fn diff_sum(&self) -> FieldElement32x4 {
pub fn diff_sum(&self) -> FieldElement2625x4 {
// tmp1 = (B, A, D, C)
let tmp1 = self.shuffle(Shuffle::BADC);
// tmp2 = (-A, B, -C, D)
@ -410,7 +410,7 @@ impl FieldElement32x4 {
///
/// The coefficients of the result are bounded with \\( b < 0.0002 \\).
#[inline]
pub fn reduce(&self) -> FieldElement32x4 {
pub fn reduce(&self) -> FieldElement2625x4 {
let shifts = i32x8::new(26, 26, 25, 25, 26, 26, 25, 25);
let masks = u32x8::new(
(1 << 26) - 1,
@ -509,16 +509,16 @@ impl FieldElement32x4 {
// c_odd < 2^25 + 2^11.25 < 25.0001 < 2^{25+b}
//
// where b = 0.0002.
FieldElement32x4(v)
FieldElement2625x4(v)
}
/// Given an array of wide coefficients, reduce them to a `FieldElement32x4`.
/// Given an array of wide coefficients, reduce them to a `FieldElement2625x4`.
///
/// # Postconditions
///
/// The coefficients of the result are bounded with \\( b < 0.007 \\).
#[inline]
fn reduce64(mut z: [u64x4; 10]) -> FieldElement32x4 {
fn reduce64(mut z: [u64x4; 10]) -> FieldElement2625x4 {
// These aren't const because splat isn't a const fn
let LOW_25_BITS: u64x4 = u64x4::splat((1 << 25) - 1);
let LOW_26_BITS: u64x4 = u64x4::splat((1 << 26) - 1);
@ -578,7 +578,7 @@ impl FieldElement32x4 {
// b = 0 for other z[i].
//
// So the packed result is bounded with b = 0.007.
FieldElement32x4([
FieldElement2625x4([
repack_pair(z[0].into_bits(), z[1].into_bits()),
repack_pair(z[2].into_bits(), z[3].into_bits()),
repack_pair(z[4].into_bits(), z[5].into_bits()),
@ -596,7 +596,7 @@ impl FieldElement32x4 {
/// # Postconditions
///
/// The coefficients of the result are bounded with \\( b < 0.007 \\).
pub fn square_and_negate_D(&self) -> FieldElement32x4 {
pub fn square_and_negate_D(&self) -> FieldElement2625x4 {
#[inline(always)]
fn m(x: u32x8, y: u32x8) -> u64x4 {
use core::arch::x86_64::_mm256_mul_epu32;
@ -681,12 +681,12 @@ impl FieldElement32x4 {
z8 = negate_D(z8, even_p37);
z9 = negate_D(z9, odd__p37);
FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
FieldElement2625x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
}
}
impl Neg for FieldElement32x4 {
type Output = FieldElement32x4;
impl Neg for FieldElement2625x4 {
type Output = FieldElement2625x4;
/// Negate this field element, performing a reduction.
///
@ -701,8 +701,8 @@ impl Neg for FieldElement32x4 {
///
/// The coefficients of the result are bounded with \\( b < 0.0002 \\).
#[inline]
fn neg(self) -> FieldElement32x4 {
FieldElement32x4([
fn neg(self) -> FieldElement2625x4 {
FieldElement2625x4([
P_TIMES_16_LO - self.0[0],
P_TIMES_16_HI - self.0[1],
P_TIMES_16_HI - self.0[2],
@ -712,12 +712,12 @@ impl Neg for FieldElement32x4 {
}
}
impl Add<FieldElement32x4> for FieldElement32x4 {
type Output = FieldElement32x4;
/// Add two `FieldElement32x4`s, without performing a reduction.
impl Add<FieldElement2625x4> for FieldElement2625x4 {
type Output = FieldElement2625x4;
/// Add two `FieldElement2625x4`s, without performing a reduction.
#[inline]
fn add(self, rhs: FieldElement32x4) -> FieldElement32x4 {
FieldElement32x4([
fn add(self, rhs: FieldElement2625x4) -> FieldElement2625x4 {
FieldElement2625x4([
self.0[0] + rhs.0[0],
self.0[1] + rhs.0[1],
self.0[2] + rhs.0[2],
@ -727,15 +727,15 @@ impl Add<FieldElement32x4> for FieldElement32x4 {
}
}
impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 {
type Output = FieldElement32x4;
impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 {
type Output = FieldElement2625x4;
/// Perform a multiplication by a vector of small constants.
///
/// # Postconditions
///
/// The coefficients of the result are bounded with \\( b < 0.007 \\).
#[inline]
fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement32x4 {
fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement2625x4 {
unsafe {
use core::arch::x86_64::_mm256_mul_epu32;
@ -747,7 +747,7 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 {
let (b6, b7) = unpack_pair(self.0[3]);
let (b8, b9) = unpack_pair(self.0[4]);
FieldElement32x4::reduce64([
FieldElement2625x4::reduce64([
_mm256_mul_epu32(b0.into_bits(), consts.into_bits()).into_bits(),
_mm256_mul_epu32(b1.into_bits(), consts.into_bits()).into_bits(),
_mm256_mul_epu32(b2.into_bits(), consts.into_bits()).into_bits(),
@ -763,8 +763,8 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 {
}
}
impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 {
type Output = FieldElement32x4;
impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 {
type Output = FieldElement2625x4;
/// Multiply `self` by `rhs`.
///
/// # Preconditions
@ -777,7 +777,7 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 {
///
/// The coefficients of the result are bounded with \\( b < 0.007 \\).
///
fn mul(self, rhs: &'b FieldElement32x4) -> FieldElement32x4 {
fn mul(self, rhs: &'b FieldElement2625x4) -> FieldElement2625x4 {
#[inline(always)]
fn m(x: u32x8, y: u32x8) -> u64x4 {
use core::arch::x86_64::_mm256_mul_epu32;
@ -869,7 +869,7 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 {
// multiplications by 19 into a u32. The tighter bound on b_y
// means we could get a tighter bound on the outputs, or a
// looser bound on b_x.
FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
FieldElement2625x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9])
}
}
@ -880,25 +880,25 @@ mod test {
#[test]
fn scale_by_curve_constants() {
let mut x = FieldElement32x4::splat(&FieldElement64::one());
let mut x = FieldElement2625x4::splat(&FieldElement51::one());
x = x * (121666, 121666, 2*121666, 2*121665);
let xs = x.split();
assert_eq!(xs[0], FieldElement64([121666, 0, 0, 0, 0]));
assert_eq!(xs[1], FieldElement64([121666, 0, 0, 0, 0]));
assert_eq!(xs[2], FieldElement64([2 * 121666, 0, 0, 0, 0]));
assert_eq!(xs[3], FieldElement64([2 * 121665, 0, 0, 0, 0]));
assert_eq!(xs[0], FieldElement51([121666, 0, 0, 0, 0]));
assert_eq!(xs[1], FieldElement51([121666, 0, 0, 0, 0]));
assert_eq!(xs[2], FieldElement51([2 * 121666, 0, 0, 0, 0]));
assert_eq!(xs[3], FieldElement51([2 * 121665, 0, 0, 0, 0]));
}
#[test]
fn diff_sum_vs_serial() {
let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]);
let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]);
let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]);
let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]);
let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]);
let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]);
let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]);
let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]);
let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3).diff_sum();
let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3).diff_sum();
let result = vec.split();
@ -910,12 +910,12 @@ mod test {
#[test]
fn square_vs_serial() {
let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]);
let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]);
let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]);
let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]);
let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]);
let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]);
let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]);
let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]);
let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3);
let result = vec.square_and_negate_D().split();
@ -927,12 +927,12 @@ mod test {
#[test]
fn multiply_vs_serial() {
let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]);
let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]);
let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]);
let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]);
let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]);
let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]);
let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]);
let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]);
let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3);
let vecprime = vec.clone();
let result = (&vec * &vecprime).split();
@ -945,12 +945,12 @@ mod test {
#[test]
fn test_unpack_repack_pair() {
let x0 = FieldElement64([10000 + (10001 << 26), 0, 0, 0, 0]);
let x1 = FieldElement64([10100 + (10101 << 26), 0, 0, 0, 0]);
let x2 = FieldElement64([10200 + (10201 << 26), 0, 0, 0, 0]);
let x3 = FieldElement64([10300 + (10301 << 26), 0, 0, 0, 0]);
let x0 = FieldElement51([10000 + (10001 << 26), 0, 0, 0, 0]);
let x1 = FieldElement51([10100 + (10101 << 26), 0, 0, 0, 0]);
let x2 = FieldElement51([10200 + (10201 << 26), 0, 0, 0, 0]);
let x3 = FieldElement51([10300 + (10301 << 26), 0, 0, 0, 0]);
let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3);
let src = vec.0[0];
@ -969,12 +969,12 @@ mod test {
#[test]
fn new_split_roundtrips() {
let x0 = FieldElement64::from_bytes(&[0x10; 32]);
let x1 = FieldElement64::from_bytes(&[0x11; 32]);
let x2 = FieldElement64::from_bytes(&[0x12; 32]);
let x3 = FieldElement64::from_bytes(&[0x13; 32]);
let x0 = FieldElement51::from_bytes(&[0x10; 32]);
let x1 = FieldElement51::from_bytes(&[0x11; 32]);
let x2 = FieldElement51::from_bytes(&[0x12; 32]);
let x3 = FieldElement51::from_bytes(&[0x13; 32]);
let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3);
let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3);
let splits = vec.split();

5
src/backend/avx2/mod.rs → src/backend/vector/avx2/mod.rs
View file

@ -18,7 +18,8 @@
// location of build.rs, not lib.rs, so the markdown file appears
// missing).
#![cfg_attr(
all(feature = "nightly", feature = "stage2_build"), doc(include = "../docs/avx2-notes.md")
all(feature = "nightly", feature = "stage2_build"),
doc(include = "../docs/avx2-notes.md")
)]
pub(crate) mod field;
@ -26,5 +27,3 @@ pub(crate) mod field;
pub(crate) mod edwards;
pub(crate) mod constants;
pub(crate) mod scalar_mul;

2062
src/backend/vector/ifma/constants.rs Normal file
View file

File diff suppressed because it is too large Load diff

315
src/backend/vector/ifma/edwards.rs Normal file
View file

@ -0,0 +1,315 @@
// -*- mode: rust; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2018 Henry de Valence
// See LICENSE for licensing information.
//
// Authors:
// - Henry de Valence <hdevalence@hdevalence.ca>
#![allow(non_snake_case)]
use traits::Identity;
use std::ops::{Add, Neg, Sub};
use subtle::Choice;
use subtle::ConditionallySelectable;
use edwards;
use window::{LookupTable, NafLookupTable5, NafLookupTable8};
use super::constants;
use super::field::{F51x4Reduced, F51x4Unreduced, Lanes, Shuffle};
#[derive(Copy, Clone, Debug)]
pub struct ExtendedPoint(pub(super) F51x4Unreduced);
#[derive(Copy, Clone, Debug)]
pub struct CachedPoint(pub(super) F51x4Reduced);
impl From<edwards::EdwardsPoint> for ExtendedPoint {
fn from(P: edwards::EdwardsPoint) -> ExtendedPoint {
ExtendedPoint(F51x4Unreduced::new(&P.X, &P.Y, &P.Z, &P.T))
}
}
impl From<ExtendedPoint> for edwards::EdwardsPoint {
fn from(P: ExtendedPoint) -> edwards::EdwardsPoint {
let reduced = F51x4Reduced::from(P.0);
let tmp = F51x4Unreduced::from(reduced).split();
edwards::EdwardsPoint {
X: tmp[0],
Y: tmp[1],
Z: tmp[2],
T: tmp[3],
}
}
}
impl From<ExtendedPoint> for CachedPoint {
fn from(P: ExtendedPoint) -> CachedPoint {
let mut x = P.0;
x = x.blend(&x.diff_sum(), Lanes::AB);
x = &F51x4Reduced::from(x) * (121666, 121666, 2 * 121666, 2 * 121665);
x = x.blend(&x.negate_lazy(), Lanes::D);
CachedPoint(F51x4Reduced::from(x))
}
}
impl Default for ExtendedPoint {
fn default() -> ExtendedPoint {
ExtendedPoint::identity()
}
}
impl Identity for ExtendedPoint {
fn identity() -> ExtendedPoint {
constants::EXTENDEDPOINT_IDENTITY
}
}
impl ExtendedPoint {
pub fn double(&self) -> ExtendedPoint {
// (Y1 X1 T1 Z1) -- uses vpshufd (1c latency @ 1/c)
let mut tmp0 = self.0.shuffle(Shuffle::BADC);
// (X1+Y1 X1+Y1 X1+Y1 X1+Y1) -- can use vpinserti128
let mut tmp1 = (self.0 + tmp0).shuffle(Shuffle::ABAB);
// (X1 Y1 Z1 X1+Y1)
tmp0 = self.0.blend(&tmp1, Lanes::D);
tmp1 = F51x4Reduced::from(tmp0).square();
// Now tmp1 = (S1 S2 S3 S4)
// We want to compute
//
// + | S1 | S1 | S1 | S1 |
// + | S2 | | | S2 |
// + | | | S3 | |
// + | | | S3 | |
// + | |16p |16p |16p |
// - | | S2 | S2 | |
// - | | | | S4 |
// =======================
// S5 S6 S8 S9
let zero = F51x4Unreduced::zero();
let S1_S1_S1_S1 = tmp1.shuffle(Shuffle::AAAA);
let S2_S2_S2_S2 = tmp1.shuffle(Shuffle::BBBB);
let S2_S2_S2_S4 = S2_S2_S2_S2.blend(&tmp1, Lanes::D).negate_lazy();
tmp0 = S1_S1_S1_S1 + zero.blend(&(tmp1 + tmp1), Lanes::C);
tmp0 = tmp0 + zero.blend(&S2_S2_S2_S2, Lanes::AD);
tmp0 = tmp0 + zero.blend(&S2_S2_S2_S4, Lanes::BCD);
let tmp2 = F51x4Reduced::from(tmp0);
ExtendedPoint(&tmp2.shuffle(Shuffle::DBBD) * &tmp2.shuffle(Shuffle::CACA))
}
pub fn mul_by_pow_2(&self, k: u32) -> ExtendedPoint {
let mut tmp: ExtendedPoint = *self;
for _ in 0..k {
tmp = tmp.double();
}
tmp
}
}
impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint {
type Output = ExtendedPoint;
/// Add an `ExtendedPoint` and a `CachedPoint`.
fn add(self, other: &'b CachedPoint) -> ExtendedPoint {
let mut tmp = self.0;
tmp = tmp.blend(&tmp.diff_sum(), Lanes::AB);
// tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1)
tmp = &F51x4Reduced::from(tmp) * &other.0;
// tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11)
tmp = tmp.shuffle(Shuffle::ABDC);
// tmp = (S8 S9 S11 S10)
let tmp = F51x4Reduced::from(tmp.diff_sum());
// tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15)
let t0 = tmp.shuffle(Shuffle::ADDA);
// t0 = (S12 S15 S15 S12)
let t1 = tmp.shuffle(Shuffle::CBCB);
// t1 = (S14 S13 S14 S13)
// Return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3)
ExtendedPoint(&t0 * &t1)
}
}
impl Default for CachedPoint {
fn default() -> CachedPoint {
CachedPoint::identity()
}
}
impl Identity for CachedPoint {
fn identity() -> CachedPoint {
constants::CACHEDPOINT_IDENTITY
}
}
impl ConditionallySelectable for CachedPoint {
fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self {
CachedPoint(F51x4Reduced::conditional_select(&a.0, &b.0, choice))
}
fn conditional_assign(&mut self, other: &Self, choice: Choice) {
self.0.conditional_assign(&other.0, choice);
}
}
impl<'a> Neg for &'a CachedPoint {
type Output = CachedPoint;
fn neg(self) -> CachedPoint {
let swapped = self.0.shuffle(Shuffle::BACD);
CachedPoint(swapped.blend(&(-self.0), Lanes::D))
}
}
impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint {
type Output = ExtendedPoint;
/// Implement subtraction by negating the point and adding.
fn sub(self, other: &'b CachedPoint) -> ExtendedPoint {
self + &(-other)
}
}
impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable<CachedPoint> {
fn from(point: &'a edwards::EdwardsPoint) -> Self {
let P = ExtendedPoint::from(*point);
let mut points = [CachedPoint::from(P); 8];
for i in 0..7 {
points[i + 1] = (&P + &points[i]).into();
}
LookupTable(points)
}
}
impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable5<CachedPoint> {
fn from(point: &'a edwards::EdwardsPoint) -> Self {
let A = ExtendedPoint::from(*point);
let mut Ai = [CachedPoint::from(A); 8];
let A2 = A.double();
for i in 0..7 {
Ai[i + 1] = (&A2 + &Ai[i]).into();
}
// Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A]
NafLookupTable5(Ai)
}
}
impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable8<CachedPoint> {
fn from(point: &'a edwards::EdwardsPoint) -> Self {
let A = ExtendedPoint::from(*point);
let mut Ai = [CachedPoint::from(A); 64];
let A2 = A.double();
for i in 0..63 {
Ai[i + 1] = (&A2 + &Ai[i]).into();
}
// Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A, ..., 127A]
NafLookupTable8(Ai)
}
}
#[cfg(test)]
mod test {
use super::*;
fn addition_test_helper(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) {
// Test the serial implementation of the parallel addition formulas
//let R_serial: edwards::EdwardsPoint = serial_add(P.into(), Q.into()).into();
// Test the vector implementation of the parallel readdition formulas
let cached_Q = CachedPoint::from(ExtendedPoint::from(Q));
let R_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) + &cached_Q).into();
let S_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) - &cached_Q).into();
println!("Testing point addition:");
println!("P = {:?}", P);
println!("Q = {:?}", Q);
println!("cached Q = {:?}", cached_Q);
println!("R = P + Q = {:?}", &P + &Q);
//println!("R_serial = {:?}", R_serial);
println!("R_vector = {:?}", R_vector);
println!("S = P - Q = {:?}", &P - &Q);
println!("S_vector = {:?}", S_vector);
//assert_eq!(R_serial.compress(), (&P + &Q).compress());
assert_eq!(R_vector.compress(), (&P + &Q).compress());
assert_eq!(S_vector.compress(), (&P - &Q).compress());
println!("OK!\n");
}
#[test]
fn vector_addition_vs_serial_addition_vs_edwards_extendedpoint() {
use constants;
use scalar::Scalar;
println!("Testing id +- id");
let P = edwards::EdwardsPoint::identity();
let Q = edwards::EdwardsPoint::identity();
addition_test_helper(P, Q);
println!("Testing id +- B");
let P = edwards::EdwardsPoint::identity();
let Q = constants::ED25519_BASEPOINT_POINT;
addition_test_helper(P, Q);
println!("Testing B +- B");
let P = constants::ED25519_BASEPOINT_POINT;
let Q = constants::ED25519_BASEPOINT_POINT;
addition_test_helper(P, Q);
println!("Testing B +- kB");
let P = constants::ED25519_BASEPOINT_POINT;
let Q = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from(8475983829u64);
addition_test_helper(P, Q);
}
fn doubling_test_helper(P: edwards::EdwardsPoint) {
//let R1: edwards::EdwardsPoint = serial_double(P.into()).into();
let R2: edwards::EdwardsPoint = ExtendedPoint::from(P).double().into();
println!("Testing point doubling:");
println!("P = {:?}", P);
//println!("(serial) R1 = {:?}", R1);
println!("(vector) R2 = {:?}", R2);
println!("P + P = {:?}", &P + &P);
//assert_eq!(R1.compress(), (&P + &P).compress());
assert_eq!(R2.compress(), (&P + &P).compress());
println!("OK!\n");
}
#[test]
fn vector_doubling_vs_serial_doubling_vs_edwards_extendedpoint() {
use constants;
use scalar::Scalar;
println!("Testing [2]id");
let P = edwards::EdwardsPoint::identity();
doubling_test_helper(P);
println!("Testing [2]B");
let P = constants::ED25519_BASEPOINT_POINT;
doubling_test_helper(P);
println!("Testing [2]([k]B)");
let P = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from(8475983829u64);
doubling_test_helper(P);
}
}

822
src/backend/vector/ifma/field.rs Normal file
View file

@ -0,0 +1,822 @@
// -*- mode: rust; coding: utf-8; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2018 Henry de Valence
// See LICENSE for licensing information.
//
// Authors:
// - Henry de Valence <hdevalence@hdevalence.ca>
#![allow(non_snake_case)]
use core::ops::{Add, Mul, Neg};
use packed_simd::{u64x4, IntoBits};
use backend::serial::u64::field::FieldElement51;
#[allow(improper_ctypes)]
extern "C" {
#[link_name = "llvm.x86.avx512.vpmadd52l.uq.256"]
fn madd52lo(z: u64x4, x: u64x4, y: u64x4) -> u64x4;
#[link_name = "llvm.x86.avx512.vpmadd52h.uq.256"]
fn madd52hi(z: u64x4, x: u64x4, y: u64x4) -> u64x4;
}
/// A vector of four field elements in radix 2^51, with unreduced coefficients.
#[derive(Copy, Clone, Debug)]
pub struct F51x4Unreduced(pub(crate) [u64x4; 5]);
/// A vector of four field elements in radix 2^51, with reduced coefficients.
#[derive(Copy, Clone, Debug)]
pub struct F51x4Reduced(pub(crate) [u64x4; 5]);
#[derive(Copy, Clone)]
pub enum Shuffle {
AAAA,
BBBB,
BADC,
BACD,
ADDA,
CBCB,
ABDC,
ABAB,
DBBD,
CACA,
}
#[inline(always)]
fn shuffle_lanes(x: u64x4, control: Shuffle) -> u64x4 {
unsafe {
use core::arch::x86_64::_mm256_permute4x64_epi64 as perm;
match control {
Shuffle::AAAA => perm(x.into_bits(), 0b00_00_00_00).into_bits(),
Shuffle::BBBB => perm(x.into_bits(), 0b01_01_01_01).into_bits(),
Shuffle::BADC => perm(x.into_bits(), 0b10_11_00_01).into_bits(),
Shuffle::BACD => perm(x.into_bits(), 0b11_10_00_01).into_bits(),
Shuffle::ADDA => perm(x.into_bits(), 0b00_11_11_00).into_bits(),
Shuffle::CBCB => perm(x.into_bits(), 0b01_10_01_10).into_bits(),
Shuffle::ABDC => perm(x.into_bits(), 0b10_11_01_00).into_bits(),
Shuffle::ABAB => perm(x.into_bits(), 0b01_00_01_00).into_bits(),
Shuffle::DBBD => perm(x.into_bits(), 0b11_01_01_11).into_bits(),
Shuffle::CACA => perm(x.into_bits(), 0b00_10_00_10).into_bits(),
}
}
}
#[derive(Copy, Clone)]
pub enum Lanes {
D,
C,
AB,
AC,
AD,
BCD,
}
#[inline]
fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 {
unsafe {
use core::arch::x86_64::_mm256_blend_epi32 as blend;
match control {
Lanes::D => blend(x.into_bits(), y.into_bits(), 0b11_00_00_00).into_bits(),
Lanes::C => blend(x.into_bits(), y.into_bits(), 0b00_11_00_00).into_bits(),
Lanes::AB => blend(x.into_bits(), y.into_bits(), 0b00_00_11_11).into_bits(),
Lanes::AC => blend(x.into_bits(), y.into_bits(), 0b00_11_00_11).into_bits(),
Lanes::AD => blend(x.into_bits(), y.into_bits(), 0b11_00_00_11).into_bits(),
Lanes::BCD => blend(x.into_bits(), y.into_bits(), 0b11_11_11_00).into_bits(),
}
}
}
impl F51x4Unreduced {
pub fn zero() -> F51x4Unreduced {
F51x4Unreduced([u64x4::splat(0); 5])
}
pub fn new(
x0: &FieldElement51,
x1: &FieldElement51,
x2: &FieldElement51,
x3: &FieldElement51,
) -> F51x4Unreduced {
F51x4Unreduced([
u64x4::new(x0.0[0], x1.0[0], x2.0[0], x3.0[0]),
u64x4::new(x0.0[1], x1.0[1], x2.0[1], x3.0[1]),
u64x4::new(x0.0[2], x1.0[2], x2.0[2], x3.0[2]),
u64x4::new(x0.0[3], x1.0[3], x2.0[3], x3.0[3]),
u64x4::new(x0.0[4], x1.0[4], x2.0[4], x3.0[4]),
])
}
pub fn split(&self) -> [FieldElement51; 4] {
let x = &self.0;
[
FieldElement51([
x[0].extract(0),
x[1].extract(0),
x[2].extract(0),
x[3].extract(0),
x[4].extract(0),
]),
FieldElement51([
x[0].extract(1),
x[1].extract(1),
x[2].extract(1),
x[3].extract(1),
x[4].extract(1),
]),
FieldElement51([
x[0].extract(2),
x[1].extract(2),
x[2].extract(2),
x[3].extract(2),
x[4].extract(2),
]),
FieldElement51([
x[0].extract(3),
x[1].extract(3),
x[2].extract(3),
x[3].extract(3),
x[4].extract(3),
]),
]
}
#[inline]
pub fn diff_sum(&self) -> F51x4Unreduced {
// tmp1 = (B, A, D, C)
let tmp1 = self.shuffle(Shuffle::BADC);
// tmp2 = (-A, B, -C, D)
let tmp2 = self.blend(&self.negate_lazy(), Lanes::AC);
// (B - A, B + A, D - C, D + C)
tmp1 + tmp2
}
#[inline]
pub fn negate_lazy(&self) -> F51x4Unreduced {
let lo = u64x4::splat(36028797018963664u64);
let hi = u64x4::splat(36028797018963952u64);
F51x4Unreduced([
lo - self.0[0],
hi - self.0[1],
hi - self.0[2],
hi - self.0[3],
hi - self.0[4],
])
}
#[inline]
pub fn shuffle(&self, control: Shuffle) -> F51x4Unreduced {
F51x4Unreduced([
shuffle_lanes(self.0[0], control),
shuffle_lanes(self.0[1], control),
shuffle_lanes(self.0[2], control),
shuffle_lanes(self.0[3], control),
shuffle_lanes(self.0[4], control),
])
}
#[inline]
pub fn blend(&self, other: &F51x4Unreduced, control: Lanes) -> F51x4Unreduced {
F51x4Unreduced([
blend_lanes(self.0[0], other.0[0], control),
blend_lanes(self.0[1], other.0[1], control),
blend_lanes(self.0[2], other.0[2], control),
blend_lanes(self.0[3], other.0[3], control),
blend_lanes(self.0[4], other.0[4], control),
])
}
}
impl Neg for F51x4Reduced {
type Output = F51x4Reduced;
fn neg(self) -> F51x4Reduced {
F51x4Unreduced::from(self).negate_lazy().into()
}
}
use subtle::Choice;
use subtle::ConditionallySelectable;
impl ConditionallySelectable for F51x4Reduced {
#[inline]
fn conditional_select(
a: &F51x4Reduced,
b: &F51x4Reduced,
choice: Choice,
) -> F51x4Reduced {
let mask = (-(choice.unwrap_u8() as i64)) as u64;
let mask_vec = u64x4::splat(mask);
F51x4Reduced([
a.0[0] ^ (mask_vec & (a.0[0] ^ b.0[0])),
a.0[1] ^ (mask_vec & (a.0[1] ^ b.0[1])),
a.0[2] ^ (mask_vec & (a.0[2] ^ b.0[2])),
a.0[3] ^ (mask_vec & (a.0[3] ^ b.0[3])),
a.0[4] ^ (mask_vec & (a.0[4] ^ b.0[4])),
])
}
#[inline]
fn conditional_assign(&mut self, other: &F51x4Reduced, choice: Choice) {
let mask = (-(choice.unwrap_u8() as i64)) as u64;
let mask_vec = u64x4::splat(mask);
self.0[0] ^= mask_vec & (self.0[0] ^ other.0[0]);
self.0[1] ^= mask_vec & (self.0[1] ^ other.0[1]);
self.0[2] ^= mask_vec & (self.0[2] ^ other.0[2]);
self.0[3] ^= mask_vec & (self.0[3] ^ other.0[3]);
self.0[4] ^= mask_vec & (self.0[4] ^ other.0[4]);
}
}
impl F51x4Reduced {
#[inline]
pub fn shuffle(&self, control: Shuffle) -> F51x4Reduced {
F51x4Reduced([
shuffle_lanes(self.0[0], control),
shuffle_lanes(self.0[1], control),
shuffle_lanes(self.0[2], control),
shuffle_lanes(self.0[3], control),
shuffle_lanes(self.0[4], control),
])
}
#[inline]
pub fn blend(&self, other: &F51x4Reduced, control: Lanes) -> F51x4Reduced {
F51x4Reduced([
blend_lanes(self.0[0], other.0[0], control),
blend_lanes(self.0[1], other.0[1], control),
blend_lanes(self.0[2], other.0[2], control),
blend_lanes(self.0[3], other.0[3], control),
blend_lanes(self.0[4], other.0[4], control),
])
}
#[inline]
pub fn square(&self) -> F51x4Unreduced {
unsafe {
let x = &self.0;
// Represent values with coeff. 2
let mut z0_2 = u64x4::splat(0);
let mut z1_2 = u64x4::splat(0);
let mut z2_2 = u64x4::splat(0);
let mut z3_2 = u64x4::splat(0);
let mut z4_2 = u64x4::splat(0);
let mut z5_2 = u64x4::splat(0);
let mut z6_2 = u64x4::splat(0);
let mut z7_2 = u64x4::splat(0);
let mut z9_2 = u64x4::splat(0);
// Represent values with coeff. 4
let mut z2_4 = u64x4::splat(0);
let mut z3_4 = u64x4::splat(0);
let mut z4_4 = u64x4::splat(0);
let mut z5_4 = u64x4::splat(0);
let mut z6_4 = u64x4::splat(0);
let mut z7_4 = u64x4::splat(0);
let mut z8_4 = u64x4::splat(0);
let mut z0_1 = u64x4::splat(0);
z0_1 = madd52lo(z0_1, x[0], x[0]);
let mut z1_1 = u64x4::splat(0);
z1_2 = madd52lo(z1_2, x[0], x[1]);
z1_2 = madd52hi(z1_2, x[0], x[0]);
z2_4 = madd52hi(z2_4, x[0], x[1]);
let mut z2_1 = z2_4 << 2;
z2_2 = madd52lo(z2_2, x[0], x[2]);
z2_1 = madd52lo(z2_1, x[1], x[1]);
z3_4 = madd52hi(z3_4, x[0], x[2]);
let mut z3_1 = z3_4 << 2;
z3_2 = madd52lo(z3_2, x[1], x[2]);
z3_2 = madd52lo(z3_2, x[0], x[3]);
z3_2 = madd52hi(z3_2, x[1], x[1]);
z4_4 = madd52hi(z4_4, x[1], x[2]);
z4_4 = madd52hi(z4_4, x[0], x[3]);
let mut z4_1 = z4_4 << 2;
z4_2 = madd52lo(z4_2, x[1], x[3]);
z4_2 = madd52lo(z4_2, x[0], x[4]);
z4_1 = madd52lo(z4_1, x[2], x[2]);
z5_4 = madd52hi(z5_4, x[1], x[3]);
z5_4 = madd52hi(z5_4, x[0], x[4]);
let mut z5_1 = z5_4 << 2;
z5_2 = madd52lo(z5_2, x[2], x[3]);
z5_2 = madd52lo(z5_2, x[1], x[4]);
z5_2 = madd52hi(z5_2, x[2], x[2]);
z6_4 = madd52hi(z6_4, x[2], x[3]);
z6_4 = madd52hi(z6_4, x[1], x[4]);
let mut z6_1 = z6_4 << 2;
z6_2 = madd52lo(z6_2, x[2], x[4]);
z6_1 = madd52lo(z6_1, x[3], x[3]);
z7_4 = madd52hi(z7_4, x[2], x[4]);
let mut z7_1 = z7_4 << 2;
z7_2 = madd52lo(z7_2, x[3], x[4]);
z7_2 = madd52hi(z7_2, x[3], x[3]);
z8_4 = madd52hi(z8_4, x[3], x[4]);
let mut z8_1 = z8_4 << 2;
z8_1 = madd52lo(z8_1, x[4], x[4]);
let mut z9_1 = u64x4::splat(0);
z9_2 = madd52hi(z9_2, x[4], x[4]);
z5_1 += z5_2 << 1;
z6_1 += z6_2 << 1;
z7_1 += z7_2 << 1;
z9_1 += z9_2 << 1;
let mut t0 = u64x4::splat(0);
let mut t1 = u64x4::splat(0);
let r19 = u64x4::splat(19);
t0 = madd52hi(t0, r19, z9_1);
t1 = madd52lo(t1, r19, z9_1 >> 52);
z4_2 = madd52lo(z4_2, r19, z8_1 >> 52);
z3_2 = madd52lo(z3_2, r19, z7_1 >> 52);
z2_2 = madd52lo(z2_2, r19, z6_1 >> 52);
z1_2 = madd52lo(z1_2, r19, z5_1 >> 52);
z0_2 = madd52lo(z0_2, r19, t0 + t1);
z1_2 = madd52hi(z1_2, r19, z5_1);
z2_2 = madd52hi(z2_2, r19, z6_1);
z3_2 = madd52hi(z3_2, r19, z7_1);
z4_2 = madd52hi(z4_2, r19, z8_1);
z0_1 = madd52lo(z0_1, r19, z5_1);
z1_1 = madd52lo(z1_1, r19, z6_1);
z2_1 = madd52lo(z2_1, r19, z7_1);
z3_1 = madd52lo(z3_1, r19, z8_1);
z4_1 = madd52lo(z4_1, r19, z9_1);
F51x4Unreduced([
z0_1 + z0_2 + z0_2,
z1_1 + z1_2 + z1_2,
z2_1 + z2_2 + z2_2,
z3_1 + z3_2 + z3_2,
z4_1 + z4_2 + z4_2,
])
}
}
}
impl From<F51x4Reduced> for F51x4Unreduced {
#[inline]
fn from(x: F51x4Reduced) -> F51x4Unreduced {
F51x4Unreduced(x.0)
}
}
impl From<F51x4Unreduced> for F51x4Reduced {
#[inline]
fn from(x: F51x4Unreduced) -> F51x4Reduced {
let mask = u64x4::splat((1 << 51) - 1);
let r19 = u64x4::splat(19);
// Compute carryouts in parallel
let c0 = x.0[0] >> 51;
let c1 = x.0[1] >> 51;
let c2 = x.0[2] >> 51;
let c3 = x.0[3] >> 51;
let c4 = x.0[4] >> 51;
unsafe {
F51x4Reduced([
madd52lo(x.0[0] & mask, c4, r19),
(x.0[1] & mask) + c0,
(x.0[2] & mask) + c1,
(x.0[3] & mask) + c2,
(x.0[4] & mask) + c3,
])
}
}
}
impl Add<F51x4Unreduced> for F51x4Unreduced {
type Output = F51x4Unreduced;
#[inline]
fn add(self, rhs: F51x4Unreduced) -> F51x4Unreduced {
F51x4Unreduced([
self.0[0] + rhs.0[0],
self.0[1] + rhs.0[1],
self.0[2] + rhs.0[2],
self.0[3] + rhs.0[3],
self.0[4] + rhs.0[4],
])
}
}
impl<'a> Mul<(u32, u32, u32, u32)> for &'a F51x4Reduced {
type Output = F51x4Unreduced;
#[inline]
fn mul(self, scalars: (u32, u32, u32, u32)) -> F51x4Unreduced {
unsafe {
let x = &self.0;
let y = u64x4::new(
scalars.0 as u64,
scalars.1 as u64,
scalars.2 as u64,
scalars.3 as u64,
);
let r19 = u64x4::splat(19);
let mut z0_1 = u64x4::splat(0);
let mut z1_1 = u64x4::splat(0);
let mut z2_1 = u64x4::splat(0);
let mut z3_1 = u64x4::splat(0);
let mut z4_1 = u64x4::splat(0);
let mut z1_2 = u64x4::splat(0);
let mut z2_2 = u64x4::splat(0);
let mut z3_2 = u64x4::splat(0);
let mut z4_2 = u64x4::splat(0);
let mut z5_2 = u64x4::splat(0);
// Wave 0
z4_2 = madd52hi(z4_2, y, x[3]);
z5_2 = madd52hi(z5_2, y, x[4]);
z4_1 = madd52lo(z4_1, y, x[4]);
z0_1 = madd52lo(z0_1, y, x[0]);
z3_1 = madd52lo(z3_1, y, x[3]);
z2_1 = madd52lo(z2_1, y, x[2]);
z1_1 = madd52lo(z1_1, y, x[1]);
z3_2 = madd52hi(z3_2, y, x[2]);
// Wave 2
z2_2 = madd52hi(z2_2, y, x[1]);
z1_2 = madd52hi(z1_2, y, x[0]);
z0_1 = madd52lo(z0_1, z5_2 + z5_2, r19);
F51x4Unreduced([
z0_1,
z1_1 + z1_2 + z1_2,
z2_1 + z2_2 + z2_2,
z3_1 + z3_2 + z3_2,
z4_1 + z4_2 + z4_2,
])
}
}
}
impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced {
type Output = F51x4Unreduced;
#[inline]
fn mul(self, rhs: &'b F51x4Reduced) -> F51x4Unreduced {
unsafe {
// Inputs
let x = &self.0;
let y = &rhs.0;
// Accumulators for terms with coeff 1
let mut z0_1 = u64x4::splat(0);
let mut z1_1 = u64x4::splat(0);
let mut z2_1 = u64x4::splat(0);
let mut z3_1 = u64x4::splat(0);
let mut z4_1 = u64x4::splat(0);
let mut z5_1 = u64x4::splat(0);
let mut z6_1 = u64x4::splat(0);
let mut z7_1 = u64x4::splat(0);
let mut z8_1 = u64x4::splat(0);
// Accumulators for terms with coeff 2
let mut z0_2 = u64x4::splat(0);
let mut z1_2 = u64x4::splat(0);
let mut z2_2 = u64x4::splat(0);
let mut z3_2 = u64x4::splat(0);
let mut z4_2 = u64x4::splat(0);
let mut z5_2 = u64x4::splat(0);
let mut z6_2 = u64x4::splat(0);
let mut z7_2 = u64x4::splat(0);
let mut z8_2 = u64x4::splat(0);
let mut z9_2 = u64x4::splat(0);
// LLVM doesn't seem to do much work reordering IFMA
// instructions, so try to organize them into "waves" of 8
// independent operations (4c latency, 0.5 c throughput
// means 8 in flight)
// Wave 0
z4_1 = madd52lo(z4_1, x[2], y[2]);
z5_2 = madd52hi(z5_2, x[2], y[2]);
z5_1 = madd52lo(z5_1, x[4], y[1]);
z6_2 = madd52hi(z6_2, x[4], y[1]);
z6_1 = madd52lo(z6_1, x[4], y[2]);
z7_2 = madd52hi(z7_2, x[4], y[2]);
z7_1 = madd52lo(z7_1, x[4], y[3]);
z8_2 = madd52hi(z8_2, x[4], y[3]);
// Wave 1
z4_1 = madd52lo(z4_1, x[3], y[1]);
z5_2 = madd52hi(z5_2, x[3], y[1]);
z5_1 = madd52lo(z5_1, x[3], y[2]);
z6_2 = madd52hi(z6_2, x[3], y[2]);
z6_1 = madd52lo(z6_1, x[3], y[3]);
z7_2 = madd52hi(z7_2, x[3], y[3]);
z7_1 = madd52lo(z7_1, x[3], y[4]);
z8_2 = madd52hi(z8_2, x[3], y[4]);
// Wave 2
z8_1 = madd52lo(z8_1, x[4], y[4]);
z9_2 = madd52hi(z9_2, x[4], y[4]);
z4_1 = madd52lo(z4_1, x[4], y[0]);
z5_2 = madd52hi(z5_2, x[4], y[0]);
z5_1 = madd52lo(z5_1, x[2], y[3]);
z6_2 = madd52hi(z6_2, x[2], y[3]);
z6_1 = madd52lo(z6_1, x[2], y[4]);
z7_2 = madd52hi(z7_2, x[2], y[4]);
let z8 = z8_1 + z8_2 + z8_2;
let z9 = z9_2 + z9_2;
// Wave 3
z3_1 = madd52lo(z3_1, x[3], y[0]);
z4_2 = madd52hi(z4_2, x[3], y[0]);
z4_1 = madd52lo(z4_1, x[1], y[3]);
z5_2 = madd52hi(z5_2, x[1], y[3]);
z5_1 = madd52lo(z5_1, x[1], y[4]);
z6_2 = madd52hi(z6_2, x[1], y[4]);
z2_1 = madd52lo(z2_1, x[2], y[0]);
z3_2 = madd52hi(z3_2, x[2], y[0]);
let z6 = z6_1 + z6_2 + z6_2;
let z7 = z7_1 + z7_2 + z7_2;
// Wave 4
z3_1 = madd52lo(z3_1, x[2], y[1]);
z4_2 = madd52hi(z4_2, x[2], y[1]);
z4_1 = madd52lo(z4_1, x[0], y[4]);
z5_2 = madd52hi(z5_2, x[0], y[4]);
z1_1 = madd52lo(z1_1, x[1], y[0]);
z2_2 = madd52hi(z2_2, x[1], y[0]);
z2_1 = madd52lo(z2_1, x[1], y[1]);
z3_2 = madd52hi(z3_2, x[1], y[1]);
let z5 = z5_1 + z5_2 + z5_2;
// Wave 5
z3_1 = madd52lo(z3_1, x[1], y[2]);
z4_2 = madd52hi(z4_2, x[1], y[2]);
z0_1 = madd52lo(z0_1, x[0], y[0]);
z1_2 = madd52hi(z1_2, x[0], y[0]);
z1_1 = madd52lo(z1_1, x[0], y[1]);
z2_1 = madd52lo(z2_1, x[0], y[2]);
z2_2 = madd52hi(z2_2, x[0], y[1]);
z3_2 = madd52hi(z3_2, x[0], y[2]);
let mut t0 = u64x4::splat(0);
let mut t1 = u64x4::splat(0);
let r19 = u64x4::splat(19);
// Wave 6
t0 = madd52hi(t0, r19, z9);
t1 = madd52lo(t1, r19, z9 >> 52);
z3_1 = madd52lo(z3_1, x[0], y[3]);
z4_2 = madd52hi(z4_2, x[0], y[3]);
z1_2 = madd52lo(z1_2, r19, z5 >> 52);
z2_2 = madd52lo(z2_2, r19, z6 >> 52);
z3_2 = madd52lo(z3_2, r19, z7 >> 52);
z0_1 = madd52lo(z0_1, r19, z5);
// Wave 7
z4_1 = madd52lo(z4_1, r19, z9);
z1_1 = madd52lo(z1_1, r19, z6);
z0_2 = madd52lo(z0_2, r19, t0 + t1);
z4_2 = madd52hi(z4_2, r19, z8);
z2_1 = madd52lo(z2_1, r19, z7);
z1_2 = madd52hi(z1_2, r19, z5);
z2_2 = madd52hi(z2_2, r19, z6);
z3_2 = madd52hi(z3_2, r19, z7);
// Wave 8
z3_1 = madd52lo(z3_1, r19, z8);
z4_2 = madd52lo(z4_2, r19, z8 >> 52);
F51x4Unreduced([
z0_1 + z0_2 + z0_2,
z1_1 + z1_2 + z1_2,
z2_1 + z2_2 + z2_2,
z3_1 + z3_2 + z3_2,
z4_1 + z4_2 + z4_2,
])
}
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn vpmadd52luq() {
let x = u64x4::splat(2);
let y = u64x4::splat(3);
let mut z = u64x4::splat(5);
z = unsafe { madd52lo(z, x, y) };
assert_eq!(z, u64x4::splat(5 + 2 * 3));
}
#[test]
fn new_split_round_trip_on_reduced_input() {
// Invert a small field element to get a big one
let a = FieldElement51([2438, 24, 243, 0, 0]).invert();
let ax4 = F51x4Unreduced::new(&a, &a, &a, &a);
let splits = ax4.split();
for i in 0..4 {
assert_eq!(a, splits[i]);
}
}
#[test]
fn new_split_round_trip_on_unreduced_input() {
// Invert a small field element to get a big one
let a = FieldElement51([2438, 24, 243, 0, 0]).invert();
// ... but now multiply it by 16 without reducing coeffs
let a16 = FieldElement51([
a.0[0] << 4,
a.0[1] << 4,
a.0[2] << 4,
a.0[3] << 4,
a.0[4] << 4,
]);
let a16x4 = F51x4Unreduced::new(&a16, &a16, &a16, &a16);
let splits = a16x4.split();
for i in 0..4 {
assert_eq!(a16, splits[i]);
}
}
#[test]
fn test_reduction() {
// Invert a small field element to get a big one
let a = FieldElement51([2438, 24, 243, 0, 0]).invert();
// ... but now multiply it by 128 without reducing coeffs
let abig = FieldElement51([
a.0[0] << 4,
a.0[1] << 4,
a.0[2] << 4,
a.0[3] << 4,
a.0[4] << 4,
]);
let abigx4: F51x4Reduced = F51x4Unreduced::new(&abig, &abig, &abig, &abig).into();
let splits = F51x4Unreduced::from(abigx4).split();
let c = &a * &FieldElement51([(1 << 4), 0, 0, 0, 0]);
for i in 0..4 {
assert_eq!(c, splits[i]);
}
}
#[test]
fn mul_matches_serial() {
// Invert a small field element to get a big one
let a = FieldElement51([2438, 24, 243, 0, 0]).invert();
let b = FieldElement51([98098, 87987897, 0, 1, 0]).invert();
let c = &a * &b;
let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into();
let bx4: F51x4Reduced = F51x4Unreduced::new(&b, &b, &b, &b).into();
let cx4 = &ax4 * &bx4;
let splits = cx4.split();
for i in 0..4 {
assert_eq!(c, splits[i]);
}
}
#[test]
fn iterated_mul_matches_serial() {
// Invert a small field element to get a big one
let a = FieldElement51([2438, 24, 243, 0, 0]).invert();
let b = FieldElement51([98098, 87987897, 0, 1, 0]).invert();
let mut c = &a * &b;
for _i in 0..1024 {
c = &a * &c;
c = &b * &c;
}
let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into();
let bx4: F51x4Reduced = F51x4Unreduced::new(&b, &b, &b, &b).into();
let mut cx4 = &ax4 * &bx4;
for _i in 0..1024 {
cx4 = &ax4 * &F51x4Reduced::from(cx4);
cx4 = &bx4 * &F51x4Reduced::from(cx4);
}
let splits = cx4.split();
for i in 0..4 {
assert_eq!(c, splits[i]);
}
}
#[test]
fn square_matches_mul() {
// Invert a small field element to get a big one
let a = FieldElement51([2438, 24, 243, 0, 0]).invert();
let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into();
let cx4 = &ax4 * &ax4;
let cx4_sq = ax4.square();
let splits = cx4.split();
let splits_sq = cx4_sq.split();
for i in 0..4 {
assert_eq!(splits_sq[i], splits[i]);
}
}
#[test]
fn iterated_square_matches_serial() {
// Invert a small field element to get a big one
let mut a = FieldElement51([2438, 24, 243, 0, 0]).invert();
let mut ax4 = F51x4Unreduced::new(&a, &a, &a, &a);
for _j in 0..1024 {
a = a.square();
ax4 = F51x4Reduced::from(ax4).square();
let splits = ax4.split();
for i in 0..4 {
assert_eq!(a, splits[i]);
}
}
}
#[test]
fn iterated_u32_mul_matches_serial() {
// Invert a small field element to get a big one
let a = FieldElement51([2438, 24, 243, 0, 0]).invert();
let b = FieldElement51([121665, 0, 0, 0, 0]);
let mut c = &a * &b;
for _i in 0..1024 {
c = &b * &c;
}
let ax4 = F51x4Unreduced::new(&a, &a, &a, &a);
let bx4 = (121665u32, 121665u32, 121665u32, 121665u32);
let mut cx4 = &F51x4Reduced::from(ax4) * bx4;
for _i in 0..1024 {
cx4 = &F51x4Reduced::from(cx4) * bx4;
}
let splits = cx4.split();
for i in 0..4 {
assert_eq!(c, splits[i]);
}
}
#[test]
fn shuffle_AAAA() {
let x0 = FieldElement51::from_bytes(&[0x10; 32]);
let x1 = FieldElement51::from_bytes(&[0x11; 32]);
let x2 = FieldElement51::from_bytes(&[0x12; 32]);
let x3 = FieldElement51::from_bytes(&[0x13; 32]);
let x = F51x4Unreduced::new(&x0, &x1, &x2, &x3);
let y = x.shuffle(Shuffle::AAAA);
let splits = y.split();
assert_eq!(splits[0], x0);
assert_eq!(splits[1], x0);
assert_eq!(splits[2], x0);
assert_eq!(splits[3], x0);
}
#[test]
fn blend_AB() {
let x0 = FieldElement51::from_bytes(&[0x10; 32]);
let x1 = FieldElement51::from_bytes(&[0x11; 32]);
let x2 = FieldElement51::from_bytes(&[0x12; 32]);
let x3 = FieldElement51::from_bytes(&[0x13; 32]);
let x = F51x4Unreduced::new(&x0, &x1, &x2, &x3);
let z = F51x4Unreduced::new(&x3, &x2, &x1, &x0);
let y = x.blend(&z, Lanes::AB);
let splits = y.split();
assert_eq!(splits[0], x3);
assert_eq!(splits[1], x2);
assert_eq!(splits[2], x2);
assert_eq!(splits[3], x3);
}
}

19
src/backend/vector/ifma/mod.rs Normal file
View file

@ -0,0 +1,19 @@
// -*- mode: rust; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2018 Henry de Valence
// See LICENSE for licensing information.
//
// Authors:
// - Henry de Valence <hdevalence@hdevalence.ca>
#![cfg_attr(
all(feature = "nightly", feature = "stage2_build"),
doc(include = "../docs/ifma-notes.md")
)]
pub mod field;
pub mod edwards;
pub mod constants;

44
src/backend/vector/mod.rs Normal file
View file

@ -0,0 +1,44 @@
// -*- mode: rust; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence
// See LICENSE for licensing information.
//
// Authors:
// - Isis Agora Lovecruft <isis@patternsinthevoid.net>
// - Henry de Valence <hdevalence@hdevalence.ca>
// Conditionally include the notes if:
// - we're on nightly (so we can include docs at all)
// - we're in stage 2 of the build.
// The latter point prevents a really silly and annoying problem,
// where the location of ".." is different depending on whether we're
// building the crate for real, or whether we're in build.rs
// generating the lookup tables (in which case we're relative to the
// location of build.rs, not lib.rs, so the markdown file appears
// missing).
#![cfg_attr(
all(feature = "nightly", feature = "stage2_build"),
doc(include = "../docs/parallel-formulas.md")
)]
#[cfg(not(any(target_feature = "avx2", target_feature = "avx512ifma",)))]
compile_error!("simd_backend selected without target_feature=+avx2 or +avx512ifma");
#[cfg(any(all(target_feature = "avx2", not(target_feature = "avx512ifma")), rustdoc))]
#[doc(cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma"))))]
pub mod avx2;
#[cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma")))]
pub(crate) use self::avx2::{
constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint,
};
#[cfg(any(target_feature = "avx512ifma", rustdoc))]
#[doc(cfg(target_feature = "avx512ifma"))]
pub mod ifma;
#[cfg(target_feature = "avx512ifma")]
pub(crate) use self::ifma::{
constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint,
};
pub mod scalar_mul;

5
src/backend/avx2/scalar_mul/mod.rs → src/backend/vector/scalar_mul/mod.rs
View file

@ -10,7 +10,10 @@
pub mod variable_base;
#[cfg(feature = "stage2_build")]
pub mod vartime_double_base;
#[cfg(feature = "alloc")]
pub mod straus;
#[cfg(feature = "alloc")]
pub mod precomputed_straus;

111
src/backend/vector/scalar_mul/precomputed_straus.rs Normal file
View file

@ -0,0 +1,111 @@
// -*- mode: rust; -*-
//
// This file is part of curve25519-dalek.
// Copyright (c) 2019 Henry de Valence.
// See LICENSE for licensing information.
//
// Authors:
// - Henry de Valence <hdevalence@hdevalence.ca>
//! Precomputation for Straus's method.
#![allow(non_snake_case)]
use core::borrow::Borrow;
use backend::vector::{CachedPoint, ExtendedPoint};
use edwards::EdwardsPoint;
use scalar::Scalar;
use traits::Identity;
use traits::VartimePrecomputedMultiscalarMul;
use window::{NafLookupTable5, NafLookupTable8};
#[allow(unused_imports)]
use prelude::*;
pub struct VartimePrecomputedStraus {
static_lookup_tables: Vec<NafLookupTable8<CachedPoint>>,
}
impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus {
type Point = EdwardsPoint;
fn new<I>(static_points: I) -> Self
where
I: IntoIterator,
I::Item: Borrow<Self::Point>,
{
Self {
static_lookup_tables: static_points
.into_iter()
.map(|P| NafLookupTable8::<CachedPoint>::from(P.borrow()))
.collect(),
}
}
fn optional_mixed_multiscalar_mul<I, J, K>(
&self,
static_scalars: I,
dynamic_scalars: J,
dynamic_points: K,
) -> Option<Self::Point>
where
I: IntoIterator,
I::Item: Borrow<Scalar>,
J: IntoIterator,
J::Item: Borrow<Scalar>,
K: IntoIterator<Item = Option<Self::Point>>,
{
let static_nafs = static_scalars
.into_iter()
.map(|c| c.borrow().non_adjacent_form(5))
.collect::<Vec<_>>();
let dynamic_nafs: Vec<_> = dynamic_scalars
.into_iter()
.map(|c| c.borrow().non_adjacent_form(5))
.collect::<Vec<_>>();
let dynamic_lookup_tables = match dynamic_points
.into_iter()
.map(|P_opt| P_opt.map(|P| NafLookupTable5::<CachedPoint>::from(&P)))
.collect::<Option<Vec<_>>>()
{
Some(x) => x,
None => return None,
};
let sp = self.static_lookup_tables.len();
let dp = dynamic_lookup_tables.len();
assert_eq!(sp, static_nafs.len());
assert_eq!(dp, dynamic_nafs.len());
// We could save some doublings by looking for the highest
// nonzero NAF coefficient, but since we might have a lot of
// them to search, it's not clear it's worthwhile to check.
let mut R = ExtendedPoint::identity();
for j in (0..255).rev() {
R = R.double();
for i in 0..dp {
let t_ij = dynamic_nafs[i][j];
if t_ij > 0 {
R = &R + &dynamic_lookup_tables[i].select(t_ij as usize);
} else if t_ij < 0 {
R = &R - &dynamic_lookup_tables[i].select(-t_ij as usize);
}
}
for i in 0..sp {
let t_ij = static_nafs[i][j];
if t_ij > 0 {
R = &R + &self.static_lookup_tables[i].select(t_ij as usize);
} else if t_ij < 0 {
R = &R - &self.static_lookup_tables[i].select(-t_ij as usize);
}
}
}
Some(R.into())
}
}

6
src/backend/avx2/scalar_mul/straus.rs → src/backend/vector/scalar_mul/straus.rs
View file

@ -14,10 +14,10 @@ use core::borrow::Borrow;
use clear_on_drop::ClearOnDrop;
use backend::avx2::edwards::{CachedPoint, ExtendedPoint};
use backend::vector::{CachedPoint, ExtendedPoint};
use edwards::EdwardsPoint;
use scalar::Scalar;
use scalar_mul::window::{LookupTable, NafLookupTable5};
use window::{LookupTable, NafLookupTable5};
use traits::{Identity, MultiscalarMul, VartimeMultiscalarMul};
#[allow(unused_imports)]
@ -33,7 +33,6 @@ use prelude::*;
/// point representation on the fly.
pub struct Straus {}
#[cfg(feature = "alloc")]
impl MultiscalarMul for Straus {
type Point = EdwardsPoint;
@ -71,7 +70,6 @@ impl MultiscalarMul for Straus {
}
}
#[cfg(feature = "alloc")]
impl VartimeMultiscalarMul for Straus {
type Point = EdwardsPoint;

8
src/backend/avx2/scalar_mul/variable_base.rs → src/backend/vector/scalar_mul/variable_base.rs
View file

@ -1,10 +1,10 @@
#![allow(non_snake_case)]
use traits::Identity;
use scalar::Scalar;
use backend::vector::{CachedPoint, ExtendedPoint};
use edwards::EdwardsPoint;
use backend::avx2::edwards::{ExtendedPoint, CachedPoint};
use scalar_mul::window::LookupTable;
use scalar::Scalar;
use traits::Identity;
use window::LookupTable;
/// Perform constant-time, variable-base scalar multiplication.
pub fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint {

10
src/backend/avx2/scalar_mul/vartime_double_base.rs → src/backend/vector/scalar_mul/vartime_double_base.rs
View file

@ -9,12 +9,12 @@
// - Henry de Valence <hdevalence@hdevalence.ca>
#![allow(non_snake_case)]
use traits::Identity;
use scalar::Scalar;
use backend::vector::BASEPOINT_ODD_LOOKUP_TABLE;
use backend::vector::{CachedPoint, ExtendedPoint};
use edwards::EdwardsPoint;
use scalar_mul::window::NafLookupTable5;
use backend::avx2::edwards::{CachedPoint, ExtendedPoint};
use backend::avx2::constants::BASEPOINT_ODD_LOOKUP_TABLE;
use scalar::Scalar;
use traits::Identity;
use window::NafLookupTable5;
/// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint.
pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint {

16
src/constants.rs
View file

@ -34,9 +34,9 @@ use montgomery::MontgomeryPoint;
use scalar::Scalar;
#[cfg(feature = "u64_backend")]
pub use backend::u64::constants::*;
pub use backend::serial::u64::constants::*;
#[cfg(feature = "u32_backend")]
pub use backend::u32::constants::*;
pub use backend::serial::u32::constants::*;
/// The Ed25519 basepoint, in `CompressedEdwardsY` format.
///
@ -151,9 +151,9 @@ mod test {
#[test]
#[cfg(feature = "u32_backend")]
fn test_d_vs_ratio() {
use backend::u32::field::FieldElement32;
let a = -&FieldElement32([121665,0,0,0,0,0,0,0,0,0]);
let b = FieldElement32([121666,0,0,0,0,0,0,0,0,0]);
use backend::serial::u32::field::FieldElement2625;
let a = -&FieldElement2625([121665,0,0,0,0,0,0,0,0,0]);
let b = FieldElement2625([121666,0,0,0,0,0,0,0,0,0]);
let d = &a * &b.invert();
let d2 = &d + &d;
assert_eq!(d, constants::EDWARDS_D);
@ -164,9 +164,9 @@ mod test {
#[test]
#[cfg(feature = "u64_backend")]
fn test_d_vs_ratio() {
use backend::u64::field::FieldElement64;
let a = -&FieldElement64([121665,0,0,0,0]);
let b = FieldElement64([121666,0,0,0,0]);
use backend::serial::u64::field::FieldElement51;
let a = -&FieldElement51([121665,0,0,0,0]);
let b = FieldElement51([121666,0,0,0,0]);
let d = &a * &b.invert();
let d2 = &d + &d;
assert_eq!(d, constants::EDWARDS_D);

161
src/edwards.rs
View file

@ -112,23 +112,34 @@ use scalar::Scalar;
use montgomery::MontgomeryPoint;
use curve_models::ProjectivePoint;
use curve_models::CompletedPoint;
use curve_models::AffineNielsPoint;
use curve_models::ProjectiveNielsPoint;
use backend::serial::curve_models::AffineNielsPoint;
use backend::serial::curve_models::CompletedPoint;
use backend::serial::curve_models::ProjectiveNielsPoint;
use backend::serial::curve_models::ProjectivePoint;
use window::LookupTable;
#[allow(unused_imports)]
use prelude::*;
use scalar_mul::window::LookupTable;
use traits::{Identity, IsIdentity};
use traits::ValidityCheck;
use traits::{Identity, IsIdentity};
#[cfg(any(feature = "alloc", feature = "std"))]
use traits::MultiscalarMul;
#[cfg(any(feature = "alloc", feature = "std"))]
use traits::VartimeMultiscalarMul;
use traits::{VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul};
#[cfg(not(all(
feature = "simd_backend",
any(target_feature = "avx2", target_feature = "avx512ifma")
)))]
use backend::serial::scalar_mul;
#[cfg(all(
feature = "simd_backend",
any(target_feature = "avx2", target_feature = "avx512ifma")
))]
use backend::vector::scalar_mul;
// ------------------------------------------------------------------------
// Compressed points
@ -439,6 +450,10 @@ impl EdwardsPoint {
/// Convert this `EdwardsPoint` on the Edwards model to the
/// corresponding `MontgomeryPoint` on the Montgomery model.
///
/// This function has one exceptional case; the identity point of
/// the Edwards curve is sent to the 2-torsion point \\((0,0)\\)
/// on the Montgomery curve.
///
/// Note that this is a one-way conversion, since the Montgomery
/// model does not retain sign information.
pub fn to_montgomery(&self) -> MontgomeryPoint {
@ -446,7 +461,7 @@ impl EdwardsPoint {
//
// The denominator is zero only when y=1, the identity point of
// the Edwards curve. Since 0.invert() = 0, in this case we
// compute u = 0, the identity point of the Montgomery line.
// compute the 2-torsion point (0,0).
let U = &self.Z + &self.Y;
let W = &self.Z - &self.Y;
let u = &U * &W.invert();
@ -576,18 +591,7 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsPoint {
/// For scalar multiplication of a basepoint,
/// `EdwardsBasepointTable` is approximately 4x faster.
fn mul(self, scalar: &'b Scalar) -> EdwardsPoint {
// If we built with AVX2, use the AVX2 backend.
#[cfg(all(feature="avx2_backend", target_feature="avx2"))]
{
use backend::avx2::scalar_mul::variable_base::mul;
mul(self, scalar)
}
// Otherwise, use the serial backend:
#[cfg(not(all(feature="avx2_backend", target_feature="avx2")))]
{
use scalar_mul::variable_base::mul;
mul(self, scalar)
}
scalar_mul::variable_base::mul(self, scalar)
}
}
@ -613,7 +617,7 @@ impl<'a, 'b> Mul<&'b EdwardsPoint> for &'a Scalar {
#[cfg(feature = "alloc")]
impl MultiscalarMul for EdwardsPoint {
type Point = EdwardsPoint;
fn multiscalar_mul<I, J>(scalars: I, points: J) -> EdwardsPoint
where
I: IntoIterator,
@ -638,21 +642,14 @@ impl MultiscalarMul for EdwardsPoint {
// size-dependent algorithm dispatch, use this as the hint.
let _size = s_lo;
// If we built with AVX2, use the AVX2 backend.
#[cfg(all(feature="avx2_backend", target_feature="avx2"))]
use backend::avx2::scalar_mul::straus::Straus;
// Otherwise, proceed as normal:
#[cfg(not(all(feature="avx2_backend", target_feature="avx2")))]
use scalar_mul::straus::Straus;
Straus::multiscalar_mul(scalars, points)
scalar_mul::straus::Straus::multiscalar_mul(scalars, points)
}
}
#[cfg(feature = "alloc")]
impl VartimeMultiscalarMul for EdwardsPoint {
type Point = EdwardsPoint;
fn optional_multiscalar_mul<I, J>(scalars: I, points: J) -> Option<EdwardsPoint>
where
I: IntoIterator,
@ -676,29 +673,56 @@ impl VartimeMultiscalarMul for EdwardsPoint {
// size-dependent algorithm dispatch, use this as the hint.
let _size = s_lo;
// If we built with AVX2, use the AVX2 backend.
#[cfg(all(feature="avx2_backend", target_feature="avx2"))]
use backend::avx2::scalar_mul::straus::Straus;
// Otherwise, proceed as normal:
#[cfg(not(all(feature="avx2_backend", target_feature="avx2")))]
use scalar_mul::straus::Straus;
scalar_mul::straus::Straus::optional_multiscalar_mul(scalars, points)
}
}
Straus::optional_multiscalar_mul(scalars, points)
/// Precomputation for variable-time multiscalar multiplication with `EdwardsPoint`s.
// This wraps the inner implementation in a facade type so that we can
// decouple stability of the inner type from the stability of the
// outer type.
#[cfg(feature = "alloc")]
pub struct VartimeEdwardsPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus);
#[cfg(feature = "alloc")]
impl VartimePrecomputedMultiscalarMul for VartimeEdwardsPrecomputation {
type Point = EdwardsPoint;
fn new<I>(static_points: I) -> Self
where
I: IntoIterator,
I::Item: Borrow<Self::Point>,
{
Self(scalar_mul::precomputed_straus::VartimePrecomputedStraus::new(static_points))
}
fn optional_mixed_multiscalar_mul<I, J, K>(
&self,
static_scalars: I,
dynamic_scalars: J,
dynamic_points: K,
) -> Option<Self::Point>
where
I: IntoIterator,
I::Item: Borrow<Scalar>,
J: IntoIterator,
J::Item: Borrow<Scalar>,
K: IntoIterator<Item = Option<Self::Point>>,
{
self.0
.optional_mixed_multiscalar_mul(static_scalars, dynamic_scalars, dynamic_points)
}
}
impl EdwardsPoint {
/// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint.
#[cfg(feature = "stage2_build")]
pub fn vartime_double_scalar_mul_basepoint(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint {
// If we built with AVX2, use the AVX2 backend.
#[cfg(all(feature="avx2_backend", target_feature="avx2"))]
use backend::avx2::scalar_mul::vartime_double_base;
// Otherwise, use the serial backend:
#[cfg(not(all(feature="avx2_backend", target_feature="avx2")))]
use scalar_mul::vartime_double_base;
vartime_double_base::mul(a, A, b)
pub fn vartime_double_scalar_mul_basepoint(
a: &Scalar,
A: &EdwardsPoint,
b: &Scalar,
) -> EdwardsPoint {
scalar_mul::vartime_double_base::mul(a, A, b)
}
}
@ -1220,6 +1244,49 @@ mod test {
assert!(P1.compress().to_bytes() == P2.compress().to_bytes());
}
#[test]
fn vartime_precomputed_vs_nonprecomputed_multiscalar() {
let mut rng = rand::thread_rng();
let B = &::constants::ED25519_BASEPOINT_TABLE;
let static_scalars = (0..128)
.map(|_| Scalar::random(&mut rng))
.collect::<Vec<_>>();
let dynamic_scalars = (0..128)
.map(|_| Scalar::random(&mut rng))
.collect::<Vec<_>>();
let check_scalar: Scalar = static_scalars
.iter()
.chain(dynamic_scalars.iter())
.map(|s| s * s)
.sum();
let static_points = static_scalars.iter().map(|s| s * B).collect::<Vec<_>>();
let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::<Vec<_>>();
let precomputation = VartimeEdwardsPrecomputation::new(static_points.iter());
let P = precomputation.vartime_mixed_multiscalar_mul(
&static_scalars,
&dynamic_scalars,
&dynamic_points,
);
use traits::VartimeMultiscalarMul;
let Q = EdwardsPoint::vartime_multiscalar_mul(
static_scalars.iter().chain(dynamic_scalars.iter()),
static_points.iter().chain(dynamic_points.iter()),
);
let R = &check_scalar * B;
assert_eq!(P.compress(), R.compress());
assert_eq!(Q.compress(), R.compress());
}
mod vartime {
use super::super::*;
use super::{A_SCALAR, B_SCALAR, A_TIMES_BASEPOINT, DOUBLE_SCALAR_MULT_RESULT};

18
src/field.rs
View file

@ -12,8 +12,8 @@
//!
//! The `curve25519_dalek::field` module provides a type alias
//! `curve25519_dalek::field::FieldElement` to a field element type
//! defined in the `backend` module; either `FieldElement64` or
//! `FieldElement32`.
//! defined in the `backend` module; either `FieldElement51` or
//! `FieldElement2625`.
//!
//! Field operations defined in terms of machine
//! operations, such as field multiplication or squaring, are defined in
@ -33,24 +33,24 @@ use constants;
use backend;
#[cfg(feature = "u64_backend")]
pub use backend::u64::field::*;
pub use backend::serial::u64::field::*;
/// A `FieldElement` represents an element of the field
/// \\( \mathbb Z / (2\^{255} - 19)\\).
///
/// The `FieldElement` type is an alias for one of the platform-specific
/// implementations.
#[cfg(feature = "u64_backend")]
pub type FieldElement = backend::u64::field::FieldElement64;
pub type FieldElement = backend::serial::u64::field::FieldElement51;
#[cfg(feature = "u32_backend")]
pub use backend::u32::field::*;
pub use backend::serial::u32::field::*;
/// A `FieldElement` represents an element of the field
/// \\( \mathbb Z / (2\^{255} - 19)\\).
///
/// The `FieldElement` type is an alias for one of the platform-specific
/// implementations.
#[cfg(feature = "u32_backend")]
pub type FieldElement = backend::u32::field::FieldElement32;
pub type FieldElement = backend::serial::u32::field::FieldElement2625;
impl Eq for FieldElement {}
@ -255,7 +255,7 @@ impl FieldElement {
(was_nonzero_square, r)
}
/// Attempt to compute `1/sqrt(self)` in constant time.
/// Attempt to compute `sqrt(1/self)` in constant time.
///
/// Convenience wrapper around `sqrt_ratio_i`.
///
@ -265,7 +265,7 @@ impl FieldElement {
///
/// - `(Choice(1), +sqrt(1/self)) ` if `self` is a nonzero square;
/// - `(Choice(0), zero) ` if `self` is zero;
/// - `(Choice(0), +sqrt(i*u/v)) ` if `self` is a nonzero nonsquare;
/// - `(Choice(0), +sqrt(i/self)) ` if `self` is a nonzero nonsquare;
///
pub fn invsqrt(&self) -> (Choice, FieldElement) {
FieldElement::sqrt_ratio_i(&FieldElement::one(), self)
@ -280,7 +280,7 @@ mod test {
/// Random element a of GF(2^255-19), from Sage
/// a = 1070314506888354081329385823235218444233221\
/// 2228051251926706380353716438957572
pub static A_BYTES: [u8; 32] =
static A_BYTES: [u8; 32] =
[ 0x04, 0xfe, 0xdf, 0x98, 0xa7, 0xfa, 0x0a, 0x68,
0x84, 0x92, 0xbd, 0x59, 0x08, 0x07, 0xa7, 0x03,
0x9e, 0xd1, 0xf6, 0xf2, 0xe1, 0xd9, 0xe2, 0xa4,

31
src/lib.rs
View file

@ -9,11 +9,17 @@
// - Henry de Valence <hdevalence@hdevalence.ca>
#![no_std]
#![cfg_attr(
any(
all(feature = "simd_backend", target_feature = "avx512ifma"),
all(feature = "nightly", rustdoc)
),
feature(simd_ffi, link_llvm_intrinsics)
)]
#![cfg_attr(feature = "nightly", feature(test))]
#![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))]
#![cfg_attr(feature = "nightly", feature(cfg_target_feature))]
#![cfg_attr(feature = "nightly", feature(external_doc))]
#![cfg_attr(feature = "nightly", feature(doc_cfg))]
// Refuse to compile if documentation is missing, but only on nightly.
//
// This means that missing docs will still fail CI, but means we can use
@ -38,21 +44,23 @@ extern crate alloc;
#[macro_use]
extern crate std;
#[cfg(all(feature = "nightly", feature = "avx2_backend"))]
#[cfg(all(feature = "nightly", feature = "packed_simd"))]
extern crate packed_simd;
extern crate rand;
extern crate clear_on_drop;
extern crate byteorder;
extern crate clear_on_drop;
pub extern crate digest;
extern crate rand_core;
#[cfg(all(test, feature = "stage2_build"))]
extern crate rand_os;
// Used for traits related to constant-time code.
extern crate subtle;
#[cfg(feature = "serde")]
extern crate serde;
#[cfg(all(test, feature = "serde"))]
extern crate bincode;
#[cfg(feature = "serde")]
extern crate serde;
// Internal macros. Must come first!
#[macro_use]
@ -90,11 +98,8 @@ pub(crate) mod field;
// Arithmetic backends (using u32, u64, etc) live here
pub(crate) mod backend;
// Internal curve models which are not part of the public API.
pub(crate) mod curve_models;
// Crate-local prelude (for alloc-dependent features like `Vec`)
pub(crate) mod prelude;
// Implementations of scalar mul algorithms live here
pub(crate) mod scalar_mul;
// Generic code for window lookups
pub(crate) mod window;

2
src/montgomery.rs
View file

@ -305,7 +305,7 @@ mod test {
use super::*;
#[cfg(feature = "rand")]
use rand::rngs::OsRng;
use rand_os::OsRng;
/// Test Montgomery -> Edwards on the X/Ed25519 basepoint
#[test]

123
src/ristretto.rs
View file

@ -164,7 +164,7 @@ use core::ops::{Add, Neg, Sub};
use core::ops::{AddAssign, SubAssign};
use core::ops::{Mul, MulAssign};
use rand::{CryptoRng, Rng};
use rand_core::{CryptoRng, RngCore};
use digest::generic_array::typenum::U64;
use digest::Digest;
@ -185,11 +185,20 @@ use prelude::*;
use scalar::Scalar;
use curve_models::CompletedPoint;
use traits::Identity;
#[cfg(any(feature = "alloc", feature = "std"))]
use traits::{MultiscalarMul, VartimeMultiscalarMul};
use traits::{MultiscalarMul, VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul};
#[cfg(not(all(
feature = "simd_backend",
any(target_feature = "avx2", target_feature = "avx512ifma")
)))]
use backend::serial::scalar_mul;
#[cfg(all(
feature = "simd_backend",
any(target_feature = "avx2", target_feature = "avx512ifma")
))]
use backend::vector::scalar_mul;
// ------------------------------------------------------------------------
// Compressed points
@ -468,8 +477,8 @@ impl RistrettoPoint {
/// ```
/// # extern crate curve25519_dalek;
/// # use curve25519_dalek::ristretto::RistrettoPoint;
/// extern crate rand;
/// use rand::rngs::OsRng;
/// extern crate rand_os;
/// use rand_os::OsRng;
///
/// # // Need fn main() here in comment so the doctest compiles
/// # // See https://doc.rust-lang.org/book/documentation.html#documentation-as-tests
@ -602,6 +611,8 @@ impl RistrettoPoint {
let N_t = &(&(&c * &(&r - &one)) * &d_minus_one_sq) - &D;
let s_sq = s.square();
use backend::serial::curve_models::CompletedPoint;
// The conversion from W_i is exactly the conversion from P1xP1.
RistrettoPoint(CompletedPoint{
X: &(&s + &s) * &D,
@ -615,7 +626,7 @@ impl RistrettoPoint {
///
/// # Inputs
///
/// * `rng`: any RNG which implements the `rand::Rng` interface.
/// * `rng`: any RNG which implements the `RngCore + CryptoRng` interface.
///
/// # Returns
///
@ -627,9 +638,9 @@ impl RistrettoPoint {
/// discrete log of the output point with respect to any other
/// point should be unknown. The map is applied twice and the
/// results are added, to ensure a uniform distribution.
pub fn random<T: Rng + CryptoRng>(rng: &mut T) -> Self {
pub fn random<T: RngCore + CryptoRng>(mut rng: T) -> Self {
let mut uniform_bytes = [0u8; 64];
rng.fill(&mut uniform_bytes);
rng.fill_bytes(&mut uniform_bytes);
RistrettoPoint::from_uniform_bytes(&uniform_bytes)
}
@ -891,8 +902,53 @@ impl VartimeMultiscalarMul for RistrettoPoint {
{
let extended_points = points.into_iter().map(|opt_P| opt_P.map(|P| P.borrow().0));
EdwardsPoint::optional_multiscalar_mul(scalars, extended_points)
.map(|P| RistrettoPoint(P))
EdwardsPoint::optional_multiscalar_mul(scalars, extended_points).map(|P| RistrettoPoint(P))
}
}
/// Precomputation for variable-time multiscalar multiplication with `RistrettoPoint`s.
// This wraps the inner implementation in a facade type so that we can
// decouple stability of the inner type from the stability of the
// outer type.
#[cfg(feature = "alloc")]
pub struct VartimeRistrettoPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus);
#[cfg(feature = "alloc")]
impl VartimePrecomputedMultiscalarMul for VartimeRistrettoPrecomputation {
type Point = RistrettoPoint;
fn new<I>(static_points: I) -> Self
where
I: IntoIterator,
I::Item: Borrow<Self::Point>,
{
Self(
scalar_mul::precomputed_straus::VartimePrecomputedStraus::new(
static_points.into_iter().map(|P| P.borrow().0),
),
)
}
fn optional_mixed_multiscalar_mul<I, J, K>(
&self,
static_scalars: I,
dynamic_scalars: J,
dynamic_points: K,
) -> Option<Self::Point>
where
I: IntoIterator,
I::Item: Borrow<Scalar>,
J: IntoIterator,
J::Item: Borrow<Scalar>,
K: IntoIterator<Item = Option<Self::Point>>,
{
self.0
.optional_mixed_multiscalar_mul(
static_scalars,
dynamic_scalars,
dynamic_points.into_iter().map(|P_opt| P_opt.map(|P| P.0)),
)
.map(|P_ed| RistrettoPoint(P_ed))
}
}
@ -1020,7 +1076,7 @@ impl Debug for RistrettoPoint {
#[cfg(all(test, feature = "stage2_build"))]
mod test {
#[cfg(feature = "rand")]
use rand::rngs::OsRng;
use rand_os::OsRng;
use scalar::Scalar;
use constants;
@ -1263,4 +1319,47 @@ mod test {
P.compress();
}
}
#[test]
fn vartime_precomputed_vs_nonprecomputed_multiscalar() {
let mut rng = rand::thread_rng();
let B = &::constants::RISTRETTO_BASEPOINT_TABLE;
let static_scalars = (0..128)
.map(|_| Scalar::random(&mut rng))
.collect::<Vec<_>>();
let dynamic_scalars = (0..128)
.map(|_| Scalar::random(&mut rng))
.collect::<Vec<_>>();
let check_scalar: Scalar = static_scalars
.iter()
.chain(dynamic_scalars.iter())
.map(|s| s * s)
.sum();
let static_points = static_scalars.iter().map(|s| s * B).collect::<Vec<_>>();
let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::<Vec<_>>();
let precomputation = VartimeRistrettoPrecomputation::new(static_points.iter());
let P = precomputation.vartime_mixed_multiscalar_mul(
&static_scalars,
&dynamic_scalars,
&dynamic_points,
);
use traits::VartimeMultiscalarMul;
let Q = RistrettoPoint::vartime_multiscalar_mul(
static_scalars.iter().chain(dynamic_scalars.iter()),
static_points.iter().chain(dynamic_points.iter()),
);
let R = &check_scalar * B;
assert_eq!(P.compress(), R.compress());
assert_eq!(Q.compress(), R.compress());
}
}

16
src/scalar.rs
View file

@ -151,7 +151,7 @@ use core::ops::{Sub, SubAssign};
#[allow(unused_imports)]
use prelude::*;
use rand::{CryptoRng, Rng};
use rand_core::{CryptoRng, RngCore};
use digest::generic_array::typenum::U64;
use digest::Digest;
@ -168,14 +168,14 @@ use constants;
/// This is a type alias for one of the scalar types in the `backend`
/// module.
#[cfg(feature = "u64_backend")]
type UnpackedScalar = backend::u64::scalar::Scalar64;
type UnpackedScalar = backend::serial::u64::scalar::Scalar52;
/// An `UnpackedScalar` represents an element of the field GF(l), optimized for speed.
///
/// This is a type alias for one of the scalar types in the `backend`
/// module.
#[cfg(feature = "u32_backend")]
type UnpackedScalar = backend::u32::scalar::Scalar32;
type UnpackedScalar = backend::serial::u32::scalar::Scalar29;
/// The `Scalar` struct holds an integer \\(s < 2\^{255} \\) which
@ -507,7 +507,7 @@ impl Scalar {
///
/// # Inputs
///
/// * `rng`: any RNG which implements the `rand::CryptoRng` interface.
/// * `rng`: any RNG which implements the `RngCore + CryptoRng` interface.
///
/// # Returns
///
@ -516,20 +516,20 @@ impl Scalar {
/// # Example
///
/// ```
/// extern crate rand;
/// extern crate rand_os;
/// # extern crate curve25519_dalek;
/// #
/// # fn main() {
/// use curve25519_dalek::scalar::Scalar;
///
/// use rand::OsRng;
/// use rand_os::OsRng;
///
/// let mut csprng: OsRng = OsRng::new().unwrap();
/// let a: Scalar = Scalar::random(&mut csprng);
/// # }
pub fn random<T: Rng + CryptoRng>(rng: &mut T) -> Self {
pub fn random<T: RngCore + CryptoRng>(mut rng: T) -> Self {
let mut scalar_bytes = [0u8; 64];
rng.fill(&mut scalar_bytes);
rng.fill_bytes(&mut scalar_bytes);
Scalar::from_bytes_mod_order_wide(&scalar_bytes)
}

142
src/traits.rs
View file

@ -219,11 +219,149 @@ pub trait VartimeMultiscalarMul {
{
Self::optional_multiscalar_mul(
scalars,
points.into_iter().map(|P| Some(P.borrow().clone()))
).unwrap()
points.into_iter().map(|P| Some(P.borrow().clone())),
)
.unwrap()
}
}
/// A trait for variable-time multiscalar multiplication with precomputation.
///
/// A general multiscalar multiplication with precomputation can be written as
/// $$
/// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m,
/// $$
/// where the \\(B_i\\) are *static* points, for which precomputation
/// is possible, and the \\(A_j\\) are *dynamic* points, for which
/// precomputation is not possible.
///
/// This trait has three methods for performing this computation:
///
/// * [`vartime_multiscalar_mul`], which handles the special case
/// where \\(n = 0\\) and there are no dynamic points;
///
/// * [`vartime_mixed_multiscalar_mul`], which takes the dynamic
/// points as already-validated `Point`s and is infallible;
///
/// * [`optional_mixed_multiscalar_mul`], which takes the dynamic
/// points as `Option<Point>`s and returns an `Option<Point>`,
/// allowing decompression to be composed into the input iterators.
///
/// All methods require that the lengths of the input iterators be
/// known and matching, as if they were `ExactSizeIterator`s. (It
/// does not require `ExactSizeIterator` only because that trait is
/// broken).
pub trait VartimePrecomputedMultiscalarMul: Sized {
/// The type of point to be multiplied, e.g., `RistrettoPoint`.
type Point: Clone;
/// Given the static points \\( B_i \\), perform precomputation
/// and return the precomputation data.
fn new<I>(static_points: I) -> Self
where
I: IntoIterator,
I::Item: Borrow<Self::Point>;
/// Given `static_scalars`, an iterator of public scalars
/// \\(b_i\\), compute
/// $$
/// Q = b_1 B_1 + \cdots + b_m B_m,
/// $$
/// where the \\(B_j\\) are the points that were supplied to `new`.
///
/// It is an error to call this function with iterators of
/// inconsistent lengths.
///
/// The trait bound aims for maximum flexibility: the input must
/// be convertable to iterators (`I: IntoIter`), and the
/// iterator's items must be `Borrow<Scalar>`, to allow iterators
/// returning either `Scalar`s or `&Scalar`s.
fn vartime_multiscalar_mul<I>(&self, static_scalars: I) -> Self::Point
where
I: IntoIterator,
I::Item: Borrow<Scalar>,
{
use core::iter;
Self::vartime_mixed_multiscalar_mul(
self,
static_scalars,
iter::empty::<Scalar>(),
iter::empty::<Self::Point>(),
)
}
/// Given `static_scalars`, an iterator of public scalars
/// \\(b_i\\), `dynamic_scalars`, an iterator of public scalars
/// \\(a_i\\), and `dynamic_points`, an iterator of points
/// \\(A_i\\), compute
/// $$
/// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m,
/// $$
/// where the \\(B_j\\) are the points that were supplied to `new`.
///
/// It is an error to call this function with iterators of
/// inconsistent lengths.
///
/// The trait bound aims for maximum flexibility: the inputs must be
/// convertable to iterators (`I: IntoIter`), and the iterator's items
/// must be `Borrow<Scalar>` (or `Borrow<Point>`), to allow
/// iterators returning either `Scalar`s or `&Scalar`s.
fn vartime_mixed_multiscalar_mul<I, J, K>(
&self,
static_scalars: I,
dynamic_scalars: J,
dynamic_points: K,
) -> Self::Point
where
I: IntoIterator,
I::Item: Borrow<Scalar>,
J: IntoIterator,
J::Item: Borrow<Scalar>,
K: IntoIterator,
K::Item: Borrow<Self::Point>,
{
Self::optional_mixed_multiscalar_mul(
self,
static_scalars,
dynamic_scalars,
dynamic_points.into_iter().map(|P| Some(P.borrow().clone())),
)
.unwrap()
}
/// Given `static_scalars`, an iterator of public scalars
/// \\(b_i\\), `dynamic_scalars`, an iterator of public scalars
/// \\(a_i\\), and `dynamic_points`, an iterator of points
/// \\(A_i\\), compute
/// $$
/// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m,
/// $$
/// where the \\(B_j\\) are the points that were supplied to `new`.
///
/// If any of the dynamic points were `None`, return `None`.
///
/// It is an error to call this function with iterators of
/// inconsistent lengths.
///
/// This function is particularly useful when verifying statements
/// involving compressed points. Accepting `Option<Point>` allows
/// inlining point decompression into the multiscalar call,
/// avoiding the need for temporary buffers.
fn optional_mixed_multiscalar_mul<I, J, K>(
&self,
static_scalars: I,
dynamic_scalars: J,
dynamic_points: K,
) -> Option<Self::Point>
where
I: IntoIterator,
I::Item: Borrow<Scalar>,
J: IntoIterator,
J::Item: Borrow<Scalar>,
K: IntoIterator<Item = Option<Self::Point>>;
}
// ------------------------------------------------------------------------
// Private Traits
// ------------------------------------------------------------------------

4
src/scalar_mul/window.rs → src/window.rs
View file

@ -22,8 +22,8 @@ use subtle::Choice;
use traits::Identity;
use edwards::EdwardsPoint;
use curve_models::ProjectiveNielsPoint;
use curve_models::AffineNielsPoint;
use backend::serial::curve_models::ProjectiveNielsPoint;
use backend::serial::curve_models::AffineNielsPoint;
/// A lookup table of precomputed multiples of a point \\(P\\), used to
/// compute \\( xP \\) for \\( -8 \leq x \leq 8 \\).