diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..317be17 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,32 @@ +# Changelog + +Entries are listed in reverse chronological order. + +## 1.1.0-pre.0 + +* Restructures the source tree into `serial` and `vector` backends. +* Adds a new IFMA backend which sets speed records. +* Adds support for precomputation for multiscalar multiplication. +* Replaces the `rand` dependency with `rand_core`. +* Generalizes trait bounds on + - `RistrettoPoint::random()` + - `Scalar::random()` + to allow owned and borrowed RNGs and to allow `RngCore` instead of + `Rng`. + +## 1.0.3 + +* Adds `ConstantTimeEq` implementation for compressed points. + +## 1.0.2 + +* Fixes a typo in the naming of variables in Ristretto formulas (no change to functionality). + +## 1.0.1 + +* Depends on the stable `2.0` version of `subtle` instead of `2.0.0-pre.0`. + +## 1.0.0 + +Initial stable release. Yanked due to a dependency mistake (see above). + diff --git a/Cargo.toml b/Cargo.toml index a437e7e..ed0cbb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "curve25519-dalek" -version = "1.0.3" +version = "1.1.0-pre.0" authors = ["Isis Lovecruft ", "Henry de Valence "] readme = "README.md" @@ -26,9 +26,11 @@ features = ["nightly"] travis-ci = { repository = "dalek-cryptography/curve25519-dalek", branch = "master"} [dev-dependencies] +rand_os = "0.1.0" sha2 = { version = "0.8", default-features = false } bincode = "1" criterion = "0.2" +rand = "0.6" [[bench]] name = "dalek_benchmarks" @@ -41,7 +43,7 @@ harness = false # match exactly, since the build.rs uses the crate itself as a library. [dependencies] -rand = { version = "0.6.0", default-features = false } +rand_core = { version = "0.3.0", default-features = false } byteorder = { version = "^1.2.3", default-features = false, features = ["i128"] } digest = { version = "0.8", default-features = false } clear_on_drop = "=0.2.3" @@ -50,7 +52,7 @@ serde = { version = "1.0", optional = true } packed_simd = { version = "0.3.0", features = ["into_bits"], optional = true } [build-dependencies] -rand = { version = "0.6.0", default-features = false } +rand_core = { version = "0.3.0", default-features = false } byteorder = { version = "^1.2.3", default-features = false, features = ["i128"] } digest = { version = "0.8", default-features = false } clear_on_drop = "=0.2.3" @@ -61,7 +63,7 @@ packed_simd = { version = "0.3.0", features = ["into_bits"], optional = true } [features] nightly = ["subtle/nightly", "clear_on_drop/nightly"] default = ["std", "u64_backend"] -std = ["alloc", "subtle/std", "rand/std"] +std = ["alloc", "subtle/std", "rand_core/std"] alloc = [] yolocrypto = [] @@ -69,9 +71,10 @@ yolocrypto = [] u32_backend = [] # The u64 backend uses u64s with u128 products. u64_backend = [] -# The AVX2 backend uses u32x8s with u64x4 products. -# It uses the u64 code for serial operations. -avx2_backend = ["nightly", "u64_backend", "packed_simd"] +# The SIMD backend uses parallel formulas, using either AVX2 or AVX512-IFMA. +simd_backend = ["nightly", "u64_backend", "packed_simd"] +# Old name for the SIMD backend, preserved for compatibility +avx2_backend = ["simd_backend"] # Signals that we're in the main build stage. This is off by default, # to signal stage 1 of the build, where build.rs loads the library diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs index 71792a3..bb47634 100644 --- a/benches/dalek_benchmarks.rs +++ b/benches/dalek_benchmarks.rs @@ -2,10 +2,12 @@ extern crate rand; use rand::rngs::OsRng; +use rand::thread_rng; #[macro_use] extern crate criterion; +use criterion::BatchSize; use criterion::Criterion; extern crate curve25519_dalek; @@ -20,14 +22,10 @@ mod edwards_benches { use super::*; use curve25519_dalek::edwards; use curve25519_dalek::edwards::EdwardsPoint; - use curve25519_dalek::traits::MultiscalarMul; - use curve25519_dalek::traits::VartimeMultiscalarMul; fn compress(c: &mut Criterion) { let B = &constants::ED25519_BASEPOINT_POINT; - c.bench_function("EdwardsPoint compression", move |b| { - b.iter(|| B.compress()) - }); + c.bench_function("EdwardsPoint compression", move |b| b.iter(|| B.compress())); } fn decompress(c: &mut Criterion) { @@ -55,25 +53,65 @@ mod edwards_benches { fn vartime_double_base_scalar_mul(c: &mut Criterion) { c.bench_function("Variable-time aA+bB, A variable, B fixed", |bench| { - let B = &constants::ED25519_BASEPOINT_POINT; - let a = Scalar::from(298374928u64).invert(); - let b = Scalar::from(897987897u64).invert(); - let A = B * (b * a); - bench.iter(|| EdwardsPoint::vartime_double_scalar_mul_basepoint(&a, &A, &b)); + let mut rng = thread_rng(); + let A = &Scalar::random(&mut rng) * &constants::ED25519_BASEPOINT_TABLE; + bench.iter_batched( + || (Scalar::random(&mut rng), Scalar::random(&mut rng)), + |(a, b)| EdwardsPoint::vartime_double_scalar_mul_basepoint(&a, &A, &b), + BatchSize::SmallInput, + ); }); } + criterion_group! { + name = edwards_benches; + config = Criterion::default(); + targets = + compress, + decompress, + consttime_fixed_base_scalar_mul, + consttime_variable_base_scalar_mul, + vartime_double_base_scalar_mul, + } +} + +mod multiscalar_benches { + use super::*; + use curve25519_dalek::edwards; + use curve25519_dalek::edwards::EdwardsPoint; + use curve25519_dalek::edwards::VartimeEdwardsPrecomputation; + use curve25519_dalek::traits::MultiscalarMul; + use curve25519_dalek::traits::VartimeMultiscalarMul; + use curve25519_dalek::traits::VartimePrecomputedMultiscalarMul; + + fn construct_scalars(n: usize) -> Vec { + let mut rng = thread_rng(); + (0..n).map(|_| Scalar::random(&mut rng)).collect() + } + + fn construct_points(n: usize) -> Vec { + let mut rng = thread_rng(); + (0..n) + .map(|_| &Scalar::random(&mut rng) * &constants::ED25519_BASEPOINT_TABLE) + .collect() + } + + fn construct(n: usize) -> (Vec, Vec) { + (construct_scalars(n), construct_points(n)) + } + fn consttime_multiscalar_mul(c: &mut Criterion) { c.bench_function_over_inputs( "Constant-time variable-base multiscalar multiplication", |b, &&size| { - let mut rng = OsRng::new().unwrap(); - let scalars: Vec = (0..size).map(|_| Scalar::random(&mut rng)).collect(); - let points: Vec = scalars - .iter() - .map(|s| s * &constants::ED25519_BASEPOINT_TABLE) - .collect(); - b.iter(|| EdwardsPoint::multiscalar_mul(&scalars, &points)); + let points = construct_points(size); + // This is supposed to be constant-time, but we might as well + // rerandomize the scalars for every call just in case. + b.iter_batched( + || construct_scalars(size), + |scalars| EdwardsPoint::multiscalar_mul(&scalars, &points), + BatchSize::SmallInput, + ); }, &MULTISCALAR_SIZES, ); @@ -83,29 +121,107 @@ mod edwards_benches { c.bench_function_over_inputs( "Variable-time variable-base multiscalar multiplication", |b, &&size| { - let mut rng = OsRng::new().unwrap(); - let scalars: Vec = (0..size).map(|_| Scalar::random(&mut rng)).collect(); - let points: Vec = scalars - .iter() - .map(|s| s * &constants::ED25519_BASEPOINT_TABLE) - .collect(); - b.iter(|| EdwardsPoint::vartime_multiscalar_mul(&scalars, &points)); + let points = construct_points(size); + // Rerandomize the scalars for every call to prevent + // false timings from better caching (e.g., the CPU + // cache lifts exactly the right table entries for the + // benchmark into the highest cache levels). + b.iter_batched( + || construct_scalars(size), + |scalars| EdwardsPoint::vartime_multiscalar_mul(&scalars, &points), + BatchSize::SmallInput, + ); }, &MULTISCALAR_SIZES, ); } - criterion_group!{ - name = edwards_benches; - config = Criterion::default(); + fn vartime_precomputed_pure_static(c: &mut Criterion) { + c.bench_function_over_inputs( + "Variable-time fixed-base multiscalar multiplication", + move |b, &&total_size| { + let static_size = total_size; + + let static_points = construct_points(static_size); + let precomp = VartimeEdwardsPrecomputation::new(&static_points); + // Rerandomize the scalars for every call to prevent + // false timings from better caching (e.g., the CPU + // cache lifts exactly the right table entries for the + // benchmark into the highest cache levels). + b.iter_batched( + || construct_scalars(static_size), + |scalars| precomp.vartime_multiscalar_mul(&scalars), + BatchSize::SmallInput, + ); + }, + &MULTISCALAR_SIZES, + ); + } + + fn vartime_precomputed_helper(c: &mut Criterion, dynamic_fraction: f64) { + let label = format!( + "Variable-time mixed-base multiscalar multiplication ({:.0}pct dyn)", + 100.0 * dynamic_fraction, + ); + c.bench_function_over_inputs( + &label, + move |b, &&total_size| { + let dynamic_size = ((total_size as f64) * dynamic_fraction) as usize; + let static_size = total_size - dynamic_size; + + let static_points = construct_points(static_size); + let dynamic_points = construct_points(dynamic_size); + let precomp = VartimeEdwardsPrecomputation::new(&static_points); + // Rerandomize the scalars for every call to prevent + // false timings from better caching (e.g., the CPU + // cache lifts exactly the right table entries for the + // benchmark into the highest cache levels). Timings + // should be independent of points so we don't + // randomize them. + b.iter_batched( + || { + ( + construct_scalars(static_size), + construct_scalars(dynamic_size), + ) + }, + |(static_scalars, dynamic_scalars)| { + precomp.vartime_mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ) + }, + BatchSize::SmallInput, + ); + }, + &MULTISCALAR_SIZES, + ); + } + + fn vartime_precomputed_00_pct_dynamic(c: &mut Criterion) { + vartime_precomputed_helper(c, 0.0); + } + + fn vartime_precomputed_20_pct_dynamic(c: &mut Criterion) { + vartime_precomputed_helper(c, 0.2); + } + + fn vartime_precomputed_50_pct_dynamic(c: &mut Criterion) { + vartime_precomputed_helper(c, 0.5); + } + + criterion_group! { + name = multiscalar_benches; + // Lower the sample size to run the benchmarks faster + config = Criterion::default().sample_size(15); targets = - compress, - decompress, - consttime_fixed_base_scalar_mul, - consttime_variable_base_scalar_mul, - vartime_double_base_scalar_mul, consttime_multiscalar_mul, vartime_multiscalar_mul, + vartime_precomputed_pure_static, + vartime_precomputed_00_pct_dynamic, + vartime_precomputed_20_pct_dynamic, + vartime_precomputed_50_pct_dynamic, } } @@ -141,7 +257,7 @@ mod ristretto_benches { ); } - criterion_group!{ + criterion_group! { name = ristretto_benches; config = Criterion::default(); targets = @@ -162,7 +278,7 @@ mod montgomery_benches { }); } - criterion_group!{ + criterion_group! { name = montgomery_benches; config = Criterion::default(); targets = montgomery_ladder, @@ -194,7 +310,7 @@ mod scalar_benches { ); } - criterion_group!{ + criterion_group! { name = scalar_benches; config = Criterion::default(); targets = @@ -208,4 +324,5 @@ criterion_main!( montgomery_benches::montgomery_benches, ristretto_benches::ristretto_benches, edwards_benches::edwards_benches, + multiscalar_benches::multiscalar_benches, ); diff --git a/build.rs b/build.rs index 284bce7..2d020e1 100644 --- a/build.rs +++ b/build.rs @@ -1,5 +1,13 @@ +#![cfg_attr( + all(feature = "simd_backend", target_feature = "avx512ifma"), + feature(simd_ffi) +)] +#![cfg_attr( + all(feature = "simd_backend", target_feature = "avx512ifma"), + feature(link_llvm_intrinsics) +)] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] -#![cfg_attr(feature = "nightly", feature(cfg_target_feature))] +#![cfg_attr(feature = "nightly", feature(doc_cfg))] #![allow(unused_variables)] #![allow(non_snake_case)] #![allow(dead_code)] @@ -10,10 +18,10 @@ extern crate byteorder; extern crate clear_on_drop; extern crate core; extern crate digest; -extern crate rand; +extern crate rand_core; extern crate subtle; -#[cfg(all(feature = "nightly", feature = "avx2_backend"))] +#[cfg(all(feature = "nightly", feature = "packed_simd"))] extern crate packed_simd; use std::env; @@ -36,35 +44,31 @@ mod macros; // Public modules -#[path = "src/scalar.rs"] -mod scalar; -#[path = "src/montgomery.rs"] -mod montgomery; -#[path = "src/edwards.rs"] -mod edwards; -#[path = "src/ristretto.rs"] -mod ristretto; #[path = "src/constants.rs"] mod constants; +#[path = "src/edwards.rs"] +mod edwards; +#[path = "src/montgomery.rs"] +mod montgomery; +#[path = "src/ristretto.rs"] +mod ristretto; +#[path = "src/scalar.rs"] +mod scalar; #[path = "src/traits.rs"] mod traits; // Internal modules -#[path = "src/field.rs"] -mod field; -#[path = "src/curve_models/mod.rs"] -mod curve_models; #[path = "src/backend/mod.rs"] mod backend; +#[path = "src/field.rs"] +mod field; #[path = "src/prelude.rs"] mod prelude; -#[path = "src/scalar_mul/mod.rs"] -mod scalar_mul; +#[path = "src/window.rs"] +mod window; use edwards::EdwardsBasepointTable; -use curve_models::AffineNielsPoint; -use scalar_mul::window::NafLookupTable8; fn main() { // Enable the "stage2_build" feature in the main build stage @@ -81,17 +85,16 @@ fn main() { format!( "\n #[cfg(feature = \"u32_backend\")] -use backend::u32::field::FieldElement32; +use backend::serial::u32::field::FieldElement2625; #[cfg(feature = \"u64_backend\")] -use backend::u64::field::FieldElement64; +use backend::serial::u64::field::FieldElement51; use edwards::EdwardsBasepointTable; -use curve_models::AffineNielsPoint; +use backend::serial::curve_models::AffineNielsPoint; -use scalar_mul::window::LookupTable; -use scalar_mul::window::NafLookupTable8; +use window::LookupTable; /// Table containing precomputed multiples of the Ed25519 basepoint \\\\(B = (x, 4/5)\\\\). pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN; @@ -101,20 +104,35 @@ pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TAB pub const ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN: EdwardsBasepointTable = {:?}; \n\n", &table - ).as_bytes(), - ).unwrap(); + ) + .as_bytes(), + ) + .unwrap(); // Now generate AFFINE_ODD_MULTIPLES_OF_BASEPOINT - let B = &constants::ED25519_BASEPOINT_POINT; - let odd_multiples = NafLookupTable8::::from(B); + // if we are going to build the serial scalar_mul backend + #[cfg(not(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") + )))] + { + use backend::serial::curve_models::AffineNielsPoint; + use window::NafLookupTable8; - f.write_all( - format!( - "\n + let B = &constants::ED25519_BASEPOINT_POINT; + let odd_multiples = NafLookupTable8::::from(B); + + f.write_all( + format!( + "\n +use window::NafLookupTable8; /// Odd multiples of the basepoint `[B, 3B, 5B, 7B, 9B, 11B, 13B, 15B, ..., 127B]`. pub(crate) const AFFINE_ODD_MULTIPLES_OF_BASEPOINT: NafLookupTable8 = {:?}; \n\n", - &odd_multiples - ).as_bytes(), - ).unwrap(); + &odd_multiples + ) + .as_bytes(), + ) + .unwrap(); + } } diff --git a/docs/avx2-notes.md b/docs/avx2-notes.md index d902650..87992b3 100644 --- a/docs/avx2-notes.md +++ b/docs/avx2-notes.md @@ -1,218 +1,4 @@ -A vectorized implementation of group operations on the twisted Edwards -form of Curve25519, using a modification of the 4-way parallel -formulas of Hisil, Wong, Carter, and Dawson. - -# Overview - -The 2008 paper [_Twisted Edwards Curves Revisited_][hwcd08] by Hisil, -Wong, Carter, and Dawson (HWCD) introduced the “extended coordinates” -and mixed-model representations which are used by most Edwards curve -implementations. - -However, they also describe 4-way parallel formulas for point addition -and doubling: a unified addition algorithm taking an effective -\\(2\mathbf M + 1\mathbf D\\), a doubling algorithm taking an -effective \\(1\mathbf M + 1\mathbf S\\), and a dedicated (i.e., for -distinct points) addition algorithm taking an effective \\(2 \mathbf M -\\). They compare these formulas with a 2-way parallel variant of the -Montgomery ladder. - -Unlike their serial formulas, which are used widely, their parallel -formulas do not seem to have been implemented in software before. The -2-way parallel Montgomery ladder was used in 2015 by Tung Chou's -`sandy2x` implementation. Curiously, however, although the [`sandy2x` -paper][sandy2x] also implements Edwards arithmetic, and cites HWCD08, -it doesn't mention their parallel Edwards formulas. -A 2015 paper by Hernández and López describes an AVX2 implementation -of X25519. Neither the paper nor the code are publicly available, but -it apparently gives only a [slight speedup][avx2trac], suggesting that -it uses a 4-way parallel Montgomery ladder rather than parallel -Edwards formulas. - -The reason may be that HWCD08 describe their formulas as operating on -four independent processors, which would make a software -implementation impractical: all of the operations are too low-latency -to effectively synchronize. But a closer inspection reveals that the -(more expensive) multiplication and squaring steps are uniform, while -the instruction divergence occurs in the (much cheaper) addition and -subtraction steps. This means that a SIMD implementation can perform -the expensive steps uniformly, and handle divergence in the -inexpensive steps using masking. - -These notes describe modifications to the original parallel formulas -to allow a SIMD implementation, and this module contains an -implementation of the modified formulas using 256-bit AVX2 vector -operations. - -# Parallel formulas in HWCD'08 - -The doubling formula is presented in the HWCD paper as follows: - -| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | -|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| | idle | idle | idle | \\( R\_1 \gets X\_1 + Y\_1 \\) | -| \\(1\mathbf S\\) | \\( R\_2 \gets X\_1\^2 \\) | \\( R\_3 \gets Y\_1\^2 \\) | \\( R\_4 \gets Z\_1\^2 \\) | \\( R\_5 \gets R\_1\^2 \\) | -| | \\( R\_6 \gets R\_2 + R\_3 \\) | \\( R\_7 \gets R\_2 - R\_3 \\) | \\( R\_4 \gets 2 R\_4 \\) | idle | -| | idle | \\( R\_1 \gets R\_4 + R\_7 \\) | idle | \\( R\_2 \gets R\_6 - R\_5 \\) | -| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_6 R\_7 \\) | \\( T\_3 \gets R\_2 R\_6 \\) | \\( Z\_3 \gets R\_1 R\_7 \\) | - -and the unified addition algorithm is presented as follows: - -| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | -|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| | \\( R\_1 \gets Y\_1 - X\_1 \\) | \\( R\_2 \gets Y\_2 - X\_2 \\) | \\( R\_3 \gets Y\_1 + X\_1 \\) | \\( R\_4 \gets Y\_2 + X\_2 \\) | -| \\(1\mathbf M\\) | \\( R\_5 \gets R\_1 R\_2 \\) | \\( R\_6 \gets R\_3 R\_4 \\) | \\( R\_7 \gets T\_1 T\_2 \\) | \\( R\_8 \gets Z\_1 Z\_2 \\) | -| \\(1\mathbf D\\) | idle | idle | \\( R\_7 \gets k R\_7 \\) | \\( R\_8 \gets 2 R\_8 \\) | -| | \\( R\_1 \gets R\_6 - R\_5 \\) | \\( R\_2 \gets R\_8 - R\_7 \\) | \\( R\_3 \gets R\_8 + R\_7 \\) | \\( R\_4 \gets R\_6 + R\_5 \\) | -| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_3 R\_4 \\) | \\( T\_3 \gets R\_1 R\_4 \\) | \\( Z\_3 \gets R\_2 R\_3 \\) | - -Here \\(\mathbf M\\) and \\(\mathbf S\\) represent the cost of -multiplication and squaring of generic field elements, \\(\mathbf D\\) -represents the cost of multiplication by a curve constant (in this -case \\( k = 2d \\)). - -Notice that the \\(1\mathbf M\\) and \\(1\mathbf S\\) steps are -uniform. The non-uniform steps are all inexpensive additions or -subtractions, with the exception of the multiplication by the curve -constant \\(k = 2d\\): -$$ -R\_7 \gets 2 d R\_7. -$$ - -HWCD suggest parallelising this step by breaking \\(k = 2d\\) into four -parts as \\(k = k_0 + 2\^n k_1 + 2\^{2n} k_2 + 2\^{3n} k_3 \\) and -computing \\(k_i R_7 \\) in parallel. This is quite awkward, but if -the curve constant is a ratio \\( d = d\_1/d\_2 \\), then projective -coordinates allow us to instead compute -$$ -(R\_5, R\_6, R\_7, R\_8) \gets (d\_2 R\_5, d\_2 R\_6, 2d\_1 R\_7, d\_2 R\_8). -$$ -This can be performed as a uniform multiplication by a vector of -constants, and if \\(d\_1, d\_2\\) are small, it is relatively -inexpensive. (This trick was suggested by Mike Hamburg). -In the Curve25519 case, we have -$$ -d = \frac{d\_1}{d\_2} = \frac{-121665}{121666}; -$$ -Since \\(2 \cdot 121666 < 2\^{18}\\), all the constants above fit (up -to sign) in 32 bits, so this can be done in parallel as four -multiplications by small constants \\( (121666, 121666, 2\cdot 121665, -2\cdot 121666) \\), followed by a negation to compute \\( - 2\cdot 121665\\). - -# Modified parallel formulas - -Using the modifications sketched above, we can write SIMD-friendly -versions of the parallel formulas as follows. To avoid confusion with -the original formulas, temporary variables are named \\(S\\) instead -of \\(R\\) and are in static single-assignment form. - -## Addition - -To add points -\\(P_1 = (X_1 : Y_1 : Z_1 : T_1) \\) -and -\\(P_2 = (X_2 : Y_2 : Z_2 : T_2 ) \\), -we compute -$$ -\begin{aligned} -(S\_0 &&,&& S\_1 &&,&& S\_2 &&,&& S\_3 ) -&\gets -(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Y\_2 - X\_2&&,&& Y\_2 + X\_2) -\\\\ -(S\_4 &&,&& S\_5 &&,&& S\_6 &&,&& S\_7 ) -&\gets -(S\_0 \cdot S\_2&&,&& S\_1 \cdot S\_3&&,&& Z\_1 \cdot Z\_2&&,&& T\_1 \cdot T\_2) -\\\\ -(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} ) -&\gets -(d\_2 \cdot S\_4 &&,&& d\_2 \cdot S\_5 &&,&& 2 d\_2 \cdot S\_6 &&,&& 2 d\_1 \cdot S\_7 ) -\\\\ -(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15}) -&\gets -(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11}) -\\\\ -(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3) -&\gets -(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13}) -\end{aligned} -$$ -to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). -This costs \\( 2\mathbf M + 1 \mathbf D\\). - -## Readdition - -If the point \\( P_2 = (X\_2 : Y\_2 : Z\_2 : T\_2) \\) is fixed, we -can cache the multiplication of the curve constants by computing -$$ -\begin{aligned} -(S\_2' &&,&& S\_3' &&,&& Z\_2' &&,&& T\_2' ) -&\gets -(d\_2 \cdot (Y\_2 - X\_2)&&,&& d\_2 \cdot (Y\_1 + X\_1)&&,&& 2d\_2 \cdot Z\_2 &&,&& 2d\_1 \cdot T\_2). -\end{aligned} -$$ -This costs \\( 1\mathbf D\\); with \\( (S\_2', S\_3', Z\_2', T\_2')\\) -in hand, the addition formulas above become -$$ -\begin{aligned} -(S\_0 &&,&& S\_1 &&,&& Z\_1 &&,&& T\_1 ) -&\gets -(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Z\_1 &&,&& T\_1) -\\\\ -(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} ) -&\gets -(S\_0 \cdot S\_2' &&,&& S\_1 \cdot S\_3'&&,&& Z\_1 \cdot Z\_2' &&,&& T\_1 \cdot T\_2') -\\\\ -(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15}) -&\gets -(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11}) -\\\\ -(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3) -&\gets -(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13}) -\end{aligned} -$$ -which costs only \\( 2\mathbf M \\). This precomputation is -essentially similar to the precomputation that HWCD suggest for their -serial formulas. Because the cost of precomputation and then -readdition is the same as addition, it's sufficient to only -implement caching and readdition. - -## Doubling - -The non-uniform portions of the (re)addition formulas have a fairly -regular structure. Unfortunately, this is not the case for the -doubling formulas, which are much less nice. - -To double a point \\( P = (X\_1 : Y\_1 : Z\_1 : T\_1) \\), we compute -$$ -\begin{aligned} -(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& S\_0) -&\gets -(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& X\_1 + Y\_1) -\\\\ -(S\_1 &&,&& S\_2 &&,&& S\_3 &&,&& S\_4 ) -&\gets -(X\_1\^2 &&,&& Y\_1\^2&&,&& Z\_1\^2 &&,&& S\_0\^2) -\\\\ -(S\_5 &&,&& S\_6 &&,&& S\_8 &&,&& S\_9 ) -&\gets -(S\_1 + S\_2 &&,&& S\_1 - S\_2 &&,&& S\_1 + 2S\_3 - S\_2 &&,&& S\_1 + S\_2 - S\_4) -\\\\ -(X\_3 &&,&& Y\_3 &&,&& Z\_3 &&,&& T\_3 ) -&\gets -(S\_8 \cdot S\_9 &&,&& S\_5 \cdot S\_6 &&,&& S\_8 \cdot S\_6 &&,&& S\_5 \cdot S\_9) -\end{aligned} -$$ -to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = [2]P\_1 \\). - -The intermediate step between the squaring and multiplication requires -a long chain of additions, but with some care and finesse, -described below, it is possible (in our case) to arrange this -computation without requiring an intermediate reduction. - -However, it does mean that the doubling formulas have proportionately -more vectorization overhead than the (re)addition formulas. The -effects of this are discussed in the comparison section below. +An AVX2 implementation of the vectorized point operation strategy. # Field element representation @@ -266,9 +52,7 @@ the form Since this breaks cleanly into two 128-bit lanes, it may be possible to adapt it to 128-bit vector instructions such as NEON without too -much difficulty. Going the other direction, to extend this to AVX512, -we could either run two point operations in parallel in lower and upper -halves of the registers, or use 2-way parallelism within a field operation. +much difficulty. # Avoiding Overflow in Doubling @@ -354,106 +138,3 @@ $$ whose right-hand sides are all bounded with \\( b < 1.75 \\) and whose left-hand sides are all bounded with \\( b < 2.5 \\), so that we can avoid any intermediate reductions. - -# Comparison to non-vectorized formulas - -In theory, the parallel Edwards formulas seem to allow a \\(4\\)-way -speedup from parallelism. However, an actual vectorized -implementation has several slowdowns that cut into this speedup. - -First, the parallel formulas can only use a \\( 32 \times 32 -\rightarrow 64 \\)-bit integer multiplier, so the speedup from -vectorization must overcome the disadvantage of losing the \\( 64 -\times 64 \rightarrow 128\\)-bit (serial) integer multiplier. The -effect of this slowdown is microarchitecture-dependent, since it -requires accounting for the total number of multiplications and -additions and their relative costs. In the future, it will probably -be possible to avoid this slowdown by using the `IFMA52` instructions, -whose parallelism is perfectly suited to these formulas. - -Second, the parallel doubling formulas incur both a theoretical and -practical slowdown. The parallel formulas described above work on the -\\( \mathbb P\^3 \\) “extended” coordinates. The \\( \mathbb P\^2 \\) -model introduced earlier by [Bernstein, Birkner, Joye, Lange, and -Peters][bbjlp08] allows slightly faster doublings, so HWCD suggest -mixing coordinate systems while performing scalar multiplication -(attributing the idea to [a 1998 paper][cmo98] by Cohen, Miyagi, and -Ono). The \\( T \\) coordinate is not required for doublings, so when -doublings are followed by doublings, its computation can be skipped. -More details on this approach and the different coordinate systems can -be found in the [`curve_models` module documentation][curve_models]. - -Unfortunately, this optimization is not compatible with the parallel -formulas, which cannot save time by skipping a single variable, so the -parallel doubling formulas do slightly more work when counting the -total number of field multiplications and squarings. - -In addition, the parallel doubling formulas have a less regular -pattern of additions and subtractions than the parallel addition -formulas, so the vectorization overhead is proportionately greater. -Both the parallel addition and parallel doubling formulas also require -some shuffling to rearrange data within the vectors, which places more -pressure on the shuffle unit than is desirable. - -This means that the speedup from using a vectorized implementation of -parallel Edwards formulas is likely to be greatest in applications -that do fewer doublings and more additions (like a large multiscalar -multiplication) rather than applications that do fewer additions and -more doublings (like a double-base scalar multiplication). - -Third, current Intel CPUs perform thermal throttling when using wide -vector instructions. A detailed description can be found in §15.26 of -[the Intel Optimization Manual][intel], but using wide vector -instructions prevents the core from operating at higher frequencies. -The core can return to the higher-frequency state after 2 -milliseconds, but this timer is reset every time high-power -instructions are used. - -Any speedup from vectorization therefore has to be weighed against a -slowdown for the next few million instructions. For a mixed workload, -where point operations are interspersed with other tasks, this can -reduce overall performance. This implementation is therefore probably -not suitable for basic applications, like signatures, but is -worthwhile for complex applications, like zero-knowledge proofs, which -do sustained work. - -For this reason, the AVX2 backend is not enabled by default, but can -be selected using the `avx2_backend` feature. - -# Future work - -There are several directions for future improvement: - -* Using the vectorized field arithmetic code to parallelize across - point operations rather than within a single point operation. This - is less flexible, but would give a speedup both from allowing use of - the faster mixed-model arithmetic and from reducing shuffle - pressure. One approach in this direction would be to implement - batched scalar-point operations using vectors of points (AoSoA - layout). This less generally useful but would give a speedup for - Bulletproofs. - -* Extending the implementation to use the full width of AVX512, either - handling the extra parallelism internally to a single point - operation (by using a 2-way parallel implementation of field - arithmetic instead of a wordsliced one), or externally, - parallelizing across point operations. Internal parallelism would - be preferable but might require too much shuffle pressure. - -* Generalizing the implementation to non-AVX2 instructions, - particularly NEON. The current point arithmetic code is written in - terms of field element vectors, which are in turn implemented using - platform SIMD vectors. It should be possible to write an alternate - implementation of the `FieldElement32x4` using NEON without changing - the point arithmetic. NEON has 128-bit vectors rather than 256-bit - vectors, but this may still be worthwhile compared to a serial - implementation. - - -[sandy2x]: https://eprint.iacr.org/2015/943.pdf -[avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28 -[hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf -[curve_models]: https://doc-internal.dalek.rs/curve25519_dalek/curve_models/index.html -[bbjlp08]: https://eprint.iacr.org/2008/013 -[cmo98]: https://link.springer.com/content/pdf/10.1007%2F3-540-49649-1_6.pdf -[intel]: https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf \ No newline at end of file diff --git a/docs/ifma-notes.md b/docs/ifma-notes.md new file mode 100644 index 0000000..c6fd3b3 --- /dev/null +++ b/docs/ifma-notes.md @@ -0,0 +1,580 @@ +An AVX512-IFMA implementation of the vectorized point operation +strategy. + +# IFMA instructions + +AVX512-IFMA is an extension to AVX-512 consisting of two instructions: + +* `vpmadd52luq`: packed multiply of unsigned 52-bit integers and add + the low 52 product bits to 64-bit accumulators; +* `vpmadd52huq`: packed multiply of unsigned 52-bit integers and add + the high 52 product bits to 64-bit accumulators; + +These operate on 64-bit lanes of their source vectors, taking the low +52 bits of each lane of each source vector, computing the 104-bit +products of each pair, and then adding either the high or low 52 bits +of the 104-bit products to the 64-bit lanes of the destination vector. +The multiplication is performed internally by reusing circuitry for +floating-point arithmetic. Although these instructions are part of +AVX512, the AVX512VL (vector length) extension (present whenever IFMA +is) allows using them with 512, 256, or 128-bit operands. + +This provides a major advantage to vectorized integer operations: +previously, vector operations could only use a \\(32 \times 32 +\rightarrow 64\\)-bit multiplier, while serial code could use a +\\(64\times 64 \rightarrow 128\\)-bit multiplier. + +## IFMA for big-integer multiplications + +A detailed example of the intended use of the IFMA instructions can be +found in a 2016 paper by Gueron and Krasnov, [_Accelerating Big +Integer Arithmetic Using Intel IFMA Extensions_][2016_gueron_krasnov]. +The basic idea is that multiplication of large integers (such as 1024, +2048, or more bits) can be performed as follows. + +First, convert a “packed” 64-bit representation +\\[ +\begin{aligned} +x &= x'_0 + x'_1 2^{64} + x'_2 2^{128} + \cdots \\\\ +y &= y'_0 + y'_1 2^{64} + y'_2 2^{128} + \cdots +\end{aligned} +\\] +into a “redundant” 52-bit representation +\\[ +\begin{aligned} +x &= x_0 + x_1 2^{52} + x_2 2^{104} + \cdots \\\\ +y &= y_0 + y_1 2^{52} + y_2 2^{104} + \cdots +\end{aligned} +\\] +with each \\(x_i, y_j\\) in a 64-bit lane. + +Writing the product as \\(z = z_0 + z_1 2^{52} + z_2 2^{104} + \cdots\\), +the “schoolbook” multiplication strategy gives +\\[ +\begin{aligned} +&z_0 &&=& x_0 & y_0 & & & & & & & & \\\\ +&z_1 &&=& x_1 & y_0 &+ x_0 & y_1 & & & & & & \\\\ +&z_2 &&=& x_2 & y_0 &+ x_1 & y_1 &+ x_0 & y_2 & & & & \\\\ +&z_3 &&=& x_3 & y_0 &+ x_2 & y_1 &+ x_1 & y_2 &+ x_0 & y_3 & & \\\\ +&z_4 &&=& \vdots\\;&\\;\vdots &+ x_3 & y_1 &+ x_2 & y_2 &+ x_1 & y_3 &+ \cdots& \\\\ +&z_5 &&=& & & \vdots\\;&\\;\vdots &+ x_3 & y_2 &+ x_2 & y_3 &+ \cdots& \\\\ +&z_6 &&=& & & & & \vdots\\;&\\;\vdots &+ x_3 & y_3 &+ \cdots& \\\\ +&z_7 &&=& & & & & & & \vdots\\;&\\;\vdots &+ \cdots& \\\\ +&\vdots&&=& & & & & & & & & \ddots& \\\\ +\end{aligned} +\\] +Notice that the product coefficient \\(z_k\\), representing the value +\\(z_k 2^{52k}\\), is the sum of all product terms +\\( +(x_i 2^{52 i}) (y_j 2^{52 j}) +\\) +with \\(k = i + j\\). +Write the IFMA operators \\(\mathrm{lo}(a,b)\\), denoting the low +\\(52\\) bits of \\(ab\\), and +\\(\mathrm{hi}(a,b)\\), denoting the high \\(52\\) bits of +\\(ab\\). +Now we can rewrite the product terms as +\\[ +\begin{aligned} +(x_i 2^{52 i}) (y_j 2^{52 j}) +&= +2^{52 (i+j)}( +\mathrm{lo}(x_i, y_j) + +\mathrm{hi}(x_i, y_j) 2^{52} +) +\\\\ +&= +\mathrm{lo}(x_i, y_j) 2^{52 (i+j)} + +\mathrm{hi}(x_i, y_j) 2^{52 (i+j+1)}. +\end{aligned} +\\] +This means that the low half of \\(x_i y_j\\) can be accumulated onto +the product limb \\(z_{i+j}\\) and the high half can be directly +accumulated onto the next-higher product limb \\(z_{i+j+1}\\) with no +additional operations. This allows rewriting the schoolbook +multiplication into the form +\\[ +\begin{aligned} +&z_0 &&=& \mathrm{lo}(x_0,&y_0) & & & & & & & & & & \\\\ +&z_1 &&=& \mathrm{lo}(x_1,&y_0) &+\mathrm{hi}(x_0,&y_0) &+\mathrm{lo}(x_0,&y_1) & & & & & & \\\\ +&z_2 &&=& \mathrm{lo}(x_2,&y_0) &+\mathrm{hi}(x_1,&y_0) &+\mathrm{lo}(x_1,&y_1) &+\mathrm{hi}(x_0,&y_1) &+\mathrm{lo}(x_0,&y_2) & & \\\\ +&z_3 &&=& \mathrm{lo}(x_3,&y_0) &+\mathrm{hi}(x_2,&y_0) &+\mathrm{lo}(x_2,&y_1) &+\mathrm{hi}(x_1,&y_1) &+\mathrm{lo}(x_1,&y_2) &+ \cdots& \\\\ +&z_4 &&=& \vdots\\;&\\;\vdots &+\mathrm{hi}(x_3,&y_0) &+\mathrm{lo}(x_3,&y_1) &+\mathrm{hi}(x_2,&y_1) &+\mathrm{lo}(x_2,&y_2) &+ \cdots& \\\\ +&z_5 &&=& & & \vdots\\;&\\;\vdots & \vdots\\;&\\;\vdots &+\mathrm{hi}(x_3,&y_1) &+\mathrm{lo}(x_3,&y_2) &+ \cdots& \\\\ +&z_6 &&=& & & & & & & \vdots\\;&\\;\vdots & \vdots\\;&\\;\vdots &+ \cdots& \\\\ +&\vdots&&=& & & & & & & & & & & \ddots& \\\\ +\end{aligned} +\\] +Gueron and Krasnov implement multiplication by constructing vectors +out of the columns of this diagram, so that the source operands for +the IFMA instructions are of the form \\((x_0, x_1, x_2, \ldots)\\) +and \\((y_i, y_i, y_i, \ldots)\\). +After performing the multiplication, +the product terms \\(z_i\\) are then repacked into a 64-bit representation. + +## An alternative strategy + +The strategy described above is aimed at big-integer multiplications, +such as 1024, 2048, or 4096 bits, which would be used for applications +like RSA. However, elliptic curve cryptography uses much smaller field +sizes, such as 256 or 384 bits, so a different strategy is needed. + +The parallel Edwards formulas provide parallelism at the level of the +formulas for curve operations. This means that instead of scanning +through the terms of the source operands and parallelizing *within* a +field element (as described above), we can arrange the computation in +product-scanning form and parallelize *across* field elements (as +described below). + +The parallel Edwards +formulas provide 4-way parallelism, so they can be implemented using +256-bit vectors using a single 64-bit lane for each element, or using +512-bit vectors using two 64-bit lanes. +The only available CPU supporting IFMA (the +i3-8121U) executes 512-bit IFMA instructions at half rate compared to +256-bit instructions, so for now there's no throughput advantage to +using 512-bit IFMA instructions, and this implementation uses 256-bit +vectors. + +To extend this to 512-bit vectors, it's only only necessary to achieve +2-way parallelism, and it's possible (with a small amount of overhead) +to create a hybrid strategy that operates entirely within 128-bit +lanes. This means that cross-lane operations can use the faster +`vpshufd` (1c latency) instead of a general shuffle instruction (3c +latency). + +# Choice of radix + +The inputs to IFMA instructions are 52 bits wide, so the radix \\(r\\) +used to represent a multiprecision integer must be \\( r \leq 52 \\). +The obvious choice is the "native" radix \\(r = 52\\). + +As described above, this choice +has the advantage that for \\(x_i, y_j \in [0,2^{52})\\), the product term +\\[ +\begin{aligned} +(x_i 2^{52 i}) (y_j 2^{52 j}) +&= +2^{52 (i+j)}( +\mathrm{lo}(x_i, y_j) + +\mathrm{hi}(x_i, y_j) 2^{52} +) +\\\\ +&= +\mathrm{lo}(x_i, y_j) 2^{52 (i+j)} + +\mathrm{hi}(x_i, y_j) 2^{52 (i+j+1)}, +\end{aligned} +\\] +so that the low and high halves of the product can be directly accumulated +onto the product limbs. +In contrast, when using a smaller radix \\(r = 52 - k\\), +the product term has the form +\\[ +\begin{aligned} +(x_i 2^{r i}) (y_j 2^{r j}) +&= +2^{r (i+j)}( +\mathrm{lo}(x_i, y_j) + +\mathrm{hi}(x_i, y_j) 2^{52} +) +\\\\ +&= +\mathrm{lo}(x_i, y_j) 2^{r (i+j)} + +( +\mathrm{hi}(x_i, y_j) 2^k +) +2^{r (i+j+1)}. +\end{aligned} +\\] +What's happening is that the product \\(x_i y_j\\) of size \\(2r\\) +bits is split not at \\(r\\) but at \\(52\\), so \\(k\\) product bits +are placed into the low half instead of the high half. This means +that the high half of the product cannot be directly accumulated onto +\\(z_{i+j+1}\\), but must first be multiplied by \\(2^k\\) (i.e., left +shifted by \\(k\\)). In addition, the low half of the product is +\\(52\\) bits large instead of \\(r\\) bits. + +## Handling offset product terms + +[Drucker and Gueron][2018_drucker_gueron] analyze the choice of radix +in the context of big-integer squaring, outlining three ways to handle +the offset product terms, before concluding that all of them are +suboptimal: + +1. Shift the results after accumulation; +2. Shift the input operands before multiplication; +3. Split the MAC operation, accumulating into a zeroed register, + shifting the result, and then adding. + +The first option is rejected because it could double-shift some +previously accumulated terms, the second doesn't work because the +inputs could become larger than \\(52\\) bits, and the third requires +additional instructions to handle the shifting and adding. + +Based on an analysis of total number of instructions, they suggest an +addition to the instruction set, which they call `FMSA` (fused +multiply-shift-add). This would shift the result according to an 8-bit +immediate value before accumulating it into the destination register. + +However, this change to the instruction set doesn't seem to be +necessary. Instead, the product terms can be grouped according to +their coefficients, accumulated together, then shifted once before +adding them to the final sum. This uses an extra register, shift, and +add, but only once per product term (accumulation target), not once +per source term (as in the Drucker-Gueron paper). + +Moreover, because IFMA instructions execute only on two ports +(presumably 0 and 1), while adds and shifts can execute on three ports +(0, 1, and 5), the adds and shifts can execute independently of the +IFMA operations, as long as there is not too much pressure on port 5. +This means that, although the total number of instructions increases, +the shifts and adds do not necessarily increase the execution time, as +long as throughput is limited by IFMA operations. + +Finally, because IFMA instructions have 4 cycle latency and 0.5/1 +cycle throughput (for 256/512 bit vectors), maximizing IFMA throughput +requires either 8 (for 256) or 4 (for 512) independent operations. So +accumulating groups of terms independently before adding them at the +end may be necessary anyways, in order to prevent long chains of +dependent instructions. + +## Advantages of a smaller radix + +Using a smaller radix has other advantages. Although radix \\(52\\) +is an unsaturated representation from the point of view of the +\\(64\\)-bit accumulators (because up to 4096 product terms can be +accumulated without carries), it's a saturated representation from the +point of view of the multiplier (since \\(52\\)-bit values are the +maximum input size). + +Because the inputs to a multiplication must have all of their limbs +bounded by \\(2^{52}\\), limbs in excess of \\(2^{52}\\) must be +reduced before they can be used as an input. The +[Gueron-Krasnov][2016_gueron_krasnov] paper suggests normalizing +values using a standard, sequential carry chain: for each limb, add +the carryin from reducing the previous limb, compute the carryout and +reduce the current limb, then move to the next limb. + +However, when using a smaller radix, such as \\(51\\), each limb can +store a carry bit and still be used as the input to a multiplication. +This means that the inputs do not need to be normalized, and instead +of using a sequential carry chain, we can compute all carryouts in +parallel, reduce all limbs in parallel, and then add the carryins in +parallel (possibly growing the limb values by one bit). + +Because the output of this partial reduction is an acceptable +multiplication input, we can "close the loop" using partial reductions +and never have to normalize to a canonical representation through the +entire computation, in contrast to the Gueron-Krasnov approach, which +converts back to a packed representation after every operation. (This +idea seems to trace back to at least as early as [this 1999 +paper][1999_walter]). + +Using \\(r = 51\\) is enough to keep a carry bit in each limb and +avoid normalizations. What about an even smaller radix? One reason +to choose a smaller radix would be to align the limb boundaries with +an inline reduction (for instance, choosing \\(r = 43\\) for the +Mersenne field \\(p = 2^{127} - 1\\)), but for \\(p = 2^{255 - 19}\\), +\\(r = 51 = 255/5\\) is the natural choice. + +# Multiplication + +The inputs to a multiplication are two field elements +\\[ +\begin{aligned} +x &= x_0 + x_1 2^{51} + x_2 2^{102} + x_3 2^{153} + x_4 2^{204} \\\\ +y &= y_0 + y_1 2^{51} + y_2 2^{102} + y_3 2^{153} + y_4 2^{204}, +\end{aligned} +\\] +with limbs in range \\([0,2^{52})\\). + +Writing the product terms as +\\[ +\begin{aligned} +z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\ + &+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459}, +\end{aligned} +\\] +a schoolbook multiplication in product scanning form takes the form +\\[ +\begin{aligned} +z_0 &= x_0 y_0 \\\\ +z_1 &= x_1 y_0 + x_0 y_1 \\\\ +z_2 &= x_2 y_0 + x_1 y_1 + x_0 y_2 \\\\ +z_3 &= x_3 y_0 + x_2 y_1 + x_1 y_2 + x_0 y_3 \\\\ +z_4 &= x_4 y_0 + x_3 y_1 + x_2 y_2 + x_1 y_3 + x_0 y_4 \\\\ +z_5 &= x_4 y_1 + x_3 y_2 + x_2 y_3 + x_1 y_4 \\\\ +z_6 &= x_4 y_2 + x_3 y_3 + x_2 y_4 \\\\ +z_7 &= x_4 y_3 + x_3 y_4 \\\\ +z_8 &= x_4 y_4 \\\\ +z_9 &= 0 \\\\ +\end{aligned} +\\] +Each term \\(x_i y_j\\) can be written in terms of IFMA operations as +\\[ +x_i y_j = \mathrm{lo}(x_i,y_j) + 2\mathrm{hi}(x_i,y_j)2^{51}. +\\] +Substituting this equation into the schoolbook multiplication, then +moving terms to eliminate the \\(2^{51}\\) factors gives +\\[ +\begin{aligned} +z_0 &= \mathrm{lo}(x_0, y_0) \\\\ + &+ \qquad 0 \\\\ +z_1 &= \mathrm{lo}(x_1, y_0) + \mathrm{lo}(x_0, y_1) \\\\ + &+ \qquad 2( \mathrm{hi}(x_0, y_0) )\\\\ +z_2 &= \mathrm{lo}(x_2, y_0) + \mathrm{lo}(x_1, y_1) + \mathrm{lo}(x_0, y_2) \\\\ + &+ \qquad 2( \mathrm{hi}(x_1, y_0) + \mathrm{hi}(x_0, y_1) )\\\\ +z_3 &= \mathrm{lo}(x_3, y_0) + \mathrm{lo}(x_2, y_1) + \mathrm{lo}(x_1, y_2) + \mathrm{lo}(x_0, y_3) \\\\ + &+ \qquad 2( \mathrm{hi}(x_2, y_0) + \mathrm{hi}(x_1, y_1) + \mathrm{hi}(x_0, y_2) )\\\\ +z_4 &= \mathrm{lo}(x_4, y_0) + \mathrm{lo}(x_3, y_1) + \mathrm{lo}(x_2, y_2) + \mathrm{lo}(x_1, y_3) + \mathrm{lo}(x_0, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_3, y_0) + \mathrm{hi}(x_2, y_1) + \mathrm{hi}(x_1, y_2) + \mathrm{hi}(x_0, y_3) )\\\\ +z_5 &= \mathrm{lo}(x_4, y_1) + \mathrm{lo}(x_3, y_2) + \mathrm{lo}(x_2, y_3) + \mathrm{lo}(x_1, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_0) + \mathrm{hi}(x_3, y_1) + \mathrm{hi}(x_2, y_2) + \mathrm{hi}(x_1, y_3) + \mathrm{hi}(x_0, y_4) )\\\\ +z_6 &= \mathrm{lo}(x_4, y_2) + \mathrm{lo}(x_3, y_3) + \mathrm{lo}(x_2, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_1) + \mathrm{hi}(x_3, y_2) + \mathrm{hi}(x_2, y_3) + \mathrm{hi}(x_1, y_4) )\\\\ +z_7 &= \mathrm{lo}(x_4, y_3) + \mathrm{lo}(x_3, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_2) + \mathrm{hi}(x_3, y_3) + \mathrm{hi}(x_2, y_4) )\\\\ +z_8 &= \mathrm{lo}(x_4, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_3) + \mathrm{hi}(x_3, y_4) )\\\\ +z_9 &= 0 \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_4) )\\\\ +\end{aligned} +\\] +As noted above, our strategy will be to multiply and accumulate the +terms with coefficient \\(2\\) separately from those with coefficient +\\(1\\), before combining them at the end. This can alternately be +thought of as accumulating product terms into a *doubly-redundant* +representation, with two limbs for each digit, before collapsing +the doubly-redundant representation by shifts and adds. + +This computation requires 25 `vpmadd52luq` and 25 `vpmadd52huq` +operations. For 256-bit vectors, IFMA operations execute on an +i3-8121U with latency 4 cycles, throughput 0.5 cycles, so executing 50 +instructions requires 25 cycles' worth of throughput. Accumulating +terms with coefficient \\(1\\) and \\(2\\) seperately means that the +longest dependency chain has length 5, so the critical path has length +20 cycles and the bottleneck is throughput. + +# Reduction modulo \\(p\\) + +The next question is how to handle the reduction modulo \\(p\\). +Because \\(p = 2^{255} - 19\\), \\(2^{255} = 19 \pmod p\\), so we can +alternately write +\\[ +\begin{aligned} +z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\ + &+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459} +\end{aligned} +\\] +as +\\[ +\begin{aligned} +z &= (z_0 + 19z_5) + (z_1 + 19z_6) 2^{51} + (z_2 + 19z_7) 2^{102} + (z_3 + 19z_8) 2^{153} + (z_4 + 19z_9) 2^{204}. +\end{aligned} +\\] +When using a \\(64 \times 64 \rightarrow 128\\)-bit multiplier, this +can be handled (as in [Ed25519][ed25519_paper]) by premultiplying +source terms by \\(19\\). Since \\(\lg(19) < 4.25\\), this increases +their size by less than \\(4.25\\) bits, and the rest of the +multiplication can be shown to work out. + +Here, we have at most \\(1\\) bit of headroom. In order to allow +premultiplication, we would need to use radix \\(2^{47}\\), which +would require six limbs instead of five. Instead, we compute the high +terms \\(z_5, \ldots, z_9\\), each using two chains of IFMA +operations, then multiply by \\(19\\) and combine with the lower terms +\\(z_0, \ldots, z_4\\). There are two ways to perform the +multiplication by \\(19\\): using more IFMA operations, or using the +`vpmullq` instruction, which computes the low \\(64\\) bits of a \\(64 +\times 64\\)-bit product. However, `vpmullq` has 15c/1.5c +latency/throughput, in contrast to the 4c/0.5c latency/throughput of +IFMA operations, so it seems like a worse choice. + +The high terms \\(z_5, \ldots, z_9\\) are sums of \\(52\\)-bit terms, +so they are larger than \\(52\\) bits. Write these terms in radix \\(52\\) as +\\[ +z_{5+i} = z_{5+i}' + z_{5+i}'' 2^{52}, \qquad z_{5+i}' < 2^{52}. +\\] +Then the contribution of \\(z_{5+i}\\), taken modulo \\(p\\), is +\\[ +\begin{aligned} +z_{5+i} 2^{255} 2^{51 i} +&= +19 (z_{5+i}' + z_{5+i}'' 2^{52}) 2^{51 i} +\\\\ +&= +19 z_{5+i}' 2^{51 i} + 2 \cdot 19 z_{5+i}'' 2^{51 (i+1)} +\\\\ +\end{aligned} +\\] +The products \\(19 z_{5+i}', 19 z_{5+i}''\\) can be written in terms of IFMA operations as +\\[ +\begin{aligned} +19 z_{5+i}' &= \mathrm{lo}(19, z_{5+i}') + 2 \mathrm{hi}(19, z_{5+i}') 2^{51}, \\\\ +19 z_{5+i}'' &= \mathrm{lo}(19, z_{5+i}'') + 2 \mathrm{hi}(19, z_{5+i}'') 2^{51}. \\\\ +\end{aligned} +\\] +Because \\(z_{5+i} < 2^{64}\\), \\(z_{5+i}'' < 2^{12} \\), so \\(19 +z_{5+i}'' < 2^{17} < 2^{52} \\) and \\(\mathrm{hi}(19, z_{5+i}'') = 0\\). +Because IFMA operations ignore the high bits of their source +operands, we do not need to compute \\(z\_{5+i}'\\) explicitly: +the high bits will be ignored. +Combining these observations, we can write +\\[ +\begin{aligned} +z_{5+i} 2^{255} 2^{51 i} +&= +19 z_{5+i}' 2^{51 i} + 2 \cdot 19 z_{5+i}'' 2^{51 (i+1)} +\\\\ +&= +\mathrm{lo}(19, z_{5+i}) 2^{51 i} +\+ 2 \mathrm{hi}(19, z_{5+i}) 2^{51 (i+1)} +\+ 2 \mathrm{lo}(19, z_{5+i}/2^{52}) 2^{51 (i+1)}. +\end{aligned} +\\] + +For \\(i = 0,1,2,3\\), this allows reducing \\(z_{5+i}\\) onto +\\(z_{i}, z_{i+1}\\), and if the low terms are computed using a +doubly-redundant representation, no additional shifts are needed to +handle the \\(2\\) coefficients. For \\(i = 4\\), there's a +complication: the contribution becomes +\\[ +\begin{aligned} +z_{9} 2^{255} 2^{204} +&= +\mathrm{lo}(19, z_{9}) 2^{204} +\+ 2 \mathrm{hi}(19, z_{9}) 2^{255} +\+ 2 \mathrm{lo}(19, z_{9}/2^{52}) 2^{255} +\\\\ +&= +\mathrm{lo}(19, z_{9}) 2^{204} +\+ 2 \mathrm{hi}(19, z_{9}) 19 +\+ 2 \mathrm{lo}(19, z_{9}/2^{52}) 19 +\\\\ +&= +\mathrm{lo}(19, z_{9}) 2^{204} +\+ 2 +\mathrm{lo}(19, \mathrm{hi}(19, z_{9}) + \mathrm{lo}(19, z_{9}/2^{52})). +\\\\ +\end{aligned} +\\] + +It would be possible to cut the number of multiplications from 3 to 2 +by carrying the high part of each \\(z_i\\) onto \\(z_{i+1}\\). This +would eliminate 5 multiplications, clearing 2.5 cycles of port +pressure, at the cost of 5 additions, adding 1.66 cycles of port +pressure. But doing this would create a dependency between terms +(e.g., \\(z_{5}\\) must be computed before the reduction of +\\(z_{6}\\) can begin), whereas with the approach above, all +contributions to all terms are computed independently, to maximize ILP +and flexibility for the processor to schedule instructions. + +This strategy performs 16 IFMA operations, adding two IFMA operations +to each of the \\(2\\)-coefficient terms and one to each of the +\\(1\\)-coefficient terms. Considering the multiplication and +reduction together, we use 66 IFMA operations, requiring 33 cycles' +throughput, while the longest chain of IFMA operations is in the +reduction of \\(z_5\\) onto \\(z_1\\), of length 7 (so 28 cycles, plus +2 cycles to combine the two parts of \\(z_5\\), and the bottleneck is +again throughput. + +Once this is done, we have computed the product terms +\\[ +z = z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204}, +\\] +without reducing the \\(z_i\\) to fit in \\(52\\) bits. Because the +overall flow of operations alternates multiplications and additions or +subtractions, we would have to perform a reduction after an addition +but before the next multiplication anyways, so there's no benefit to +fully reducing the limbs at the end of a multiplication. Instead, we +leave them unreduced, and track the reduction state using the type +system to ensure that unreduced limbs are not accidentally used as an +input to a multiplication. + +# Squaring + +Squaring operates similarly to multiplication, but with the +possibility to combine identical terms. +As before, we write the input as +\\[ +\begin{aligned} +x &= x_0 + x_1 2^{51} + x_2 2^{102} + x_3 2^{153} + x_4 2^{204} +\end{aligned} +\\] +with limbs in range \\([0,2^{52})\\). +Writing the product terms as +\\[ +\begin{aligned} +z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\ + &+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459}, +\end{aligned} +\\] +a schoolbook squaring in product scanning form takes the form +\\[ +\begin{aligned} +z_0 &= x_0 x_0 \\\\ +z_1 &= 2 x_1 x_0 \\\\ +z_2 &= 2 x_2 x_0 + x_1 x_1 \\\\ +z_3 &= 2 x_3 x_0 + 2 x_2 x_1 \\\\ +z_4 &= 2 x_4 x_0 + 2 x_3 x_1 + x_2 x_2 \\\\ +z_5 &= 2 x_4 x_1 + 2 x_3 x_2 \\\\ +z_6 &= 2 x_4 x_2 + x_3 x_3 \\\\ +z_7 &= 2 x_4 x_3 \\\\ +z_8 &= x_4 x_4 \\\\ +z_9 &= 0 \\\\ +\end{aligned} +\\] +As before, we write \\(x_i x_j\\) as +\\[ +x_i x_j = \mathrm{lo}(x_i,x_j) + 2\mathrm{hi}(x_i,x_j)2^{51}, +\\] +and substitute to obtain +\\[ +\begin{aligned} +z_0 &= \mathrm{lo}(x_0, x_0) + 0 \\\\ +z_1 &= 2 \mathrm{lo}(x_1, x_0) + 2 \mathrm{hi}(x_0, x_0) \\\\ +z_2 &= 2 \mathrm{lo}(x_2, x_0) + \mathrm{lo}(x_1, x_1) + 4 \mathrm{hi}(x_1, x_0) \\\\ +z_3 &= 2 \mathrm{lo}(x_3, x_0) + 2 \mathrm{lo}(x_2, x_1) + 4 \mathrm{hi}(x_2, x_0) + 2 \mathrm{hi}(x_1, x_1) \\\\ +z_4 &= 2 \mathrm{lo}(x_4, x_0) + 2 \mathrm{lo}(x_3, x_1) + \mathrm{lo}(x_2, x_2) + 4 \mathrm{hi}(x_3, x_0) + 4 \mathrm{hi}(x_2, x_1) \\\\ +z_5 &= 2 \mathrm{lo}(x_4, x_1) + 2 \mathrm{lo}(x_3, x_2) + 4 \mathrm{hi}(x_4, x_0) + 4 \mathrm{hi}(x_3, x_1) + 2 \mathrm{hi}(x_2, x_2) \\\\ +z_6 &= 2 \mathrm{lo}(x_4, x_2) + \mathrm{lo}(x_3, x_3) + 4 \mathrm{hi}(x_4, x_1) + 4 \mathrm{hi}(x_3, x_2) \\\\ +z_7 &= 2 \mathrm{lo}(x_4, x_3) + 4 \mathrm{hi}(x_4, x_2) + 2 \mathrm{hi}(x_3, x_3) \\\\ +z_8 &= \mathrm{lo}(x_4, x_4) + 4 \mathrm{hi}(x_4, x_3) \\\\ +z_9 &= 0 + 2 \mathrm{hi}(x_4, x_4) \\\\ +\end{aligned} +\\] +To implement these, we group terms by their coefficient, computing +those with coefficient \\(2\\) on set of IFMA chains, and on another +set of chains, we begin with coefficient-\\(4\\) terms, then shift +left before continuing with the coefficient-\\(1\\) terms. +The reduction strategy is the same as for multiplication. + +# Future improvements + +LLVM won't use blend operations on [256-bit vectors yet][llvm_blend], +so there's a bunch of blend instructions that could be omitted. + +Although the multiplications and squarings are much faster, there's no +speedup to the additions and subtractions, so there are diminishing +returns. In fact, the complications in the doubling formulas mean +that doubling is actually slower than readdition. This also suggests +that moving to 512-bit vectors won't be much help for a strategy aimed +at parallelism within a group operation, so to extract performance +gains from 512-bit vectors it will probably be necessary to create a +parallel-friendly multiscalar multiplication algorithm. This could +also help with reducing shuffle pressure. + +The squaring implementation could probably be optimized, but without +`perf` support on Cannonlake it's difficult to make actual +measurements. + +Another improvement would be to implement vectorized square root +computations, which would allow creating an iterator adaptor for point +decompression that bunched decompression operations and executed them +in parallel. This would accelerate batch verification. + +[2016_gueron_krasnov]: https://ieeexplore.ieee.org/document/7563269 +[2018_drucker_gueron]: https://eprint.iacr.org/2018/335 +[1999_walter]: https://pdfs.semanticscholar.org/0e6a/3e8f30b63b556679f5dff2cbfdfe9523f4fa.pdf +[ed25519_paper]: https://ed25519.cr.yp.to/ed25519-20110926.pdf +[llvm_blend]: https://bugs.llvm.org/show_bug.cgi?id=38343 diff --git a/docs/parallel-formulas.md b/docs/parallel-formulas.md new file mode 100644 index 0000000..22f59cd --- /dev/null +++ b/docs/parallel-formulas.md @@ -0,0 +1,333 @@ +Vectorized implementations of field and point operations, using a +modification of the 4-way parallel formulas of Hisil, Wong, Carter, +and Dawson. + +These notes explain the parallel formulas and our strategy for using +them with SIMD operations. There are two backend implementations: one +using AVX2, and the other using AVX512-IFMA. + +# Overview + +The 2008 paper [_Twisted Edwards Curves Revisited_][hwcd08] by Hisil, +Wong, Carter, and Dawson (HWCD) introduced the “extended coordinates” +and mixed-model representations which are used by most Edwards curve +implementations. + +However, they also describe 4-way parallel formulas for point addition +and doubling: a unified addition algorithm taking an effective +\\(2\mathbf M + 1\mathbf D\\), a doubling algorithm taking an +effective \\(1\mathbf M + 1\mathbf S\\), and a dedicated (i.e., for +distinct points) addition algorithm taking an effective \\(2 \mathbf M +\\). They compare these formulas with a 2-way parallel variant of the +Montgomery ladder. + +Unlike their serial formulas, which are used widely, their parallel +formulas do not seem to have been implemented in software before. The +2-way parallel Montgomery ladder was used in 2015 by Tung Chou's +`sandy2x` implementation. Curiously, however, although the [`sandy2x` +paper][sandy2x] also implements Edwards arithmetic, and cites HWCD08, +it doesn't mention their parallel Edwards formulas. +A 2015 paper by Hernández and López describes an AVX2 implementation +of X25519. Neither the paper nor the code are publicly available, but +it apparently gives only a [slight speedup][avx2trac], suggesting that +it uses a 4-way parallel Montgomery ladder rather than parallel +Edwards formulas. + +The reason may be that HWCD08 describe their formulas as operating on +four independent processors, which would make a software +implementation impractical: all of the operations are too low-latency +to effectively synchronize. But a closer inspection reveals that the +(more expensive) multiplication and squaring steps are uniform, while +the instruction divergence occurs in the (much cheaper) addition and +subtraction steps. This means that a SIMD implementation can perform +the expensive steps uniformly, and handle divergence in the +inexpensive steps using masking. + +These notes describe modifications to the original parallel formulas +to allow a SIMD implementation, and this module contains +implementations of the modified formulas targeting either AVX2 or +AVX512-IFMA. + +# Parallel formulas in HWCD'08 + +The doubling formula is presented in the HWCD paper as follows: + +| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | +|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| | idle | idle | idle | \\( R\_1 \gets X\_1 + Y\_1 \\) | +| \\(1\mathbf S\\) | \\( R\_2 \gets X\_1\^2 \\) | \\( R\_3 \gets Y\_1\^2 \\) | \\( R\_4 \gets Z\_1\^2 \\) | \\( R\_5 \gets R\_1\^2 \\) | +| | \\( R\_6 \gets R\_2 + R\_3 \\) | \\( R\_7 \gets R\_2 - R\_3 \\) | \\( R\_4 \gets 2 R\_4 \\) | idle | +| | idle | \\( R\_1 \gets R\_4 + R\_7 \\) | idle | \\( R\_2 \gets R\_6 - R\_5 \\) | +| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_6 R\_7 \\) | \\( T\_3 \gets R\_2 R\_6 \\) | \\( Z\_3 \gets R\_1 R\_7 \\) | + +and the unified addition algorithm is presented as follows: + +| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | +|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| | \\( R\_1 \gets Y\_1 - X\_1 \\) | \\( R\_2 \gets Y\_2 - X\_2 \\) | \\( R\_3 \gets Y\_1 + X\_1 \\) | \\( R\_4 \gets Y\_2 + X\_2 \\) | +| \\(1\mathbf M\\) | \\( R\_5 \gets R\_1 R\_2 \\) | \\( R\_6 \gets R\_3 R\_4 \\) | \\( R\_7 \gets T\_1 T\_2 \\) | \\( R\_8 \gets Z\_1 Z\_2 \\) | +| \\(1\mathbf D\\) | idle | idle | \\( R\_7 \gets k R\_7 \\) | \\( R\_8 \gets 2 R\_8 \\) | +| | \\( R\_1 \gets R\_6 - R\_5 \\) | \\( R\_2 \gets R\_8 - R\_7 \\) | \\( R\_3 \gets R\_8 + R\_7 \\) | \\( R\_4 \gets R\_6 + R\_5 \\) | +| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_3 R\_4 \\) | \\( T\_3 \gets R\_1 R\_4 \\) | \\( Z\_3 \gets R\_2 R\_3 \\) | + +Here \\(\mathbf M\\) and \\(\mathbf S\\) represent the cost of +multiplication and squaring of generic field elements, \\(\mathbf D\\) +represents the cost of multiplication by a curve constant (in this +case \\( k = 2d \\)). + +Notice that the \\(1\mathbf M\\) and \\(1\mathbf S\\) steps are +uniform. The non-uniform steps are all inexpensive additions or +subtractions, with the exception of the multiplication by the curve +constant \\(k = 2d\\): +$$ +R\_7 \gets 2 d R\_7. +$$ + +HWCD suggest parallelising this step by breaking \\(k = 2d\\) into four +parts as \\(k = k_0 + 2\^n k_1 + 2\^{2n} k_2 + 2\^{3n} k_3 \\) and +computing \\(k_i R_7 \\) in parallel. This is quite awkward, but if +the curve constant is a ratio \\( d = d\_1/d\_2 \\), then projective +coordinates allow us to instead compute +$$ +(R\_5, R\_6, R\_7, R\_8) \gets (d\_2 R\_5, d\_2 R\_6, 2d\_1 R\_7, d\_2 R\_8). +$$ +This can be performed as a uniform multiplication by a vector of +constants, and if \\(d\_1, d\_2\\) are small, it is relatively +inexpensive. (This trick was suggested by Mike Hamburg). +In the Curve25519 case, we have +$$ +d = \frac{d\_1}{d\_2} = \frac{-121665}{121666}; +$$ +Since \\(2 \cdot 121666 < 2\^{18}\\), all the constants above fit (up +to sign) in 32 bits, so this can be done in parallel as four +multiplications by small constants \\( (121666, 121666, 2\cdot 121665, +2\cdot 121666) \\), followed by a negation to compute \\( - 2\cdot 121665\\). + +# Modified parallel formulas + +Using the modifications sketched above, we can write SIMD-friendly +versions of the parallel formulas as follows. To avoid confusion with +the original formulas, temporary variables are named \\(S\\) instead +of \\(R\\) and are in static single-assignment form. + +## Addition + +To add points +\\(P_1 = (X_1 : Y_1 : Z_1 : T_1) \\) +and +\\(P_2 = (X_2 : Y_2 : Z_2 : T_2 ) \\), +we compute +$$ +\begin{aligned} +(S\_0 &&,&& S\_1 &&,&& S\_2 &&,&& S\_3 ) +&\gets +(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Y\_2 - X\_2&&,&& Y\_2 + X\_2) +\\\\ +(S\_4 &&,&& S\_5 &&,&& S\_6 &&,&& S\_7 ) +&\gets +(S\_0 \cdot S\_2&&,&& S\_1 \cdot S\_3&&,&& Z\_1 \cdot Z\_2&&,&& T\_1 \cdot T\_2) +\\\\ +(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} ) +&\gets +(d\_2 \cdot S\_4 &&,&& d\_2 \cdot S\_5 &&,&& 2 d\_2 \cdot S\_6 &&,&& 2 d\_1 \cdot S\_7 ) +\\\\ +(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15}) +&\gets +(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11}) +\\\\ +(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3) +&\gets +(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13}) +\end{aligned} +$$ +to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). +This costs \\( 2\mathbf M + 1 \mathbf D\\). + +## Readdition + +If the point \\( P_2 = (X\_2 : Y\_2 : Z\_2 : T\_2) \\) is fixed, we +can cache the multiplication of the curve constants by computing +$$ +\begin{aligned} +(S\_2' &&,&& S\_3' &&,&& Z\_2' &&,&& T\_2' ) +&\gets +(d\_2 \cdot (Y\_2 - X\_2)&&,&& d\_2 \cdot (Y\_1 + X\_1)&&,&& 2d\_2 \cdot Z\_2 &&,&& 2d\_1 \cdot T\_2). +\end{aligned} +$$ +This costs \\( 1\mathbf D\\); with \\( (S\_2', S\_3', Z\_2', T\_2')\\) +in hand, the addition formulas above become +$$ +\begin{aligned} +(S\_0 &&,&& S\_1 &&,&& Z\_1 &&,&& T\_1 ) +&\gets +(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Z\_1 &&,&& T\_1) +\\\\ +(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} ) +&\gets +(S\_0 \cdot S\_2' &&,&& S\_1 \cdot S\_3'&&,&& Z\_1 \cdot Z\_2' &&,&& T\_1 \cdot T\_2') +\\\\ +(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15}) +&\gets +(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11}) +\\\\ +(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3) +&\gets +(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13}) +\end{aligned} +$$ +which costs only \\( 2\mathbf M \\). This precomputation is +essentially similar to the precomputation that HWCD suggest for their +serial formulas. Because the cost of precomputation and then +readdition is the same as addition, it's sufficient to only +implement caching and readdition. + +## Doubling + +The non-uniform portions of the (re)addition formulas have a fairly +regular structure. Unfortunately, this is not the case for the +doubling formulas, which are much less nice. + +To double a point \\( P = (X\_1 : Y\_1 : Z\_1 : T\_1) \\), we compute +$$ +\begin{aligned} +(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& S\_0) +&\gets +(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& X\_1 + Y\_1) +\\\\ +(S\_1 &&,&& S\_2 &&,&& S\_3 &&,&& S\_4 ) +&\gets +(X\_1\^2 &&,&& Y\_1\^2&&,&& Z\_1\^2 &&,&& S\_0\^2) +\\\\ +(S\_5 &&,&& S\_6 &&,&& S\_8 &&,&& S\_9 ) +&\gets +(S\_1 + S\_2 &&,&& S\_1 - S\_2 &&,&& S\_1 + 2S\_3 - S\_2 &&,&& S\_1 + S\_2 - S\_4) +\\\\ +(X\_3 &&,&& Y\_3 &&,&& Z\_3 &&,&& T\_3 ) +&\gets +(S\_8 \cdot S\_9 &&,&& S\_5 \cdot S\_6 &&,&& S\_8 \cdot S\_6 &&,&& S\_5 \cdot S\_9) +\end{aligned} +$$ +to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = [2]P\_1 \\). + +The intermediate step between the squaring and multiplication requires +a long chain of additions. For the IFMA-based implementation, this is not a problem; for the AVX2-based implementation, it is, but with some care and finesse, it's possible to arrange the computation without requiring an intermediate reduction. + +# Implementation + +These formulas aren't specific to a particular representation of field +element vectors, whose optimum choice is determined by the details of +the instruction set. However, it's not possible to perfectly separate +the implementation of the field element vectors from the +implementation of the point operations. Instead, the [`avx2`] and +[`ifma`] backends provide `ExtendedPoint` and `CachedPoint` types, and +the [`scalar_mul`] code uses one of the backend types by a type alias. + +# Comparison to non-vectorized formulas + +In theory, the parallel Edwards formulas seem to allow a \\(4\\)-way +speedup from parallelism. However, an actual vectorized +implementation has several slowdowns that cut into this speedup. + +First, the parallel formulas can only use the available vector +multiplier. For AVX2, this is a \\( 32 \times 32 \rightarrow 64 +\\)-bit integer multiplier, so the speedup from vectorization must +overcome the disadvantage of losing the \\( 64 \times 64 \rightarrow +128\\)-bit (serial) integer multiplier. The effect of this slowdown +is microarchitecture-dependent, since it requires accounting for the +total number of multiplications and additions and their relative +costs. IFMA allows using a \\( 52 \times 52 \rightarrow 104 \\)-bit +multiplier, but the high and low halves need to be computed +separately, and the reduction requires extra work because it's not +possible to pre-multiply by \\(19\\). + +Second, the parallel doubling formulas incur both a theoretical and +practical slowdown. The parallel formulas described above work on the +\\( \mathbb P\^3 \\) “extended” coordinates. The \\( \mathbb P\^2 \\) +model introduced earlier by [Bernstein, Birkner, Joye, Lange, and +Peters][bbjlp08] allows slightly faster doublings, so HWCD suggest +mixing coordinate systems while performing scalar multiplication +(attributing the idea to [a 1998 paper][cmo98] by Cohen, Miyagi, and +Ono). The \\( T \\) coordinate is not required for doublings, so when +doublings are followed by doublings, its computation can be skipped. +More details on this approach and the different coordinate systems can +be found in the [`curve_models` module documentation][curve_models]. + +Unfortunately, this optimization is not compatible with the parallel +formulas, which cannot save time by skipping a single variable, so the +parallel doubling formulas do slightly more work when counting the +total number of field multiplications and squarings. + +In addition, the parallel doubling formulas have a less regular +pattern of additions and subtractions than the parallel addition +formulas, so the vectorization overhead is proportionately greater. +Both the parallel addition and parallel doubling formulas also require +some shuffling to rearrange data within the vectors, which places more +pressure on the shuffle unit than is desirable. + +This means that the speedup from using a vectorized implementation of +parallel Edwards formulas is likely to be greatest in applications +that do fewer doublings and more additions (like a large multiscalar +multiplication) rather than applications that do fewer additions and +more doublings (like a double-base scalar multiplication). + +Third, Amdahl's law says that the speedup is limited to the portion +which can be parallelized. Normally, the field multiplications +dominate the cost of point operations, but with the IFMA backend, the +multiplications are so fast that the non-parallel additions end up as +a significant portion of the total time. + +Fourth, current Intel CPUs perform thermal throttling when using wide +vector instructions. A detailed description can be found in §15.26 of +[the Intel Optimization Manual][intel], but using wide vector +instructions prevents the core from operating at higher frequencies. +The core can return to the higher-frequency state after 2 +milliseconds, but this timer is reset every time high-power +instructions are used. + +Any speedup from vectorization therefore has to be weighed against a +slowdown for the next few million instructions. For a mixed workload, +where point operations are interspersed with other tasks, this can +reduce overall performance. This implementation is therefore probably +not suitable for basic applications, like signatures, but is +worthwhile for complex applications, like zero-knowledge proofs, which +do sustained work. + +# Future work + +There are several directions for future improvement: + +* Using the vectorized field arithmetic code to parallelize across + point operations rather than within a single point operation. This + is less flexible, but would give a speedup both from allowing use of + the faster mixed-model arithmetic and from reducing shuffle + pressure. One approach in this direction would be to implement + batched scalar-point operations using vectors of points (AoSoA + layout). This less generally useful but would give a speedup for + Bulletproofs. + +* Extending the IFMA implementation to use the full width of AVX512, + either handling the extra parallelism internally to a single point + operation (by using a 2-way parallel implementation of field + arithmetic instead of a wordsliced one), or externally, + parallelizing across point operations. Internal parallelism would + be preferable but might require too much shuffle pressure. For now, + the only available CPU which runs IFMA operations executes them at + 256-bits wide anyways, so this isn't yet important. + +* Generalizing the implementation to NEON instructions. The current + point arithmetic code is written in terms of field element vectors, + which are in turn implemented using platform SIMD vectors. It + should be possible to write an alternate implementation of the + `FieldElement2625x4` using NEON without changing the point + arithmetic. NEON has 128-bit vectors rather than 256-bit vectors, + but this may still be worthwhile compared to a serial + implementation. + + +[sandy2x]: https://eprint.iacr.org/2015/943.pdf +[avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28 +[hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf +[curve_models]: https://doc-internal.dalek.rs/curve25519_dalek/curve_models/index.html +[bbjlp08]: https://eprint.iacr.org/2008/013 +[cmo98]: https://link.springer.com/content/pdf/10.1007%2F3-540-49649-1_6.pdf +[intel]: https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 75fa6c0..33d5bc1 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -10,29 +10,53 @@ //! Pluggable implementations for different architectures. //! -//! The naming of the `u32` and `u64` modules is somewhat unfortunate, -//! since these are also the names of primitive types. Since types have -//! a different namespace than modules, this isn't a problem to the -//! compiler, but it could cause confusion. +//! The backend code is split into two parts: a serial backend, +//! and a vector backend. //! -//! However, it's unlikely that the names of those modules would be -//! brought into scope directly, instead of used as -//! `backend::u32::field` or similar. Unfortunately we can't use -//! `32bit` since identifiers can't start with letters, and the backends -//! do use `u32`/`u64`, so this seems like a least-bad option. +//! The [`serial`] backend contains 32- and 64-bit implementations of +//! field arithmetic and scalar arithmetic, as well as implementations +//! of point operations using the mixed-model strategy (passing +//! between different curve models depending on the operation). +//! +//! The [`vector`] backend contains implementations of vectorized +//! field arithmetic, used to implement point operations using a novel +//! implementation strategy derived from parallel formulas of Hisil, +//! Wong, Carter, and Dawson. +//! +//! Because the two strategies give rise to different curve models, +//! it's not possible to reuse exactly the same scalar multiplication +//! code (or to write it generically), so both serial and vector +//! backends contain matching implementations of scalar multiplication +//! algorithms. These are intended to be selected by a `#[cfg]`-based +//! type alias. +//! +//! The [`vector`] backend is selected by the `simd_backend` cargo +//! feature; it uses the [`serial`] backend for non-vectorized operations. -#[cfg(not(any(feature = "u32_backend", feature = "u64_backend", feature = "avx2_backend")))] +#[cfg(not(any( + feature = "u32_backend", + feature = "u64_backend", + feature = "simd_backend", +)))] compile_error!( "no curve25519-dalek backend cargo feature enabled! \ - please enable one of: u32_backend, u64_backend, avx2_backend" + please enable one of: u32_backend, u64_backend, simd_backend" ); -#[cfg(feature = "u32_backend")] -pub mod u32; - -#[cfg(feature = "u64_backend")] -pub mod u64; - -#[cfg(all(feature = "avx2_backend", target_feature = "avx2"))] -pub mod avx2; +pub mod serial; +#[cfg(any( + all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") + ), + all(feature = "nightly", rustdoc) +))] +#[cfg_attr( + feature = "nightly", + doc(cfg(any(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") + )))) +)] +pub mod vector; diff --git a/src/curve_models/mod.rs b/src/backend/serial/curve_models/mod.rs similarity index 100% rename from src/curve_models/mod.rs rename to src/backend/serial/curve_models/mod.rs diff --git a/src/backend/serial/mod.rs b/src/backend/serial/mod.rs new file mode 100644 index 0000000..f7476e8 --- /dev/null +++ b/src/backend/serial/mod.rs @@ -0,0 +1,43 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +//! Serial implementations of field, scalar, point arithmetic. +//! +//! When the vector backend is disabled, the crate uses the +//! mixed-model strategy for implementing point operations and scalar +//! multiplication; see the [`curve_models`](self::curve_models) and +//! [`scalar_mul`](self::scalar_mul) documentation for more +//! information. +//! +//! When the vector backend is enabled, the field and scalar +//! implementations are still used for non-vectorized operations. +//! +//! Note: at this time the `u32` and `u64` backends cannot be built +//! together. + +#[cfg(not(any(feature = "u32_backend", feature = "u64_backend")))] +compile_error!( + "no curve25519-dalek backend cargo feature enabled! \ + please enable one of: u32_backend, u64_backend" +); + +#[cfg(feature = "u32_backend")] +pub mod u32; + +#[cfg(feature = "u64_backend")] +pub mod u64; + +pub mod curve_models; + +#[cfg(not(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +)))] +pub mod scalar_mul; diff --git a/src/scalar_mul/mod.rs b/src/backend/serial/scalar_mul/mod.rs similarity index 90% rename from src/scalar_mul/mod.rs rename to src/backend/serial/scalar_mul/mod.rs index 3ebe25f..bec874b 100644 --- a/src/scalar_mul/mod.rs +++ b/src/backend/serial/scalar_mul/mod.rs @@ -16,11 +16,13 @@ //! scalar multiplication implementations, since it only uses one //! curve model. -pub mod window; - pub mod variable_base; #[cfg(feature = "stage2_build")] pub mod vartime_double_base; +#[cfg(feature = "alloc")] pub mod straus; + +#[cfg(feature = "alloc")] +pub mod precomputed_straus; diff --git a/src/backend/serial/scalar_mul/precomputed_straus.rs b/src/backend/serial/scalar_mul/precomputed_straus.rs new file mode 100644 index 0000000..4019b14 --- /dev/null +++ b/src/backend/serial/scalar_mul/precomputed_straus.rs @@ -0,0 +1,114 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2019 Henry de Valence. +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +//! Precomputation for Straus's method. + +#![allow(non_snake_case)] + +use core::borrow::Borrow; + +use backend::serial::curve_models::{ + AffineNielsPoint, CompletedPoint, ProjectiveNielsPoint, ProjectivePoint, +}; +use edwards::EdwardsPoint; +use scalar::Scalar; +use traits::Identity; +use traits::VartimePrecomputedMultiscalarMul; +use window::{NafLookupTable5, NafLookupTable8}; + +#[allow(unused_imports)] +use prelude::*; + +pub struct VartimePrecomputedStraus { + static_lookup_tables: Vec>, +} + +impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self { + static_lookup_tables: static_points + .into_iter() + .map(|P| NafLookupTable8::::from(P.borrow())) + .collect(), + } + } + + fn optional_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Option + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator>, + { + let static_nafs = static_scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form(5)) + .collect::>(); + let dynamic_nafs: Vec<_> = dynamic_scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form(5)) + .collect::>(); + + let dynamic_lookup_tables = match dynamic_points + .into_iter() + .map(|P_opt| P_opt.map(|P| NafLookupTable5::::from(&P))) + .collect::>>() + { + Some(x) => x, + None => return None, + }; + + let sp = self.static_lookup_tables.len(); + let dp = dynamic_lookup_tables.len(); + assert_eq!(sp, static_nafs.len()); + assert_eq!(dp, dynamic_nafs.len()); + + // We could save some doublings by looking for the highest + // nonzero NAF coefficient, but since we might have a lot of + // them to search, it's not clear it's worthwhile to check. + let mut S = ProjectivePoint::identity(); + for j in (0..255).rev() { + let mut R: CompletedPoint = S.double(); + + for i in 0..dp { + let t_ij = dynamic_nafs[i][j]; + if t_ij > 0 { + R = &R.to_extended() + &dynamic_lookup_tables[i].select(t_ij as usize); + } else if t_ij < 0 { + R = &R.to_extended() - &dynamic_lookup_tables[i].select(-t_ij as usize); + } + } + + for i in 0..sp { + let t_ij = static_nafs[i][j]; + if t_ij > 0 { + R = &R.to_extended() + &self.static_lookup_tables[i].select(t_ij as usize); + } else if t_ij < 0 { + R = &R.to_extended() - &self.static_lookup_tables[i].select(-t_ij as usize); + } + } + + S = R.to_projective(); + } + + Some(S.to_extended()) + } +} diff --git a/src/scalar_mul/straus.rs b/src/backend/serial/scalar_mul/straus.rs similarity index 92% rename from src/scalar_mul/straus.rs rename to src/backend/serial/scalar_mul/straus.rs index 0053570..4053ea3 100644 --- a/src/scalar_mul/straus.rs +++ b/src/backend/serial/scalar_mul/straus.rs @@ -12,16 +12,11 @@ #![allow(non_snake_case)] -#[cfg(any(feature = "alloc", feature = "std"))] use core::borrow::Borrow; -#[cfg(any(feature = "alloc", feature = "std"))] use edwards::EdwardsPoint; -#[cfg(any(feature = "alloc", feature = "std"))] use scalar::Scalar; -#[cfg(any(feature = "alloc", feature = "std"))] use traits::MultiscalarMul; -#[cfg(any(feature = "alloc", feature = "std"))] use traits::VartimeMultiscalarMul; #[allow(unused_imports)] @@ -48,10 +43,8 @@ use prelude::*; /// /// [solution]: https://www.jstor.org/stable/2310929 /// [problem]: https://www.jstor.org/stable/2312273 -#[cfg(any(feature = "alloc", feature = "std"))] pub struct Straus {} -#[cfg(feature = "alloc")] impl MultiscalarMul for Straus { type Point = EdwardsPoint; @@ -115,8 +108,8 @@ impl MultiscalarMul for Straus { { use clear_on_drop::ClearOnDrop; - use curve_models::ProjectiveNielsPoint; - use scalar_mul::window::LookupTable; + use backend::serial::curve_models::ProjectiveNielsPoint; + use window::LookupTable; use traits::Identity; let lookup_tables: Vec<_> = points @@ -148,7 +141,6 @@ impl MultiscalarMul for Straus { } } -#[cfg(feature = "alloc")] impl VartimeMultiscalarMul for Straus { type Point = EdwardsPoint; @@ -167,8 +159,8 @@ impl VartimeMultiscalarMul for Straus { I::Item: Borrow, J: IntoIterator>, { - use curve_models::{CompletedPoint, ProjectiveNielsPoint, ProjectivePoint}; - use scalar_mul::window::NafLookupTable5; + use backend::serial::curve_models::{CompletedPoint, ProjectiveNielsPoint, ProjectivePoint}; + use window::NafLookupTable5; use traits::Identity; let nafs: Vec<_> = scalars diff --git a/src/scalar_mul/variable_base.rs b/src/backend/serial/scalar_mul/variable_base.rs similarity index 92% rename from src/scalar_mul/variable_base.rs rename to src/backend/serial/scalar_mul/variable_base.rs index 9569bd5..588a84e 100644 --- a/src/scalar_mul/variable_base.rs +++ b/src/backend/serial/scalar_mul/variable_base.rs @@ -3,8 +3,8 @@ use traits::Identity; use scalar::Scalar; use edwards::EdwardsPoint; -use curve_models::ProjectiveNielsPoint; -use scalar_mul::window::LookupTable; +use backend::serial::curve_models::ProjectiveNielsPoint; +use window::LookupTable; /// Perform constant-time, variable-base scalar multiplication. pub(crate) fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint { diff --git a/src/scalar_mul/vartime_double_base.rs b/src/backend/serial/scalar_mul/vartime_double_base.rs similarity index 93% rename from src/scalar_mul/vartime_double_base.rs rename to src/backend/serial/scalar_mul/vartime_double_base.rs index 86022d2..d95151f 100644 --- a/src/scalar_mul/vartime_double_base.rs +++ b/src/backend/serial/scalar_mul/vartime_double_base.rs @@ -13,8 +13,8 @@ use constants; use traits::Identity; use scalar::Scalar; use edwards::EdwardsPoint; -use curve_models::{ProjectiveNielsPoint, ProjectivePoint}; -use scalar_mul::window::NafLookupTable5; +use backend::serial::curve_models::{ProjectiveNielsPoint, ProjectivePoint}; +use window::NafLookupTable5; /// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { diff --git a/src/backend/serial/u32/constants.rs b/src/backend/serial/u32/constants.rs new file mode 100644 index 0000000..f5102f8 --- /dev/null +++ b/src/backend/serial/u32/constants.rs @@ -0,0 +1,149 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +//! This module contains various constants (such as curve parameters +//! and useful field elements like `sqrt(-1)`), as well as +//! lookup tables of pre-computed points. + +use backend::serial::u32::field::FieldElement2625; +use backend::serial::u32::scalar::Scalar29; +use edwards::EdwardsPoint; + +/// Edwards `d` value, equal to `-121665/121666 mod p`. +pub(crate) const EDWARDS_D: FieldElement2625 = FieldElement2625([ + 56195235, 13857412, 51736253, 6949390, 114729, + 24766616, 60832955, 30306712, 48412415, 21499315, +]); + +/// Edwards `2*d` value, equal to `2*(-121665/121666) mod p`. +pub(crate) const EDWARDS_D2: FieldElement2625 = FieldElement2625([ + 45281625, 27714825, 36363642, 13898781, 229458, + 15978800, 54557047, 27058993, 29715967, 9444199, +]); + +/// `= sqrt(a*d - 1)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. +pub(crate) const SQRT_AD_MINUS_ONE: FieldElement2625 = FieldElement2625([ + 24849947, 33400850, 43495378, 6347714, 46036536, + 32887293, 41837720, 18186727, 66238516, 14525638, +]); + +/// `= 1/sqrt(a-d)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. +pub(crate) const INVSQRT_A_MINUS_D: FieldElement2625 = FieldElement2625([ + 6111466, 4156064, 39310137, 12243467, 41204824, + 120896, 20826367, 26493656, 6093567, 31568420, +]); + +/// Precomputed value of one of the square roots of -1 (mod p) +pub(crate) const SQRT_M1: FieldElement2625 = FieldElement2625([ + 34513072, 25610706, 9377949, 3500415, 12389472, + 33281959, 41962654, 31548777, 326685, 11406482, +]); + +/// `APLUS2_OVER_FOUR` is (A+2)/4. (This is used internally within the Montgomery ladder.) +pub(crate) const APLUS2_OVER_FOUR: FieldElement2625 = FieldElement2625([ + 121666, 0, 0, 0, 0, 0, 0, 0, 0, 0 +]); + +/// `L` is the order of base point, i.e. 2^252 + +/// 27742317777372353535851937790883648493 +pub(crate) const L: Scalar29 = Scalar29([ 0x1cf5d3ed, 0x009318d2, 0x1de73596, 0x1df3bd45, + 0x0000014d, 0x00000000, 0x00000000, 0x00000000, + 0x00100000 ]); + +/// `L` * `LFACTOR` = -1 (mod 2^29) +pub(crate) const LFACTOR: u32 = 0x12547e1b; + +/// `R` = R % L where R = 2^261 +pub(crate) const R: Scalar29 = Scalar29([ 0x114df9ed, 0x1a617303, 0x0f7c098c, 0x16793167, + 0x1ffd656e, 0x1fffffff, 0x1fffffff, 0x1fffffff, + 0x000fffff ]); + +/// `RR` = (R^2) % L where R = 2^261 +pub(crate) const RR: Scalar29 = Scalar29([ 0x0b5f9d12, 0x1e141b17, 0x158d7f3d, 0x143f3757, + 0x1972d781, 0x042feb7c, 0x1ceec73d, 0x1e184d1e, + 0x0005046d ]); + +/// The Ed25519 basepoint, as an `EdwardsPoint`. +/// +/// This is called `_POINT` to distinguish it from +/// `ED25519_BASEPOINT_TABLE`, which should be used for scalar +/// multiplication (it's much faster). +pub const ED25519_BASEPOINT_POINT: EdwardsPoint = EdwardsPoint{ + X: FieldElement2625([52811034, 25909283, 16144682, 17082669, 27570973, 30858332, 40966398, 8378388, 20764389, 8758491]), + Y: FieldElement2625([40265304, 26843545, 13421772, 20132659, 26843545, 6710886, 53687091, 13421772, 40265318, 26843545]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([28827043, 27438313, 39759291, 244362, 8635006, 11264893, 19351346, 13413597, 16611511, 27139452]), +}; + +/// The 8-torsion subgroup \\(\mathcal E [8]\\). +/// +/// In the case of Curve25519, it is cyclic; the \\(i\\)-th element of +/// the array is \\([i]P\\), where \\(P\\) is a point of order \\(8\\) +/// generating \\(\mathcal E[8]\\). +/// +/// Thus \\(\mathcal E[8]\\) is the points indexed by `0,2,4,6`, and +/// \\(\mathcal E[2]\\) is the points indexed by `0,4`. +/// The Ed25519 basepoint has y = 4/5. This is called `_POINT` to +/// distinguish it from `_TABLE`, which should be used for scalar +/// multiplication (it's much faster). +pub const EIGHT_TORSION: [EdwardsPoint; 8] = EIGHT_TORSION_INNER_DOC_HIDDEN; + +/// Inner item used to hide limb constants from cargo doc output. +#[doc(hidden)] +pub const EIGHT_TORSION_INNER_DOC_HIDDEN: [EdwardsPoint; 8] = [ + EdwardsPoint{ + X: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Y: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + }, + EdwardsPoint{ + X: FieldElement2625([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]), + Y: FieldElement2625([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251]) + }, + EdwardsPoint{ + X: FieldElement2625([32595773, 7943725, 57730914, 30054016, 54719391, 272472, 25146209, 2005654, 66782178, 22147949]), + Y: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + }, + EdwardsPoint{ + X: FieldElement2625([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]), + Y: FieldElement2625([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180]) + }, + EdwardsPoint{ + X: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Y: FieldElement2625([67108844, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + }, + EdwardsPoint{ + X: FieldElement2625([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]), + Y: FieldElement2625([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251]) + }, + EdwardsPoint{ + X: FieldElement2625([34513072, 25610706, 9377949, 3500415, 12389472, 33281959, 41962654, 31548777, 326685, 11406482]), + Y: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + }, + EdwardsPoint{ + X: FieldElement2625([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]), + Y: FieldElement2625([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180]) + }, +]; diff --git a/src/backend/u32/field.rs b/src/backend/serial/u32/field.rs similarity index 87% rename from src/backend/u32/field.rs rename to src/backend/serial/u32/field.rs index 5f2fe1d..8a7ca72 100644 --- a/src/backend/u32/field.rs +++ b/src/backend/serial/u32/field.rs @@ -24,7 +24,7 @@ use core::ops::{Sub, SubAssign}; use subtle::Choice; use subtle::ConditionallySelectable; -/// A `FieldElement32` represents an element of the field +/// A `FieldElement2625` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// /// In the 32-bit implementation, a `FieldElement` is represented in @@ -41,44 +41,44 @@ use subtle::ConditionallySelectable; /// # Note /// /// The `curve25519_dalek::field` module provides a type alias -/// `curve25519_dalek::field::FieldElement` to either `FieldElement64` -/// or `FieldElement32`. +/// `curve25519_dalek::field::FieldElement` to either `FieldElement51` +/// or `FieldElement2625`. /// -/// The backend-specific type `FieldElement32` should not be used +/// The backend-specific type `FieldElement2625` should not be used /// outside of the `curve25519_dalek::field` module. #[derive(Copy, Clone)] -pub struct FieldElement32(pub (crate) [u32; 10]); +pub struct FieldElement2625(pub (crate) [u32; 10]); -impl Debug for FieldElement32 { +impl Debug for FieldElement2625 { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - write!(f, "FieldElement32({:?})", &self.0[..]) + write!(f, "FieldElement2625({:?})", &self.0[..]) } } -impl<'b> AddAssign<&'b FieldElement32> for FieldElement32 { - fn add_assign(&mut self, _rhs: &'b FieldElement32) { +impl<'b> AddAssign<&'b FieldElement2625> for FieldElement2625 { + fn add_assign(&mut self, _rhs: &'b FieldElement2625) { for i in 0..10 { self.0[i] += _rhs.0[i]; } } } -impl<'a, 'b> Add<&'b FieldElement32> for &'a FieldElement32 { - type Output = FieldElement32; - fn add(self, _rhs: &'b FieldElement32) -> FieldElement32 { +impl<'a, 'b> Add<&'b FieldElement2625> for &'a FieldElement2625 { + type Output = FieldElement2625; + fn add(self, _rhs: &'b FieldElement2625) -> FieldElement2625 { let mut output = *self; output += _rhs; output } } -impl<'b> SubAssign<&'b FieldElement32> for FieldElement32 { - fn sub_assign(&mut self, _rhs: &'b FieldElement32) { - // See comment in FieldElement64::Sub +impl<'b> SubAssign<&'b FieldElement2625> for FieldElement2625 { + fn sub_assign(&mut self, _rhs: &'b FieldElement2625) { + // See comment in FieldElement51::Sub // // Compute a - b as ((a + 2^4 * p) - b) to avoid underflow. let b = &_rhs.0; - self.0 = FieldElement32::reduce([ + self.0 = FieldElement2625::reduce([ ((self.0[0] + (0x3ffffed << 4)) - b[0]) as u64, ((self.0[1] + (0x1ffffff << 4)) - b[1]) as u64, ((self.0[2] + (0x3ffffff << 4)) - b[2]) as u64, @@ -93,25 +93,25 @@ impl<'b> SubAssign<&'b FieldElement32> for FieldElement32 { } } -impl<'a, 'b> Sub<&'b FieldElement32> for &'a FieldElement32 { - type Output = FieldElement32; - fn sub(self, _rhs: &'b FieldElement32) -> FieldElement32 { +impl<'a, 'b> Sub<&'b FieldElement2625> for &'a FieldElement2625 { + type Output = FieldElement2625; + fn sub(self, _rhs: &'b FieldElement2625) -> FieldElement2625 { let mut output = *self; output -= _rhs; output } } -impl<'b> MulAssign<&'b FieldElement32> for FieldElement32 { - fn mul_assign(&mut self, _rhs: &'b FieldElement32) { - let result = (self as &FieldElement32) * _rhs; +impl<'b> MulAssign<&'b FieldElement2625> for FieldElement2625 { + fn mul_assign(&mut self, _rhs: &'b FieldElement2625) { + let result = (self as &FieldElement2625) * _rhs; self.0 = result.0; } } -impl<'a, 'b> Mul<&'b FieldElement32> for &'a FieldElement32 { - type Output = FieldElement32; - fn mul(self, _rhs: &'b FieldElement32) -> FieldElement32 { +impl<'a, 'b> Mul<&'b FieldElement2625> for &'a FieldElement2625 { + type Output = FieldElement2625; + fn mul(self, _rhs: &'b FieldElement2625) -> FieldElement2625 { /// Helper function to multiply two 32-bit integers with 64 bits /// of output. #[inline(always)] @@ -206,26 +206,26 @@ impl<'a, 'b> Mul<&'b FieldElement32> for &'a FieldElement32 { // // So z[0] fits into a u64 if 51 + 2*b + lg(249) < 64 // if b < 2.5. - FieldElement32::reduce([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) + FieldElement2625::reduce([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } } -impl<'a> Neg for &'a FieldElement32 { - type Output = FieldElement32; - fn neg(self) -> FieldElement32 { +impl<'a> Neg for &'a FieldElement2625 { + type Output = FieldElement2625; + fn neg(self) -> FieldElement2625 { let mut output = *self; output.negate(); output } } -impl ConditionallySelectable for FieldElement32 { +impl ConditionallySelectable for FieldElement2625 { fn conditional_select( - a: &FieldElement32, - b: &FieldElement32, + a: &FieldElement2625, + b: &FieldElement2625, choice: Choice, - ) -> FieldElement32 { - FieldElement32([ + ) -> FieldElement2625 { + FieldElement2625([ u32::conditional_select(&a.0[0], &b.0[0], choice), u32::conditional_select(&a.0[1], &b.0[1], choice), u32::conditional_select(&a.0[2], &b.0[2], choice), @@ -239,7 +239,7 @@ impl ConditionallySelectable for FieldElement32 { ]) } - fn conditional_assign(&mut self, other: &FieldElement32, choice: Choice) { + fn conditional_assign(&mut self, other: &FieldElement2625, choice: Choice) { self.0[0].conditional_assign(&other.0[0], choice); self.0[1].conditional_assign(&other.0[1], choice); self.0[2].conditional_assign(&other.0[2], choice); @@ -252,7 +252,7 @@ impl ConditionallySelectable for FieldElement32 { self.0[9].conditional_assign(&other.0[9], choice); } - fn conditional_swap(a: &mut FieldElement32, b: &mut FieldElement32, choice: Choice) { + fn conditional_swap(a: &mut FieldElement2625, b: &mut FieldElement2625, choice: Choice) { u32::conditional_swap(&mut a.0[0], &mut b.0[0], choice); u32::conditional_swap(&mut a.0[1], &mut b.0[1], choice); u32::conditional_swap(&mut a.0[2], &mut b.0[2], choice); @@ -266,11 +266,11 @@ impl ConditionallySelectable for FieldElement32 { } } -impl FieldElement32 { +impl FieldElement2625 { /// Invert the sign of this field element pub fn negate(&mut self) { // Compute -b as ((2^4 * p) - b) to avoid underflow. - let neg = FieldElement32::reduce([ + let neg = FieldElement2625::reduce([ ((0x3ffffed << 4) - self.0[0]) as u64, ((0x1ffffff << 4) - self.0[1]) as u64, ((0x3ffffff << 4) - self.0[2]) as u64, @@ -286,25 +286,25 @@ impl FieldElement32 { } /// Construct zero. - pub fn zero() -> FieldElement32 { - FieldElement32([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) + pub fn zero() -> FieldElement2625 { + FieldElement2625([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) } /// Construct one. - pub fn one() -> FieldElement32 { - FieldElement32([ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) + pub fn one() -> FieldElement2625 { + FieldElement2625([ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) } /// Construct -1. - pub fn minus_one() -> FieldElement32 { - FieldElement32([ + pub fn minus_one() -> FieldElement2625 { + FieldElement2625([ 0x3ffffec, 0x1ffffff, 0x3ffffff, 0x1ffffff, 0x3ffffff, 0x1ffffff, 0x3ffffff, 0x1ffffff, 0x3ffffff, 0x1ffffff, ]) } /// Given `k > 0`, return `self^(2^k)`. - pub fn pow2k(&self, k: u32) -> FieldElement32 { + pub fn pow2k(&self, k: u32) -> FieldElement2625 { debug_assert!( k > 0 ); let mut z = self.square(); for _ in 1..k { @@ -314,12 +314,12 @@ impl FieldElement32 { } /// Given unreduced coefficients `z[0], ..., z[9]` of any size, - /// carry and reduce them mod p to obtain a `FieldElement32` + /// carry and reduce them mod p to obtain a `FieldElement2625` /// whose coefficients have excess `b < 0.007`. /// /// In other words, each coefficient of the result is bounded by /// either `2^(25 + 0.007)` or `2^(26 + 0.007)`, as appropriate. - fn reduce(mut z: [u64; 10]) -> FieldElement32 { + fn reduce(mut z: [u64; 10]) -> FieldElement2625 { const LOW_25_BITS: u64 = (1 << 25) - 1; const LOW_26_BITS: u64 = (1 << 26) - 1; @@ -361,13 +361,13 @@ impl FieldElement32 { // < 2^25.007 (good enough) // and we're done. - FieldElement32([ + FieldElement2625([ z[0] as u32, z[1] as u32, z[2] as u32, z[3] as u32, z[4] as u32, z[5] as u32, z[6] as u32, z[7] as u32, z[8] as u32, z[9] as u32, ]) } - /// Load a `FieldElement64` from the low 255 bits of a 256-bit + /// Load a `FieldElement51` from the low 255 bits of a 256-bit /// input. /// /// # Warning @@ -378,7 +378,7 @@ impl FieldElement32 { /// encoding of every field element should decode, re-encode to /// the canonical encoding, and check that the input was /// canonical. - pub fn from_bytes(data: &[u8; 32]) -> FieldElement32 { //FeFromBytes + pub fn from_bytes(data: &[u8; 32]) -> FieldElement2625 { //FeFromBytes #[inline] fn load3(b: &[u8]) -> u64 { (b[0] as u64) | ((b[1] as u64) << 8) | ((b[2] as u64) << 16) @@ -402,16 +402,16 @@ impl FieldElement32 { h[8] = load3(&data[26..]) << 4; h[9] = (load3(&data[29..]) & LOW_23_BITS) << 2; - FieldElement32::reduce(h) + FieldElement2625::reduce(h) } - /// Serialize this `FieldElement64` to a 32-byte array. The + /// Serialize this `FieldElement51` to a 32-byte array. The /// encoding is canonical. pub fn to_bytes(&self) -> [u8; 32] { let inp = &self.0; // Reduce the value represented by `in` to the range [0,2*p) - let mut h: [u32; 10] = FieldElement32::reduce([ + let mut h: [u32; 10] = FieldElement2625::reduce([ // XXX this cast is annoying inp[0] as u64, inp[1] as u64, inp[2] as u64, inp[3] as u64, inp[4] as u64, inp[5] as u64, inp[6] as u64, inp[7] as u64, inp[8] as u64, inp[9] as u64, @@ -554,16 +554,16 @@ impl FieldElement32 { } /// Compute `self^2`. - pub fn square(&self) -> FieldElement32 { - FieldElement32::reduce(self.square_inner()) + pub fn square(&self) -> FieldElement2625 { + FieldElement2625::reduce(self.square_inner()) } /// Compute `2*self^2`. - pub fn square2(&self) -> FieldElement32 { + pub fn square2(&self) -> FieldElement2625 { let mut coeffs = self.square_inner(); for i in 0..self.0.len() { coeffs[i] += coeffs[i]; } - FieldElement32::reduce(coeffs) + FieldElement2625::reduce(coeffs) } } diff --git a/src/backend/u32/mod.rs b/src/backend/serial/u32/mod.rs similarity index 100% rename from src/backend/u32/mod.rs rename to src/backend/serial/u32/mod.rs diff --git a/src/backend/u32/scalar.rs b/src/backend/serial/u32/scalar.rs similarity index 87% rename from src/backend/u32/scalar.rs rename to src/backend/serial/u32/scalar.rs index 6c3bb51..8dce8a3 100644 --- a/src/backend/u32/scalar.rs +++ b/src/backend/serial/u32/scalar.rs @@ -15,24 +15,24 @@ use core::ops::{Index, IndexMut}; use constants; -/// The `Scalar32` struct represents an element in ℤ/lℤ as 9 29-bit limbs +/// The `Scalar29` struct represents an element in ℤ/lℤ as 9 29-bit limbs #[derive(Copy,Clone)] -pub struct Scalar32(pub [u32; 9]); +pub struct Scalar29(pub [u32; 9]); -impl Debug for Scalar32 { +impl Debug for Scalar29 { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - write!(f, "Scalar32: {:?}", &self.0[..]) + write!(f, "Scalar29: {:?}", &self.0[..]) } } -impl Index for Scalar32 { +impl Index for Scalar29 { type Output = u32; fn index(&self, _index: usize) -> &u32 { &(self.0[_index]) } } -impl IndexMut for Scalar32 { +impl IndexMut for Scalar29 { fn index_mut(&mut self, _index: usize) -> &mut u32 { &mut (self.0[_index]) } @@ -44,14 +44,14 @@ fn m(x: u32, y: u32) -> u64 { (x as u64) * (y as u64) } -impl Scalar32 { +impl Scalar29 { /// Return the zero scalar. - pub fn zero() -> Scalar32 { - Scalar32([0,0,0,0,0,0,0,0,0]) + pub fn zero() -> Scalar29 { + Scalar29([0,0,0,0,0,0,0,0,0]) } /// Unpack a 32 byte / 256 bit scalar into 9 29-bit limbs. - pub fn from_bytes(bytes: &[u8; 32]) -> Scalar32 { + pub fn from_bytes(bytes: &[u8; 32]) -> Scalar29 { let mut words = [0u32; 8]; for i in 0..8 { for j in 0..4 { @@ -61,7 +61,7 @@ impl Scalar32 { let mask = (1u32 << 29) - 1; let top_mask = (1u32 << 24) - 1; - let mut s = Scalar32::zero(); + let mut s = Scalar29::zero(); s[ 0] = words[0] & mask; s[ 1] = ((words[0] >> 29) | (words[1] << 3)) & mask; @@ -77,7 +77,7 @@ impl Scalar32 { } /// Reduce a 64 byte / 512 bit scalar mod l. - pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar32 { + pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar29 { let mut words = [0u32; 16]; for i in 0..16 { for j in 0..4 { @@ -86,8 +86,8 @@ impl Scalar32 { } let mask = (1u32 << 29) - 1; - let mut lo = Scalar32::zero(); - let mut hi = Scalar32::zero(); + let mut lo = Scalar29::zero(); + let mut hi = Scalar29::zero(); lo[0] = words[ 0] & mask; lo[1] = ((words[ 0] >> 29) | (words[ 1] << 3)) & mask; @@ -108,13 +108,13 @@ impl Scalar32 { hi[7] = ((words[14] >> 16) | (words[15] << 16)) & mask; hi[8] = words[15] >> 13 ; - lo = Scalar32::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo - hi = Scalar32::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R + lo = Scalar29::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo + hi = Scalar29::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R - Scalar32::add(&hi, &lo) // (hi * R) + lo + Scalar29::add(&hi, &lo) // (hi * R) + lo } - /// Pack the limbs of this `Scalar32` into 32 bytes. + /// Pack the limbs of this `Scalar29` into 32 bytes. pub fn to_bytes(&self) -> [u8; 32] { let mut s = [0u8; 32]; @@ -155,8 +155,8 @@ impl Scalar32 { } /// Compute `a + b` (mod l). - pub fn add(a: &Scalar32, b: &Scalar32) -> Scalar32 { - let mut sum = Scalar32::zero(); + pub fn add(a: &Scalar29, b: &Scalar29) -> Scalar29 { + let mut sum = Scalar29::zero(); let mask = (1u32 << 29) - 1; // a + b @@ -167,12 +167,12 @@ impl Scalar32 { } // subtract l if the sum is >= l - Scalar32::sub(&sum, &constants::L) + Scalar29::sub(&sum, &constants::L) } /// Compute `a - b` (mod l). - pub fn sub(a: &Scalar32, b: &Scalar32) -> Scalar32 { - let mut difference = Scalar32::zero(); + pub fn sub(a: &Scalar29, b: &Scalar29) -> Scalar29 { + let mut difference = Scalar29::zero(); let mask = (1u32 << 29) - 1; // a - b @@ -197,7 +197,7 @@ impl Scalar32 { /// /// This is implemented with a one-level refined Karatsuba decomposition #[inline(always)] - pub (crate) fn mul_internal(a: &Scalar32, b: &Scalar32) -> [u64; 17] { + pub (crate) fn mul_internal(a: &Scalar29, b: &Scalar29) -> [u64; 17] { let mut z = [0u64; 17]; z[0] = m(a[0],b[0]); // c00 @@ -254,7 +254,7 @@ impl Scalar32 { /// Compute `a^2`. #[inline(always)] - fn square_internal(a: &Scalar32) -> [u64; 17] { + fn square_internal(a: &Scalar29) -> [u64; 17] { let aa = [ a[0]*2, a[1]*2, @@ -289,7 +289,7 @@ impl Scalar32 { /// Compute `limbs/R` (mod l), where R is the Montgomery modulus 2^261 #[inline(always)] - pub (crate) fn montgomery_reduce(limbs: &[u64; 17]) -> Scalar32 { + pub (crate) fn montgomery_reduce(limbs: &[u64; 17]) -> Scalar29 { #[inline(always)] fn part1(sum: u64) -> (u64, u32) { @@ -329,49 +329,49 @@ impl Scalar32 { let r8 = carry as u32; // result may be >= l, so attempt to subtract l - Scalar32::sub(&Scalar32([r0,r1,r2,r3,r4,r5,r6,r7,r8]), l) + Scalar29::sub(&Scalar29([r0,r1,r2,r3,r4,r5,r6,r7,r8]), l) } /// Compute `a * b` (mod l). #[inline(never)] - pub fn mul(a: &Scalar32, b: &Scalar32) -> Scalar32 { - let ab = Scalar32::montgomery_reduce(&Scalar32::mul_internal(a, b)); - Scalar32::montgomery_reduce(&Scalar32::mul_internal(&ab, &constants::RR)) + pub fn mul(a: &Scalar29, b: &Scalar29) -> Scalar29 { + let ab = Scalar29::montgomery_reduce(&Scalar29::mul_internal(a, b)); + Scalar29::montgomery_reduce(&Scalar29::mul_internal(&ab, &constants::RR)) } /// Compute `a^2` (mod l). #[inline(never)] #[allow(dead_code)] // XXX we don't expose square() via the Scalar API - pub fn square(&self) -> Scalar32 { - let aa = Scalar32::montgomery_reduce(&Scalar32::square_internal(self)); - Scalar32::montgomery_reduce(&Scalar32::mul_internal(&aa, &constants::RR)) + pub fn square(&self) -> Scalar29 { + let aa = Scalar29::montgomery_reduce(&Scalar29::square_internal(self)); + Scalar29::montgomery_reduce(&Scalar29::mul_internal(&aa, &constants::RR)) } /// Compute `(a * b) / R` (mod l), where R is the Montgomery modulus 2^261 #[inline(never)] - pub fn montgomery_mul(a: &Scalar32, b: &Scalar32) -> Scalar32 { - Scalar32::montgomery_reduce(&Scalar32::mul_internal(a, b)) + pub fn montgomery_mul(a: &Scalar29, b: &Scalar29) -> Scalar29 { + Scalar29::montgomery_reduce(&Scalar29::mul_internal(a, b)) } /// Compute `(a^2) / R` (mod l) in Montgomery form, where R is the Montgomery modulus 2^261 #[inline(never)] - pub fn montgomery_square(&self) -> Scalar32 { - Scalar32::montgomery_reduce(&Scalar32::square_internal(self)) + pub fn montgomery_square(&self) -> Scalar29 { + Scalar29::montgomery_reduce(&Scalar29::square_internal(self)) } - /// Puts a Scalar32 in to Montgomery form, i.e. computes `a*R (mod l)` + /// Puts a Scalar29 in to Montgomery form, i.e. computes `a*R (mod l)` #[inline(never)] - pub fn to_montgomery(&self) -> Scalar32 { - Scalar32::montgomery_mul(self, &constants::RR) + pub fn to_montgomery(&self) -> Scalar29 { + Scalar29::montgomery_mul(self, &constants::RR) } - /// Takes a Scalar32 out of Montgomery form, i.e. computes `a/R (mod l)` - pub fn from_montgomery(&self) -> Scalar32 { + /// Takes a Scalar29 out of Montgomery form, i.e. computes `a/R (mod l)` + pub fn from_montgomery(&self) -> Scalar29 { let mut limbs = [0u64; 17]; for i in 0..9 { limbs[i] = self[i] as u64; } - Scalar32::montgomery_reduce(&limbs) + Scalar29::montgomery_reduce(&limbs) } } @@ -385,69 +385,69 @@ mod test { /// x = 2^253-1 = 14474011154664524427946373126085988481658748083205070504932198000989141204991 /// x = 7237005577332262213973186563042994240801631723825162898930247062703686954002 mod l /// x = 5147078182513738803124273553712992179887200054963030844803268920753008712037*R mod l in Montgomery form - pub static X: Scalar32 = Scalar32( + pub static X: Scalar29 = Scalar29( [0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x001fffff]); /// x^2 = 3078544782642840487852506753550082162405942681916160040940637093560259278169 mod l - pub static XX: Scalar32 = Scalar32( + pub static XX: Scalar29 = Scalar29( [0x00217559, 0x000b3401, 0x103ff43b, 0x1462a62c, 0x1d6f9f38, 0x18e7a42f, 0x09a3dcee, 0x008dbe18, 0x0006ce65]); /// x^2 = 2912514428060642753613814151688322857484807845836623976981729207238463947987*R mod l in Montgomery form - pub static XX_MONT: Scalar32 = Scalar32( + pub static XX_MONT: Scalar29 = Scalar29( [0x152b4d2e, 0x0571d53b, 0x1da6d964, 0x188663b6, 0x1d1b5f92, 0x19d50e3f, 0x12306c29, 0x0c6f26fe, 0x00030edb]); /// y = 6145104759870991071742105800796537629880401874866217824609283457819451087098 - pub static Y: Scalar32 = Scalar32( + pub static Y: Scalar29 = Scalar29( [0x1e1458fa, 0x165ba838, 0x1d787b36, 0x0e577f3a, 0x1d2baf06, 0x1d689a19, 0x1fff3047, 0x117704ab, 0x000d9601]); /// x*y = 36752150652102274958925982391442301741 - pub static XY: Scalar32 = Scalar32( + pub static XY: Scalar29 = Scalar29( [0x0ba7632d, 0x017736bb, 0x15c76138, 0x0c69daa1, 0x000001ba, 0x00000000, 0x00000000, 0x00000000, 0x00000000]); /// x*y = 3783114862749659543382438697751927473898937741870308063443170013240655651591*R mod l in Montgomery form - pub static XY_MONT: Scalar32 = Scalar32( + pub static XY_MONT: Scalar29 = Scalar29( [0x077b51e1, 0x1c64e119, 0x02a19ef5, 0x18d2129e, 0x00de0430, 0x045a7bc8, 0x04cfc7c9, 0x1c002681, 0x000bdc1c]); /// a = 2351415481556538453565687241199399922945659411799870114962672658845158063753 - pub static A: Scalar32 = Scalar32( + pub static A: Scalar29 = Scalar29( [0x07b3be89, 0x02291b60, 0x14a99f03, 0x07dc3787, 0x0a782aae, 0x16262525, 0x0cfdb93f, 0x13f5718d, 0x000532da]); /// b = 4885590095775723760407499321843594317911456947580037491039278279440296187236 - pub static B: Scalar32 = Scalar32( + pub static B: Scalar29 = Scalar29( [0x15421564, 0x1e69fd72, 0x093d9692, 0x161785be, 0x1587d69f, 0x09d9dada, 0x130246c0, 0x0c0a8e72, 0x000acd25]); /// a+b = 0 /// a-b = 4702830963113076907131374482398799845891318823599740229925345317690316127506 - pub static AB: Scalar32 = Scalar32( + pub static AB: Scalar29 = Scalar29( [0x0f677d12, 0x045236c0, 0x09533e06, 0x0fb86f0f, 0x14f0555c, 0x0c4c4a4a, 0x19fb727f, 0x07eae31a, 0x000a65b5]); // c = (2^512 - 1) % l = 1627715501170711445284395025044413883736156588369414752970002579683115011840 - pub static C: Scalar32 = Scalar32( + pub static C: Scalar29 = Scalar29( [0x049c0f00, 0x00308f1a, 0x0164d1e9, 0x1c374ed1, 0x1be65d00, 0x19e90bfa, 0x08f73bb1, 0x036f8613, 0x00039941]); #[test] fn mul_max() { - let res = Scalar32::mul(&X, &X); + let res = Scalar29::mul(&X, &X); for i in 0..9 { assert!(res[i] == XX[i]); } @@ -463,7 +463,7 @@ mod test { #[test] fn montgomery_mul_max() { - let res = Scalar32::montgomery_mul(&X, &X); + let res = Scalar29::montgomery_mul(&X, &X); for i in 0..9 { assert!(res[i] == XX_MONT[i]); } @@ -479,7 +479,7 @@ mod test { #[test] fn mul() { - let res = Scalar32::mul(&X, &Y); + let res = Scalar29::mul(&X, &Y); for i in 0..9 { assert!(res[i] == XY[i]); } @@ -487,7 +487,7 @@ mod test { #[test] fn montgomery_mul() { - let res = Scalar32::montgomery_mul(&X, &Y); + let res = Scalar29::montgomery_mul(&X, &Y); for i in 0..9 { assert!(res[i] == XY_MONT[i]); } @@ -495,8 +495,8 @@ mod test { #[test] fn add() { - let res = Scalar32::add(&A, &B); - let zero = Scalar32::zero(); + let res = Scalar29::add(&A, &B); + let zero = Scalar29::zero(); for i in 0..9 { assert!(res[i] == zero[i]); } @@ -504,7 +504,7 @@ mod test { #[test] fn sub() { - let res = Scalar32::sub(&A, &B); + let res = Scalar29::sub(&A, &B); for i in 0..9 { assert!(res[i] == AB[i]); } @@ -513,7 +513,7 @@ mod test { #[test] fn from_bytes_wide() { let bignum = [255u8; 64]; // 2^512 - 1 - let reduced = Scalar32::from_bytes_wide(&bignum); + let reduced = Scalar29::from_bytes_wide(&bignum); for i in 0..9 { assert!(reduced[i] == C[i]); } diff --git a/src/backend/u64/constants.rs b/src/backend/serial/u64/constants.rs similarity index 59% rename from src/backend/u64/constants.rs rename to src/backend/serial/u64/constants.rs index 0ac7fe3..a5c1f07 100644 --- a/src/backend/u64/constants.rs +++ b/src/backend/serial/u64/constants.rs @@ -10,43 +10,43 @@ //! This module contains backend-specific constant values, such as the 64-bit limbs of curve constants. -use backend::u64::field::FieldElement64; -use backend::u64::scalar::Scalar64; +use backend::serial::u64::field::FieldElement51; +use backend::serial::u64::scalar::Scalar52; use edwards::EdwardsPoint; /// Edwards `d` value, equal to `-121665/121666 mod p`. -pub(crate) const EDWARDS_D: FieldElement64 = FieldElement64([929955233495203, 466365720129213, 1662059464998953, 2033849074728123, 1442794654840575]); +pub(crate) const EDWARDS_D: FieldElement51 = FieldElement51([929955233495203, 466365720129213, 1662059464998953, 2033849074728123, 1442794654840575]); /// Edwards `2*d` value, equal to `2*(-121665/121666) mod p`. -pub(crate) const EDWARDS_D2: FieldElement64 = FieldElement64([1859910466990425, 932731440258426, 1072319116312658, 1815898335770999, 633789495995903]); +pub(crate) const EDWARDS_D2: FieldElement51 = FieldElement51([1859910466990425, 932731440258426, 1072319116312658, 1815898335770999, 633789495995903]); /// `= sqrt(a*d - 1)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. -pub(crate) const SQRT_AD_MINUS_ONE: FieldElement64 = FieldElement64([ +pub(crate) const SQRT_AD_MINUS_ONE: FieldElement51 = FieldElement51([ 2241493124984347, 425987919032274, 2207028919301688, 1220490630685848, 974799131293748 ]); /// `= 1/sqrt(a-d)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. -pub(crate) const INVSQRT_A_MINUS_D: FieldElement64 = FieldElement64([ +pub(crate) const INVSQRT_A_MINUS_D: FieldElement51 = FieldElement51([ 278908739862762, 821645201101625, 8113234426968, 1777959178193151, 2118520810568447 ]); /// Precomputed value of one of the square roots of -1 (mod p) -pub(crate) const SQRT_M1: FieldElement64 = FieldElement64([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]); +pub(crate) const SQRT_M1: FieldElement51 = FieldElement51([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]); /// `APLUS2_OVER_FOUR` is (A+2)/4. (This is used internally within the Montgomery ladder.) -pub(crate) const APLUS2_OVER_FOUR: FieldElement64 = FieldElement64([121666, 0, 0, 0, 0]); +pub(crate) const APLUS2_OVER_FOUR: FieldElement51 = FieldElement51([121666, 0, 0, 0, 0]); /// `L` is the order of base point, i.e. 2^252 + 27742317777372353535851937790883648493 -pub(crate) const L: Scalar64 = Scalar64([ 0x0002631a5cf5d3ed, 0x000dea2f79cd6581, 0x000000000014def9, 0x0000000000000000, 0x0000100000000000 ]); +pub(crate) const L: Scalar52 = Scalar52([ 0x0002631a5cf5d3ed, 0x000dea2f79cd6581, 0x000000000014def9, 0x0000000000000000, 0x0000100000000000 ]); /// `L` * `LFACTOR` = -1 (mod 2^52) pub(crate) const LFACTOR: u64 = 0x51da312547e1b; /// `R` = R % L where R = 2^260 -pub(crate) const R: Scalar64 = Scalar64([ 0x000f48bd6721e6ed, 0x0003bab5ac67e45a, 0x000fffffeb35e51b, 0x000fffffffffffff, 0x00000fffffffffff ]); +pub(crate) const R: Scalar52 = Scalar52([ 0x000f48bd6721e6ed, 0x0003bab5ac67e45a, 0x000fffffeb35e51b, 0x000fffffffffffff, 0x00000fffffffffff ]); /// `RR` = (R^2) % L where R = 2^260 -pub(crate) const RR: Scalar64 = Scalar64([ 0x0009d265e952d13b, 0x000d63c715bea69f, 0x0005be65cb687604, 0x0003dceec73d217f, 0x000009411b7c309a ]); +pub(crate) const RR: Scalar52 = Scalar52([ 0x0009d265e952d13b, 0x000d63c715bea69f, 0x0005be65cb687604, 0x0003dceec73d217f, 0x000009411b7c309a ]); /// The Ed25519 basepoint, as an `EdwardsPoint`. /// @@ -54,10 +54,10 @@ pub(crate) const RR: Scalar64 = Scalar64([ 0x0009d265e952d13b, 0x000d63c715bea69 /// `ED25519_BASEPOINT_TABLE`, which should be used for scalar /// multiplication (it's much faster). pub const ED25519_BASEPOINT_POINT: EdwardsPoint = EdwardsPoint{ - X: FieldElement64([1738742601995546, 1146398526822698, 2070867633025821, 562264141797630, 587772402128613]), - Y: FieldElement64([1801439850948184, 1351079888211148, 450359962737049, 900719925474099, 1801439850948198]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([1841354044333475, 16398895984059, 755974180946558, 900171276175154, 1821297809914039]), + X: FieldElement51([1738742601995546, 1146398526822698, 2070867633025821, 562264141797630, 587772402128613]), + Y: FieldElement51([1801439850948184, 1351079888211148, 450359962737049, 900719925474099, 1801439850948198]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([1841354044333475, 16398895984059, 755974180946558, 900171276175154, 1821297809914039]), }; /// The 8-torsion subgroup \\(\mathcal E [8]\\). @@ -74,58 +74,58 @@ pub const EIGHT_TORSION: [EdwardsPoint; 8] = EIGHT_TORSION_INNER_DOC_HIDDEN; #[doc(hidden)] pub const EIGHT_TORSION_INNER_DOC_HIDDEN: [EdwardsPoint; 8] = [ EdwardsPoint { - X: FieldElement64([0, 0, 0, 0, 0]), - Y: FieldElement64([1, 0, 0, 0, 0]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([0, 0, 0, 0, 0]), + X: FieldElement51([0, 0, 0, 0, 0]), + Y: FieldElement51([1, 0, 0, 0, 0]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([0, 0, 0, 0, 0]), } , EdwardsPoint { - X: FieldElement64([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]), - Y: FieldElement64([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]), + X: FieldElement51([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]), + Y: FieldElement51([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]), } , EdwardsPoint { - X: FieldElement64([533094393274173, 2016890930128738, 18285341111199, 134597186663265, 1486323764102114]), - Y: FieldElement64([0, 0, 0, 0, 0]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([0, 0, 0, 0, 0]), + X: FieldElement51([533094393274173, 2016890930128738, 18285341111199, 134597186663265, 1486323764102114]), + Y: FieldElement51([0, 0, 0, 0, 0]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([0, 0, 0, 0, 0]), } , EdwardsPoint { - X: FieldElement64([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]), - Y: FieldElement64([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]), + X: FieldElement51([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]), + Y: FieldElement51([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]), } , EdwardsPoint { - X: FieldElement64([0, 0, 0, 0, 0]), - Y: FieldElement64([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([0, 0, 0, 0, 0]), + X: FieldElement51([0, 0, 0, 0, 0]), + Y: FieldElement51([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([0, 0, 0, 0, 0]), } , EdwardsPoint { - X: FieldElement64([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]), - Y: FieldElement64([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]), + X: FieldElement51([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]), + Y: FieldElement51([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]), } , EdwardsPoint { - X: FieldElement64([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]), - Y: FieldElement64([0, 0, 0, 0, 0]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([0, 0, 0, 0, 0]), + X: FieldElement51([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]), + Y: FieldElement51([0, 0, 0, 0, 0]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([0, 0, 0, 0, 0]), } , EdwardsPoint { - X: FieldElement64([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]), - Y: FieldElement64([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]), + X: FieldElement51([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]), + Y: FieldElement51([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]), } ]; diff --git a/src/backend/u64/field.rs b/src/backend/serial/u64/field.rs similarity index 86% rename from src/backend/u64/field.rs rename to src/backend/serial/u64/field.rs index ffe1a48..bdf440c 100644 --- a/src/backend/u64/field.rs +++ b/src/backend/serial/u64/field.rs @@ -20,7 +20,7 @@ use core::ops::{Sub, SubAssign}; use subtle::Choice; use subtle::ConditionallySelectable; -/// A `FieldElement64` represents an element of the field +/// A `FieldElement51` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// /// In the 64-bit implementation, a `FieldElement` is represented in @@ -30,57 +30,57 @@ use subtle::ConditionallySelectable; /// # Note /// /// The `curve25519_dalek::field` module provides a type alias -/// `curve25519_dalek::field::FieldElement` to either `FieldElement64` -/// or `FieldElement32`. +/// `curve25519_dalek::field::FieldElement` to either `FieldElement51` +/// or `FieldElement2625`. /// -/// The backend-specific type `FieldElement64` should not be used +/// The backend-specific type `FieldElement51` should not be used /// outside of the `curve25519_dalek::field` module. #[derive(Copy, Clone)] -pub struct FieldElement64(pub (crate) [u64; 5]); +pub struct FieldElement51(pub (crate) [u64; 5]); -impl Debug for FieldElement64 { +impl Debug for FieldElement51 { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - write!(f, "FieldElement64({:?})", &self.0[..]) + write!(f, "FieldElement51({:?})", &self.0[..]) } } -impl<'b> AddAssign<&'b FieldElement64> for FieldElement64 { - fn add_assign(&mut self, _rhs: &'b FieldElement64) { +impl<'b> AddAssign<&'b FieldElement51> for FieldElement51 { + fn add_assign(&mut self, _rhs: &'b FieldElement51) { for i in 0..5 { self.0[i] += _rhs.0[i]; } } } -impl<'a, 'b> Add<&'b FieldElement64> for &'a FieldElement64 { - type Output = FieldElement64; - fn add(self, _rhs: &'b FieldElement64) -> FieldElement64 { +impl<'a, 'b> Add<&'b FieldElement51> for &'a FieldElement51 { + type Output = FieldElement51; + fn add(self, _rhs: &'b FieldElement51) -> FieldElement51 { let mut output = *self; output += _rhs; output } } -impl<'b> SubAssign<&'b FieldElement64> for FieldElement64 { - fn sub_assign(&mut self, _rhs: &'b FieldElement64) { - let result = (self as &FieldElement64) - _rhs; +impl<'b> SubAssign<&'b FieldElement51> for FieldElement51 { + fn sub_assign(&mut self, _rhs: &'b FieldElement51) { + let result = (self as &FieldElement51) - _rhs; self.0 = result.0; } } -impl<'a, 'b> Sub<&'b FieldElement64> for &'a FieldElement64 { - type Output = FieldElement64; - fn sub(self, _rhs: &'b FieldElement64) -> FieldElement64 { +impl<'a, 'b> Sub<&'b FieldElement51> for &'a FieldElement51 { + type Output = FieldElement51; + fn sub(self, _rhs: &'b FieldElement51) -> FieldElement51 { // To avoid underflow, first add a multiple of p. // Choose 16*p = p << 4 to be larger than 54-bit _rhs. // // If we could statically track the bitlengths of the limbs - // of every FieldElement64, we could choose a multiple of p + // of every FieldElement51, we could choose a multiple of p // just bigger than _rhs and avoid having to do a reduction. // // Since we don't yet have type-level integers to do this, we // have to add an explicit reduction call here. - FieldElement64::reduce([ + FieldElement51::reduce([ (self.0[0] + 36028797018963664u64) - _rhs.0[0], (self.0[1] + 36028797018963952u64) - _rhs.0[1], (self.0[2] + 36028797018963952u64) - _rhs.0[2], @@ -90,16 +90,16 @@ impl<'a, 'b> Sub<&'b FieldElement64> for &'a FieldElement64 { } } -impl<'b> MulAssign<&'b FieldElement64> for FieldElement64 { - fn mul_assign(&mut self, _rhs: &'b FieldElement64) { - let result = (self as &FieldElement64) * _rhs; +impl<'b> MulAssign<&'b FieldElement51> for FieldElement51 { + fn mul_assign(&mut self, _rhs: &'b FieldElement51) { + let result = (self as &FieldElement51) * _rhs; self.0 = result.0; } } -impl<'a, 'b> Mul<&'b FieldElement64> for &'a FieldElement64 { - type Output = FieldElement64; - fn mul(self, _rhs: &'b FieldElement64) -> FieldElement64 { +impl<'a, 'b> Mul<&'b FieldElement51> for &'a FieldElement51 { + type Output = FieldElement51; + fn mul(self, _rhs: &'b FieldElement51) -> FieldElement51 { /// Helper function to multiply two 64-bit integers with 128 /// bits of output. #[inline(always)] @@ -196,26 +196,26 @@ impl<'a, 'b> Mul<&'b FieldElement64> for &'a FieldElement64 { out[0] &= LOW_51_BIT_MASK; // Now out[i] < 2^(51 + epsilon) for all i. - FieldElement64(out) + FieldElement51(out) } } -impl<'a> Neg for &'a FieldElement64 { - type Output = FieldElement64; - fn neg(self) -> FieldElement64 { +impl<'a> Neg for &'a FieldElement51 { + type Output = FieldElement51; + fn neg(self) -> FieldElement51 { let mut output = *self; output.negate(); output } } -impl ConditionallySelectable for FieldElement64 { +impl ConditionallySelectable for FieldElement51 { fn conditional_select( - a: &FieldElement64, - b: &FieldElement64, + a: &FieldElement51, + b: &FieldElement51, choice: Choice, - ) -> FieldElement64 { - FieldElement64([ + ) -> FieldElement51 { + FieldElement51([ u64::conditional_select(&a.0[0], &b.0[0], choice), u64::conditional_select(&a.0[1], &b.0[1], choice), u64::conditional_select(&a.0[2], &b.0[2], choice), @@ -224,7 +224,7 @@ impl ConditionallySelectable for FieldElement64 { ]) } - fn conditional_swap(a: &mut FieldElement64, b: &mut FieldElement64, choice: Choice) { + fn conditional_swap(a: &mut FieldElement51, b: &mut FieldElement51, choice: Choice) { u64::conditional_swap(&mut a.0[0], &mut b.0[0], choice); u64::conditional_swap(&mut a.0[1], &mut b.0[1], choice); u64::conditional_swap(&mut a.0[2], &mut b.0[2], choice); @@ -232,7 +232,7 @@ impl ConditionallySelectable for FieldElement64 { u64::conditional_swap(&mut a.0[4], &mut b.0[4], choice); } - fn conditional_assign(&mut self, other: &FieldElement64, choice: Choice) { + fn conditional_assign(&mut self, other: &FieldElement51, choice: Choice) { self.0[0].conditional_assign(&other.0[0], choice); self.0[1].conditional_assign(&other.0[1], choice); self.0[2].conditional_assign(&other.0[2], choice); @@ -241,11 +241,11 @@ impl ConditionallySelectable for FieldElement64 { } } -impl FieldElement64 { +impl FieldElement51 { /// Invert the sign of this field element pub fn negate(&mut self) { // See commentary in the Sub impl - let neg = FieldElement64::reduce([ + let neg = FieldElement51::reduce([ 36028797018963664u64 - self.0[0], 36028797018963952u64 - self.0[1], 36028797018963952u64 - self.0[2], @@ -256,23 +256,23 @@ impl FieldElement64 { } /// Construct zero. - pub fn zero() -> FieldElement64 { - FieldElement64([ 0, 0, 0, 0, 0 ]) + pub fn zero() -> FieldElement51 { + FieldElement51([ 0, 0, 0, 0, 0 ]) } /// Construct one. - pub fn one() -> FieldElement64 { - FieldElement64([ 1, 0, 0, 0, 0 ]) + pub fn one() -> FieldElement51 { + FieldElement51([ 1, 0, 0, 0, 0 ]) } /// Construct -1. - pub fn minus_one() -> FieldElement64 { - FieldElement64([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]) + pub fn minus_one() -> FieldElement51 { + FieldElement51([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]) } /// Given 64-bit input limbs, reduce to enforce the bound 2^(51 + epsilon). #[inline(always)] - fn reduce(mut limbs: [u64; 5]) -> FieldElement64 { + fn reduce(mut limbs: [u64; 5]) -> FieldElement51 { const LOW_51_BIT_MASK: u64 = (1u64 << 51) - 1; // Since the input limbs are bounded by 2^64, the biggest @@ -304,10 +304,10 @@ impl FieldElement64 { limbs[3] += c2; limbs[4] += c3; - FieldElement64(limbs) + FieldElement51(limbs) } - /// Load a `FieldElement64` from the low 255 bits of a 256-bit + /// Load a `FieldElement51` from the low 255 bits of a 256-bit /// input. /// /// # Warning @@ -319,7 +319,7 @@ impl FieldElement64 { /// the canonical encoding, and check that the input was /// canonical. /// - pub fn from_bytes(bytes: &[u8; 32]) -> FieldElement64 { + pub fn from_bytes(bytes: &[u8; 32]) -> FieldElement51 { let load8 = |input: &[u8]| -> u64 { (input[0] as u64) | ((input[1] as u64) << 8) @@ -332,7 +332,7 @@ impl FieldElement64 { }; let low_51_bit_mask = (1u64 << 51) - 1; - FieldElement64( + FieldElement51( // load bits [ 0, 64), no shift [ load8(&bytes[ 0..]) & low_51_bit_mask // load bits [ 48,112), shift to [ 51,112) @@ -346,7 +346,7 @@ impl FieldElement64 { ]) } - /// Serialize this `FieldElement64` to a 32-byte array. The + /// Serialize this `FieldElement51` to a 32-byte array. The /// encoding is canonical. pub fn to_bytes(&self) -> [u8; 32] { // Let h = limbs[0] + limbs[1]*2^51 + ... + limbs[4]*2^204. @@ -365,7 +365,7 @@ impl FieldElement64 { // Therefore q can be computed as the carry bit of h + 19. // First, reduce the limbs to ensure h < 2*p. - let mut limbs = FieldElement64::reduce(self.0).0; + let mut limbs = FieldElement51::reduce(self.0).0; let mut q = (limbs[0] + 19) >> 51; q = (limbs[1] + q) >> 51; @@ -433,7 +433,7 @@ impl FieldElement64 { } /// Given `k > 0`, return `self^(2^k)`. - pub fn pow2k(&self, mut k: u32) -> FieldElement64 { + pub fn pow2k(&self, mut k: u32) -> FieldElement51 { debug_assert!( k > 0 ); @@ -535,16 +535,16 @@ impl FieldElement64 { } } - FieldElement64(a) + FieldElement51(a) } /// Returns the square of this field element. - pub fn square(&self) -> FieldElement64 { + pub fn square(&self) -> FieldElement51 { self.pow2k(1) } /// Returns 2 times the square of this field element. - pub fn square2(&self) -> FieldElement64 { + pub fn square2(&self) -> FieldElement51 { let mut square = self.pow2k(1); for i in 0..5 { square.0[i] *= 2; diff --git a/src/backend/u64/mod.rs b/src/backend/serial/u64/mod.rs similarity index 100% rename from src/backend/u64/mod.rs rename to src/backend/serial/u64/mod.rs diff --git a/src/backend/u64/scalar.rs b/src/backend/serial/u64/scalar.rs similarity index 82% rename from src/backend/u64/scalar.rs rename to src/backend/serial/u64/scalar.rs index e14440b..e9031b0 100644 --- a/src/backend/u64/scalar.rs +++ b/src/backend/serial/u64/scalar.rs @@ -16,25 +16,25 @@ use core::ops::{Index, IndexMut}; use constants; -/// The `Scalar64` struct represents an element in +/// The `Scalar52` struct represents an element in /// \\(\mathbb Z / \ell \mathbb Z\\) as 5 \\(52\\)-bit limbs. #[derive(Copy,Clone)] -pub struct Scalar64(pub [u64; 5]); +pub struct Scalar52(pub [u64; 5]); -impl Debug for Scalar64 { +impl Debug for Scalar52 { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - write!(f, "Scalar64: {:?}", &self.0[..]) + write!(f, "Scalar52: {:?}", &self.0[..]) } } -impl Index for Scalar64 { +impl Index for Scalar52 { type Output = u64; fn index(&self, _index: usize) -> &u64 { &(self.0[_index]) } } -impl IndexMut for Scalar64 { +impl IndexMut for Scalar52 { fn index_mut(&mut self, _index: usize) -> &mut u64 { &mut (self.0[_index]) } @@ -46,14 +46,14 @@ fn m(x: u64, y: u64) -> u128 { (x as u128) * (y as u128) } -impl Scalar64 { +impl Scalar52 { /// Return the zero scalar - pub fn zero() -> Scalar64 { - Scalar64([0,0,0,0,0]) + pub fn zero() -> Scalar52 { + Scalar52([0,0,0,0,0]) } /// Unpack a 32 byte / 256 bit scalar into 5 52-bit limbs. - pub fn from_bytes(bytes: &[u8; 32]) -> Scalar64 { + pub fn from_bytes(bytes: &[u8; 32]) -> Scalar52 { let mut words = [0u64; 4]; for i in 0..4 { for j in 0..8 { @@ -63,7 +63,7 @@ impl Scalar64 { let mask = (1u64 << 52) - 1; let top_mask = (1u64 << 48) - 1; - let mut s = Scalar64::zero(); + let mut s = Scalar52::zero(); s[ 0] = words[0] & mask; s[ 1] = ((words[0] >> 52) | (words[1] << 12)) & mask; @@ -75,7 +75,7 @@ impl Scalar64 { } /// Reduce a 64 byte / 512 bit scalar mod l - pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar64 { + pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar52 { let mut words = [0u64; 8]; for i in 0..8 { for j in 0..8 { @@ -84,8 +84,8 @@ impl Scalar64 { } let mask = (1u64 << 52) - 1; - let mut lo = Scalar64::zero(); - let mut hi = Scalar64::zero(); + let mut lo = Scalar52::zero(); + let mut hi = Scalar52::zero(); lo[0] = words[ 0] & mask; lo[1] = ((words[ 0] >> 52) | (words[ 1] << 12)) & mask; @@ -98,13 +98,13 @@ impl Scalar64 { hi[3] = ((words[ 6] >> 32) | (words[ 7] << 32)) & mask; hi[4] = words[ 7] >> 20 ; - lo = Scalar64::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo - hi = Scalar64::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R + lo = Scalar52::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo + hi = Scalar52::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R - Scalar64::add(&hi, &lo) + Scalar52::add(&hi, &lo) } - /// Pack the limbs of this `Scalar64` into 32 bytes + /// Pack the limbs of this `Scalar52` into 32 bytes pub fn to_bytes(&self) -> [u8; 32] { let mut s = [0u8; 32]; @@ -145,8 +145,8 @@ impl Scalar64 { } /// Compute `a + b` (mod l) - pub fn add(a: &Scalar64, b: &Scalar64) -> Scalar64 { - let mut sum = Scalar64::zero(); + pub fn add(a: &Scalar52, b: &Scalar52) -> Scalar52 { + let mut sum = Scalar52::zero(); let mask = (1u64 << 52) - 1; // a + b @@ -157,12 +157,12 @@ impl Scalar64 { } // subtract l if the sum is >= l - Scalar64::sub(&sum, &constants::L) + Scalar52::sub(&sum, &constants::L) } /// Compute `a - b` (mod l) - pub fn sub(a: &Scalar64, b: &Scalar64) -> Scalar64 { - let mut difference = Scalar64::zero(); + pub fn sub(a: &Scalar52, b: &Scalar52) -> Scalar52 { + let mut difference = Scalar52::zero(); let mask = (1u64 << 52) - 1; // a - b @@ -185,7 +185,7 @@ impl Scalar64 { /// Compute `a * b` #[inline(always)] - pub (crate) fn mul_internal(a: &Scalar64, b: &Scalar64) -> [u128; 9] { + pub (crate) fn mul_internal(a: &Scalar52, b: &Scalar52) -> [u128; 9] { let mut z = [0u128; 9]; z[0] = m(a[0],b[0]); @@ -203,7 +203,7 @@ impl Scalar64 { /// Compute `a^2` #[inline(always)] - fn square_internal(a: &Scalar64) -> [u128; 9] { + fn square_internal(a: &Scalar52) -> [u128; 9] { let aa = [ a[0]*2, a[1]*2, @@ -226,7 +226,7 @@ impl Scalar64 { /// Compute `limbs/R` (mod l), where R is the Montgomery modulus 2^260 #[inline(always)] - pub (crate) fn montgomery_reduce(limbs: &[u128; 9]) -> Scalar64 { + pub (crate) fn montgomery_reduce(limbs: &[u128; 9]) -> Scalar52 { #[inline(always)] fn part1(sum: u128) -> (u128, u64) { @@ -258,50 +258,50 @@ impl Scalar64 { let r4 = carry as u64; // result may be >= l, so attempt to subtract l - Scalar64::sub(&Scalar64([r0,r1,r2,r3,r4]), l) + Scalar52::sub(&Scalar52([r0,r1,r2,r3,r4]), l) } /// Compute `a * b` (mod l) #[inline(never)] - pub fn mul(a: &Scalar64, b: &Scalar64) -> Scalar64 { - let ab = Scalar64::montgomery_reduce(&Scalar64::mul_internal(a, b)); - Scalar64::montgomery_reduce(&Scalar64::mul_internal(&ab, &constants::RR)) + pub fn mul(a: &Scalar52, b: &Scalar52) -> Scalar52 { + let ab = Scalar52::montgomery_reduce(&Scalar52::mul_internal(a, b)); + Scalar52::montgomery_reduce(&Scalar52::mul_internal(&ab, &constants::RR)) } /// Compute `a^2` (mod l) #[inline(never)] #[allow(dead_code)] // XXX we don't expose square() via the Scalar API - pub fn square(&self) -> Scalar64 { - let aa = Scalar64::montgomery_reduce(&Scalar64::square_internal(self)); - Scalar64::montgomery_reduce(&Scalar64::mul_internal(&aa, &constants::RR)) + pub fn square(&self) -> Scalar52 { + let aa = Scalar52::montgomery_reduce(&Scalar52::square_internal(self)); + Scalar52::montgomery_reduce(&Scalar52::mul_internal(&aa, &constants::RR)) } /// Compute `(a * b) / R` (mod l), where R is the Montgomery modulus 2^260 #[inline(never)] - pub fn montgomery_mul(a: &Scalar64, b: &Scalar64) -> Scalar64 { - Scalar64::montgomery_reduce(&Scalar64::mul_internal(a, b)) + pub fn montgomery_mul(a: &Scalar52, b: &Scalar52) -> Scalar52 { + Scalar52::montgomery_reduce(&Scalar52::mul_internal(a, b)) } /// Compute `(a^2) / R` (mod l) in Montgomery form, where R is the Montgomery modulus 2^260 #[inline(never)] - pub fn montgomery_square(&self) -> Scalar64 { - Scalar64::montgomery_reduce(&Scalar64::square_internal(self)) + pub fn montgomery_square(&self) -> Scalar52 { + Scalar52::montgomery_reduce(&Scalar52::square_internal(self)) } - /// Puts a Scalar64 in to Montgomery form, i.e. computes `a*R (mod l)` + /// Puts a Scalar52 in to Montgomery form, i.e. computes `a*R (mod l)` #[inline(never)] - pub fn to_montgomery(&self) -> Scalar64 { - Scalar64::montgomery_mul(self, &constants::RR) + pub fn to_montgomery(&self) -> Scalar52 { + Scalar52::montgomery_mul(self, &constants::RR) } - /// Takes a Scalar64 out of Montgomery form, i.e. computes `a/R (mod l)` + /// Takes a Scalar52 out of Montgomery form, i.e. computes `a/R (mod l)` #[inline(never)] - pub fn from_montgomery(&self) -> Scalar64 { + pub fn from_montgomery(&self) -> Scalar52 { let mut limbs = [0u128; 9]; for i in 0..5 { limbs[i] = self[i] as u128; } - Scalar64::montgomery_reduce(&limbs) + Scalar52::montgomery_reduce(&limbs) } } @@ -316,59 +316,59 @@ mod test { /// x = 14474011154664524427946373126085988481658748083205070504932198000989141204991 /// x = 7237005577332262213973186563042994240801631723825162898930247062703686954002 mod l /// x = 3057150787695215392275360544382990118917283750546154083604586903220563173085*R mod l in Montgomery form - pub static X: Scalar64 = Scalar64( + pub static X: Scalar52 = Scalar52( [0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x00001fffffffffff]); /// x^2 = 3078544782642840487852506753550082162405942681916160040940637093560259278169 mod l - pub static XX: Scalar64 = Scalar64( + pub static XX: Scalar52 = Scalar52( [0x0001668020217559, 0x000531640ffd0ec0, 0x00085fd6f9f38a31, 0x000c268f73bb1cf4, 0x000006ce65046df0]); /// x^2 = 4413052134910308800482070043710297189082115023966588301924965890668401540959*R mod l in Montgomery form - pub static XX_MONT: Scalar64 = Scalar64( + pub static XX_MONT: Scalar52 = Scalar52( [0x000c754eea569a5c, 0x00063b6ed36cb215, 0x0008ffa36bf25886, 0x000e9183614e7543, 0x0000061db6c6f26f]); /// y = 6145104759870991071742105800796537629880401874866217824609283457819451087098 - pub static Y: Scalar64 = Scalar64( + pub static Y: Scalar52 = Scalar52( [0x000b75071e1458fa, 0x000bf9d75e1ecdac, 0x000433d2baf0672b, 0x0005fffcc11fad13, 0x00000d96018bb825]); /// x*y = 36752150652102274958925982391442301741 mod l - pub static XY: Scalar64 = Scalar64( + pub static XY: Scalar52 = Scalar52( [0x000ee6d76ba7632d, 0x000ed50d71d84e02, 0x00000000001ba634, 0x0000000000000000, 0x0000000000000000]); /// x*y = 658448296334113745583381664921721413881518248721417041768778176391714104386*R mod l in Montgomery form - pub static XY_MONT: Scalar64 = Scalar64( + pub static XY_MONT: Scalar52 = Scalar52( [0x0006d52bf200cfd5, 0x00033fb1d7021570, 0x000f201bc07139d8, 0x0001267e3e49169e, 0x000007b839c00268]); /// a = 2351415481556538453565687241199399922945659411799870114962672658845158063753 - pub static A: Scalar64 = Scalar64( + pub static A: Scalar52 = Scalar52( [0x0005236c07b3be89, 0x0001bc3d2a67c0c4, 0x000a4aa782aae3ee, 0x0006b3f6e4fec4c4, 0x00000532da9fab8c]); /// b = 4885590095775723760407499321843594317911456947580037491039278279440296187236 - pub static B: Scalar64 = Scalar64( + pub static B: Scalar52 = Scalar52( [0x000d3fae55421564, 0x000c2df24f65a4bc, 0x0005b5587d69fb0b, 0x00094c091b013b3b, 0x00000acd25605473]); /// a+b = 0 /// a-b = 4702830963113076907131374482398799845891318823599740229925345317690316127506 - pub static AB: Scalar64 = Scalar64( + pub static AB: Scalar52 = Scalar52( [0x000a46d80f677d12, 0x0003787a54cf8188, 0x0004954f0555c7dc, 0x000d67edc9fd8989, 0x00000a65b53f5718]); // c = (2^512 - 1) % l = 1627715501170711445284395025044413883736156588369414752970002579683115011840 - pub static C: Scalar64 = Scalar64( + pub static C: Scalar52 = Scalar52( [0x000611e3449c0f00, 0x000a768859347a40, 0x0007f5be65d00e1b, 0x0009a3dceec73d21, 0x00000399411b7c30]); #[test] fn mul_max() { - let res = Scalar64::mul(&X, &X); + let res = Scalar52::mul(&X, &X); for i in 0..5 { assert!(res[i] == XX[i]); } @@ -384,7 +384,7 @@ mod test { #[test] fn montgomery_mul_max() { - let res = Scalar64::montgomery_mul(&X, &X); + let res = Scalar52::montgomery_mul(&X, &X); for i in 0..5 { assert!(res[i] == XX_MONT[i]); } @@ -400,7 +400,7 @@ mod test { #[test] fn mul() { - let res = Scalar64::mul(&X, &Y); + let res = Scalar52::mul(&X, &Y); for i in 0..5 { assert!(res[i] == XY[i]); } @@ -408,7 +408,7 @@ mod test { #[test] fn montgomery_mul() { - let res = Scalar64::montgomery_mul(&X, &Y); + let res = Scalar52::montgomery_mul(&X, &Y); for i in 0..5 { assert!(res[i] == XY_MONT[i]); } @@ -416,8 +416,8 @@ mod test { #[test] fn add() { - let res = Scalar64::add(&A, &B); - let zero = Scalar64::zero(); + let res = Scalar52::add(&A, &B); + let zero = Scalar52::zero(); for i in 0..5 { assert!(res[i] == zero[i]); } @@ -425,7 +425,7 @@ mod test { #[test] fn sub() { - let res = Scalar64::sub(&A, &B); + let res = Scalar52::sub(&A, &B); for i in 0..5 { assert!(res[i] == AB[i]); } @@ -434,7 +434,7 @@ mod test { #[test] fn from_bytes_wide() { let bignum = [255u8; 64]; // 2^512 - 1 - let reduced = Scalar64::from_bytes_wide(&bignum); + let reduced = Scalar52::from_bytes_wide(&bignum); println!("{:?}", reduced); for i in 0..5 { assert!(reduced[i] == C[i]); diff --git a/src/backend/u32/constants.rs b/src/backend/u32/constants.rs deleted file mode 100644 index e0e0525..0000000 --- a/src/backend/u32/constants.rs +++ /dev/null @@ -1,149 +0,0 @@ -// -*- mode: rust; -*- -// -// This file is part of curve25519-dalek. -// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence -// See LICENSE for licensing information. -// -// Authors: -// - Isis Agora Lovecruft -// - Henry de Valence - -//! This module contains various constants (such as curve parameters -//! and useful field elements like `sqrt(-1)`), as well as -//! lookup tables of pre-computed points. - -use backend::u32::field::FieldElement32; -use backend::u32::scalar::Scalar32; -use edwards::EdwardsPoint; - -/// Edwards `d` value, equal to `-121665/121666 mod p`. -pub(crate) const EDWARDS_D: FieldElement32 = FieldElement32([ - 56195235, 13857412, 51736253, 6949390, 114729, - 24766616, 60832955, 30306712, 48412415, 21499315, -]); - -/// Edwards `2*d` value, equal to `2*(-121665/121666) mod p`. -pub(crate) const EDWARDS_D2: FieldElement32 = FieldElement32([ - 45281625, 27714825, 36363642, 13898781, 229458, - 15978800, 54557047, 27058993, 29715967, 9444199, -]); - -/// `= sqrt(a*d - 1)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. -pub(crate) const SQRT_AD_MINUS_ONE: FieldElement32 = FieldElement32([ - 24849947, 33400850, 43495378, 6347714, 46036536, - 32887293, 41837720, 18186727, 66238516, 14525638, -]); - -/// `= 1/sqrt(a-d)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. -pub(crate) const INVSQRT_A_MINUS_D: FieldElement32 = FieldElement32([ - 6111466, 4156064, 39310137, 12243467, 41204824, - 120896, 20826367, 26493656, 6093567, 31568420, -]); - -/// Precomputed value of one of the square roots of -1 (mod p) -pub(crate) const SQRT_M1: FieldElement32 = FieldElement32([ - 34513072, 25610706, 9377949, 3500415, 12389472, - 33281959, 41962654, 31548777, 326685, 11406482, -]); - -/// `APLUS2_OVER_FOUR` is (A+2)/4. (This is used internally within the Montgomery ladder.) -pub(crate) const APLUS2_OVER_FOUR: FieldElement32 = FieldElement32([ - 121666, 0, 0, 0, 0, 0, 0, 0, 0, 0 -]); - -/// `L` is the order of base point, i.e. 2^252 + -/// 27742317777372353535851937790883648493 -pub(crate) const L: Scalar32 = Scalar32([ 0x1cf5d3ed, 0x009318d2, 0x1de73596, 0x1df3bd45, - 0x0000014d, 0x00000000, 0x00000000, 0x00000000, - 0x00100000 ]); - -/// `L` * `LFACTOR` = -1 (mod 2^29) -pub(crate) const LFACTOR: u32 = 0x12547e1b; - -/// `R` = R % L where R = 2^261 -pub(crate) const R: Scalar32 = Scalar32([ 0x114df9ed, 0x1a617303, 0x0f7c098c, 0x16793167, - 0x1ffd656e, 0x1fffffff, 0x1fffffff, 0x1fffffff, - 0x000fffff ]); - -/// `RR` = (R^2) % L where R = 2^261 -pub(crate) const RR: Scalar32 = Scalar32([ 0x0b5f9d12, 0x1e141b17, 0x158d7f3d, 0x143f3757, - 0x1972d781, 0x042feb7c, 0x1ceec73d, 0x1e184d1e, - 0x0005046d ]); - -/// The Ed25519 basepoint, as an `EdwardsPoint`. -/// -/// This is called `_POINT` to distinguish it from -/// `ED25519_BASEPOINT_TABLE`, which should be used for scalar -/// multiplication (it's much faster). -pub const ED25519_BASEPOINT_POINT: EdwardsPoint = EdwardsPoint{ - X: FieldElement32([52811034, 25909283, 16144682, 17082669, 27570973, 30858332, 40966398, 8378388, 20764389, 8758491]), - Y: FieldElement32([40265304, 26843545, 13421772, 20132659, 26843545, 6710886, 53687091, 13421772, 40265318, 26843545]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([28827043, 27438313, 39759291, 244362, 8635006, 11264893, 19351346, 13413597, 16611511, 27139452]), -}; - -/// The 8-torsion subgroup \\(\mathcal E [8]\\). -/// -/// In the case of Curve25519, it is cyclic; the \\(i\\)-th element of -/// the array is \\([i]P\\), where \\(P\\) is a point of order \\(8\\) -/// generating \\(\mathcal E[8]\\). -/// -/// Thus \\(\mathcal E[8]\\) is the points indexed by `0,2,4,6`, and -/// \\(\mathcal E[2]\\) is the points indexed by `0,4`. -/// The Ed25519 basepoint has y = 4/5. This is called `_POINT` to -/// distinguish it from `_TABLE`, which should be used for scalar -/// multiplication (it's much faster). -pub const EIGHT_TORSION: [EdwardsPoint; 8] = EIGHT_TORSION_INNER_DOC_HIDDEN; - -/// Inner item used to hide limb constants from cargo doc output. -#[doc(hidden)] -pub const EIGHT_TORSION_INNER_DOC_HIDDEN: [EdwardsPoint; 8] = [ - EdwardsPoint{ - X: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Y: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) - }, - EdwardsPoint{ - X: FieldElement32([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]), - Y: FieldElement32([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251]) - }, - EdwardsPoint{ - X: FieldElement32([32595773, 7943725, 57730914, 30054016, 54719391, 272472, 25146209, 2005654, 66782178, 22147949]), - Y: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) - }, - EdwardsPoint{ - X: FieldElement32([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]), - Y: FieldElement32([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180]) - }, - EdwardsPoint{ - X: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Y: FieldElement32([67108844, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) - }, - EdwardsPoint{ - X: FieldElement32([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]), - Y: FieldElement32([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251]) - }, - EdwardsPoint{ - X: FieldElement32([34513072, 25610706, 9377949, 3500415, 12389472, 33281959, 41962654, 31548777, 326685, 11406482]), - Y: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) - }, - EdwardsPoint{ - X: FieldElement32([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]), - Y: FieldElement32([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180]) - }, -]; diff --git a/src/backend/avx2/constants.rs b/src/backend/vector/avx2/constants.rs similarity index 96% rename from src/backend/avx2/constants.rs rename to src/backend/vector/avx2/constants.rs index 76a644a..614286c 100644 --- a/src/backend/avx2/constants.rs +++ b/src/backend/vector/avx2/constants.rs @@ -12,12 +12,12 @@ use packed_simd::u32x8; -use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; -use backend::avx2::field::FieldElement32x4; -use scalar_mul::window::NafLookupTable8; +use backend::vector::avx2::edwards::{CachedPoint, ExtendedPoint}; +use backend::vector::avx2::field::FieldElement2625x4; +use window::NafLookupTable8; /// The identity element as an `ExtendedPoint`. -pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldElement32x4([ +pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldElement2625x4([ u32x8::new(0, 1, 0, 0, 1, 0, 0, 0), u32x8::splat(0), u32x8::splat(0), @@ -26,7 +26,7 @@ pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldEle ])); /// The identity element as a `CachedPoint`. -pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement32x4([ +pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement2625x4([ u32x8::new(121647, 121666, 0, 0, 243332, 67108845, 0, 33554431), u32x8::new(67108864, 0, 33554431, 0, 0, 67108863, 0, 33554431), u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), @@ -35,7 +35,7 @@ pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement32 ])); /// The low limbs of (2p, 2p, 2p, 2p), so that -/// ```no_run +/// ```ascii,no_run /// (2p, 2p, 2p, 2p) = [P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI] /// ``` pub(crate) static P_TIMES_2_LO: u32x8 = u32x8::new( @@ -50,7 +50,7 @@ pub(crate) static P_TIMES_2_LO: u32x8 = u32x8::new( ); /// The high limbs of (2p, 2p, 2p, 2p), so that -/// ```no_run +/// ```ascii,no_run /// (2p, 2p, 2p, 2p) = [P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI] /// ``` pub(crate) static P_TIMES_2_HI: u32x8 = u32x8::new( @@ -65,7 +65,7 @@ pub(crate) static P_TIMES_2_HI: u32x8 = u32x8::new( ); /// The low limbs of (16p, 16p, 16p, 16p), so that -/// ```no_run +/// ```ascii,no_run /// (16p, 16p, 16p, 16p) = [P_TIMES_16_LO, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI] /// ``` pub(crate) static P_TIMES_16_LO: u32x8 = u32x8::new( @@ -80,7 +80,7 @@ pub(crate) static P_TIMES_16_LO: u32x8 = u32x8::new( ); /// The high limbs of (16p, 16p, 16p, 16p), so that -/// ```no_run +/// ```ascii,no_run /// (16p, 16p, 16p, 16p) = [P_TIMES_16_LO, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI] /// ``` pub(crate) static P_TIMES_16_HI: u32x8 = u32x8::new( @@ -96,7 +96,7 @@ pub(crate) static P_TIMES_16_HI: u32x8 = u32x8::new( /// Odd multiples of the Ed25519 basepoint: pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = NafLookupTable8([ - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 3571425, 10045002, @@ -148,7 +148,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4846528, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 47099681, 31447946, @@ -200,7 +200,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 31366585, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 18147205, 62587998, @@ -252,7 +252,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 31948344, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 10625852, 15193821, @@ -304,7 +304,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 11531760, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 22176662, 3984313, @@ -356,7 +356,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 9686767, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 21157200, 39156966, @@ -408,7 +408,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 17395963, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 63153652, 32195955, @@ -460,7 +460,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10289708, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 1401265, 58846825, @@ -512,7 +512,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 11976112, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 39849808, 44781685, @@ -564,7 +564,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 27313245, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 58237774, 15917425, @@ -616,7 +616,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 22487864, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 12671134, 56419053, @@ -668,7 +668,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 2606889, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 55082775, 45300503, @@ -720,7 +720,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 5956424, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 8211442, 8014184, @@ -772,7 +772,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 1824195, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 59402443, 17056879, @@ -824,7 +824,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 11687259, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 12521903, 26014045, @@ -876,7 +876,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 32973409, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 30654745, 51286025, @@ -928,7 +928,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 32228854, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 49518649, 59119280, @@ -980,7 +980,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 27003505, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 28286608, 10767548, @@ -1032,7 +1032,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 18292949, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 13869851, 31448904, @@ -1084,7 +1084,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 17277037, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 43287109, 27900723, @@ -1136,7 +1136,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 30748046, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 16441817, 36111849, @@ -1188,7 +1188,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10930179, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 28319289, 40734650, @@ -1240,7 +1240,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10938429, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 8373273, 49064494, @@ -1292,7 +1292,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 15812027, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 30741269, 38648744, @@ -1344,7 +1344,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10839820, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 18672548, 57660959, @@ -1396,7 +1396,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 30432268, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 12179834, 41005450, @@ -1448,7 +1448,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 24862543, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 59523541, 62195428, @@ -1500,7 +1500,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 27449522, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 19770733, 26478685, @@ -1552,7 +1552,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 1776722, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 36719806, 20827965, @@ -1604,7 +1604,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 29334408, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 42307254, 57217102, @@ -1656,7 +1656,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 11292096, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 7071115, 46444090, @@ -1708,7 +1708,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 26285185, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 963440, 63742255, @@ -1760,7 +1760,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4928058, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 63655588, 17883670, @@ -1812,7 +1812,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 32462955, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 39158670, 15322548, @@ -1864,7 +1864,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 20307815, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 2863373, 40876242, @@ -1916,7 +1916,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 21388876, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 59276548, 49972346, @@ -1968,7 +1968,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 27257051, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 52951491, 66542164, @@ -2020,7 +2020,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 26001714, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 66783087, 5234346, @@ -2072,7 +2072,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10065424, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 42822326, 57678669, @@ -2124,7 +2124,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4170709, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 26535281, 60238317, @@ -2176,7 +2176,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 33286062, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 54863941, 67016431, @@ -2228,7 +2228,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 347423, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 41391822, 34336880, @@ -2280,7 +2280,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4942942, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 31956192, 59570132, @@ -2332,7 +2332,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 32932252, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 29885826, 51028067, @@ -2384,7 +2384,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 9101885, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 43331297, 18431341, @@ -2436,7 +2436,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 201203, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 20548943, 14334571, @@ -2488,7 +2488,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 25177079, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 41186817, 46681702, @@ -2540,7 +2540,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 7976478, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 46509314, 55327128, @@ -2592,7 +2592,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 22687008, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 15091184, 32550863, @@ -2644,7 +2644,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 542137, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 62776018, 32835413, @@ -2696,7 +2696,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 21024049, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 60835961, 48209103, @@ -2748,7 +2748,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 20924342, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 18135013, 20358426, @@ -2800,7 +2800,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 14572399, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 10785787, 46564798, @@ -2852,7 +2852,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 24110612, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 39907267, 45940262, @@ -2904,7 +2904,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 29853825, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 66303987, 36060363, @@ -2956,7 +2956,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 23261841, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 51218008, 5070126, @@ -3008,7 +3008,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 18326047, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 64176557, 3912400, @@ -3060,7 +3060,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 12655016, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 38372660, 57101970, @@ -3112,7 +3112,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 13421882, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 22917795, 22088359, @@ -3164,7 +3164,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 13175986, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 16185025, 61537525, @@ -3216,7 +3216,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 21409233, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 59411973, 57437124, @@ -3268,7 +3268,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4211851, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 6757410, 65455566, @@ -3320,7 +3320,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10524446, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 1497507, 33054449, @@ -3372,7 +3372,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 29687002, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 35889734, 23047226, diff --git a/src/backend/avx2/edwards.rs b/src/backend/vector/avx2/edwards.rs similarity index 94% rename from src/backend/avx2/edwards.rs rename to src/backend/vector/avx2/edwards.rs index 18ecfde..675797d 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/vector/avx2/edwards.rs @@ -41,12 +41,12 @@ use subtle::Choice; use subtle::ConditionallySelectable; use edwards; -use scalar_mul::window::{LookupTable, NafLookupTable5, NafLookupTable8}; +use window::{LookupTable, NafLookupTable5, NafLookupTable8}; use traits::Identity; -use backend::avx2::field::{FieldElement32x4, Lanes, Shuffle}; -use backend::avx2::constants; +use super::constants; +use super::field::{FieldElement2625x4, Lanes, Shuffle}; /// A point on Curve25519, using parallel Edwards formulas for curve /// operations. @@ -56,11 +56,11 @@ use backend::avx2::constants; /// The coefficients of an `ExtendedPoint` are bounded with /// \\( b < 0.007 \\). #[derive(Copy, Clone, Debug)] -pub struct ExtendedPoint(pub(super) FieldElement32x4); +pub struct ExtendedPoint(pub(super) FieldElement2625x4); impl From for ExtendedPoint { fn from(P: edwards::EdwardsPoint) -> ExtendedPoint { - ExtendedPoint(FieldElement32x4::new(&P.X, &P.Y, &P.Z, &P.T)) + ExtendedPoint(FieldElement2625x4::new(&P.X, &P.Y, &P.Z, &P.T)) } } @@ -78,7 +78,7 @@ impl From for edwards::EdwardsPoint { impl ConditionallySelectable for ExtendedPoint { fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self { - ExtendedPoint(FieldElement32x4::conditional_select(&a.0, &b.0, choice)) + ExtendedPoint(FieldElement2625x4::conditional_select(&a.0, &b.0, choice)) } fn conditional_assign(&mut self, other: &Self, choice: Choice) { @@ -133,7 +133,7 @@ impl ExtendedPoint { // ======================= // S5 S6 S8 S9 - let zero = FieldElement32x4::zero(); + let zero = FieldElement2625x4::zero(); let S_1 = tmp1.shuffle(Shuffle::AAAA); let S_2 = tmp1.shuffle(Shuffle::BBBB); @@ -181,7 +181,7 @@ impl ExtendedPoint { /// As long as the `CachedPoint` is not repeatedly negated, its /// coefficients will be bounded with \\( b < 1.0 \\). #[derive(Copy, Clone, Debug)] -pub struct CachedPoint(pub(super) FieldElement32x4); +pub struct CachedPoint(pub(super) FieldElement2625x4); impl From for CachedPoint { fn from(P: ExtendedPoint) -> CachedPoint { @@ -190,7 +190,7 @@ impl From for CachedPoint { x = x.blend(x.diff_sum(), Lanes::AB); // x = (X1 - Y1, X2 + Y2, Z2, T2) = (S2 S3 Z2 T2) - x = x * (121666, 121666, 2*121666, 2*121665); + x = x * (121666, 121666, 2 * 121666, 2 * 121665); // x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2) x = x.blend(-x, Lanes::D); @@ -215,7 +215,7 @@ impl Identity for CachedPoint { impl ConditionallySelectable for CachedPoint { fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self { - CachedPoint(FieldElement32x4::conditional_select(&a.0, &b.0, choice)) + CachedPoint(FieldElement2625x4::conditional_select(&a.0, &b.0, choice)) } fn conditional_assign(&mut self, other: &Self, choice: Choice) { @@ -247,7 +247,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { // coefficients grow by one bit. So on input, `self` is // bounded with `b < 0.007` and `other` is bounded with // `b < 1.0`. - + let mut tmp = self.0; tmp = tmp.blend(tmp.diff_sum(), Lanes::AB); @@ -329,7 +329,7 @@ mod test { use super::*; fn serial_add(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) -> edwards::EdwardsPoint { - use backend::u64::field::FieldElement64; + use backend::serial::u64::field::FieldElement51; let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); let (X2, Y2, Z2, T2) = (Q.X, Q.Y, Q.Z, Q.T); @@ -360,10 +360,10 @@ mod test { print_var!(S7); println!(""); - let S8 = &S4 * &FieldElement64([ 121666,0,0,0,0]); // R5 - let S9 = &S5 * &FieldElement64([ 121666,0,0,0,0]); // R6 - let S10 = &S6 * &FieldElement64([2*121666,0,0,0,0]); // R8 - let S11 = &S7 * &(-&FieldElement64([2*121665,0,0,0,0])); // R7 + let S8 = &S4 * &FieldElement51([ 121666,0,0,0,0]); // R5 + let S9 = &S5 * &FieldElement51([ 121666,0,0,0,0]); // R6 + let S10 = &S6 * &FieldElement51([2*121666,0,0,0,0]); // R8 + let S11 = &S7 * &(-&FieldElement51([2*121665,0,0,0,0])); // R7 print_var!(S8); print_var!(S9); print_var!(S10); diff --git a/src/backend/avx2/field.rs b/src/backend/vector/avx2/field.rs similarity index 87% rename from src/backend/avx2/field.rs rename to src/backend/vector/avx2/field.rs index 27ba558..e7ee916 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/vector/avx2/field.rs @@ -11,14 +11,14 @@ //! An implementation of 4-way vectorized 32bit field arithmetic using //! AVX2. //! -//! The `FieldElement32x4` struct provides a vector of four field +//! The `FieldElement2625x4` struct provides a vector of four field //! elements, implemented using AVX2 operations. Its API is designed //! to abstract away the platform-dependent details, so that point //! arithmetic can be implemented only in terms of a vector of field //! elements. //! //! At this level, the API is optimized for speed and not safety. The -//! `FieldElement32x4` does not always perform reductions. The pre- +//! `FieldElement2625x4` does not always perform reductions. The pre- //! and post-conditions on the bounds of the coefficients are //! documented for each method, but it is the caller's responsibility //! to ensure that there are no overflows. @@ -42,15 +42,15 @@ const D_LANES64: u8 = 0b11_00_00_00; use core::ops::{Add, Mul, Neg}; use packed_simd::{i32x8, u32x8, u64x4, IntoBits}; -use backend::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO}; -use backend::u64::field::FieldElement64; +use backend::vector::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO}; +use backend::serial::u64::field::FieldElement51; /// Unpack 32-bit lanes into 64-bit lanes: -/// ``` +/// ```ascii,no_run /// (a0, b0, a1, b1, c0, d0, c1, d1) /// ``` /// into -/// ``` +/// ```ascii,no_run /// (a0, 0, b0, 0, c0, 0, d0, 0) /// (a1, 0, b1, 0, c1, 0, d1, 0) /// ``` @@ -69,12 +69,12 @@ fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { } /// Repack 64-bit lanes into 32-bit lanes: -/// ``` +/// ```ascii,no_run /// (a0, 0, b0, 0, c0, 0, d0, 0) /// (a1, 0, b1, 0, c1, 0, d1, 0) /// ``` /// into -/// ``` +/// ```ascii,no_run /// (a0, b0, a1, b1, c0, d0, c1, d1) /// ``` #[inline(always)] @@ -97,11 +97,11 @@ fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { } /// The `Lanes` enum represents a subset of the lanes `A,B,C,D` of a -/// `FieldElement32x4`. +/// `FieldElement2625x4`. /// /// It's used to specify blend operations without /// having to know details about the data layout of the -/// `FieldElement32x4`. +/// `FieldElement2625x4`. #[derive(Copy, Clone, Debug)] pub enum Lanes { C, @@ -114,7 +114,7 @@ pub enum Lanes { ABCD, } -/// The `Shuffle` enum represents a shuffle of a `FieldElement32x4`. +/// The `Shuffle` enum represents a shuffle of a `FieldElement2625x4`. /// /// The enum variants are named by what they do to a vector \\( /// (A,B,C,D) \\); for instance, `Shuffle::BADC` turns \\( (A, B, C, @@ -135,26 +135,26 @@ pub enum Shuffle { /// A vector of four field elements. /// -/// Each operation on a `FieldElement32x4` has documented effects on +/// Each operation on a `FieldElement2625x4` has documented effects on /// the bounds of the coefficients. This API is designed for speed /// and not safety; it is the caller's responsibility to ensure that /// the post-conditions of one operation are compatible with the /// pre-conditions of the next. #[derive(Clone, Copy, Debug)] -pub struct FieldElement32x4(pub(crate) [u32x8; 5]); +pub struct FieldElement2625x4(pub(crate) [u32x8; 5]); use subtle::Choice; use subtle::ConditionallySelectable; -impl ConditionallySelectable for FieldElement32x4 { +impl ConditionallySelectable for FieldElement2625x4 { fn conditional_select( - a: &FieldElement32x4, - b: &FieldElement32x4, + a: &FieldElement2625x4, + b: &FieldElement2625x4, choice: Choice, - ) -> FieldElement32x4 { + ) -> FieldElement2625x4 { let mask = (-(choice.unwrap_u8() as i32)) as u32; let mask_vec = u32x8::splat(mask); - FieldElement32x4([ + FieldElement2625x4([ a.0[0] ^ (mask_vec & (a.0[0] ^ b.0[0])), a.0[1] ^ (mask_vec & (a.0[1] ^ b.0[1])), a.0[2] ^ (mask_vec & (a.0[2] ^ b.0[2])), @@ -165,7 +165,7 @@ impl ConditionallySelectable for FieldElement32x4 { fn conditional_assign( &mut self, - other: &FieldElement32x4, + other: &FieldElement2625x4, choice: Choice, ) { let mask = (-(choice.unwrap_u8() as i32)) as u32; @@ -178,11 +178,11 @@ impl ConditionallySelectable for FieldElement32x4 { } } -impl FieldElement32x4 { +impl FieldElement2625x4 { /// Split this vector into an array of four (serial) field /// elements. - pub fn split(&self) -> [FieldElement64; 4] { - let mut out = [FieldElement64::zero(); 4]; + pub fn split(&self) -> [FieldElement51; 4] { + let mut out = [FieldElement51::zero(); 4]; for i in 0..5 { let a_2i = self.0[i].extract(0) as u64; // let b_2i = self.0[i].extract(1) as u64; // @@ -208,7 +208,7 @@ impl FieldElement32x4 { /// that when this function is inlined, LLVM is able to lower the /// shuffle using an immediate. #[inline] - pub fn shuffle(&self, control: Shuffle) -> FieldElement32x4 { + pub fn shuffle(&self, control: Shuffle) -> FieldElement2625x4 { #[inline(always)] fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 { unsafe { @@ -233,7 +233,7 @@ impl FieldElement32x4 { } } - FieldElement32x4([ + FieldElement2625x4([ shuffle_lanes(self.0[0], control), shuffle_lanes(self.0[1], control), shuffle_lanes(self.0[2], control), @@ -248,7 +248,7 @@ impl FieldElement32x4 { /// that this function can be inlined and LLVM can lower it to a /// blend instruction using an immediate. #[inline] - pub fn blend(&self, other: FieldElement32x4, control: Lanes) -> FieldElement32x4 { + pub fn blend(&self, other: FieldElement2625x4, control: Lanes) -> FieldElement2625x4 { #[inline(always)] fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 { unsafe { @@ -310,7 +310,7 @@ impl FieldElement32x4 { } } - FieldElement32x4([ + FieldElement2625x4([ blend_lanes(self.0[0], other.0[0], control), blend_lanes(self.0[1], other.0[1], control), blend_lanes(self.0[2], other.0[2], control), @@ -320,26 +320,26 @@ impl FieldElement32x4 { } /// Construct a vector of zeros. - pub fn zero() -> FieldElement32x4 { - FieldElement32x4([u32x8::splat(0); 5]) + pub fn zero() -> FieldElement2625x4 { + FieldElement2625x4([u32x8::splat(0); 5]) } /// Convenience wrapper around `new(x,x,x,x)`. - pub fn splat(x: &FieldElement64) -> FieldElement32x4 { - FieldElement32x4::new(x, x, x, x) + pub fn splat(x: &FieldElement51) -> FieldElement2625x4 { + FieldElement2625x4::new(x, x, x, x) } - /// Create a `FieldElement32x4` from four `FieldElement64`s. + /// Create a `FieldElement2625x4` from four `FieldElement51`s. /// /// # Postconditions /// - /// The resulting `FieldElement32x4` is bounded with \\( b < 0.0002 \\). + /// The resulting `FieldElement2625x4` is bounded with \\( b < 0.0002 \\). pub fn new( - x0: &FieldElement64, - x1: &FieldElement64, - x2: &FieldElement64, - x3: &FieldElement64, - ) -> FieldElement32x4 { + x0: &FieldElement51, + x1: &FieldElement51, + x2: &FieldElement51, + x3: &FieldElement51, + ) -> FieldElement2625x4 { let mut buf = [u32x8::splat(0); 5]; let low_26_bits = (1 << 26) - 1; for i in 0..5 { @@ -355,10 +355,10 @@ impl FieldElement32x4 { buf[i] = u32x8::new(a_2i, b_2i, a_2i_1, b_2i_1, c_2i, d_2i, c_2i_1, d_2i_1); } - // We don't know that the original `FieldElement64`s were + // We don't know that the original `FieldElement51`s were // fully reduced, so the odd limbs may exceed 2^25. // Reduce them to be sure. - FieldElement32x4(buf).reduce() + FieldElement2625x4(buf).reduce() } /// Given \\((A,B,C,D)\\), compute \\((-A,-B,-C,-D)\\), without @@ -372,11 +372,11 @@ impl FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 1 \\). #[inline] - pub fn negate_lazy(&self) -> FieldElement32x4 { + pub fn negate_lazy(&self) -> FieldElement2625x4 { // The limbs of self are bounded with b < 0.999, while the // smallest limb of 2*p is 67108845 > 2^{26+0.9999}, so // underflows are not possible. - FieldElement32x4([ + FieldElement2625x4([ P_TIMES_2_LO - self.0[0], P_TIMES_2_HI - self.0[1], P_TIMES_2_HI - self.0[2], @@ -395,7 +395,7 @@ impl FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 1.6 \\). #[inline] - pub fn diff_sum(&self) -> FieldElement32x4 { + pub fn diff_sum(&self) -> FieldElement2625x4 { // tmp1 = (B, A, D, C) let tmp1 = self.shuffle(Shuffle::BADC); // tmp2 = (-A, B, -C, D) @@ -410,7 +410,7 @@ impl FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 0.0002 \\). #[inline] - pub fn reduce(&self) -> FieldElement32x4 { + pub fn reduce(&self) -> FieldElement2625x4 { let shifts = i32x8::new(26, 26, 25, 25, 26, 26, 25, 25); let masks = u32x8::new( (1 << 26) - 1, @@ -509,16 +509,16 @@ impl FieldElement32x4 { // c_odd < 2^25 + 2^11.25 < 25.0001 < 2^{25+b} // // where b = 0.0002. - FieldElement32x4(v) + FieldElement2625x4(v) } - /// Given an array of wide coefficients, reduce them to a `FieldElement32x4`. + /// Given an array of wide coefficients, reduce them to a `FieldElement2625x4`. /// /// # Postconditions /// /// The coefficients of the result are bounded with \\( b < 0.007 \\). #[inline] - fn reduce64(mut z: [u64x4; 10]) -> FieldElement32x4 { + fn reduce64(mut z: [u64x4; 10]) -> FieldElement2625x4 { // These aren't const because splat isn't a const fn let LOW_25_BITS: u64x4 = u64x4::splat((1 << 25) - 1); let LOW_26_BITS: u64x4 = u64x4::splat((1 << 26) - 1); @@ -578,7 +578,7 @@ impl FieldElement32x4 { // b = 0 for other z[i]. // // So the packed result is bounded with b = 0.007. - FieldElement32x4([ + FieldElement2625x4([ repack_pair(z[0].into_bits(), z[1].into_bits()), repack_pair(z[2].into_bits(), z[3].into_bits()), repack_pair(z[4].into_bits(), z[5].into_bits()), @@ -596,7 +596,7 @@ impl FieldElement32x4 { /// # Postconditions /// /// The coefficients of the result are bounded with \\( b < 0.007 \\). - pub fn square_and_negate_D(&self) -> FieldElement32x4 { + pub fn square_and_negate_D(&self) -> FieldElement2625x4 { #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { use core::arch::x86_64::_mm256_mul_epu32; @@ -681,12 +681,12 @@ impl FieldElement32x4 { z8 = negate_D(z8, even_p37); z9 = negate_D(z9, odd__p37); - FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) + FieldElement2625x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } } -impl Neg for FieldElement32x4 { - type Output = FieldElement32x4; +impl Neg for FieldElement2625x4 { + type Output = FieldElement2625x4; /// Negate this field element, performing a reduction. /// @@ -701,8 +701,8 @@ impl Neg for FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 0.0002 \\). #[inline] - fn neg(self) -> FieldElement32x4 { - FieldElement32x4([ + fn neg(self) -> FieldElement2625x4 { + FieldElement2625x4([ P_TIMES_16_LO - self.0[0], P_TIMES_16_HI - self.0[1], P_TIMES_16_HI - self.0[2], @@ -712,12 +712,12 @@ impl Neg for FieldElement32x4 { } } -impl Add for FieldElement32x4 { - type Output = FieldElement32x4; - /// Add two `FieldElement32x4`s, without performing a reduction. +impl Add for FieldElement2625x4 { + type Output = FieldElement2625x4; + /// Add two `FieldElement2625x4`s, without performing a reduction. #[inline] - fn add(self, rhs: FieldElement32x4) -> FieldElement32x4 { - FieldElement32x4([ + fn add(self, rhs: FieldElement2625x4) -> FieldElement2625x4 { + FieldElement2625x4([ self.0[0] + rhs.0[0], self.0[1] + rhs.0[1], self.0[2] + rhs.0[2], @@ -727,15 +727,15 @@ impl Add for FieldElement32x4 { } } -impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 { - type Output = FieldElement32x4; +impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 { + type Output = FieldElement2625x4; /// Perform a multiplication by a vector of small constants. /// /// # Postconditions /// /// The coefficients of the result are bounded with \\( b < 0.007 \\). #[inline] - fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement32x4 { + fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement2625x4 { unsafe { use core::arch::x86_64::_mm256_mul_epu32; @@ -747,7 +747,7 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 { let (b6, b7) = unpack_pair(self.0[3]); let (b8, b9) = unpack_pair(self.0[4]); - FieldElement32x4::reduce64([ + FieldElement2625x4::reduce64([ _mm256_mul_epu32(b0.into_bits(), consts.into_bits()).into_bits(), _mm256_mul_epu32(b1.into_bits(), consts.into_bits()).into_bits(), _mm256_mul_epu32(b2.into_bits(), consts.into_bits()).into_bits(), @@ -763,8 +763,8 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 { } } -impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { - type Output = FieldElement32x4; +impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 { + type Output = FieldElement2625x4; /// Multiply `self` by `rhs`. /// /// # Preconditions @@ -777,7 +777,7 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 0.007 \\). /// - fn mul(self, rhs: &'b FieldElement32x4) -> FieldElement32x4 { + fn mul(self, rhs: &'b FieldElement2625x4) -> FieldElement2625x4 { #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { use core::arch::x86_64::_mm256_mul_epu32; @@ -869,7 +869,7 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { // multiplications by 19 into a u32. The tighter bound on b_y // means we could get a tighter bound on the outputs, or a // looser bound on b_x. - FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) + FieldElement2625x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } } @@ -880,25 +880,25 @@ mod test { #[test] fn scale_by_curve_constants() { - let mut x = FieldElement32x4::splat(&FieldElement64::one()); + let mut x = FieldElement2625x4::splat(&FieldElement51::one()); x = x * (121666, 121666, 2*121666, 2*121665); let xs = x.split(); - assert_eq!(xs[0], FieldElement64([121666, 0, 0, 0, 0])); - assert_eq!(xs[1], FieldElement64([121666, 0, 0, 0, 0])); - assert_eq!(xs[2], FieldElement64([2 * 121666, 0, 0, 0, 0])); - assert_eq!(xs[3], FieldElement64([2 * 121665, 0, 0, 0, 0])); + assert_eq!(xs[0], FieldElement51([121666, 0, 0, 0, 0])); + assert_eq!(xs[1], FieldElement51([121666, 0, 0, 0, 0])); + assert_eq!(xs[2], FieldElement51([2 * 121666, 0, 0, 0, 0])); + assert_eq!(xs[3], FieldElement51([2 * 121665, 0, 0, 0, 0])); } #[test] fn diff_sum_vs_serial() { - let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); - let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); - let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); - let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); + let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3).diff_sum(); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3).diff_sum(); let result = vec.split(); @@ -910,12 +910,12 @@ mod test { #[test] fn square_vs_serial() { - let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); - let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); - let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); - let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); + let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3); let result = vec.square_and_negate_D().split(); @@ -927,12 +927,12 @@ mod test { #[test] fn multiply_vs_serial() { - let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); - let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); - let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); - let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); + let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3); let vecprime = vec.clone(); let result = (&vec * &vecprime).split(); @@ -945,12 +945,12 @@ mod test { #[test] fn test_unpack_repack_pair() { - let x0 = FieldElement64([10000 + (10001 << 26), 0, 0, 0, 0]); - let x1 = FieldElement64([10100 + (10101 << 26), 0, 0, 0, 0]); - let x2 = FieldElement64([10200 + (10201 << 26), 0, 0, 0, 0]); - let x3 = FieldElement64([10300 + (10301 << 26), 0, 0, 0, 0]); + let x0 = FieldElement51([10000 + (10001 << 26), 0, 0, 0, 0]); + let x1 = FieldElement51([10100 + (10101 << 26), 0, 0, 0, 0]); + let x2 = FieldElement51([10200 + (10201 << 26), 0, 0, 0, 0]); + let x3 = FieldElement51([10300 + (10301 << 26), 0, 0, 0, 0]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3); let src = vec.0[0]; @@ -969,12 +969,12 @@ mod test { #[test] fn new_split_roundtrips() { - let x0 = FieldElement64::from_bytes(&[0x10; 32]); - let x1 = FieldElement64::from_bytes(&[0x11; 32]); - let x2 = FieldElement64::from_bytes(&[0x12; 32]); - let x3 = FieldElement64::from_bytes(&[0x13; 32]); + let x0 = FieldElement51::from_bytes(&[0x10; 32]); + let x1 = FieldElement51::from_bytes(&[0x11; 32]); + let x2 = FieldElement51::from_bytes(&[0x12; 32]); + let x3 = FieldElement51::from_bytes(&[0x13; 32]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3); let splits = vec.split(); diff --git a/src/backend/avx2/mod.rs b/src/backend/vector/avx2/mod.rs similarity index 87% rename from src/backend/avx2/mod.rs rename to src/backend/vector/avx2/mod.rs index 10886af..0b30085 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/vector/avx2/mod.rs @@ -18,7 +18,8 @@ // location of build.rs, not lib.rs, so the markdown file appears // missing). #![cfg_attr( - all(feature = "nightly", feature = "stage2_build"), doc(include = "../docs/avx2-notes.md") + all(feature = "nightly", feature = "stage2_build"), + doc(include = "../docs/avx2-notes.md") )] pub(crate) mod field; @@ -26,5 +27,3 @@ pub(crate) mod field; pub(crate) mod edwards; pub(crate) mod constants; - -pub(crate) mod scalar_mul; diff --git a/src/backend/vector/ifma/constants.rs b/src/backend/vector/ifma/constants.rs new file mode 100644 index 0000000..257fb54 --- /dev/null +++ b/src/backend/vector/ifma/constants.rs @@ -0,0 +1,2062 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2018 Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +//! This module contains constants used by the IFMA backend. + +use packed_simd::u64x4; + +use window::NafLookupTable8; + +use super::edwards::{CachedPoint, ExtendedPoint}; +use super::field::{F51x4Reduced, F51x4Unreduced}; + +/// The identity element as an `ExtendedPoint`. +pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(F51x4Unreduced([ + u64x4::new(0, 1, 1, 0), + u64x4::new(0, 0, 0, 0), + u64x4::new(0, 0, 0, 0), + u64x4::new(0, 0, 0, 0), + u64x4::new(0, 0, 0, 0), +])); + +/// The identity element as a `CachedPoint`. +pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(F51x4Reduced([ + u64x4::new(121647, 121666, 243332, 2251799813685229), + u64x4::new(2251799813685248, 0, 0, 2251799813685247), + u64x4::new(2251799813685247, 0, 0, 2251799813685247), + u64x4::new(2251799813685247, 0, 0, 2251799813685247), + u64x4::new(2251799813685247, 0, 0, 2251799813685247), +])); + +/// Odd multiples of the Ed25519 basepoint: +pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = NafLookupTable8([ + CachedPoint(F51x4Reduced([ + u64x4::new(1277522120965857, 73557767439946, 243332, 1943719795065404), + u64x4::new(108375142003455, 341984820733594, 0, 2097709862669256), + u64x4::new(150073485536043, 750646439938056, 0, 581130035634455), + u64x4::new(2149983732744869, 1903255931888577, 0, 646644904824193), + u64x4::new(291045673509296, 1060034214701851, 0, 325245010451737), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1970681836121889, + 1660307753655178, + 1077207637163462, + 1436413309977108, + ), + u64x4::new( + 158785710838757, + 919645875412951, + 174577133496574, + 2213787394009350, + ), + u64x4::new( + 1017606396438281, + 1240932851489554, + 918203302506967, + 1239827708070863, + ), + u64x4::new( + 1748989883612327, + 1745367742532782, + 1168385548387, + 1211387683826673, + ), + u64x4::new( + 799349980018733, + 1471088235739693, + 1505351346057417, + 2104975925096407, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 171437462972293, + 36016853025886, + 1184164975342640, + 1633525003912147, + ), + u64x4::new( + 2113383632509037, + 1946216474924125, + 1884174984466256, + 1373317790955847, + ), + u64x4::new( + 791293623466401, + 1796466048084189, + 444977763198796, + 629823271230872, + ), + u64x4::new( + 1093217720067380, + 2157024270666135, + 238122980108466, + 806820763806847, + ), + u64x4::new( + 793658959468458, + 368578641413741, + 11592529764159, + 2144017075993471, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1538027396670268, + 1588896993892061, + 675619548648376, + 788373514423313, + ), + u64x4::new( + 1987517656073805, + 1940987929951188, + 666993851697339, + 2040540928108427, + ), + u64x4::new( + 375514548584082, + 1726008037083790, + 1070069155000872, + 570111103756303, + ), + u64x4::new( + 772223645372213, + 2123395244967674, + 868238486911408, + 1846639042240362, + ), + u64x4::new( + 872865734460736, + 32277956842850, + 1701451131455402, + 773883376061880, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1845177363882902, + 275858237213625, + 1052127336883600, + 171072805852218, + ), + u64x4::new( + 139016783952609, + 462699304987089, + 430046471494974, + 410922720999257, + ), + u64x4::new( + 846403935976337, + 243817706931454, + 971825428236901, + 571800039596794, + ), + u64x4::new( + 807642685434918, + 1933536976438782, + 812324278898440, + 688391556487313, + ), + u64x4::new( + 76239450396192, + 629532732688863, + 1833302026979779, + 650067934544499, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1373931604989264, + 331159264656614, + 364391529321767, + 874765630865409, + ), + u64x4::new( + 2109908262150241, + 473400816504190, + 91544045127333, + 976307977609515, + ), + u64x4::new( + 330175435673491, + 2126511895885904, + 1022944071588421, + 2158480209801463, + ), + u64x4::new( + 1305666795527971, + 162063591028664, + 2193154870675382, + 1789166662611800, + ), + u64x4::new( + 817858592500508, + 1672743239440202, + 859976879916778, + 1167423340862516, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 274334925170164, + 565841102587251, + 603083835949120, + 607539210240861, + ), + u64x4::new( + 196754662972649, + 1339063476699167, + 1406077076979491, + 896902435668469, + ), + u64x4::new( + 397962210956733, + 174839587476217, + 1381082665748936, + 175195877334136, + ), + u64x4::new( + 717429432748391, + 1635309821746318, + 363374010274647, + 882908746261699, + ), + u64x4::new( + 600946602802781, + 1946596133370711, + 1532135183320341, + 690530671668253, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2074443704000945, + 2163534804938345, + 425423840926528, + 1100826171404853, + ), + u64x4::new( + 111700142796101, + 1456893872751964, + 1186145518682968, + 2192182627706116, + ), + u64x4::new( + 1848722121856066, + 2123239575044749, + 1323870754599272, + 883211262889775, + ), + u64x4::new( + 938263017712916, + 689670293631396, + 183944529557576, + 501908638166580, + ), + u64x4::new( + 2170571907220631, + 36636756989655, + 1875035480138608, + 803703278398018, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1053429956874064, + 1636640618139765, + 1556890827801070, + 2142720579528828, + ), + u64x4::new( + 1814240918422814, + 692326274601777, + 1054896561802157, + 2025454041705534, + ), + u64x4::new( + 2109495823888757, + 1287497869997176, + 194170063200096, + 621116840113213, + ), + u64x4::new( + 2156505873679998, + 2197064359737385, + 1312887672223536, + 369862818895912, + ), + u64x4::new( + 977381163563657, + 1878897311974033, + 2144566861359744, + 1832960882773351, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1266492498289486, + 1301524759372145, + 324789537938521, + 442710471023019, + ), + u64x4::new( + 1232722320001345, + 1191193089162455, + 176474006074813, + 2158950213252857, + ), + u64x4::new( + 1901782191467749, + 494791441598902, + 1820415815322129, + 854954583485223, + ), + u64x4::new( + 1511383667649702, + 792536415032464, + 2027741263854728, + 1727944381044738, + ), + u64x4::new( + 606355788891204, + 1670687521471220, + 582824350365415, + 1509135066079912, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1079942762813598, + 2015830004785901, + 479916361323351, + 1907956590950158, + ), + u64x4::new( + 2053400302939156, + 1319799126867070, + 19493088767391, + 1908755581402373, + ), + u64x4::new( + 2235858054780980, + 885832711204321, + 810332865560178, + 103174191215441, + ), + u64x4::new( + 1843466881032833, + 355511728384038, + 693846715794114, + 186545012724117, + ), + u64x4::new( + 1661758432892509, + 1491022339899281, + 698941123765263, + 174945407208560, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1075933251927831, + 400263885306647, + 1308157532880528, + 347933379126665, + ), + u64x4::new( + 673811632329433, + 1584860147186478, + 271778891257244, + 498194055154207, + ), + u64x4::new( + 703783427747558, + 1051624728592032, + 1371463103351544, + 230351033002960, + ), + u64x4::new( + 860729466483372, + 421647596766583, + 1520613871336707, + 635298775280054, + ), + u64x4::new( + 1168352891728845, + 1691216293752089, + 1799491997061519, + 399728882318504, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 420156727446514, + 1483649215777128, + 165508610199900, + 1918121104840431, + ), + u64x4::new( + 2129902293682427, + 730952770435213, + 2184527544565390, + 1939880362232986, + ), + u64x4::new( + 1771978364905086, + 510975579746524, + 927564335219142, + 177574146260558, + ), + u64x4::new( + 2164104536437514, + 1532598873799015, + 406875369182421, + 1367005937406517, + ), + u64x4::new( + 35073200082587, + 1981124717036219, + 1854087014063833, + 122419694385217, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1963785875777739, + 411497142699119, + 1974557512687408, + 1268304422747183, + ), + u64x4::new( + 762752575978150, + 1443822019541748, + 1331556159904338, + 377726798263780, + ), + u64x4::new( + 825953972847841, + 353487068141356, + 1955697322427207, + 2048226560172078, + ), + u64x4::new( + 1482378558684434, + 657691905625918, + 923870001994493, + 1694132799397736, + ), + u64x4::new( + 1643904759603122, + 170495566698285, + 1218312703413378, + 784318735038131, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 939230507241903, + 2238763473105245, + 1827325199528162, + 1153939339775538, + ), + u64x4::new( + 38544505283339, + 258889431497015, + 351721979677947, + 1357907379592829, + ), + u64x4::new( + 1393974676373341, + 1131355528938676, + 473104915298872, + 978783482501776, + ), + u64x4::new( + 2131516168980501, + 2113911780991092, + 1477027502354261, + 542884524860340, + ), + u64x4::new( + 1029606261349423, + 64226378557628, + 1669131167474348, + 2212808057234874, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1423176501543193, + 163313632579593, + 2220495688893001, + 2220041045291870, + ), + u64x4::new( + 1111834224023697, + 1026815658023689, + 1404605100939775, + 1412149108248227, + ), + u64x4::new( + 1542537854906076, + 1270288391129127, + 991419278941933, + 1824939809581980, + ), + u64x4::new( + 1142003215657891, + 525980550896367, + 1508270666157963, + 917719462309053, + ), + u64x4::new( + 400851268057105, + 1620818232405188, + 1251478578139510, + 2162841805361886, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2125383272208441, + 1368790097335984, + 11813369275978, + 639513785921674, + ), + u64x4::new( + 2200806265616284, + 1041996387620216, + 1275149397833084, + 1723371028064068, + ), + u64x4::new( + 603720163891275, + 2135593511176153, + 2049641644431548, + 1198460677818310, + ), + u64x4::new( + 1862491879401621, + 2008116580769441, + 626566325260235, + 1058308304975798, + ), + u64x4::new( + 628557314314858, + 1075323332046522, + 1631772244117095, + 1812174547405683, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1222773123817104, + 363276129291452, + 796237592807883, + 1914425291893078, + ), + u64x4::new( + 1721259057429088, + 734941709009373, + 1553365830564638, + 1492120931079419, + ), + u64x4::new( + 1009354843273686, + 293884504384873, + 1050281954944357, + 134132942667344, + ), + u64x4::new( + 23119363298711, + 1694754778833445, + 1725925193393496, + 1738396998222001, + ), + u64x4::new( + 1753692057254667, + 118428526447110, + 840961387840295, + 1227619055408558, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1004186117579547, + 508771992330056, + 1426571663072421, + 2238524171903259, + ), + u64x4::new( + 744764613007812, + 398885442368825, + 2047459490294949, + 2141797621077959, + ), + u64x4::new( + 4556204156489, + 1708213022802363, + 1071381560923933, + 393474529142567, + ), + u64x4::new( + 350116198213005, + 945907227204695, + 168267474358731, + 1801504420122711, + ), + u64x4::new( + 728788674520360, + 1262722049156121, + 455259596607008, + 1159442365834489, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2226818917892677, + 185673745808179, + 2240952219732549, + 324137961621908, + ), + u64x4::new( + 1659527641857410, + 973964060249383, + 1349692151487730, + 1172743533370593, + ), + u64x4::new( + 310591478467746, + 2123977244137170, + 774562885265820, + 430035546191685, + ), + u64x4::new( + 2150863173197992, + 2101978317708856, + 193592648406011, + 1375328504508580, + ), + u64x4::new( + 1946235834250479, + 121741431658675, + 1004342690620100, + 2063466488599450, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 463079632200153, + 40415275714025, + 545935352782679, + 1458043501600908, + ), + u64x4::new( + 783771976559993, + 880839641726471, + 1782028201271831, + 41664413404590, + ), + u64x4::new( + 985129151724159, + 187728621410000, + 16620051933318, + 378011085567733, + ), + u64x4::new( + 1820372198168638, + 905710046480679, + 1912961774249737, + 1868135861067161, + ), + u64x4::new( + 474460473983187, + 1455684425673661, + 652771171116843, + 733511920760779, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1088886980746809, + 1660218575261626, + 527921875040240, + 915086639857889, + ), + u64x4::new( + 1814735788528175, + 1586698876186367, + 2040856637532862, + 405684812785624, + ), + u64x4::new( + 658578559700999, + 1751442070931114, + 1293623371490094, + 715026719042518, + ), + u64x4::new( + 382156225644820, + 897982285504960, + 577673183555858, + 1158728558309719, + ), + u64x4::new( + 1865791902475663, + 124491617513788, + 758484125168765, + 734065580770143, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 330985690350617, + 2214424721795630, + 973374650780848, + 1507267060932964, + ), + u64x4::new( + 1733823971011290, + 1730742552292995, + 669018866977489, + 604527664126146, + ), + u64x4::new( + 1082092498645474, + 1029182053935309, + 756799947765834, + 1764720030308351, + ), + u64x4::new( + 969912105693756, + 38116887248276, + 2148030115687613, + 995140534653865, + ), + u64x4::new( + 2154373397460354, + 298128883464656, + 479587543632539, + 1061127201140779, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 843064865526549, + 2019481782959016, + 1873125524281672, + 2013330239022371, + ), + u64x4::new( + 1192932403815186, + 1818108671859220, + 1247005102016258, + 1210577394628058, + ), + u64x4::new( + 132359273326717, + 795492788299178, + 1235924489372816, + 891705064411550, + ), + u64x4::new( + 1425833709104858, + 152114045731085, + 991347902581315, + 1387773338707683, + ), + u64x4::new( + 48024203807922, + 157005564892977, + 1474053161953744, + 727448023498345, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1076621484026788, + 1309917234320927, + 1786998180233659, + 1595497085944737, + ), + u64x4::new( + 1737334672694726, + 2038133716999447, + 1929061192400917, + 620544235219084, + ), + u64x4::new( + 1550527313469747, + 329096759623509, + 1585214659209474, + 693419841748324, + ), + u64x4::new( + 1450010875912315, + 2085047082180569, + 757421110771886, + 389367139787400, + ), + u64x4::new( + 781339490566117, + 132941783448971, + 258650459725225, + 2042274962585613, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 859638991542650, + 2249840007426442, + 1138753070862357, + 793751342318913, + ), + u64x4::new( + 2133476133447306, + 1027010646129239, + 436851910892865, + 866949948830344, + ), + u64x4::new( + 1936003572431223, + 531513680252193, + 1929877059408416, + 830585477662503, + ), + u64x4::new( + 1460760405777960, + 686673748420916, + 275475330051554, + 1581792376993692, + ), + u64x4::new( + 894482039456784, + 1801274480988632, + 16407898635278, + 1668497039215206, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 258585746227669, + 936490904651492, + 1826793887434108, + 1201219990633823, + ), + u64x4::new( + 979462791643635, + 461762372210187, + 218708929991480, + 1378150755760178, + ), + u64x4::new( + 642542170229970, + 787135445552820, + 371168855880557, + 182642566486693, + ), + u64x4::new( + 1152277399721904, + 1726910452705576, + 1452393215705343, + 2117799581546845, + ), + u64x4::new( + 1211265143925330, + 14373046151823, + 1745528818271507, + 1842106288572078, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 635154614562157, + 1956763034454109, + 509123035953043, + 445727657534780, + ), + u64x4::new( + 2072765509783252, + 1282639891593570, + 1075086397362049, + 722996110178195, + ), + u64x4::new( + 1385572918825603, + 1190035835509576, + 218317841176013, + 1047865370756924, + ), + u64x4::new( + 473991569426488, + 1910588123704592, + 1338270051770806, + 401676861680875, + ), + u64x4::new( + 992455353618436, + 126422733426929, + 1955248037756399, + 119233843022643, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1555272991526078, + 2214378187116349, + 366893798097444, + 1401502118355702, + ), + u64x4::new( + 1157229521930713, + 2144787187506262, + 1681597469697840, + 847499096518697, + ), + u64x4::new( + 1872802655800758, + 1027119609820793, + 1137278714788290, + 1664750301179485, + ), + u64x4::new( + 1091289858897030, + 910126419483563, + 1101920147235731, + 597083075893952, + ), + u64x4::new( + 1711011533670315, + 185206680336278, + 1620960612579784, + 1968598849170880, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 73077300235958, + 257216723095630, + 466947267713785, + 847105214181598, + ), + u64x4::new( + 1322905631406309, + 407458059314731, + 230045063190376, + 923800751267786, + ), + u64x4::new( + 1146027205000415, + 1541328763727623, + 768510249199119, + 1630223587589059, + ), + u64x4::new( + 1930368769879433, + 1376145403022159, + 1898149855343131, + 1709421930518180, + ), + u64x4::new( + 633944191571764, + 58314960742839, + 2050971151574988, + 757799756090059, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 361576929158539, + 1035682890165818, + 160945739362874, + 266975208626222, + ), + u64x4::new( + 1635371797076046, + 2106722851965197, + 451585919077206, + 6692426667180, + ), + u64x4::new( + 175820543533852, + 2057511393764025, + 1531846543720469, + 1648320903946519, + ), + u64x4::new( + 947461770620940, + 1107335044817620, + 1725565474111216, + 2182263619949220, + ), + u64x4::new( + 726444888601221, + 1379664085279206, + 1517215633290417, + 1763968936542507, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 686545355846512, + 1712283265573167, + 1743509592736302, + 1653906616429153, + ), + u64x4::new( + 985108805667149, + 2244347650874753, + 1304749057936860, + 321846134330589, + ), + u64x4::new( + 296321076156886, + 1717929256240029, + 450933772486425, + 2015536856431605, + ), + u64x4::new( + 1690393512821866, + 646913049470189, + 2198650647576397, + 1230646705710442, + ), + u64x4::new( + 601961913448442, + 878806578800541, + 620497587492381, + 330716414244629, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 631510982676132, + 1755753187697174, + 1596201246674299, + 2197888384902121, + ), + u64x4::new( + 626957678275745, + 1447583371478595, + 1375375216702128, + 1443613232818823, + ), + u64x4::new( + 1962997804660501, + 1051744123184519, + 1002558639300437, + 1237313314603385, + ), + u64x4::new( + 2118828335274995, + 226398203764759, + 889099617161107, + 1620967117678504, + ), + u64x4::new( + 227261019362935, + 2046897556746842, + 591524060355369, + 2178552047369691, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1375403119051662, + 222313965014452, + 539873444241395, + 213198095917915, + ), + u64x4::new( + 1436952871599114, + 1229749762725246, + 1174441562267670, + 265367077740349, + ), + u64x4::new( + 11107426165917, + 985954476039181, + 1147329112365579, + 1133931640328107, + ), + u64x4::new( + 585235055006843, + 699515259687482, + 299559608721134, + 2134819767146767, + ), + u64x4::new( + 1376401105588528, + 391412107507860, + 302743651807545, + 1362834426455518, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1802940904616205, + 1615132760193234, + 869321663313735, + 666494072545310, + ), + u64x4::new( + 1452849320020701, + 1472716813676364, + 472862999490802, + 359937983286145, + ), + u64x4::new( + 1221198323133843, + 491718521756528, + 1387135774113906, + 793779904904008, + ), + u64x4::new( + 1032129287829151, + 30730741946697, + 217603185195068, + 2118169309744162, + ), + u64x4::new( + 225899335574721, + 1767553399797342, + 881082465669982, + 1435383196392870, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1127093564374276, + 2245188499702906, + 1250041622887441, + 2179324911668149, + ), + u64x4::new( + 908019210866875, + 1879900391060964, + 1355047706206597, + 647218945377302, + ), + u64x4::new( + 1616265604422592, + 2134336781521657, + 1157711219915601, + 1227494173135033, + ), + u64x4::new( + 136450294813355, + 1984543542455033, + 1199486053011083, + 33687889941331, + ), + u64x4::new( + 1053447012707371, + 68239344331930, + 537448158443925, + 1829189783369646, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 996806463322563, + 2043104667851348, + 1110361398300309, + 1218740346887957, + ), + u64x4::new( + 399141907016839, + 1307691109658227, + 532535384961264, + 896201194398872, + ), + u64x4::new( + 111705272106160, + 1790972382466021, + 1159338112559144, + 303544352897203, + ), + u64x4::new( + 1036600573322969, + 1457119922663674, + 334117653665514, + 460023361701263, + ), + u64x4::new( + 1363773215189933, + 1915594049343802, + 1661249423378694, + 1744945551969247, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 3093919631215, + 574886478077610, + 1704446919728971, + 250093147254210, + ), + u64x4::new( + 1387413348737796, + 360142717826981, + 2116185073015983, + 474541388374100, + ), + u64x4::new( + 1632539630892580, + 1332404016215719, + 2145297637794728, + 1289783723173504, + ), + u64x4::new( + 1030244179060173, + 579782698595797, + 1062365251139982, + 677149839815546, + ), + u64x4::new( + 6671539419876, + 1426937459653775, + 406942403696343, + 675479224223817, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 271984148441782, + 1708099625818957, + 1499011822959235, + 516808451044836, + ), + u64x4::new( + 1124847751346323, + 2038336022958449, + 1721698491022600, + 705944403212572, + ), + u64x4::new( + 85459783780275, + 1715213099986669, + 1728445509034791, + 730657630359717, + ), + u64x4::new( + 1185034652652387, + 755472578204310, + 476118360897817, + 1800434542785310, + ), + u64x4::new( + 1815589628676941, + 491778500674079, + 1547664984392513, + 279891608681267, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2036337168672113, + 1730787524684269, + 639134121311693, + 698060925015524, + ), + u64x4::new( + 315211075189491, + 1329055848835358, + 688621136402134, + 1271193060119448, + ), + u64x4::new( + 1697984374314012, + 459330773536457, + 305481314707918, + 61676911066002, + ), + u64x4::new( + 2166631826859191, + 2105217187401781, + 937587962768434, + 357397435365683, + ), + u64x4::new( + 1206757093145471, + 1287847622009294, + 1951336140421622, + 2233789834777410, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 82144190081093, + 1568417433687791, + 907555979158442, + 2037855062523867, + ), + u64x4::new( + 1225315484058853, + 315317868015613, + 1765025920288384, + 175223259828436, + ), + u64x4::new( + 1215010304871271, + 662713408454950, + 429517658575616, + 991062684008811, + ), + u64x4::new( + 993837615254894, + 1485561584889450, + 2001836754226476, + 1915943063896801, + ), + u64x4::new( + 818895101625673, + 1342479472068804, + 1380235330010671, + 23315169761453, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1500726307559118, + 956166860173424, + 512663951564436, + 1940180717699824, + ), + u64x4::new( + 1789521472720825, + 779456898652427, + 2035063615853504, + 863582140589407, + ), + u64x4::new( + 634508890793787, + 1748041666732214, + 259642099961634, + 1294936839797812, + ), + u64x4::new( + 2183334898697038, + 2197242820694806, + 2217225409073703, + 992633998226449, + ), + u64x4::new( + 2197077498155916, + 1562008797791883, + 1395088759904208, + 331715244679294, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 186854731652320, + 284389440026580, + 1252175415119400, + 1025377410100223, + ), + u64x4::new( + 1578732129417607, + 898645497852382, + 2237766074482974, + 1939197790303592, + ), + u64x4::new( + 1438830390640145, + 1682452015845597, + 1108441197232223, + 1984134492898664, + ), + u64x4::new( + 282668727301669, + 1609018289552856, + 390363439795705, + 1138459124667912, + ), + u64x4::new( + 18889015928490, + 532489638086725, + 324621535996080, + 2210046082697453, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2041327051605378, + 2244037852176483, + 2116336876147147, + 9616672544864, + ), + u64x4::new( + 969847387559191, + 1059119127679639, + 1764630094670633, + 364568045311834, + ), + u64x4::new( + 505938893153679, + 2075421412172902, + 326984153045666, + 1959549727324704, + ), + u64x4::new( + 1088715617911260, + 13917085151028, + 950568481355929, + 23687195265771, + ), + u64x4::new( + 1798284568673198, + 808382292203333, + 2214698741961545, + 610817203275867, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1731488929623777, + 1158815615106413, + 1491090861948525, + 1428384712900962, + ), + u64x4::new( + 722237139522457, + 1514290328911535, + 1366197913116230, + 1519472657321210, + ), + u64x4::new( + 246028966932273, + 1888239319448405, + 423720022211163, + 455243905681470, + ), + u64x4::new( + 738323403716001, + 1758018973481179, + 1180718299482318, + 1008495946606708, + ), + u64x4::new( + 334959381596119, + 1704599537529481, + 2172191232106896, + 13502508918495, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 273393076768079, + 427388720298603, + 1071733376018227, + 1715429388968611, + ), + u64x4::new( + 751776629892313, + 1965239102856011, + 541955408230119, + 831043488876080, + ), + u64x4::new( + 643718536393104, + 390543998404644, + 2176730661486279, + 499459234889079, + ), + u64x4::new( + 1482404333915009, + 865527293526285, + 507957951411713, + 216456252558825, + ), + u64x4::new( + 2210281256300231, + 1519357818277551, + 1257866936775246, + 1689605217672864, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2135395168187905, + 2214400157568614, + 2032983817870823, + 1124945109072647, + ), + u64x4::new( + 1602820011758145, + 906675633903289, + 782700735390986, + 2067218823525601, + ), + u64x4::new( + 786785748926382, + 1433583123655616, + 905839404290873, + 2249680349963778, + ), + u64x4::new( + 1940824582370584, + 1610961256326291, + 285307858781375, + 1755588655461194, + ), + u64x4::new( + 233682812055333, + 2146114223476434, + 41132209533476, + 535292431776371, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 600257696476418, + 18449221564824, + 1422209458591138, + 239571584769716, + ), + u64x4::new( + 2056372917056980, + 1155290566623531, + 1252473955568148, + 1276690716882081, + ), + u64x4::new( + 246974369025311, + 658117221519903, + 2000380937898441, + 1351183273924850, + ), + u64x4::new( + 1803747363753112, + 1736801515030186, + 2025633577199091, + 603378480769167, + ), + u64x4::new( + 57348749438551, + 1893551220299655, + 657926732731806, + 1522499384853705, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 591809128842736, + 284860517232591, + 27436696863545, + 886306697195798, + ), + u64x4::new( + 2113192175751749, + 1405882509906423, + 561316282804847, + 835573846576266, + ), + u64x4::new( + 94407289485409, + 1781534171669004, + 2098782516531528, + 598529921520053, + ), + u64x4::new( + 1860137004504786, + 2197323407480349, + 1516772733981532, + 961740253777086, + ), + u64x4::new( + 1484139612868217, + 1593557644636881, + 838834937143441, + 36382198263380, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1165898865828562, + 1153420815042389, + 1068625028915785, + 1945927229911090, + ), + u64x4::new( + 843454394017146, + 571029655293754, + 386282254545998, + 1804608237584150, + ), + u64x4::new( + 370552451091100, + 1279105656351124, + 1864742949668631, + 2093071521726981, + ), + u64x4::new( + 1872542389052198, + 1679083953574330, + 349872262454465, + 1470311090717925, + ), + u64x4::new( + 685345654160323, + 319718985807814, + 1359932285384164, + 1410900103316331, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2083666668832889, + 314624387816655, + 1496694646480345, + 1946728950459189, + ), + u64x4::new( + 1579153761571203, + 508771185291380, + 1002249659402007, + 551517831173801, + ), + u64x4::new( + 2132371471626150, + 1988122278556533, + 1552195130653890, + 1327637750292755, + ), + u64x4::new( + 118937099181527, + 382610380973142, + 634951529106471, + 382740054041699, + ), + u64x4::new( + 801287519643470, + 87822941589258, + 1908825350108451, + 1404208826499115, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 330347226380261, + 672119116965146, + 1761510370768005, + 1959200302484704, + ), + u64x4::new( + 1631876583009250, + 1684917718484264, + 1027256947805920, + 2174612545251129, + ), + u64x4::new( + 636668855699872, + 625187713984839, + 265886954766790, + 167898557908504, + ), + u64x4::new( + 1210974548180860, + 2051308710365526, + 907620584086428, + 1081788677970850, + ), + u64x4::new( + 621792955460854, + 1450945504745382, + 1666728650687828, + 977937146451674, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 24725936182267, + 2226765032752574, + 2036560083102883, + 2002351185719584, + ), + u64x4::new( + 1620080779405308, + 1493220053370419, + 2245691691038916, + 1152182628629603, + ), + u64x4::new( + 317928527147500, + 1855194218440212, + 979380281964169, + 861442286685289, + ), + u64x4::new( + 393308472784625, + 486143087279967, + 1234071346236405, + 777748237119399, + ), + u64x4::new( + 43850412814718, + 1497656407486446, + 744128331046695, + 1618035787321792, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1670169946550211, + 1230951698726438, + 806586940221293, + 23159779184607, + ), + u64x4::new( + 634011340979302, + 764182085034744, + 731065727766955, + 1737985776442180, + ), + u64x4::new( + 240492712141842, + 73976435954441, + 162810587166835, + 697230894340912, + ), + u64x4::new( + 1299745598348388, + 1359436039694544, + 1856609816731554, + 25228008461513, + ), + u64x4::new( + 2180690501932381, + 2161211192848458, + 87069466793408, + 2003456332883860, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1106932458043379, + 1675181364231371, + 1681785724775243, + 131824742557210, + ), + u64x4::new( + 1671649414647169, + 1827849994880670, + 1097958057111899, + 701956891169434, + ), + u64x4::new( + 2095539283710881, + 591029812888096, + 1699571518315654, + 1297589045812566, + ), + u64x4::new( + 1345612272298537, + 2166754730876055, + 2047982622154948, + 1785222806258129, + ), + u64x4::new( + 2181915268829890, + 1895697064378670, + 1288412327355885, + 1561075738281368, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 741330264098392, + 357073519729966, + 1603572339180975, + 433572083688575, + ), + u64x4::new( + 699685108971208, + 1719650727634959, + 1941668009419214, + 870374958347891, + ), + u64x4::new( + 385971389331537, + 11655507719711, + 94814615497633, + 515572102810609, + ), + u64x4::new( + 1396688200590426, + 1518748475144123, + 162386454324368, + 2083303971579002, + ), + u64x4::new( + 1511688632419263, + 251584258592336, + 545345887993880, + 1229840230314160, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1298668855706029, + 2017860934939344, + 2224150456036391, + 1925926576297971, + ), + u64x4::new( + 259522963883544, + 1312469129541229, + 1647530465049600, + 1113737129047154, + ), + u64x4::new( + 733193298663145, + 2115712816303403, + 897628702762311, + 116440277571901, + ), + u64x4::new( + 1998719395229750, + 1662774553684237, + 194395608126452, + 98796702872301, + ), + u64x4::new( + 2226158244229144, + 91961728239158, + 526869903032152, + 849263805316773, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 472779569333556, + 854477760843410, + 2070906720349401, + 734613359834689, + ), + u64x4::new( + 1771897100487404, + 1604024196006064, + 319699348925383, + 437152129592623, + ), + u64x4::new( + 627618365135361, + 1768642666037955, + 588564169143939, + 35295037750744, + ), + u64x4::new( + 220241884231278, + 319104161410840, + 1048165719448798, + 1583931089774347, + ), + u64x4::new( + 166479451884333, + 1623611819962804, + 59990366193679, + 900727256046987, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1944687327687331, + 1328410791053991, + 2083980670913902, + 609396833380574, + ), + u64x4::new( + 1907563845734496, + 1385619047697883, + 869817384774457, + 106642388505109, + ), + u64x4::new( + 1006516581737154, + 1561918369633937, + 1921172883211450, + 2216650451558824, + ), + u64x4::new( + 1780506017391778, + 233064930371847, + 1332962603425752, + 1380075261612354, + ), + u64x4::new( + 1907624789747741, + 1310065402098523, + 1838275780706825, + 884225500782782, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 198729830692545, + 100156148743413, + 2140568641558859, + 2220606475942394, + ), + u64x4::new( + 1108788217903741, + 1706330932366163, + 2050449866410661, + 684907598542847, + ), + u64x4::new( + 1101958322366646, + 659427843062405, + 253899933868173, + 896574852821269, + ), + u64x4::new( + 1157052140740658, + 440541103447032, + 2173354981480949, + 604768603561932, + ), + u64x4::new( + 961238337866054, + 830849154351308, + 1643852412409441, + 1436749321770368, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 784870637473285, + 1180234052037572, + 2086951602998715, + 419328169540373, + ), + u64x4::new( + 1966862397394559, + 788036164772123, + 2024355635709481, + 1471696676696146, + ), + u64x4::new( + 1468884300957205, + 1408016588131185, + 2229595828577885, + 240413942963547, + ), + u64x4::new( + 1481791691942441, + 970648959691160, + 1635500996148197, + 2236917233261585, + ), + u64x4::new( + 31660820731028, + 801794768903647, + 1069092619607344, + 282652554845923, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 911659428682786, + 762502588057038, + 1311399152500807, + 1966922911783311, + ), + u64x4::new( + 1229849228728540, + 258161307933217, + 2140796867375541, + 1569345075547911, + ), + u64x4::new( + 1487354676143742, + 1818317546165791, + 811033554173350, + 1768788663337616, + ), + u64x4::new( + 450017165913234, + 962535873747168, + 2099104262993585, + 503030952485785, + ), + u64x4::new( + 1259958681304518, + 479589250923541, + 1503904042161640, + 706283657294305, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 794562643024291, + 198670993088241, + 1678984629358943, + 273399517554618, + ), + u64x4::new( + 188458991574433, + 1389872130156447, + 1461868931574746, + 795140878721432, + ), + u64x4::new( + 624046647169653, + 630363741191019, + 911018499983500, + 1410140563046579, + ), + u64x4::new( + 1675056174405076, + 632544713589250, + 795454163559811, + 1535271563341780, + ), + u64x4::new( + 25504547444781, + 812510098987855, + 51290042016232, + 1992260991700127, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 269968325452358, + 470932785179706, + 1684444304834150, + 1027482126748243, + ), + u64x4::new( + 457941065342419, + 2117377568137882, + 1209423706730905, + 2192403099717071, + ), + u64x4::new( + 1899046404863678, + 1359500336071762, + 1492389156724726, + 1455627081827750, + ), + u64x4::new( + 2016101061876546, + 1967000012916571, + 582539481696050, + 1197538178790094, + ), + u64x4::new( + 639684852217504, + 1799941252757449, + 1470016556327743, + 846111828965901, + ), + ])), +]); diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs new file mode 100644 index 0000000..cc2b241 --- /dev/null +++ b/src/backend/vector/ifma/edwards.rs @@ -0,0 +1,315 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2018 Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +#![allow(non_snake_case)] + +use traits::Identity; + +use std::ops::{Add, Neg, Sub}; + +use subtle::Choice; +use subtle::ConditionallySelectable; + +use edwards; +use window::{LookupTable, NafLookupTable5, NafLookupTable8}; + +use super::constants; +use super::field::{F51x4Reduced, F51x4Unreduced, Lanes, Shuffle}; + +#[derive(Copy, Clone, Debug)] +pub struct ExtendedPoint(pub(super) F51x4Unreduced); + +#[derive(Copy, Clone, Debug)] +pub struct CachedPoint(pub(super) F51x4Reduced); + +impl From for ExtendedPoint { + fn from(P: edwards::EdwardsPoint) -> ExtendedPoint { + ExtendedPoint(F51x4Unreduced::new(&P.X, &P.Y, &P.Z, &P.T)) + } +} + +impl From for edwards::EdwardsPoint { + fn from(P: ExtendedPoint) -> edwards::EdwardsPoint { + let reduced = F51x4Reduced::from(P.0); + let tmp = F51x4Unreduced::from(reduced).split(); + edwards::EdwardsPoint { + X: tmp[0], + Y: tmp[1], + Z: tmp[2], + T: tmp[3], + } + } +} + +impl From for CachedPoint { + fn from(P: ExtendedPoint) -> CachedPoint { + let mut x = P.0; + + x = x.blend(&x.diff_sum(), Lanes::AB); + x = &F51x4Reduced::from(x) * (121666, 121666, 2 * 121666, 2 * 121665); + x = x.blend(&x.negate_lazy(), Lanes::D); + + CachedPoint(F51x4Reduced::from(x)) + } +} + +impl Default for ExtendedPoint { + fn default() -> ExtendedPoint { + ExtendedPoint::identity() + } +} + +impl Identity for ExtendedPoint { + fn identity() -> ExtendedPoint { + constants::EXTENDEDPOINT_IDENTITY + } +} + +impl ExtendedPoint { + pub fn double(&self) -> ExtendedPoint { + // (Y1 X1 T1 Z1) -- uses vpshufd (1c latency @ 1/c) + let mut tmp0 = self.0.shuffle(Shuffle::BADC); + + // (X1+Y1 X1+Y1 X1+Y1 X1+Y1) -- can use vpinserti128 + let mut tmp1 = (self.0 + tmp0).shuffle(Shuffle::ABAB); + + // (X1 Y1 Z1 X1+Y1) + tmp0 = self.0.blend(&tmp1, Lanes::D); + + tmp1 = F51x4Reduced::from(tmp0).square(); + // Now tmp1 = (S1 S2 S3 S4) + + // We want to compute + // + // + | S1 | S1 | S1 | S1 | + // + | S2 | | | S2 | + // + | | | S3 | | + // + | | | S3 | | + // + | |16p |16p |16p | + // - | | S2 | S2 | | + // - | | | | S4 | + // ======================= + // S5 S6 S8 S9 + + let zero = F51x4Unreduced::zero(); + + let S1_S1_S1_S1 = tmp1.shuffle(Shuffle::AAAA); + let S2_S2_S2_S2 = tmp1.shuffle(Shuffle::BBBB); + + let S2_S2_S2_S4 = S2_S2_S2_S2.blend(&tmp1, Lanes::D).negate_lazy(); + + tmp0 = S1_S1_S1_S1 + zero.blend(&(tmp1 + tmp1), Lanes::C); + tmp0 = tmp0 + zero.blend(&S2_S2_S2_S2, Lanes::AD); + tmp0 = tmp0 + zero.blend(&S2_S2_S2_S4, Lanes::BCD); + + let tmp2 = F51x4Reduced::from(tmp0); + + ExtendedPoint(&tmp2.shuffle(Shuffle::DBBD) * &tmp2.shuffle(Shuffle::CACA)) + } + + pub fn mul_by_pow_2(&self, k: u32) -> ExtendedPoint { + let mut tmp: ExtendedPoint = *self; + for _ in 0..k { + tmp = tmp.double(); + } + tmp + } +} + +impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { + type Output = ExtendedPoint; + + /// Add an `ExtendedPoint` and a `CachedPoint`. + fn add(self, other: &'b CachedPoint) -> ExtendedPoint { + let mut tmp = self.0; + + tmp = tmp.blend(&tmp.diff_sum(), Lanes::AB); + // tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1) + + tmp = &F51x4Reduced::from(tmp) * &other.0; + // tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11) + + tmp = tmp.shuffle(Shuffle::ABDC); + // tmp = (S8 S9 S11 S10) + + let tmp = F51x4Reduced::from(tmp.diff_sum()); + // tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) + + let t0 = tmp.shuffle(Shuffle::ADDA); + // t0 = (S12 S15 S15 S12) + let t1 = tmp.shuffle(Shuffle::CBCB); + // t1 = (S14 S13 S14 S13) + + // Return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) + ExtendedPoint(&t0 * &t1) + } +} + +impl Default for CachedPoint { + fn default() -> CachedPoint { + CachedPoint::identity() + } +} + +impl Identity for CachedPoint { + fn identity() -> CachedPoint { + constants::CACHEDPOINT_IDENTITY + } +} + +impl ConditionallySelectable for CachedPoint { + fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self { + CachedPoint(F51x4Reduced::conditional_select(&a.0, &b.0, choice)) + } + + fn conditional_assign(&mut self, other: &Self, choice: Choice) { + self.0.conditional_assign(&other.0, choice); + } +} + +impl<'a> Neg for &'a CachedPoint { + type Output = CachedPoint; + + fn neg(self) -> CachedPoint { + let swapped = self.0.shuffle(Shuffle::BACD); + CachedPoint(swapped.blend(&(-self.0), Lanes::D)) + } +} + +impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint { + type Output = ExtendedPoint; + + /// Implement subtraction by negating the point and adding. + fn sub(self, other: &'b CachedPoint) -> ExtendedPoint { + self + &(-other) + } +} + +impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable { + fn from(point: &'a edwards::EdwardsPoint) -> Self { + let P = ExtendedPoint::from(*point); + let mut points = [CachedPoint::from(P); 8]; + for i in 0..7 { + points[i + 1] = (&P + &points[i]).into(); + } + LookupTable(points) + } +} + +impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable5 { + fn from(point: &'a edwards::EdwardsPoint) -> Self { + let A = ExtendedPoint::from(*point); + let mut Ai = [CachedPoint::from(A); 8]; + let A2 = A.double(); + for i in 0..7 { + Ai[i + 1] = (&A2 + &Ai[i]).into(); + } + // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A] + NafLookupTable5(Ai) + } +} + +impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable8 { + fn from(point: &'a edwards::EdwardsPoint) -> Self { + let A = ExtendedPoint::from(*point); + let mut Ai = [CachedPoint::from(A); 64]; + let A2 = A.double(); + for i in 0..63 { + Ai[i + 1] = (&A2 + &Ai[i]).into(); + } + // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A, ..., 127A] + NafLookupTable8(Ai) + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn addition_test_helper(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) { + // Test the serial implementation of the parallel addition formulas + //let R_serial: edwards::EdwardsPoint = serial_add(P.into(), Q.into()).into(); + + // Test the vector implementation of the parallel readdition formulas + let cached_Q = CachedPoint::from(ExtendedPoint::from(Q)); + let R_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) + &cached_Q).into(); + let S_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) - &cached_Q).into(); + + println!("Testing point addition:"); + println!("P = {:?}", P); + println!("Q = {:?}", Q); + println!("cached Q = {:?}", cached_Q); + println!("R = P + Q = {:?}", &P + &Q); + //println!("R_serial = {:?}", R_serial); + println!("R_vector = {:?}", R_vector); + println!("S = P - Q = {:?}", &P - &Q); + println!("S_vector = {:?}", S_vector); + //assert_eq!(R_serial.compress(), (&P + &Q).compress()); + assert_eq!(R_vector.compress(), (&P + &Q).compress()); + assert_eq!(S_vector.compress(), (&P - &Q).compress()); + println!("OK!\n"); + } + + #[test] + fn vector_addition_vs_serial_addition_vs_edwards_extendedpoint() { + use constants; + use scalar::Scalar; + + println!("Testing id +- id"); + let P = edwards::EdwardsPoint::identity(); + let Q = edwards::EdwardsPoint::identity(); + addition_test_helper(P, Q); + + println!("Testing id +- B"); + let P = edwards::EdwardsPoint::identity(); + let Q = constants::ED25519_BASEPOINT_POINT; + addition_test_helper(P, Q); + + println!("Testing B +- B"); + let P = constants::ED25519_BASEPOINT_POINT; + let Q = constants::ED25519_BASEPOINT_POINT; + addition_test_helper(P, Q); + + println!("Testing B +- kB"); + let P = constants::ED25519_BASEPOINT_POINT; + let Q = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from(8475983829u64); + addition_test_helper(P, Q); + } + + fn doubling_test_helper(P: edwards::EdwardsPoint) { + //let R1: edwards::EdwardsPoint = serial_double(P.into()).into(); + let R2: edwards::EdwardsPoint = ExtendedPoint::from(P).double().into(); + println!("Testing point doubling:"); + println!("P = {:?}", P); + //println!("(serial) R1 = {:?}", R1); + println!("(vector) R2 = {:?}", R2); + println!("P + P = {:?}", &P + &P); + //assert_eq!(R1.compress(), (&P + &P).compress()); + assert_eq!(R2.compress(), (&P + &P).compress()); + println!("OK!\n"); + } + + #[test] + fn vector_doubling_vs_serial_doubling_vs_edwards_extendedpoint() { + use constants; + use scalar::Scalar; + + println!("Testing [2]id"); + let P = edwards::EdwardsPoint::identity(); + doubling_test_helper(P); + + println!("Testing [2]B"); + let P = constants::ED25519_BASEPOINT_POINT; + doubling_test_helper(P); + + println!("Testing [2]([k]B)"); + let P = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from(8475983829u64); + doubling_test_helper(P); + } +} diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs new file mode 100644 index 0000000..d220a5a --- /dev/null +++ b/src/backend/vector/ifma/field.rs @@ -0,0 +1,822 @@ +// -*- mode: rust; coding: utf-8; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2018 Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +#![allow(non_snake_case)] + +use core::ops::{Add, Mul, Neg}; +use packed_simd::{u64x4, IntoBits}; + +use backend::serial::u64::field::FieldElement51; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.vpmadd52l.uq.256"] + fn madd52lo(z: u64x4, x: u64x4, y: u64x4) -> u64x4; + #[link_name = "llvm.x86.avx512.vpmadd52h.uq.256"] + fn madd52hi(z: u64x4, x: u64x4, y: u64x4) -> u64x4; +} + +/// A vector of four field elements in radix 2^51, with unreduced coefficients. +#[derive(Copy, Clone, Debug)] +pub struct F51x4Unreduced(pub(crate) [u64x4; 5]); + +/// A vector of four field elements in radix 2^51, with reduced coefficients. +#[derive(Copy, Clone, Debug)] +pub struct F51x4Reduced(pub(crate) [u64x4; 5]); + +#[derive(Copy, Clone)] +pub enum Shuffle { + AAAA, + BBBB, + BADC, + BACD, + ADDA, + CBCB, + ABDC, + ABAB, + DBBD, + CACA, +} + +#[inline(always)] +fn shuffle_lanes(x: u64x4, control: Shuffle) -> u64x4 { + unsafe { + use core::arch::x86_64::_mm256_permute4x64_epi64 as perm; + + match control { + Shuffle::AAAA => perm(x.into_bits(), 0b00_00_00_00).into_bits(), + Shuffle::BBBB => perm(x.into_bits(), 0b01_01_01_01).into_bits(), + Shuffle::BADC => perm(x.into_bits(), 0b10_11_00_01).into_bits(), + Shuffle::BACD => perm(x.into_bits(), 0b11_10_00_01).into_bits(), + Shuffle::ADDA => perm(x.into_bits(), 0b00_11_11_00).into_bits(), + Shuffle::CBCB => perm(x.into_bits(), 0b01_10_01_10).into_bits(), + Shuffle::ABDC => perm(x.into_bits(), 0b10_11_01_00).into_bits(), + Shuffle::ABAB => perm(x.into_bits(), 0b01_00_01_00).into_bits(), + Shuffle::DBBD => perm(x.into_bits(), 0b11_01_01_11).into_bits(), + Shuffle::CACA => perm(x.into_bits(), 0b00_10_00_10).into_bits(), + } + } +} + +#[derive(Copy, Clone)] +pub enum Lanes { + D, + C, + AB, + AC, + AD, + BCD, +} + +#[inline] +fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 { + unsafe { + use core::arch::x86_64::_mm256_blend_epi32 as blend; + + match control { + Lanes::D => blend(x.into_bits(), y.into_bits(), 0b11_00_00_00).into_bits(), + Lanes::C => blend(x.into_bits(), y.into_bits(), 0b00_11_00_00).into_bits(), + Lanes::AB => blend(x.into_bits(), y.into_bits(), 0b00_00_11_11).into_bits(), + Lanes::AC => blend(x.into_bits(), y.into_bits(), 0b00_11_00_11).into_bits(), + Lanes::AD => blend(x.into_bits(), y.into_bits(), 0b11_00_00_11).into_bits(), + Lanes::BCD => blend(x.into_bits(), y.into_bits(), 0b11_11_11_00).into_bits(), + } + } +} + +impl F51x4Unreduced { + pub fn zero() -> F51x4Unreduced { + F51x4Unreduced([u64x4::splat(0); 5]) + } + + pub fn new( + x0: &FieldElement51, + x1: &FieldElement51, + x2: &FieldElement51, + x3: &FieldElement51, + ) -> F51x4Unreduced { + F51x4Unreduced([ + u64x4::new(x0.0[0], x1.0[0], x2.0[0], x3.0[0]), + u64x4::new(x0.0[1], x1.0[1], x2.0[1], x3.0[1]), + u64x4::new(x0.0[2], x1.0[2], x2.0[2], x3.0[2]), + u64x4::new(x0.0[3], x1.0[3], x2.0[3], x3.0[3]), + u64x4::new(x0.0[4], x1.0[4], x2.0[4], x3.0[4]), + ]) + } + + pub fn split(&self) -> [FieldElement51; 4] { + let x = &self.0; + [ + FieldElement51([ + x[0].extract(0), + x[1].extract(0), + x[2].extract(0), + x[3].extract(0), + x[4].extract(0), + ]), + FieldElement51([ + x[0].extract(1), + x[1].extract(1), + x[2].extract(1), + x[3].extract(1), + x[4].extract(1), + ]), + FieldElement51([ + x[0].extract(2), + x[1].extract(2), + x[2].extract(2), + x[3].extract(2), + x[4].extract(2), + ]), + FieldElement51([ + x[0].extract(3), + x[1].extract(3), + x[2].extract(3), + x[3].extract(3), + x[4].extract(3), + ]), + ] + } + + #[inline] + pub fn diff_sum(&self) -> F51x4Unreduced { + // tmp1 = (B, A, D, C) + let tmp1 = self.shuffle(Shuffle::BADC); + // tmp2 = (-A, B, -C, D) + let tmp2 = self.blend(&self.negate_lazy(), Lanes::AC); + // (B - A, B + A, D - C, D + C) + tmp1 + tmp2 + } + + #[inline] + pub fn negate_lazy(&self) -> F51x4Unreduced { + let lo = u64x4::splat(36028797018963664u64); + let hi = u64x4::splat(36028797018963952u64); + F51x4Unreduced([ + lo - self.0[0], + hi - self.0[1], + hi - self.0[2], + hi - self.0[3], + hi - self.0[4], + ]) + } + + #[inline] + pub fn shuffle(&self, control: Shuffle) -> F51x4Unreduced { + F51x4Unreduced([ + shuffle_lanes(self.0[0], control), + shuffle_lanes(self.0[1], control), + shuffle_lanes(self.0[2], control), + shuffle_lanes(self.0[3], control), + shuffle_lanes(self.0[4], control), + ]) + } + + #[inline] + pub fn blend(&self, other: &F51x4Unreduced, control: Lanes) -> F51x4Unreduced { + F51x4Unreduced([ + blend_lanes(self.0[0], other.0[0], control), + blend_lanes(self.0[1], other.0[1], control), + blend_lanes(self.0[2], other.0[2], control), + blend_lanes(self.0[3], other.0[3], control), + blend_lanes(self.0[4], other.0[4], control), + ]) + } +} + +impl Neg for F51x4Reduced { + type Output = F51x4Reduced; + + fn neg(self) -> F51x4Reduced { + F51x4Unreduced::from(self).negate_lazy().into() + } +} + +use subtle::Choice; +use subtle::ConditionallySelectable; + +impl ConditionallySelectable for F51x4Reduced { + #[inline] + fn conditional_select( + a: &F51x4Reduced, + b: &F51x4Reduced, + choice: Choice, + ) -> F51x4Reduced { + let mask = (-(choice.unwrap_u8() as i64)) as u64; + let mask_vec = u64x4::splat(mask); + F51x4Reduced([ + a.0[0] ^ (mask_vec & (a.0[0] ^ b.0[0])), + a.0[1] ^ (mask_vec & (a.0[1] ^ b.0[1])), + a.0[2] ^ (mask_vec & (a.0[2] ^ b.0[2])), + a.0[3] ^ (mask_vec & (a.0[3] ^ b.0[3])), + a.0[4] ^ (mask_vec & (a.0[4] ^ b.0[4])), + ]) + } + + #[inline] + fn conditional_assign(&mut self, other: &F51x4Reduced, choice: Choice) { + let mask = (-(choice.unwrap_u8() as i64)) as u64; + let mask_vec = u64x4::splat(mask); + self.0[0] ^= mask_vec & (self.0[0] ^ other.0[0]); + self.0[1] ^= mask_vec & (self.0[1] ^ other.0[1]); + self.0[2] ^= mask_vec & (self.0[2] ^ other.0[2]); + self.0[3] ^= mask_vec & (self.0[3] ^ other.0[3]); + self.0[4] ^= mask_vec & (self.0[4] ^ other.0[4]); + } +} + +impl F51x4Reduced { + #[inline] + pub fn shuffle(&self, control: Shuffle) -> F51x4Reduced { + F51x4Reduced([ + shuffle_lanes(self.0[0], control), + shuffle_lanes(self.0[1], control), + shuffle_lanes(self.0[2], control), + shuffle_lanes(self.0[3], control), + shuffle_lanes(self.0[4], control), + ]) + } + + #[inline] + pub fn blend(&self, other: &F51x4Reduced, control: Lanes) -> F51x4Reduced { + F51x4Reduced([ + blend_lanes(self.0[0], other.0[0], control), + blend_lanes(self.0[1], other.0[1], control), + blend_lanes(self.0[2], other.0[2], control), + blend_lanes(self.0[3], other.0[3], control), + blend_lanes(self.0[4], other.0[4], control), + ]) + } + + #[inline] + pub fn square(&self) -> F51x4Unreduced { + unsafe { + let x = &self.0; + + // Represent values with coeff. 2 + let mut z0_2 = u64x4::splat(0); + let mut z1_2 = u64x4::splat(0); + let mut z2_2 = u64x4::splat(0); + let mut z3_2 = u64x4::splat(0); + let mut z4_2 = u64x4::splat(0); + let mut z5_2 = u64x4::splat(0); + let mut z6_2 = u64x4::splat(0); + let mut z7_2 = u64x4::splat(0); + let mut z9_2 = u64x4::splat(0); + + // Represent values with coeff. 4 + let mut z2_4 = u64x4::splat(0); + let mut z3_4 = u64x4::splat(0); + let mut z4_4 = u64x4::splat(0); + let mut z5_4 = u64x4::splat(0); + let mut z6_4 = u64x4::splat(0); + let mut z7_4 = u64x4::splat(0); + let mut z8_4 = u64x4::splat(0); + + let mut z0_1 = u64x4::splat(0); + z0_1 = madd52lo(z0_1, x[0], x[0]); + + let mut z1_1 = u64x4::splat(0); + z1_2 = madd52lo(z1_2, x[0], x[1]); + z1_2 = madd52hi(z1_2, x[0], x[0]); + + z2_4 = madd52hi(z2_4, x[0], x[1]); + let mut z2_1 = z2_4 << 2; + z2_2 = madd52lo(z2_2, x[0], x[2]); + z2_1 = madd52lo(z2_1, x[1], x[1]); + + z3_4 = madd52hi(z3_4, x[0], x[2]); + let mut z3_1 = z3_4 << 2; + z3_2 = madd52lo(z3_2, x[1], x[2]); + z3_2 = madd52lo(z3_2, x[0], x[3]); + z3_2 = madd52hi(z3_2, x[1], x[1]); + + z4_4 = madd52hi(z4_4, x[1], x[2]); + z4_4 = madd52hi(z4_4, x[0], x[3]); + let mut z4_1 = z4_4 << 2; + z4_2 = madd52lo(z4_2, x[1], x[3]); + z4_2 = madd52lo(z4_2, x[0], x[4]); + z4_1 = madd52lo(z4_1, x[2], x[2]); + + z5_4 = madd52hi(z5_4, x[1], x[3]); + z5_4 = madd52hi(z5_4, x[0], x[4]); + let mut z5_1 = z5_4 << 2; + z5_2 = madd52lo(z5_2, x[2], x[3]); + z5_2 = madd52lo(z5_2, x[1], x[4]); + z5_2 = madd52hi(z5_2, x[2], x[2]); + + z6_4 = madd52hi(z6_4, x[2], x[3]); + z6_4 = madd52hi(z6_4, x[1], x[4]); + let mut z6_1 = z6_4 << 2; + z6_2 = madd52lo(z6_2, x[2], x[4]); + z6_1 = madd52lo(z6_1, x[3], x[3]); + + z7_4 = madd52hi(z7_4, x[2], x[4]); + let mut z7_1 = z7_4 << 2; + z7_2 = madd52lo(z7_2, x[3], x[4]); + z7_2 = madd52hi(z7_2, x[3], x[3]); + + z8_4 = madd52hi(z8_4, x[3], x[4]); + let mut z8_1 = z8_4 << 2; + z8_1 = madd52lo(z8_1, x[4], x[4]); + + let mut z9_1 = u64x4::splat(0); + z9_2 = madd52hi(z9_2, x[4], x[4]); + + z5_1 += z5_2 << 1; + z6_1 += z6_2 << 1; + z7_1 += z7_2 << 1; + z9_1 += z9_2 << 1; + + let mut t0 = u64x4::splat(0); + let mut t1 = u64x4::splat(0); + let r19 = u64x4::splat(19); + + t0 = madd52hi(t0, r19, z9_1); + t1 = madd52lo(t1, r19, z9_1 >> 52); + + z4_2 = madd52lo(z4_2, r19, z8_1 >> 52); + z3_2 = madd52lo(z3_2, r19, z7_1 >> 52); + z2_2 = madd52lo(z2_2, r19, z6_1 >> 52); + z1_2 = madd52lo(z1_2, r19, z5_1 >> 52); + + z0_2 = madd52lo(z0_2, r19, t0 + t1); + z1_2 = madd52hi(z1_2, r19, z5_1); + z2_2 = madd52hi(z2_2, r19, z6_1); + z3_2 = madd52hi(z3_2, r19, z7_1); + z4_2 = madd52hi(z4_2, r19, z8_1); + + z0_1 = madd52lo(z0_1, r19, z5_1); + z1_1 = madd52lo(z1_1, r19, z6_1); + z2_1 = madd52lo(z2_1, r19, z7_1); + z3_1 = madd52lo(z3_1, r19, z8_1); + z4_1 = madd52lo(z4_1, r19, z9_1); + + F51x4Unreduced([ + z0_1 + z0_2 + z0_2, + z1_1 + z1_2 + z1_2, + z2_1 + z2_2 + z2_2, + z3_1 + z3_2 + z3_2, + z4_1 + z4_2 + z4_2, + ]) + } + } +} + +impl From for F51x4Unreduced { + #[inline] + fn from(x: F51x4Reduced) -> F51x4Unreduced { + F51x4Unreduced(x.0) + } +} + +impl From for F51x4Reduced { + #[inline] + fn from(x: F51x4Unreduced) -> F51x4Reduced { + let mask = u64x4::splat((1 << 51) - 1); + let r19 = u64x4::splat(19); + + // Compute carryouts in parallel + let c0 = x.0[0] >> 51; + let c1 = x.0[1] >> 51; + let c2 = x.0[2] >> 51; + let c3 = x.0[3] >> 51; + let c4 = x.0[4] >> 51; + + unsafe { + F51x4Reduced([ + madd52lo(x.0[0] & mask, c4, r19), + (x.0[1] & mask) + c0, + (x.0[2] & mask) + c1, + (x.0[3] & mask) + c2, + (x.0[4] & mask) + c3, + ]) + } + } +} + +impl Add for F51x4Unreduced { + type Output = F51x4Unreduced; + #[inline] + fn add(self, rhs: F51x4Unreduced) -> F51x4Unreduced { + F51x4Unreduced([ + self.0[0] + rhs.0[0], + self.0[1] + rhs.0[1], + self.0[2] + rhs.0[2], + self.0[3] + rhs.0[3], + self.0[4] + rhs.0[4], + ]) + } +} + +impl<'a> Mul<(u32, u32, u32, u32)> for &'a F51x4Reduced { + type Output = F51x4Unreduced; + #[inline] + fn mul(self, scalars: (u32, u32, u32, u32)) -> F51x4Unreduced { + unsafe { + let x = &self.0; + let y = u64x4::new( + scalars.0 as u64, + scalars.1 as u64, + scalars.2 as u64, + scalars.3 as u64, + ); + let r19 = u64x4::splat(19); + + let mut z0_1 = u64x4::splat(0); + let mut z1_1 = u64x4::splat(0); + let mut z2_1 = u64x4::splat(0); + let mut z3_1 = u64x4::splat(0); + let mut z4_1 = u64x4::splat(0); + let mut z1_2 = u64x4::splat(0); + let mut z2_2 = u64x4::splat(0); + let mut z3_2 = u64x4::splat(0); + let mut z4_2 = u64x4::splat(0); + let mut z5_2 = u64x4::splat(0); + + // Wave 0 + z4_2 = madd52hi(z4_2, y, x[3]); + z5_2 = madd52hi(z5_2, y, x[4]); + z4_1 = madd52lo(z4_1, y, x[4]); + z0_1 = madd52lo(z0_1, y, x[0]); + z3_1 = madd52lo(z3_1, y, x[3]); + z2_1 = madd52lo(z2_1, y, x[2]); + z1_1 = madd52lo(z1_1, y, x[1]); + z3_2 = madd52hi(z3_2, y, x[2]); + + // Wave 2 + z2_2 = madd52hi(z2_2, y, x[1]); + z1_2 = madd52hi(z1_2, y, x[0]); + z0_1 = madd52lo(z0_1, z5_2 + z5_2, r19); + + F51x4Unreduced([ + z0_1, + z1_1 + z1_2 + z1_2, + z2_1 + z2_2 + z2_2, + z3_1 + z3_2 + z3_2, + z4_1 + z4_2 + z4_2, + ]) + } + } +} + +impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { + type Output = F51x4Unreduced; + #[inline] + fn mul(self, rhs: &'b F51x4Reduced) -> F51x4Unreduced { + unsafe { + // Inputs + let x = &self.0; + let y = &rhs.0; + + // Accumulators for terms with coeff 1 + let mut z0_1 = u64x4::splat(0); + let mut z1_1 = u64x4::splat(0); + let mut z2_1 = u64x4::splat(0); + let mut z3_1 = u64x4::splat(0); + let mut z4_1 = u64x4::splat(0); + let mut z5_1 = u64x4::splat(0); + let mut z6_1 = u64x4::splat(0); + let mut z7_1 = u64x4::splat(0); + let mut z8_1 = u64x4::splat(0); + + // Accumulators for terms with coeff 2 + let mut z0_2 = u64x4::splat(0); + let mut z1_2 = u64x4::splat(0); + let mut z2_2 = u64x4::splat(0); + let mut z3_2 = u64x4::splat(0); + let mut z4_2 = u64x4::splat(0); + let mut z5_2 = u64x4::splat(0); + let mut z6_2 = u64x4::splat(0); + let mut z7_2 = u64x4::splat(0); + let mut z8_2 = u64x4::splat(0); + let mut z9_2 = u64x4::splat(0); + + // LLVM doesn't seem to do much work reordering IFMA + // instructions, so try to organize them into "waves" of 8 + // independent operations (4c latency, 0.5 c throughput + // means 8 in flight) + + // Wave 0 + z4_1 = madd52lo(z4_1, x[2], y[2]); + z5_2 = madd52hi(z5_2, x[2], y[2]); + z5_1 = madd52lo(z5_1, x[4], y[1]); + z6_2 = madd52hi(z6_2, x[4], y[1]); + z6_1 = madd52lo(z6_1, x[4], y[2]); + z7_2 = madd52hi(z7_2, x[4], y[2]); + z7_1 = madd52lo(z7_1, x[4], y[3]); + z8_2 = madd52hi(z8_2, x[4], y[3]); + + // Wave 1 + z4_1 = madd52lo(z4_1, x[3], y[1]); + z5_2 = madd52hi(z5_2, x[3], y[1]); + z5_1 = madd52lo(z5_1, x[3], y[2]); + z6_2 = madd52hi(z6_2, x[3], y[2]); + z6_1 = madd52lo(z6_1, x[3], y[3]); + z7_2 = madd52hi(z7_2, x[3], y[3]); + z7_1 = madd52lo(z7_1, x[3], y[4]); + z8_2 = madd52hi(z8_2, x[3], y[4]); + + // Wave 2 + z8_1 = madd52lo(z8_1, x[4], y[4]); + z9_2 = madd52hi(z9_2, x[4], y[4]); + z4_1 = madd52lo(z4_1, x[4], y[0]); + z5_2 = madd52hi(z5_2, x[4], y[0]); + z5_1 = madd52lo(z5_1, x[2], y[3]); + z6_2 = madd52hi(z6_2, x[2], y[3]); + z6_1 = madd52lo(z6_1, x[2], y[4]); + z7_2 = madd52hi(z7_2, x[2], y[4]); + + let z8 = z8_1 + z8_2 + z8_2; + let z9 = z9_2 + z9_2; + + // Wave 3 + z3_1 = madd52lo(z3_1, x[3], y[0]); + z4_2 = madd52hi(z4_2, x[3], y[0]); + z4_1 = madd52lo(z4_1, x[1], y[3]); + z5_2 = madd52hi(z5_2, x[1], y[3]); + z5_1 = madd52lo(z5_1, x[1], y[4]); + z6_2 = madd52hi(z6_2, x[1], y[4]); + z2_1 = madd52lo(z2_1, x[2], y[0]); + z3_2 = madd52hi(z3_2, x[2], y[0]); + + let z6 = z6_1 + z6_2 + z6_2; + let z7 = z7_1 + z7_2 + z7_2; + + // Wave 4 + z3_1 = madd52lo(z3_1, x[2], y[1]); + z4_2 = madd52hi(z4_2, x[2], y[1]); + z4_1 = madd52lo(z4_1, x[0], y[4]); + z5_2 = madd52hi(z5_2, x[0], y[4]); + z1_1 = madd52lo(z1_1, x[1], y[0]); + z2_2 = madd52hi(z2_2, x[1], y[0]); + z2_1 = madd52lo(z2_1, x[1], y[1]); + z3_2 = madd52hi(z3_2, x[1], y[1]); + + let z5 = z5_1 + z5_2 + z5_2; + + // Wave 5 + z3_1 = madd52lo(z3_1, x[1], y[2]); + z4_2 = madd52hi(z4_2, x[1], y[2]); + z0_1 = madd52lo(z0_1, x[0], y[0]); + z1_2 = madd52hi(z1_2, x[0], y[0]); + z1_1 = madd52lo(z1_1, x[0], y[1]); + z2_1 = madd52lo(z2_1, x[0], y[2]); + z2_2 = madd52hi(z2_2, x[0], y[1]); + z3_2 = madd52hi(z3_2, x[0], y[2]); + + let mut t0 = u64x4::splat(0); + let mut t1 = u64x4::splat(0); + let r19 = u64x4::splat(19); + + // Wave 6 + t0 = madd52hi(t0, r19, z9); + t1 = madd52lo(t1, r19, z9 >> 52); + z3_1 = madd52lo(z3_1, x[0], y[3]); + z4_2 = madd52hi(z4_2, x[0], y[3]); + z1_2 = madd52lo(z1_2, r19, z5 >> 52); + z2_2 = madd52lo(z2_2, r19, z6 >> 52); + z3_2 = madd52lo(z3_2, r19, z7 >> 52); + z0_1 = madd52lo(z0_1, r19, z5); + + // Wave 7 + z4_1 = madd52lo(z4_1, r19, z9); + z1_1 = madd52lo(z1_1, r19, z6); + z0_2 = madd52lo(z0_2, r19, t0 + t1); + z4_2 = madd52hi(z4_2, r19, z8); + z2_1 = madd52lo(z2_1, r19, z7); + z1_2 = madd52hi(z1_2, r19, z5); + z2_2 = madd52hi(z2_2, r19, z6); + z3_2 = madd52hi(z3_2, r19, z7); + + // Wave 8 + z3_1 = madd52lo(z3_1, r19, z8); + z4_2 = madd52lo(z4_2, r19, z8 >> 52); + + F51x4Unreduced([ + z0_1 + z0_2 + z0_2, + z1_1 + z1_2 + z1_2, + z2_1 + z2_2 + z2_2, + z3_1 + z3_2 + z3_2, + z4_1 + z4_2 + z4_2, + ]) + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn vpmadd52luq() { + let x = u64x4::splat(2); + let y = u64x4::splat(3); + let mut z = u64x4::splat(5); + + z = unsafe { madd52lo(z, x, y) }; + + assert_eq!(z, u64x4::splat(5 + 2 * 3)); + } + + #[test] + fn new_split_round_trip_on_reduced_input() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + + let ax4 = F51x4Unreduced::new(&a, &a, &a, &a); + let splits = ax4.split(); + + for i in 0..4 { + assert_eq!(a, splits[i]); + } + } + + #[test] + fn new_split_round_trip_on_unreduced_input() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + // ... but now multiply it by 16 without reducing coeffs + let a16 = FieldElement51([ + a.0[0] << 4, + a.0[1] << 4, + a.0[2] << 4, + a.0[3] << 4, + a.0[4] << 4, + ]); + + let a16x4 = F51x4Unreduced::new(&a16, &a16, &a16, &a16); + let splits = a16x4.split(); + + for i in 0..4 { + assert_eq!(a16, splits[i]); + } + } + + #[test] + fn test_reduction() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + // ... but now multiply it by 128 without reducing coeffs + let abig = FieldElement51([ + a.0[0] << 4, + a.0[1] << 4, + a.0[2] << 4, + a.0[3] << 4, + a.0[4] << 4, + ]); + + let abigx4: F51x4Reduced = F51x4Unreduced::new(&abig, &abig, &abig, &abig).into(); + + let splits = F51x4Unreduced::from(abigx4).split(); + let c = &a * &FieldElement51([(1 << 4), 0, 0, 0, 0]); + + for i in 0..4 { + assert_eq!(c, splits[i]); + } + } + + #[test] + fn mul_matches_serial() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + let b = FieldElement51([98098, 87987897, 0, 1, 0]).invert(); + let c = &a * &b; + + let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into(); + let bx4: F51x4Reduced = F51x4Unreduced::new(&b, &b, &b, &b).into(); + let cx4 = &ax4 * &bx4; + + let splits = cx4.split(); + + for i in 0..4 { + assert_eq!(c, splits[i]); + } + } + + #[test] + fn iterated_mul_matches_serial() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + let b = FieldElement51([98098, 87987897, 0, 1, 0]).invert(); + let mut c = &a * &b; + for _i in 0..1024 { + c = &a * &c; + c = &b * &c; + } + + let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into(); + let bx4: F51x4Reduced = F51x4Unreduced::new(&b, &b, &b, &b).into(); + let mut cx4 = &ax4 * &bx4; + for _i in 0..1024 { + cx4 = &ax4 * &F51x4Reduced::from(cx4); + cx4 = &bx4 * &F51x4Reduced::from(cx4); + } + + let splits = cx4.split(); + + for i in 0..4 { + assert_eq!(c, splits[i]); + } + } + + #[test] + fn square_matches_mul() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + + let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into(); + let cx4 = &ax4 * &ax4; + let cx4_sq = ax4.square(); + + let splits = cx4.split(); + let splits_sq = cx4_sq.split(); + + for i in 0..4 { + assert_eq!(splits_sq[i], splits[i]); + } + } + + #[test] + fn iterated_square_matches_serial() { + // Invert a small field element to get a big one + let mut a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + let mut ax4 = F51x4Unreduced::new(&a, &a, &a, &a); + for _j in 0..1024 { + a = a.square(); + ax4 = F51x4Reduced::from(ax4).square(); + + let splits = ax4.split(); + for i in 0..4 { + assert_eq!(a, splits[i]); + } + } + } + + #[test] + fn iterated_u32_mul_matches_serial() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + let b = FieldElement51([121665, 0, 0, 0, 0]); + let mut c = &a * &b; + for _i in 0..1024 { + c = &b * &c; + } + + let ax4 = F51x4Unreduced::new(&a, &a, &a, &a); + let bx4 = (121665u32, 121665u32, 121665u32, 121665u32); + let mut cx4 = &F51x4Reduced::from(ax4) * bx4; + for _i in 0..1024 { + cx4 = &F51x4Reduced::from(cx4) * bx4; + } + + let splits = cx4.split(); + + for i in 0..4 { + assert_eq!(c, splits[i]); + } + } + + #[test] + fn shuffle_AAAA() { + let x0 = FieldElement51::from_bytes(&[0x10; 32]); + let x1 = FieldElement51::from_bytes(&[0x11; 32]); + let x2 = FieldElement51::from_bytes(&[0x12; 32]); + let x3 = FieldElement51::from_bytes(&[0x13; 32]); + + let x = F51x4Unreduced::new(&x0, &x1, &x2, &x3); + + let y = x.shuffle(Shuffle::AAAA); + let splits = y.split(); + + assert_eq!(splits[0], x0); + assert_eq!(splits[1], x0); + assert_eq!(splits[2], x0); + assert_eq!(splits[3], x0); + } + + #[test] + fn blend_AB() { + let x0 = FieldElement51::from_bytes(&[0x10; 32]); + let x1 = FieldElement51::from_bytes(&[0x11; 32]); + let x2 = FieldElement51::from_bytes(&[0x12; 32]); + let x3 = FieldElement51::from_bytes(&[0x13; 32]); + + let x = F51x4Unreduced::new(&x0, &x1, &x2, &x3); + let z = F51x4Unreduced::new(&x3, &x2, &x1, &x0); + + let y = x.blend(&z, Lanes::AB); + let splits = y.split(); + + assert_eq!(splits[0], x3); + assert_eq!(splits[1], x2); + assert_eq!(splits[2], x2); + assert_eq!(splits[3], x3); + } +} diff --git a/src/backend/vector/ifma/mod.rs b/src/backend/vector/ifma/mod.rs new file mode 100644 index 0000000..996f941 --- /dev/null +++ b/src/backend/vector/ifma/mod.rs @@ -0,0 +1,19 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2018 Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +#![cfg_attr( + all(feature = "nightly", feature = "stage2_build"), + doc(include = "../docs/ifma-notes.md") +)] + +pub mod field; + +pub mod edwards; + +pub mod constants; diff --git a/src/backend/vector/mod.rs b/src/backend/vector/mod.rs new file mode 100644 index 0000000..d034e6c --- /dev/null +++ b/src/backend/vector/mod.rs @@ -0,0 +1,44 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +// Conditionally include the notes if: +// - we're on nightly (so we can include docs at all) +// - we're in stage 2 of the build. +// The latter point prevents a really silly and annoying problem, +// where the location of ".." is different depending on whether we're +// building the crate for real, or whether we're in build.rs +// generating the lookup tables (in which case we're relative to the +// location of build.rs, not lib.rs, so the markdown file appears +// missing). +#![cfg_attr( + all(feature = "nightly", feature = "stage2_build"), + doc(include = "../docs/parallel-formulas.md") +)] + +#[cfg(not(any(target_feature = "avx2", target_feature = "avx512ifma",)))] +compile_error!("simd_backend selected without target_feature=+avx2 or +avx512ifma"); + +#[cfg(any(all(target_feature = "avx2", not(target_feature = "avx512ifma")), rustdoc))] +#[doc(cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma"))))] +pub mod avx2; +#[cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma")))] +pub(crate) use self::avx2::{ + constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint, +}; + +#[cfg(any(target_feature = "avx512ifma", rustdoc))] +#[doc(cfg(target_feature = "avx512ifma"))] +pub mod ifma; +#[cfg(target_feature = "avx512ifma")] +pub(crate) use self::ifma::{ + constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint, +}; + +pub mod scalar_mul; diff --git a/src/backend/avx2/scalar_mul/mod.rs b/src/backend/vector/scalar_mul/mod.rs similarity index 81% rename from src/backend/avx2/scalar_mul/mod.rs rename to src/backend/vector/scalar_mul/mod.rs index d3d14e3..5c8734d 100644 --- a/src/backend/avx2/scalar_mul/mod.rs +++ b/src/backend/vector/scalar_mul/mod.rs @@ -10,7 +10,10 @@ pub mod variable_base; -#[cfg(feature = "stage2_build")] pub mod vartime_double_base; +#[cfg(feature = "alloc")] pub mod straus; + +#[cfg(feature = "alloc")] +pub mod precomputed_straus; diff --git a/src/backend/vector/scalar_mul/precomputed_straus.rs b/src/backend/vector/scalar_mul/precomputed_straus.rs new file mode 100644 index 0000000..49d1be4 --- /dev/null +++ b/src/backend/vector/scalar_mul/precomputed_straus.rs @@ -0,0 +1,111 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2019 Henry de Valence. +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +//! Precomputation for Straus's method. + +#![allow(non_snake_case)] + +use core::borrow::Borrow; + +use backend::vector::{CachedPoint, ExtendedPoint}; +use edwards::EdwardsPoint; +use scalar::Scalar; +use traits::Identity; +use traits::VartimePrecomputedMultiscalarMul; +use window::{NafLookupTable5, NafLookupTable8}; + +#[allow(unused_imports)] +use prelude::*; + + +pub struct VartimePrecomputedStraus { + static_lookup_tables: Vec>, +} + +impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self { + static_lookup_tables: static_points + .into_iter() + .map(|P| NafLookupTable8::::from(P.borrow())) + .collect(), + } + } + + fn optional_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Option + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator>, + { + let static_nafs = static_scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form(5)) + .collect::>(); + let dynamic_nafs: Vec<_> = dynamic_scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form(5)) + .collect::>(); + + let dynamic_lookup_tables = match dynamic_points + .into_iter() + .map(|P_opt| P_opt.map(|P| NafLookupTable5::::from(&P))) + .collect::>>() + { + Some(x) => x, + None => return None, + }; + + let sp = self.static_lookup_tables.len(); + let dp = dynamic_lookup_tables.len(); + assert_eq!(sp, static_nafs.len()); + assert_eq!(dp, dynamic_nafs.len()); + + // We could save some doublings by looking for the highest + // nonzero NAF coefficient, but since we might have a lot of + // them to search, it's not clear it's worthwhile to check. + let mut R = ExtendedPoint::identity(); + for j in (0..255).rev() { + R = R.double(); + + for i in 0..dp { + let t_ij = dynamic_nafs[i][j]; + if t_ij > 0 { + R = &R + &dynamic_lookup_tables[i].select(t_ij as usize); + } else if t_ij < 0 { + R = &R - &dynamic_lookup_tables[i].select(-t_ij as usize); + } + } + + for i in 0..sp { + let t_ij = static_nafs[i][j]; + if t_ij > 0 { + R = &R + &self.static_lookup_tables[i].select(t_ij as usize); + } else if t_ij < 0 { + R = &R - &self.static_lookup_tables[i].select(-t_ij as usize); + } + } + } + + Some(R.into()) + } +} diff --git a/src/backend/avx2/scalar_mul/straus.rs b/src/backend/vector/scalar_mul/straus.rs similarity index 95% rename from src/backend/avx2/scalar_mul/straus.rs rename to src/backend/vector/scalar_mul/straus.rs index eed9084..506693d 100644 --- a/src/backend/avx2/scalar_mul/straus.rs +++ b/src/backend/vector/scalar_mul/straus.rs @@ -14,10 +14,10 @@ use core::borrow::Borrow; use clear_on_drop::ClearOnDrop; -use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; +use backend::vector::{CachedPoint, ExtendedPoint}; use edwards::EdwardsPoint; use scalar::Scalar; -use scalar_mul::window::{LookupTable, NafLookupTable5}; +use window::{LookupTable, NafLookupTable5}; use traits::{Identity, MultiscalarMul, VartimeMultiscalarMul}; #[allow(unused_imports)] @@ -33,7 +33,6 @@ use prelude::*; /// point representation on the fly. pub struct Straus {} -#[cfg(feature = "alloc")] impl MultiscalarMul for Straus { type Point = EdwardsPoint; @@ -71,7 +70,6 @@ impl MultiscalarMul for Straus { } } -#[cfg(feature = "alloc")] impl VartimeMultiscalarMul for Straus { type Point = EdwardsPoint; diff --git a/src/backend/avx2/scalar_mul/variable_base.rs b/src/backend/vector/scalar_mul/variable_base.rs similarity index 91% rename from src/backend/avx2/scalar_mul/variable_base.rs rename to src/backend/vector/scalar_mul/variable_base.rs index 6f36bc7..f53c4a0 100644 --- a/src/backend/avx2/scalar_mul/variable_base.rs +++ b/src/backend/vector/scalar_mul/variable_base.rs @@ -1,10 +1,10 @@ #![allow(non_snake_case)] -use traits::Identity; -use scalar::Scalar; +use backend::vector::{CachedPoint, ExtendedPoint}; use edwards::EdwardsPoint; -use backend::avx2::edwards::{ExtendedPoint, CachedPoint}; -use scalar_mul::window::LookupTable; +use scalar::Scalar; +use traits::Identity; +use window::LookupTable; /// Perform constant-time, variable-base scalar multiplication. pub fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint { diff --git a/src/backend/avx2/scalar_mul/vartime_double_base.rs b/src/backend/vector/scalar_mul/vartime_double_base.rs similarity index 90% rename from src/backend/avx2/scalar_mul/vartime_double_base.rs rename to src/backend/vector/scalar_mul/vartime_double_base.rs index b3d6333..44d92f7 100644 --- a/src/backend/avx2/scalar_mul/vartime_double_base.rs +++ b/src/backend/vector/scalar_mul/vartime_double_base.rs @@ -9,12 +9,12 @@ // - Henry de Valence #![allow(non_snake_case)] -use traits::Identity; -use scalar::Scalar; +use backend::vector::BASEPOINT_ODD_LOOKUP_TABLE; +use backend::vector::{CachedPoint, ExtendedPoint}; use edwards::EdwardsPoint; -use scalar_mul::window::NafLookupTable5; -use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; -use backend::avx2::constants::BASEPOINT_ODD_LOOKUP_TABLE; +use scalar::Scalar; +use traits::Identity; +use window::NafLookupTable5; /// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { diff --git a/src/constants.rs b/src/constants.rs index 51833ac..b21ee3a 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -34,9 +34,9 @@ use montgomery::MontgomeryPoint; use scalar::Scalar; #[cfg(feature = "u64_backend")] -pub use backend::u64::constants::*; +pub use backend::serial::u64::constants::*; #[cfg(feature = "u32_backend")] -pub use backend::u32::constants::*; +pub use backend::serial::u32::constants::*; /// The Ed25519 basepoint, in `CompressedEdwardsY` format. /// @@ -151,9 +151,9 @@ mod test { #[test] #[cfg(feature = "u32_backend")] fn test_d_vs_ratio() { - use backend::u32::field::FieldElement32; - let a = -&FieldElement32([121665,0,0,0,0,0,0,0,0,0]); - let b = FieldElement32([121666,0,0,0,0,0,0,0,0,0]); + use backend::serial::u32::field::FieldElement2625; + let a = -&FieldElement2625([121665,0,0,0,0,0,0,0,0,0]); + let b = FieldElement2625([121666,0,0,0,0,0,0,0,0,0]); let d = &a * &b.invert(); let d2 = &d + &d; assert_eq!(d, constants::EDWARDS_D); @@ -164,9 +164,9 @@ mod test { #[test] #[cfg(feature = "u64_backend")] fn test_d_vs_ratio() { - use backend::u64::field::FieldElement64; - let a = -&FieldElement64([121665,0,0,0,0]); - let b = FieldElement64([121666,0,0,0,0]); + use backend::serial::u64::field::FieldElement51; + let a = -&FieldElement51([121665,0,0,0,0]); + let b = FieldElement51([121666,0,0,0,0]); let d = &a * &b.invert(); let d2 = &d + &d; assert_eq!(d, constants::EDWARDS_D); diff --git a/src/edwards.rs b/src/edwards.rs index b750573..10b4614 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -112,23 +112,34 @@ use scalar::Scalar; use montgomery::MontgomeryPoint; -use curve_models::ProjectivePoint; -use curve_models::CompletedPoint; -use curve_models::AffineNielsPoint; -use curve_models::ProjectiveNielsPoint; +use backend::serial::curve_models::AffineNielsPoint; +use backend::serial::curve_models::CompletedPoint; +use backend::serial::curve_models::ProjectiveNielsPoint; +use backend::serial::curve_models::ProjectivePoint; + +use window::LookupTable; #[allow(unused_imports)] use prelude::*; -use scalar_mul::window::LookupTable; - -use traits::{Identity, IsIdentity}; use traits::ValidityCheck; +use traits::{Identity, IsIdentity}; #[cfg(any(feature = "alloc", feature = "std"))] use traits::MultiscalarMul; #[cfg(any(feature = "alloc", feature = "std"))] -use traits::VartimeMultiscalarMul; +use traits::{VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul}; + +#[cfg(not(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +)))] +use backend::serial::scalar_mul; +#[cfg(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +))] +use backend::vector::scalar_mul; // ------------------------------------------------------------------------ // Compressed points @@ -439,6 +450,10 @@ impl EdwardsPoint { /// Convert this `EdwardsPoint` on the Edwards model to the /// corresponding `MontgomeryPoint` on the Montgomery model. /// + /// This function has one exceptional case; the identity point of + /// the Edwards curve is sent to the 2-torsion point \\((0,0)\\) + /// on the Montgomery curve. + /// /// Note that this is a one-way conversion, since the Montgomery /// model does not retain sign information. pub fn to_montgomery(&self) -> MontgomeryPoint { @@ -446,7 +461,7 @@ impl EdwardsPoint { // // The denominator is zero only when y=1, the identity point of // the Edwards curve. Since 0.invert() = 0, in this case we - // compute u = 0, the identity point of the Montgomery line. + // compute the 2-torsion point (0,0). let U = &self.Z + &self.Y; let W = &self.Z - &self.Y; let u = &U * &W.invert(); @@ -576,18 +591,7 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsPoint { /// For scalar multiplication of a basepoint, /// `EdwardsBasepointTable` is approximately 4x faster. fn mul(self, scalar: &'b Scalar) -> EdwardsPoint { - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - { - use backend::avx2::scalar_mul::variable_base::mul; - mul(self, scalar) - } - // Otherwise, use the serial backend: - #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - { - use scalar_mul::variable_base::mul; - mul(self, scalar) - } + scalar_mul::variable_base::mul(self, scalar) } } @@ -613,7 +617,7 @@ impl<'a, 'b> Mul<&'b EdwardsPoint> for &'a Scalar { #[cfg(feature = "alloc")] impl MultiscalarMul for EdwardsPoint { type Point = EdwardsPoint; - + fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint where I: IntoIterator, @@ -638,21 +642,14 @@ impl MultiscalarMul for EdwardsPoint { // size-dependent algorithm dispatch, use this as the hint. let _size = s_lo; - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::avx2::scalar_mul::straus::Straus; - // Otherwise, proceed as normal: - #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use scalar_mul::straus::Straus; - - Straus::multiscalar_mul(scalars, points) + scalar_mul::straus::Straus::multiscalar_mul(scalars, points) } } #[cfg(feature = "alloc")] impl VartimeMultiscalarMul for EdwardsPoint { type Point = EdwardsPoint; - + fn optional_multiscalar_mul(scalars: I, points: J) -> Option where I: IntoIterator, @@ -676,29 +673,56 @@ impl VartimeMultiscalarMul for EdwardsPoint { // size-dependent algorithm dispatch, use this as the hint. let _size = s_lo; - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::avx2::scalar_mul::straus::Straus; - // Otherwise, proceed as normal: - #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use scalar_mul::straus::Straus; + scalar_mul::straus::Straus::optional_multiscalar_mul(scalars, points) + } +} - Straus::optional_multiscalar_mul(scalars, points) +/// Precomputation for variable-time multiscalar multiplication with `EdwardsPoint`s. +// This wraps the inner implementation in a facade type so that we can +// decouple stability of the inner type from the stability of the +// outer type. +#[cfg(feature = "alloc")] +pub struct VartimeEdwardsPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus); + +#[cfg(feature = "alloc")] +impl VartimePrecomputedMultiscalarMul for VartimeEdwardsPrecomputation { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self(scalar_mul::precomputed_straus::VartimePrecomputedStraus::new(static_points)) + } + + fn optional_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Option + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator>, + { + self.0 + .optional_mixed_multiscalar_mul(static_scalars, dynamic_scalars, dynamic_points) } } impl EdwardsPoint { /// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. #[cfg(feature = "stage2_build")] - pub fn vartime_double_scalar_mul_basepoint(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::avx2::scalar_mul::vartime_double_base; - // Otherwise, use the serial backend: - #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use scalar_mul::vartime_double_base; - - vartime_double_base::mul(a, A, b) + pub fn vartime_double_scalar_mul_basepoint( + a: &Scalar, + A: &EdwardsPoint, + b: &Scalar, + ) -> EdwardsPoint { + scalar_mul::vartime_double_base::mul(a, A, b) } } @@ -1220,6 +1244,49 @@ mod test { assert!(P1.compress().to_bytes() == P2.compress().to_bytes()); } + #[test] + fn vartime_precomputed_vs_nonprecomputed_multiscalar() { + let mut rng = rand::thread_rng(); + + let B = &::constants::ED25519_BASEPOINT_TABLE; + + let static_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let dynamic_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let check_scalar: Scalar = static_scalars + .iter() + .chain(dynamic_scalars.iter()) + .map(|s| s * s) + .sum(); + + let static_points = static_scalars.iter().map(|s| s * B).collect::>(); + let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::>(); + + let precomputation = VartimeEdwardsPrecomputation::new(static_points.iter()); + + let P = precomputation.vartime_mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ); + + use traits::VartimeMultiscalarMul; + let Q = EdwardsPoint::vartime_multiscalar_mul( + static_scalars.iter().chain(dynamic_scalars.iter()), + static_points.iter().chain(dynamic_points.iter()), + ); + + let R = &check_scalar * B; + + assert_eq!(P.compress(), R.compress()); + assert_eq!(Q.compress(), R.compress()); + } + mod vartime { use super::super::*; use super::{A_SCALAR, B_SCALAR, A_TIMES_BASEPOINT, DOUBLE_SCALAR_MULT_RESULT}; diff --git a/src/field.rs b/src/field.rs index 4448695..e154f24 100644 --- a/src/field.rs +++ b/src/field.rs @@ -12,8 +12,8 @@ //! //! The `curve25519_dalek::field` module provides a type alias //! `curve25519_dalek::field::FieldElement` to a field element type -//! defined in the `backend` module; either `FieldElement64` or -//! `FieldElement32`. +//! defined in the `backend` module; either `FieldElement51` or +//! `FieldElement2625`. //! //! Field operations defined in terms of machine //! operations, such as field multiplication or squaring, are defined in @@ -33,24 +33,24 @@ use constants; use backend; #[cfg(feature = "u64_backend")] -pub use backend::u64::field::*; +pub use backend::serial::u64::field::*; /// A `FieldElement` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// /// The `FieldElement` type is an alias for one of the platform-specific /// implementations. #[cfg(feature = "u64_backend")] -pub type FieldElement = backend::u64::field::FieldElement64; +pub type FieldElement = backend::serial::u64::field::FieldElement51; #[cfg(feature = "u32_backend")] -pub use backend::u32::field::*; +pub use backend::serial::u32::field::*; /// A `FieldElement` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// /// The `FieldElement` type is an alias for one of the platform-specific /// implementations. #[cfg(feature = "u32_backend")] -pub type FieldElement = backend::u32::field::FieldElement32; +pub type FieldElement = backend::serial::u32::field::FieldElement2625; impl Eq for FieldElement {} @@ -255,7 +255,7 @@ impl FieldElement { (was_nonzero_square, r) } - /// Attempt to compute `1/sqrt(self)` in constant time. + /// Attempt to compute `sqrt(1/self)` in constant time. /// /// Convenience wrapper around `sqrt_ratio_i`. /// @@ -265,7 +265,7 @@ impl FieldElement { /// /// - `(Choice(1), +sqrt(1/self)) ` if `self` is a nonzero square; /// - `(Choice(0), zero) ` if `self` is zero; - /// - `(Choice(0), +sqrt(i*u/v)) ` if `self` is a nonzero nonsquare; + /// - `(Choice(0), +sqrt(i/self)) ` if `self` is a nonzero nonsquare; /// pub fn invsqrt(&self) -> (Choice, FieldElement) { FieldElement::sqrt_ratio_i(&FieldElement::one(), self) @@ -280,7 +280,7 @@ mod test { /// Random element a of GF(2^255-19), from Sage /// a = 1070314506888354081329385823235218444233221\ /// 2228051251926706380353716438957572 - pub static A_BYTES: [u8; 32] = + static A_BYTES: [u8; 32] = [ 0x04, 0xfe, 0xdf, 0x98, 0xa7, 0xfa, 0x0a, 0x68, 0x84, 0x92, 0xbd, 0x59, 0x08, 0x07, 0xa7, 0x03, 0x9e, 0xd1, 0xf6, 0xf2, 0xe1, 0xd9, 0xe2, 0xa4, diff --git a/src/lib.rs b/src/lib.rs index 4f52c96..07c3bef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,11 +9,17 @@ // - Henry de Valence #![no_std] - +#![cfg_attr( + any( + all(feature = "simd_backend", target_feature = "avx512ifma"), + all(feature = "nightly", rustdoc) + ), + feature(simd_ffi, link_llvm_intrinsics) +)] +#![cfg_attr(feature = "nightly", feature(test))] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] -#![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "nightly", feature(external_doc))] - +#![cfg_attr(feature = "nightly", feature(doc_cfg))] // Refuse to compile if documentation is missing, but only on nightly. // // This means that missing docs will still fail CI, but means we can use @@ -38,21 +44,23 @@ extern crate alloc; #[macro_use] extern crate std; -#[cfg(all(feature = "nightly", feature = "avx2_backend"))] +#[cfg(all(feature = "nightly", feature = "packed_simd"))] extern crate packed_simd; -extern crate rand; -extern crate clear_on_drop; extern crate byteorder; +extern crate clear_on_drop; pub extern crate digest; +extern crate rand_core; +#[cfg(all(test, feature = "stage2_build"))] +extern crate rand_os; // Used for traits related to constant-time code. extern crate subtle; -#[cfg(feature = "serde")] -extern crate serde; #[cfg(all(test, feature = "serde"))] extern crate bincode; +#[cfg(feature = "serde")] +extern crate serde; // Internal macros. Must come first! #[macro_use] @@ -90,11 +98,8 @@ pub(crate) mod field; // Arithmetic backends (using u32, u64, etc) live here pub(crate) mod backend; -// Internal curve models which are not part of the public API. -pub(crate) mod curve_models; - // Crate-local prelude (for alloc-dependent features like `Vec`) pub(crate) mod prelude; -// Implementations of scalar mul algorithms live here -pub(crate) mod scalar_mul; +// Generic code for window lookups +pub(crate) mod window; diff --git a/src/montgomery.rs b/src/montgomery.rs index da3c9a5..0cd6f8a 100644 --- a/src/montgomery.rs +++ b/src/montgomery.rs @@ -305,7 +305,7 @@ mod test { use super::*; #[cfg(feature = "rand")] - use rand::rngs::OsRng; + use rand_os::OsRng; /// Test Montgomery -> Edwards on the X/Ed25519 basepoint #[test] diff --git a/src/ristretto.rs b/src/ristretto.rs index dd1b3a5..a7b0a27 100644 --- a/src/ristretto.rs +++ b/src/ristretto.rs @@ -164,7 +164,7 @@ use core::ops::{Add, Neg, Sub}; use core::ops::{AddAssign, SubAssign}; use core::ops::{Mul, MulAssign}; -use rand::{CryptoRng, Rng}; +use rand_core::{CryptoRng, RngCore}; use digest::generic_array::typenum::U64; use digest::Digest; @@ -185,11 +185,20 @@ use prelude::*; use scalar::Scalar; -use curve_models::CompletedPoint; - use traits::Identity; #[cfg(any(feature = "alloc", feature = "std"))] -use traits::{MultiscalarMul, VartimeMultiscalarMul}; +use traits::{MultiscalarMul, VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul}; + +#[cfg(not(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +)))] +use backend::serial::scalar_mul; +#[cfg(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +))] +use backend::vector::scalar_mul; // ------------------------------------------------------------------------ // Compressed points @@ -468,8 +477,8 @@ impl RistrettoPoint { /// ``` /// # extern crate curve25519_dalek; /// # use curve25519_dalek::ristretto::RistrettoPoint; - /// extern crate rand; - /// use rand::rngs::OsRng; + /// extern crate rand_os; + /// use rand_os::OsRng; /// /// # // Need fn main() here in comment so the doctest compiles /// # // See https://doc.rust-lang.org/book/documentation.html#documentation-as-tests @@ -602,6 +611,8 @@ impl RistrettoPoint { let N_t = &(&(&c * &(&r - &one)) * &d_minus_one_sq) - &D; let s_sq = s.square(); + use backend::serial::curve_models::CompletedPoint; + // The conversion from W_i is exactly the conversion from P1xP1. RistrettoPoint(CompletedPoint{ X: &(&s + &s) * &D, @@ -615,7 +626,7 @@ impl RistrettoPoint { /// /// # Inputs /// - /// * `rng`: any RNG which implements the `rand::Rng` interface. + /// * `rng`: any RNG which implements the `RngCore + CryptoRng` interface. /// /// # Returns /// @@ -627,9 +638,9 @@ impl RistrettoPoint { /// discrete log of the output point with respect to any other /// point should be unknown. The map is applied twice and the /// results are added, to ensure a uniform distribution. - pub fn random(rng: &mut T) -> Self { + pub fn random(mut rng: T) -> Self { let mut uniform_bytes = [0u8; 64]; - rng.fill(&mut uniform_bytes); + rng.fill_bytes(&mut uniform_bytes); RistrettoPoint::from_uniform_bytes(&uniform_bytes) } @@ -891,8 +902,53 @@ impl VartimeMultiscalarMul for RistrettoPoint { { let extended_points = points.into_iter().map(|opt_P| opt_P.map(|P| P.borrow().0)); - EdwardsPoint::optional_multiscalar_mul(scalars, extended_points) - .map(|P| RistrettoPoint(P)) + EdwardsPoint::optional_multiscalar_mul(scalars, extended_points).map(|P| RistrettoPoint(P)) + } +} + +/// Precomputation for variable-time multiscalar multiplication with `RistrettoPoint`s. +// This wraps the inner implementation in a facade type so that we can +// decouple stability of the inner type from the stability of the +// outer type. +#[cfg(feature = "alloc")] +pub struct VartimeRistrettoPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus); + +#[cfg(feature = "alloc")] +impl VartimePrecomputedMultiscalarMul for VartimeRistrettoPrecomputation { + type Point = RistrettoPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self( + scalar_mul::precomputed_straus::VartimePrecomputedStraus::new( + static_points.into_iter().map(|P| P.borrow().0), + ), + ) + } + + fn optional_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Option + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator>, + { + self.0 + .optional_mixed_multiscalar_mul( + static_scalars, + dynamic_scalars, + dynamic_points.into_iter().map(|P_opt| P_opt.map(|P| P.0)), + ) + .map(|P_ed| RistrettoPoint(P_ed)) } } @@ -1020,7 +1076,7 @@ impl Debug for RistrettoPoint { #[cfg(all(test, feature = "stage2_build"))] mod test { #[cfg(feature = "rand")] - use rand::rngs::OsRng; + use rand_os::OsRng; use scalar::Scalar; use constants; @@ -1263,4 +1319,47 @@ mod test { P.compress(); } } + + #[test] + fn vartime_precomputed_vs_nonprecomputed_multiscalar() { + let mut rng = rand::thread_rng(); + + let B = &::constants::RISTRETTO_BASEPOINT_TABLE; + + let static_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let dynamic_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let check_scalar: Scalar = static_scalars + .iter() + .chain(dynamic_scalars.iter()) + .map(|s| s * s) + .sum(); + + let static_points = static_scalars.iter().map(|s| s * B).collect::>(); + let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::>(); + + let precomputation = VartimeRistrettoPrecomputation::new(static_points.iter()); + + let P = precomputation.vartime_mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ); + + use traits::VartimeMultiscalarMul; + let Q = RistrettoPoint::vartime_multiscalar_mul( + static_scalars.iter().chain(dynamic_scalars.iter()), + static_points.iter().chain(dynamic_points.iter()), + ); + + let R = &check_scalar * B; + + assert_eq!(P.compress(), R.compress()); + assert_eq!(Q.compress(), R.compress()); + } } diff --git a/src/scalar.rs b/src/scalar.rs index 2e4d256..81a78ee 100644 --- a/src/scalar.rs +++ b/src/scalar.rs @@ -151,7 +151,7 @@ use core::ops::{Sub, SubAssign}; #[allow(unused_imports)] use prelude::*; -use rand::{CryptoRng, Rng}; +use rand_core::{CryptoRng, RngCore}; use digest::generic_array::typenum::U64; use digest::Digest; @@ -168,14 +168,14 @@ use constants; /// This is a type alias for one of the scalar types in the `backend` /// module. #[cfg(feature = "u64_backend")] -type UnpackedScalar = backend::u64::scalar::Scalar64; +type UnpackedScalar = backend::serial::u64::scalar::Scalar52; /// An `UnpackedScalar` represents an element of the field GF(l), optimized for speed. /// /// This is a type alias for one of the scalar types in the `backend` /// module. #[cfg(feature = "u32_backend")] -type UnpackedScalar = backend::u32::scalar::Scalar32; +type UnpackedScalar = backend::serial::u32::scalar::Scalar29; /// The `Scalar` struct holds an integer \\(s < 2\^{255} \\) which @@ -507,7 +507,7 @@ impl Scalar { /// /// # Inputs /// - /// * `rng`: any RNG which implements the `rand::CryptoRng` interface. + /// * `rng`: any RNG which implements the `RngCore + CryptoRng` interface. /// /// # Returns /// @@ -516,20 +516,20 @@ impl Scalar { /// # Example /// /// ``` - /// extern crate rand; + /// extern crate rand_os; /// # extern crate curve25519_dalek; /// # /// # fn main() { /// use curve25519_dalek::scalar::Scalar; /// - /// use rand::OsRng; + /// use rand_os::OsRng; /// /// let mut csprng: OsRng = OsRng::new().unwrap(); /// let a: Scalar = Scalar::random(&mut csprng); /// # } - pub fn random(rng: &mut T) -> Self { + pub fn random(mut rng: T) -> Self { let mut scalar_bytes = [0u8; 64]; - rng.fill(&mut scalar_bytes); + rng.fill_bytes(&mut scalar_bytes); Scalar::from_bytes_mod_order_wide(&scalar_bytes) } diff --git a/src/traits.rs b/src/traits.rs index 8db963a..5297aa4 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -219,11 +219,149 @@ pub trait VartimeMultiscalarMul { { Self::optional_multiscalar_mul( scalars, - points.into_iter().map(|P| Some(P.borrow().clone())) - ).unwrap() + points.into_iter().map(|P| Some(P.borrow().clone())), + ) + .unwrap() } } +/// A trait for variable-time multiscalar multiplication with precomputation. +/// +/// A general multiscalar multiplication with precomputation can be written as +/// $$ +/// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, +/// $$ +/// where the \\(B_i\\) are *static* points, for which precomputation +/// is possible, and the \\(A_j\\) are *dynamic* points, for which +/// precomputation is not possible. +/// +/// This trait has three methods for performing this computation: +/// +/// * [`vartime_multiscalar_mul`], which handles the special case +/// where \\(n = 0\\) and there are no dynamic points; +/// +/// * [`vartime_mixed_multiscalar_mul`], which takes the dynamic +/// points as already-validated `Point`s and is infallible; +/// +/// * [`optional_mixed_multiscalar_mul`], which takes the dynamic +/// points as `Option`s and returns an `Option`, +/// allowing decompression to be composed into the input iterators. +/// +/// All methods require that the lengths of the input iterators be +/// known and matching, as if they were `ExactSizeIterator`s. (It +/// does not require `ExactSizeIterator` only because that trait is +/// broken). +pub trait VartimePrecomputedMultiscalarMul: Sized { + /// The type of point to be multiplied, e.g., `RistrettoPoint`. + type Point: Clone; + + /// Given the static points \\( B_i \\), perform precomputation + /// and return the precomputation data. + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow; + + /// Given `static_scalars`, an iterator of public scalars + /// \\(b_i\\), compute + /// $$ + /// Q = b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// The trait bound aims for maximum flexibility: the input must + /// be convertable to iterators (`I: IntoIter`), and the + /// iterator's items must be `Borrow`, to allow iterators + /// returning either `Scalar`s or `&Scalar`s. + fn vartime_multiscalar_mul(&self, static_scalars: I) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + { + use core::iter; + + Self::vartime_mixed_multiscalar_mul( + self, + static_scalars, + iter::empty::(), + iter::empty::(), + ) + } + + /// Given `static_scalars`, an iterator of public scalars + /// \\(b_i\\), `dynamic_scalars`, an iterator of public scalars + /// \\(a_i\\), and `dynamic_points`, an iterator of points + /// \\(A_i\\), compute + /// $$ + /// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// The trait bound aims for maximum flexibility: the inputs must be + /// convertable to iterators (`I: IntoIter`), and the iterator's items + /// must be `Borrow` (or `Borrow`), to allow + /// iterators returning either `Scalar`s or `&Scalar`s. + fn vartime_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + Self::optional_mixed_multiscalar_mul( + self, + static_scalars, + dynamic_scalars, + dynamic_points.into_iter().map(|P| Some(P.borrow().clone())), + ) + .unwrap() + } + + /// Given `static_scalars`, an iterator of public scalars + /// \\(b_i\\), `dynamic_scalars`, an iterator of public scalars + /// \\(a_i\\), and `dynamic_points`, an iterator of points + /// \\(A_i\\), compute + /// $$ + /// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// If any of the dynamic points were `None`, return `None`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// This function is particularly useful when verifying statements + /// involving compressed points. Accepting `Option` allows + /// inlining point decompression into the multiscalar call, + /// avoiding the need for temporary buffers. + fn optional_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Option + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator>; +} + // ------------------------------------------------------------------------ // Private Traits // ------------------------------------------------------------------------ diff --git a/src/scalar_mul/window.rs b/src/window.rs similarity index 98% rename from src/scalar_mul/window.rs rename to src/window.rs index d5ad2d6..08f2d78 100644 --- a/src/scalar_mul/window.rs +++ b/src/window.rs @@ -22,8 +22,8 @@ use subtle::Choice; use traits::Identity; use edwards::EdwardsPoint; -use curve_models::ProjectiveNielsPoint; -use curve_models::AffineNielsPoint; +use backend::serial::curve_models::ProjectiveNielsPoint; +use backend::serial::curve_models::AffineNielsPoint; /// A lookup table of precomputed multiples of a point \\(P\\), used to /// compute \\( xP \\) for \\( -8 \leq x \leq 8 \\).