From d6b83894286dee4e8bd56b6b7ffa8986579afc13 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sun, 25 Mar 2018 17:10:30 -0700 Subject: [PATCH 01/16] Use `criterion.rs` instead of libtest for benchmarks. Since Criterion can only benchmark public API, these changes just drop all internal benchmarks (e.g., benchmarks for field operations). But those are usually microbenchmarks whose meaning is kind of questionable anyways, so I don't think this is a big loss. The `bench` feature disappears, since Criterion works on stable Rust. --- .travis.yml | 12 +-- Cargo.toml | 6 +- README.md | 10 +- benches/dalek_benchmarks.rs | 207 ++++++++++++++++++++++++++++++++++++ src/backend/avx2/edwards.rs | 118 -------------------- src/backend/avx2/field.rs | 16 --- src/backend/u32/scalar.rs | 35 ------ src/backend/u64/scalar.rs | 35 ------ src/edwards.rs | 150 -------------------------- src/field.rs | 47 -------- src/lib.rs | 4 - src/montgomery.rs | 19 ---- src/ristretto.rs | 51 --------- src/scalar.rs | 29 ----- 14 files changed, 223 insertions(+), 516 deletions(-) create mode 100644 benches/dalek_benchmarks.rs diff --git a/.travis.yml b/.travis.yml index a598e32..56cad2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,8 @@ env: - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='serde' - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='nightly' - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='yolocrypto nightly' - - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly bench' - - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly bench' + - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly' + - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly' - TEST_COMMAND=build EXTRA_FLAGS=--no-default-features FEATURES='' matrix: @@ -21,13 +21,13 @@ matrix: # run benchmarks, which causes dalek not to build on stable. See # https://github.com/isislovecruft/curve25519-dalek/pull/38#issuecomment-286027562 - rust: stable - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly bench' + env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly' - rust: beta - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly bench' + env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly' - rust: stable - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly bench' + env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly' - rust: beta - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly bench' + env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly' # Test nightly features, such as radix_51, only on nightly. - rust: stable env: TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='nightly' diff --git a/Cargo.toml b/Cargo.toml index c9ff5ee..72eda7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,11 @@ travis-ci = { repository = "dalek-cryptography/curve25519-dalek", branch = "mast [dev-dependencies] sha2 = "0.7" serde_cbor = "0.6" +criterion = "0.2" + +[[bench]] +name = "dalek_benchmarks" +harness = false # Note: we generate precomputed tables by building the crate twice: once as # part of build.rs, and then once "for real". @@ -60,7 +65,6 @@ default = ["std"] std = ["rand", "subtle/std"] alloc = [] yolocrypto = ["avx2_backend"] -bench = [] # Radix-51 arithmetic using u128 radix_51 = [] # Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation. diff --git a/README.md b/README.md index 1ac09aa..1ac0bfe 100644 --- a/README.md +++ b/README.md @@ -66,13 +66,12 @@ Curve arithmetic is implemented using one of the following backends: * an experimental AVX2 backend, available using the `yolocrypto` feature when compiling for a target with `target_feature=+avx2`. -By default, the benchmarks are not compiled without the `bench` -feature. Benchmarks can be run via: +Benchmarks are run using [`criterion.rs`][criterion]: ```sh -cargo bench --features="bench" # u32 backend -cargo bench --features="bench nightly" # u64 backend -cargo bench --features="bench nightly yolocrypto" # u64 or avx2 if available +cargo bench # u32 backend +cargo bench --features="nightly" # u64 backend +cargo bench --features="nightly yolocrypto" # u64 or avx2 if available ``` The `yolocrypto` feature enables experimental features. The name `yolocrypto` @@ -120,3 +119,4 @@ contributions. [contributing]: https://github.com/dalek-cryptography/curve25519-dalek/blob/master/CONTRIBUTING.md [docs-external]: https://doc.dalek.rs/curve25519_dalek/ [docs-internal]: https://doc-internal.dalek.rs/curve25519_dalek/ +[criterion]: https://github.com/japaric/criterion.rs diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs new file mode 100644 index 0000000..81c9056 --- /dev/null +++ b/benches/dalek_benchmarks.rs @@ -0,0 +1,207 @@ +#![allow(non_snake_case)] + +extern crate rand; +use rand::OsRng; + +#[macro_use] +extern crate criterion; + +use criterion::Criterion; + +extern crate curve25519_dalek; + +use curve25519_dalek::constants; +use curve25519_dalek::scalar::Scalar; + +static MULTISCALAR_SIZES: [usize; 13] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 384, 512, 768, 1024]; + +mod edwards_benches { + use super::*; + use curve25519_dalek::edwards::{self, EdwardsPoint}; + + fn compress(c: &mut Criterion) { + let B = &constants::ED25519_BASEPOINT_POINT; + c.bench_function("EdwardsPoint compression", move |b| { + b.iter(|| B.compress()) + }); + } + + fn decompress(c: &mut Criterion) { + let B_comp = &constants::ED25519_BASEPOINT_COMPRESSED; + c.bench_function("EdwardsPoint decompression", move |b| { + b.iter(|| B_comp.decompress().unwrap()) + }); + } + + fn consttime_fixed_base_scalar_mul(c: &mut Criterion) { + let B = &constants::ED25519_BASEPOINT_TABLE; + let s = Scalar::from_u64(897987897).invert(); + c.bench_function("Constant-time fixed-base scalar mul", move |b| { + b.iter(|| B * &s) + }); + } + + fn consttime_variable_base_scalar_mul(c: &mut Criterion) { + let B = &constants::ED25519_BASEPOINT_POINT; + let s = Scalar::from_u64(897987897).invert(); + c.bench_function("Constant-time variable-base scalar mul", move |b| { + b.iter(|| B * &s) + }); + } + + fn vartime_double_base_scalar_mul(c: &mut Criterion) { + c.bench_function("Variable-time aA+bB, A variable, B fixed", |bench| { + let B = &constants::ED25519_BASEPOINT_POINT; + let a = Scalar::from_u64(298374928).invert(); + let b = Scalar::from_u64(897987897).invert(); + let A = B * (b * a); + bench.iter(|| edwards::vartime::double_scalar_mul_basepoint(&a, &A, &b)); + }); + } + + fn consttime_multiscalar_mul(c: &mut Criterion) { + c.bench_function_over_inputs( + "Constant-time variable-base multiscalar multiplication", + |b, &&size| { + let mut rng = OsRng::new().unwrap(); + let scalars: Vec = (0..size).map(|_| Scalar::random(&mut rng)).collect(); + let points: Vec = scalars + .iter() + .map(|s| s * &constants::ED25519_BASEPOINT_TABLE) + .collect(); + b.iter(|| edwards::multiscalar_mul(&scalars, &points)); + }, + &MULTISCALAR_SIZES, + ); + } + + fn vartime_multiscalar_mul(c: &mut Criterion) { + c.bench_function_over_inputs( + "Variable-time variable-base multiscalar multiplication", + |b, &&size| { + let mut rng = OsRng::new().unwrap(); + let scalars: Vec = (0..size).map(|_| Scalar::random(&mut rng)).collect(); + let points: Vec = scalars + .iter() + .map(|s| s * &constants::ED25519_BASEPOINT_TABLE) + .collect(); + b.iter(|| edwards::vartime::multiscalar_mul(&scalars, &points)); + }, + &MULTISCALAR_SIZES, + ); + } + + criterion_group!{ + name = edwards_benches; + config = Criterion::default(); + targets = + compress, + decompress, + consttime_fixed_base_scalar_mul, + consttime_variable_base_scalar_mul, + vartime_double_base_scalar_mul, + consttime_multiscalar_mul, + vartime_multiscalar_mul, + } +} + +mod ristretto_benches { + use super::*; + use curve25519_dalek::ristretto::RistrettoPoint; + + fn compress(c: &mut Criterion) { + c.bench_function("RistrettoPoint compression", |b| { + let B = &constants::RISTRETTO_BASEPOINT_POINT; + b.iter(|| B.compress()) + }); + } + + fn decompress(c: &mut Criterion) { + c.bench_function("RistrettoPoint decompression", |b| { + let B_comp = &constants::RISTRETTO_BASEPOINT_COMPRESSED; + b.iter(|| B_comp.decompress().unwrap()) + }); + } + + fn double_and_compress_batch(c: &mut Criterion) { + c.bench_function_over_inputs( + "Batch Ristretto double-and-encode", + |b, &&size| { + let mut rng = OsRng::new().unwrap(); + let points: Vec = (0..size) + .map(|_| RistrettoPoint::random(&mut rng)) + .collect(); + b.iter(|| RistrettoPoint::double_and_compress_batch(&points)); + }, + &MULTISCALAR_SIZES, + ); + } + + criterion_group!{ + name = ristretto_benches; + config = Criterion::default(); + targets = + compress, + decompress, + double_and_compress_batch, + } +} + +mod montgomery_benches { + use super::*; + + fn montgomery_ladder(c: &mut Criterion) { + c.bench_function("Montgomery pseudomultiplication", |b| { + let B = constants::X25519_BASEPOINT; + let s = Scalar::from_u64(897987897).invert(); + b.iter(|| B * s); + }); + } + + criterion_group!{ + name = montgomery_benches; + config = Criterion::default(); + targets = montgomery_ladder, + } +} + +mod scalar_benches { + use super::*; + + fn scalar_inversion(c: &mut Criterion) { + c.bench_function("Scalar inversion", |b| { + let s = Scalar::from_u64(897987897).invert(); + b.iter(|| s.invert()); + }); + } + + fn batch_scalar_inversion(c: &mut Criterion) { + c.bench_function_over_inputs( + "Batch scalar inversion", + |b, &&size| { + let mut rng = OsRng::new().unwrap(); + let scalars: Vec = (0..size).map(|_| Scalar::random(&mut rng)).collect(); + b.iter(|| { + let mut s = scalars.clone(); + Scalar::batch_invert(&mut s); + }); + }, + &MULTISCALAR_SIZES, + ); + } + + criterion_group!{ + name = scalar_benches; + config = Criterion::default(); + targets = + scalar_inversion, + batch_scalar_inversion, + } +} + +criterion_main!( + scalar_benches::scalar_benches, + montgomery_benches::montgomery_benches, + ristretto_benches::ristretto_benches, + edwards_benches::edwards_benches, +); diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 575619a..7f564d7 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -919,121 +919,3 @@ mod test { } } } - -#[cfg(all(test, feature = "bench"))] -mod bench { - use test::Bencher; - use rand::OsRng; - use super::*; - - use constants; - use scalar::Scalar; - - #[bench] - fn conversion_into__avx2_format(b: &mut Bencher) { - let B = constants::ED25519_BASEPOINT_POINT; - - b.iter(|| ExtendedPoint::from(B)); - } - - #[bench] - fn conversion_outof_avx2_format(b: &mut Bencher) { - let B = constants::ED25519_BASEPOINT_POINT; - let B_avx2 = ExtendedPoint::from(B); - - b.iter(|| edwards::EdwardsPoint::from(B_avx2)); - } - - #[bench] - fn point_readdition(b: &mut Bencher) { - let B = &constants::ED25519_BASEPOINT_TABLE; - let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); - let Q = ExtendedPoint::from(B * &Scalar::from_u64(98932328)); - let Q_cached = CachedPoint::from(Q); - - b.iter(|| &P + &Q_cached ); - } - - #[bench] - fn point_addition(b: &mut Bencher) { - let B = &constants::ED25519_BASEPOINT_TABLE; - let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); - let Q = ExtendedPoint::from(B * &Scalar::from_u64(98932328)); - - b.iter(|| &P + &Q ); - } - - #[bench] - fn point_doubling(b: &mut Bencher) { - let B = &constants::ED25519_BASEPOINT_TABLE; - let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); - - b.iter(|| P.double() ); - } - - #[bench] - fn scalar_mul(b: &mut Bencher) { - let B = &constants::ED25519_BASEPOINT_TABLE; - let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); - let s = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); - - b.iter(|| &P * &s ); - } - - #[bench] - fn basepoint_table_creation(b: &mut Bencher) { - let B = ExtendedPoint::from(constants::ED25519_BASEPOINT_POINT); - - b.iter(|| EdwardsBasepointTable::create(&B) ); - } - - #[bench] - fn basepoint_mult(b: &mut Bencher) { - let B = ExtendedPoint::from(constants::ED25519_BASEPOINT_POINT); - let table = EdwardsBasepointTable::create(&B); - let s = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); - - b.iter(|| &table * &s ); - } - - #[bench] - fn ten_fold_scalar_mul(b: &mut Bencher) { - let mut csprng: OsRng = OsRng::new().unwrap(); - // Create 10 random scalars - let scalars: Vec<_> = (0..10).map(|_| Scalar::random(&mut csprng)).collect(); - // Create 10 points (by doing scalar mults) - let B = &constants::ED25519_BASEPOINT_TABLE; - let points: Vec<_> = scalars.iter().map(|s| B * s).collect(); - - b.iter(|| multiscalar_mul(&scalars, &points)); - } - - mod vartime { - use super::super::*; - use super::{constants, Bencher, OsRng}; - - #[bench] - fn double_scalar_mul(b: &mut Bencher) { - let mut csprng: OsRng = OsRng::new().unwrap(); - // Create 2 random scalars - let s1 = Scalar::random(&mut csprng); - let s2 = Scalar::random(&mut csprng); - let P = &s1 * &constants::ED25519_BASEPOINT_TABLE; - - b.iter(|| vartime::double_scalar_mul_basepoint(&s2, &P, &s1) ); - } - - #[bench] - fn ten_fold_scalar_mul(b: &mut Bencher) { - let mut csprng: OsRng = OsRng::new().unwrap(); - // Create 10 random scalars - let scalars: Vec<_> = (0..10).map(|_| Scalar::random(&mut csprng)).collect(); - // Create 10 points (by doing scalar mults) - let B = &constants::ED25519_BASEPOINT_TABLE; - let points: Vec<_> = scalars.iter().map(|s| B * s).collect(); - - b.iter(|| vartime::multiscalar_mul(&scalars, &points)); - } - } -} - diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index d8b7883..125aa55 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -671,20 +671,4 @@ mod test { assert_eq!(x2, splits[2]); assert_eq!(x3, splits[3]); } - } - -#[cfg(all(test, feature = "bench"))] -mod bench { - use test::Bencher; - use super::*; - - #[bench] - fn multiply(b: &mut Bencher) { - let vec = FieldElement32x4::splat(&FieldElement64::zero()); - let vecprime = vec.clone(); - - b.iter(|| &vec * &vecprime ); - } -} - diff --git a/src/backend/u32/scalar.rs b/src/backend/u32/scalar.rs index af88b79..6c3bb51 100644 --- a/src/backend/u32/scalar.rs +++ b/src/backend/u32/scalar.rs @@ -519,38 +519,3 @@ mod test { } } } - - -#[cfg(all(test, feature = "bench"))] -mod bench { - use test::Bencher; - - use super::*; - use super::test::{X, Y}; - - #[bench] - fn square(b: &mut Bencher) { - b.iter(|| X.square()); - } - - #[bench] - fn mul(b: &mut Bencher) { - b.iter(|| Scalar32::mul(&X, &Y)); - } - - #[bench] - fn montgomery_square(b: &mut Bencher) { - b.iter(|| X.montgomery_square()); - } - - #[bench] - fn montgomery_mul(b: &mut Bencher) { - b.iter(|| Scalar32::montgomery_mul(&X, &Y)); - } - - #[bench] - fn from_bytes_wide(b: &mut Bencher) { - let bignum = [255u8; 64]; // 2^512 - 1 - b.iter(|| Scalar32::from_bytes_wide(&bignum)); - } -} diff --git a/src/backend/u64/scalar.rs b/src/backend/u64/scalar.rs index c5d46d3..e14440b 100644 --- a/src/backend/u64/scalar.rs +++ b/src/backend/u64/scalar.rs @@ -441,38 +441,3 @@ mod test { } } } - - -#[cfg(all(test, feature = "bench"))] -mod bench { - use test::Bencher; - - use super::*; - use super::test::{X, Y}; - - #[bench] - fn square(b: &mut Bencher) { - b.iter(|| X.square()); - } - - #[bench] - fn mul(b: &mut Bencher) { - b.iter(|| Scalar64::mul(&X, &Y)); - } - - #[bench] - fn montgomery_square(b: &mut Bencher) { - b.iter(|| X.montgomery_square()); - } - - #[bench] - fn montgomery_mul(b: &mut Bencher) { - b.iter(|| Scalar64::montgomery_mul(&X, &Y)); - } - - #[bench] - fn from_bytes_wide(b: &mut Bencher) { - let bignum = [255u8; 64]; // 2^512 - 1 - b.iter(|| Scalar64::from_bytes_wide(&bignum)); - } -} diff --git a/src/edwards.rs b/src/edwards.rs index 33d299f..bcc4db6 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -1384,153 +1384,3 @@ mod test { assert!(parsed.is_err()); } } - -// ------------------------------------------------------------------------ -// Benchmarks -// ------------------------------------------------------------------------ - -#[cfg(all(test, feature = "bench"))] -mod bench { - use rand::OsRng; - use test::Bencher; - use constants; - use super::*; - use super::test::A_SCALAR; - - #[bench] - fn edwards_decompress(b: &mut Bencher) { - let B = &constants::ED25519_BASEPOINT_COMPRESSED; - b.iter(|| B.decompress().unwrap()); - } - - #[bench] - fn edwards_compress(b: &mut Bencher) { - let B = &constants::ED25519_BASEPOINT_POINT; - b.iter(|| B.compress()); - } - - #[bench] - #[cfg(feature="precomputed_tables")] - fn basepoint_mult(b: &mut Bencher) { - let B = &constants::ED25519_BASEPOINT_TABLE; - b.iter(|| B * &A_SCALAR); - } - - #[bench] - fn scalar_mul(b: &mut Bencher) { - let B = &constants::ED25519_BASEPOINT_POINT; - b.iter(|| B * &A_SCALAR); - } - - #[bench] - #[cfg(feature="precomputed_tables")] - fn bench_select_precomputed_point(b: &mut Bencher) { - use test::black_box; - let table = &constants::ED25519_BASEPOINT_TABLE.0[0]; - b.iter(|| table.select(black_box(5)) ); - } - - #[bench] - fn add_extended_and_projective_niels_output_completed(b: &mut Bencher) { - let p1 = constants::ED25519_BASEPOINT_POINT; - let p2 = constants::ED25519_BASEPOINT_POINT.to_projective_niels(); - - b.iter(|| &p1 + &p2); - } - - #[bench] - fn add_extended_and_projective_niels_output_extended(b: &mut Bencher) { - let p1 = constants::ED25519_BASEPOINT_POINT; - let p2 = constants::ED25519_BASEPOINT_POINT.to_projective_niels(); - - b.iter(|| (&p1 + &p2).to_extended()); - } - - #[bench] - fn add_extended_and_affine_niels_output_completed(b: &mut Bencher) { - let p1 = constants::ED25519_BASEPOINT_POINT; - let p2 = constants::ED25519_BASEPOINT_POINT.to_affine_niels(); - - b.iter(|| &p1 + &p2); - } - - #[bench] - fn add_extended_and_affine_niels_output_extended(b: &mut Bencher) { - let p1 = constants::ED25519_BASEPOINT_POINT; - let p2 = constants::ED25519_BASEPOINT_POINT.to_affine_niels(); - - b.iter(|| (&p1 + &p2).to_extended()); - } - - #[bench] - fn projective_double_output_completed(b: &mut Bencher) { - let p1 = constants::ED25519_BASEPOINT_POINT.to_projective(); - - b.iter(|| p1.double()); - } - - #[bench] - fn extended_double_output_extended(b: &mut Bencher) { - let p1 = constants::ED25519_BASEPOINT_POINT; - - b.iter(|| p1.double()); - } - - #[bench] - fn mul_by_cofactor(b: &mut Bencher) { - let p1 = constants::ED25519_BASEPOINT_POINT; - - b.iter(|| p1.mul_by_cofactor()); - } - - #[bench] - #[cfg(feature="precomputed_tables")] - fn create_basepoint_table(b: &mut Bencher) { - let aB = &constants::ED25519_BASEPOINT_TABLE * &A_SCALAR; - b.iter(|| EdwardsBasepointTable::create(&aB)); - } - - #[bench] - #[cfg(feature="precomputed_tables")] - fn ten_fold_scalar_mul(b: &mut Bencher) { - let mut csprng: OsRng = OsRng::new().unwrap(); - // Create 10 random scalars - let scalars: Vec<_> = (0..10).map(|_| Scalar::random(&mut csprng)).collect(); - // Create 10 points (by doing scalar mults) - let B = &constants::ED25519_BASEPOINT_TABLE; - let points: Vec<_> = scalars.iter().map(|s| B * &s).collect(); - - b.iter(|| multiscalar_mul(&scalars, &points)); - } - - mod vartime { - use super::super::*; - use super::super::test::{A_SCALAR, B_SCALAR, A_TIMES_BASEPOINT}; - use super::{Bencher, OsRng}; - - #[bench] - fn bench_double_scalar_mul_basepoint(b: &mut Bencher) { - let A = A_TIMES_BASEPOINT.decompress().unwrap(); - b.iter(|| vartime::double_scalar_mul_basepoint(&A_SCALAR, &A, &B_SCALAR)); - } - - #[bench] - #[cfg(feature="precomputed_tables")] - fn ten_fold_scalar_mul(b: &mut Bencher) { - let mut csprng: OsRng = OsRng::new().unwrap(); - // Create 10 random scalars - let scalars: Vec<_> = (0..10).map(|_| Scalar::random(&mut csprng)).collect(); - // Create 10 points (by doing scalar mults) - let B = &constants::ED25519_BASEPOINT_TABLE; - let points: Vec<_> = scalars.iter().map(|s| B * &s).collect(); - - // XXX Currently Rust's benchmarking implementation doesn't - // allow you to specify a sequence of random inputs, but only - // many trials of the same input. - // - // Since this is a variable-time function, this means the - // benchmark is only useful as a ballpark measurement. - b.iter(|| vartime::multiscalar_mul(&scalars, &points)); - } - } -} diff --git a/src/field.rs b/src/field.rs index 6b599f4..81251f9 100644 --- a/src/field.rs +++ b/src/field.rs @@ -497,50 +497,3 @@ mod test { } } } - -#[cfg(all(test, feature = "bench"))] -mod bench { - use test::Bencher; - - use super::*; - use super::test::A_BYTES; - - #[bench] - fn fieldelement_a_mul_a(b: &mut Bencher) { - let a = FieldElement::from_bytes(&A_BYTES); - b.iter(|| &a * &a); - } - - #[bench] - fn fieldelement_a_sq(b: &mut Bencher) { - let a = FieldElement::from_bytes(&A_BYTES); - b.iter(|| a.square()); - } - - #[bench] - fn fieldelement_a_inv(b: &mut Bencher) { - let a = FieldElement::from_bytes(&A_BYTES); - b.iter(|| a.invert()); - } - - #[bench] - fn batch_16_inv(b: &mut Bencher) { - let a = FieldElement::from_bytes(&A_BYTES); - let mut a_vec = vec![a; 16]; - b.iter(|| FieldElement::batch_invert(&mut a_vec)); - } - - #[bench] - fn batch_128_inv(b: &mut Bencher) { - let a = FieldElement::from_bytes(&A_BYTES); - let mut a_vec = vec![a; 128]; - b.iter(|| FieldElement::batch_invert(&mut a_vec)); - } - - #[bench] - fn batch_1024_inv(b: &mut Bencher) { - let a = FieldElement::from_bytes(&A_BYTES); - let mut a_vec = vec![a; 1024]; - b.iter(|| FieldElement::batch_invert(&mut a_vec)); - } -} diff --git a/src/lib.rs b/src/lib.rs index 5e9c854..bb788c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,6 @@ #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(feature = "alloc", feature(alloc))] -#![cfg_attr(feature = "bench", feature(test))] #![cfg_attr(feature = "nightly", feature(i128_type))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] @@ -41,9 +40,6 @@ extern crate alloc; extern crate clear_on_drop; -#[cfg(all(test, feature = "bench"))] -extern crate test; - #[cfg(feature = "yolocrypto")] extern crate stdsimd; diff --git a/src/montgomery.rs b/src/montgomery.rs index 81337c5..8be9ec0 100644 --- a/src/montgomery.rs +++ b/src/montgomery.rs @@ -347,22 +347,3 @@ mod test { assert_eq!(result, expected.to_montgomery()) } } - -#[cfg(all(test, feature = "bench"))] -#[cfg(feature="precomputed_tables")] -mod bench { - use rand::OsRng; - use constants::ED25519_BASEPOINT_TABLE; - use constants::X25519_BASEPOINT; - use test::Bencher; - use super::*; - - #[bench] - fn montgomery_ladder(b: &mut Bencher) { - let mut csprng: OsRng = OsRng::new().unwrap(); - let s: Scalar = Scalar::random(&mut csprng); - let P: MontgomeryPoint = (&Scalar::random(&mut csprng) * &ED25519_BASEPOINT_TABLE).to_montgomery(); - - b.iter(|| s * P); - } -} diff --git a/src/ristretto.rs b/src/ristretto.rs index 0426766..f2fcce0 100644 --- a/src/ristretto.rs +++ b/src/ristretto.rs @@ -1499,54 +1499,3 @@ mod test { } } } - -#[cfg(all(test, feature = "bench"))] -mod bench { - use rand::OsRng; - use test::Bencher; - - use super::*; - - #[bench] - #[cfg(feature="precomputed_tables")] - fn decompression(b: &mut Bencher) { - let mut rng = OsRng::new().unwrap(); - let B = &constants::RISTRETTO_BASEPOINT_TABLE; - let P = B * &Scalar::random(&mut rng); - let P_compressed = P.compress(); - b.iter(|| P_compressed.decompress().unwrap()); - } - - #[bench] - #[cfg(feature="precomputed_tables")] - fn compression(b: &mut Bencher) { - let mut rng = OsRng::new().unwrap(); - let B = &constants::RISTRETTO_BASEPOINT_TABLE; - let P = B * &Scalar::random(&mut rng); - b.iter(|| P.compress()); - } - - fn double_and_compress_n_random_points(n: usize, b: &mut Bencher) { - let mut rng = OsRng::new().unwrap(); - - let points: Vec = - (0..n).map(|_| RistrettoPoint::random(&mut rng)).collect(); - - b.iter(|| RistrettoPoint::double_and_compress_batch(&points) ); - } - - #[bench] - fn double_and_compress_16_random_points(b: &mut Bencher) { - double_and_compress_n_random_points(16, b); - } - - #[bench] - fn double_and_compress_128_random_points(b: &mut Bencher) { - double_and_compress_n_random_points(128, b); - } - - #[bench] - fn double_and_compress_1024_random_points(b: &mut Bencher) { - double_and_compress_n_random_points(1024, b); - } -} diff --git a/src/scalar.rs b/src/scalar.rs index db17193..7dae1e1 100644 --- a/src/scalar.rs +++ b/src/scalar.rs @@ -972,32 +972,3 @@ mod test { Scalar::batch_invert(&mut xs); } } - -#[cfg(all(test, feature = "bench"))] -mod bench { - use rand::OsRng; - use test::Bencher; - - use super::*; - use super::test::{X}; - - #[bench] - fn reduce(b: &mut Bencher) { - let unreduced = Scalar::from_bits([0xff; 32]); - - b.iter(|| unreduced.reduce()); - } - - #[bench] - fn scalar_random(b: &mut Bencher) { - let mut csprng: OsRng = OsRng::new().unwrap(); - - b.iter(|| Scalar::random(&mut csprng)); - } - - #[bench] - fn invert(b: &mut Bencher) { - let x = X.unpack(); - b.iter(|| x.invert()); - } -} From 76a8d43a04b0ed0b523d3c3b94fae9cd9432acc4 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 20 Mar 2018 12:10:39 -0700 Subject: [PATCH 02/16] Create a new `scalar_mul` module hierarchy. This should contain generic implementations of scalar multiplication algorithms that can be used with multiple backends. The goal is to move the existing scalar multiplication code into this submodule, then call it from the user-facing API. This can also contain code for things we can't do now, like multiscalar multiplication with precomputation. --- build.rs | 4 +++- src/backend/avx2/edwards.rs | 2 +- src/curve_models/mod.rs | 2 -- src/edwards.rs | 2 +- src/lib.rs | 8 ++++++++ src/scalar_mul/mod.rs | 11 +++++++++++ src/{curve_models => scalar_mul}/window.rs | 0 7 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 src/scalar_mul/mod.rs rename src/{curve_models => scalar_mul}/window.rs (100%) diff --git a/build.rs b/build.rs index 1c8be62..5eb523b 100644 --- a/build.rs +++ b/build.rs @@ -54,6 +54,8 @@ mod field; mod curve_models; #[path="src/backend/mod.rs"] mod backend; +#[path="src/scalar_mul/mod.rs"] +mod scalar_mul; use edwards::EdwardsBasepointTable; @@ -77,9 +79,9 @@ use backend::u32::field::FieldElement32; use edwards::EdwardsBasepointTable; -use curve_models::window::LookupTable; use curve_models::AffineNielsPoint; +use scalar_mul::window::LookupTable; /// Table containing precomputed multiples of the Ed25519 basepoint \\\\(B = (x, 4/5)\\\\). pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN; diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 7f564d7..74b0797 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -24,7 +24,7 @@ use subtle::Choice; use edwards; use scalar::Scalar; -use curve_models::window::LookupTable; +use scalar_mul::window::LookupTable; use traits::Identity; diff --git a/src/curve_models/mod.rs b/src/curve_models/mod.rs index 59426fa..a88c37d 100644 --- a/src/curve_models/mod.rs +++ b/src/curve_models/mod.rs @@ -135,8 +135,6 @@ use field::FieldElement; use edwards::EdwardsPoint; use traits::ValidityCheck; -pub mod window; - // ------------------------------------------------------------------------ // Internal point representations // ------------------------------------------------------------------------ diff --git a/src/edwards.rs b/src/edwards.rs index bcc4db6..439d6e6 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -117,7 +117,7 @@ use curve_models::CompletedPoint; use curve_models::AffineNielsPoint; use curve_models::ProjectiveNielsPoint; -use curve_models::window::LookupTable; +use scalar_mul::window::LookupTable; use traits::{Identity, IsIdentity}; use traits::ValidityCheck; diff --git a/src/lib.rs b/src/lib.rs index bb788c4..ffe0009 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,14 +67,19 @@ pub(crate) mod macros; // Scalar arithmetic mod l = 2^252 + ..., the order of the Ristretto group pub mod scalar; + // Point operations on the Montgomery form of Curve25519 pub mod montgomery; + // Point operations on the Edwards form of Curve25519 pub mod edwards; + // Group operations on the Ristretto group pub mod ristretto; + // Useful constants, like the Ed25519 basepoint pub mod constants; + // External (and internal) traits. pub mod traits; @@ -90,3 +95,6 @@ pub(crate) mod backend; // Internal curve models which are not part of the public API. pub(crate) mod curve_models; + +// Implementations of scalar mul algorithms live here +pub(crate) mod scalar_mul; diff --git a/src/scalar_mul/mod.rs b/src/scalar_mul/mod.rs new file mode 100644 index 0000000..bc3ef90 --- /dev/null +++ b/src/scalar_mul/mod.rs @@ -0,0 +1,11 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +pub mod window; diff --git a/src/curve_models/window.rs b/src/scalar_mul/window.rs similarity index 100% rename from src/curve_models/window.rs rename to src/scalar_mul/window.rs From ac739a3edddb60d8c6fc2e71cbafdf7cdb011092 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 26 Mar 2018 15:01:48 -0700 Subject: [PATCH 03/16] Split out constant-time variable-base scalar mul. The serial (`u32`/`u64`) implementations use a multiple curve models, passing between extended and projective coordinates when performing addition and doubling (respectively). But the AVX2 backend doesn't, so in order to write a single scalar mult implementation, we have to either abstract over the curve models or have two implementations. A generic solution is possible but extremely unreadable: the scalar mul implementation would be parameterized over the point types used by the serial implementations, with many where clauses describing how the types relate. The AVX2 types could then be substituted in the appropriate places. Instead we just duplicate the code into the `avx2` backend. --- src/backend/avx2/edwards.rs | 114 ------------------- src/backend/avx2/mod.rs | 2 + src/backend/avx2/scalar_mul/mod.rs | 11 ++ src/backend/avx2/scalar_mul/variable_base.rs | 34 ++++++ src/edwards.rs | 41 ++----- src/scalar_mul/mod.rs | 2 + src/scalar_mul/variable_base.rs | 32 ++++++ 7 files changed, 90 insertions(+), 146 deletions(-) create mode 100644 src/backend/avx2/scalar_mul/mod.rs create mode 100644 src/backend/avx2/scalar_mul/variable_base.rs create mode 100644 src/scalar_mul/variable_base.rs diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 74b0797..5aad787 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -371,40 +371,6 @@ impl From for LookupTable { } } -impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { - type Output = ExtendedPoint; - /// Scalar multiplication: compute `scalar * self`. - /// - /// Uses a window of size 4. - fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { - // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] - let lookup_table = LookupTable::::from(*self); - - // Setting s = scalar, compute - // - // s = s_0 + s_1*16^1 + ... + s_63*16^63, - // - // with `-8 ≤ s_i < 8` for `0 ≤ i < 63` and `-8 ≤ s_63 ≤ 8`. - let scalar_digits = scalar.to_radix_16(); - - // Compute s*P as - // - // s*P = P*(s_0 + s_1*16^1 + s_2*16^2 + ... + s_63*16^63) - // s*P = P*s_0 + P*s_1*16^1 + P*s_2*16^2 + ... + P*s_63*16^63 - // s*P = P*s_0 + 16*(P*s_1 + 16*(P*s_2 + 16*( ... + P*s_63)...)) - // - // We sum right-to-left. - let mut Q = ExtendedPoint::identity(); - for i in (0..64).rev() { - // Q = 16*Q - Q = Q.mul_by_pow_2(4); - // Q += P*s_i - Q = &Q + &lookup_table.select(scalar_digits[i]); - } - Q - } -} - #[derive(Clone)] pub struct EdwardsBasepointTable(pub [LookupTable; 32]); @@ -739,7 +705,6 @@ mod test { assert_eq!(R1.compress(), edwards::EdwardsPoint::identity().compress()); } - #[test] fn vector_addition_vs_serial_addition_vs_edwards_extendedpoint() { use constants; @@ -839,83 +804,4 @@ mod test { let P = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from_u64(8475983829); doubling_test_helper(P); } - - #[test] - fn identity_trait_vs_edwards_identity() { - let id1: edwards::EdwardsPoint = ExtendedPoint::identity().into(); - let id2: edwards::EdwardsPoint = edwards::EdwardsPoint::identity(); - assert_eq!(id1.compress(), id2.compress()); - } - - #[test] - fn neg_vs_edwards_neg() { - let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); - let Bneg = -&B; - assert_eq!(edwards::EdwardsPoint::from(Bneg).compress(), - (-&constants::ED25519_BASEPOINT_POINT).compress()); - } - - #[test] - fn scalar_mul_vs_edwards_scalar_mul() { - let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); - // some random bytes - let s = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); - - let R1 = edwards::EdwardsPoint::from(&B * &s); - let R2 = &constants::ED25519_BASEPOINT_TABLE * &s; - - assert_eq!(R1.compress(), R2.compress()); - } - - #[test] - fn scalar_mul_vs_basepoint_table_scalar_mul() { - let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); - let B_table = EdwardsBasepointTable::create(&B); - // some random bytes - let s = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); - - let P1 = &B * &s; - let P2 = &B_table * &s; - - assert_eq!(edwards::EdwardsPoint::from(P1).compress(), - edwards::EdwardsPoint::from(P2).compress()); - } - - #[test] - fn multiscalar_mul_vs_adding_scalar_muls() { - let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); - let s1 = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); - let s2 = Scalar::from_bits([165, 30, 79, 89, 58, 24, 195, 245, 248, 146, 203, 236, 119, 43, 64, 119, 196, 111, 188, 251, 248, 53, 234, 59, 215, 28, 218, 13, 59, 120, 14, 4]); - - let P1 = &B * &s2; - let P2 = &B * &s1; - - let R = &(&P1 * &s1) + &(&P2 * &s2); - - let R_multiscalar = multiscalar_mul(&[s1, s2], &[P1.into(), P2.into()]); - - assert_eq!(edwards::EdwardsPoint::from(R).compress(), - R_multiscalar.compress()); - } - - mod vartime { - use super::*; - - #[test] - fn multiscalar_mul_vs_adding_scalar_muls() { - let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); - let s1 = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); - let s2 = Scalar::from_bits([165, 30, 79, 89, 58, 24, 195, 245, 248, 146, 203, 236, 119, 43, 64, 119, 196, 111, 188, 251, 248, 53, 234, 59, 215, 28, 218, 13, 59, 120, 14, 4]); - - let P1 = &B * &s2; - let P2 = &B * &s1; - - let R = &(&P1 * &s1) + &(&P2 * &s2); - - let R_multiscalar = vartime::multiscalar_mul(&[s1, s2], &[P1.into(), P2.into()]); - - assert_eq!(edwards::EdwardsPoint::from(R).compress(), - R_multiscalar.compress()); - } - } } diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index ebae72f..bd3e860 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -479,3 +479,5 @@ pub(crate) mod field; pub(crate) mod edwards; pub(crate) mod constants; + +pub(crate) mod scalar_mul; diff --git a/src/backend/avx2/scalar_mul/mod.rs b/src/backend/avx2/scalar_mul/mod.rs new file mode 100644 index 0000000..54f5214 --- /dev/null +++ b/src/backend/avx2/scalar_mul/mod.rs @@ -0,0 +1,11 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +pub mod variable_base; diff --git a/src/backend/avx2/scalar_mul/variable_base.rs b/src/backend/avx2/scalar_mul/variable_base.rs new file mode 100644 index 0000000..e5261fb --- /dev/null +++ b/src/backend/avx2/scalar_mul/variable_base.rs @@ -0,0 +1,34 @@ +#![allow(non_snake_case)] + +use traits::Identity; +use scalar::Scalar; +use edwards::EdwardsPoint; +use backend::avx2::edwards::{ExtendedPoint, CachedPoint}; +use scalar_mul::window::LookupTable; + +/// Perform constant-time, variable-base scalar multiplication. +pub fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint { + // XXX combine these conversions + let avx2_point = ExtendedPoint::from(*point); + // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] + let lookup_table = LookupTable::::from(avx2_point); + // Setting s = scalar, compute + // + // s = s_0 + s_1*16^1 + ... + s_63*16^63, + // + // with `-8 ≤ s_i < 8` for `0 ≤ i < 63` and `-8 ≤ s_63 ≤ 8`. + let scalar_digits = scalar.to_radix_16(); + // Compute s*P as + // + // s*P = P*(s_0 + s_1*16^1 + s_2*16^2 + ... + s_63*16^63) + // s*P = P*s_0 + P*s_1*16^1 + P*s_2*16^2 + ... + P*s_63*16^63 + // s*P = P*s_0 + 16*(P*s_1 + 16*(P*s_2 + 16*( ... + P*s_63)...)) + // + // We sum right-to-left. + let mut Q = ExtendedPoint::identity(); + for i in (0..64).rev() { + Q = Q.mul_by_pow_2(4); + Q = &Q + &lookup_table.select(scalar_digits[i]); + } + Q.into() +} diff --git a/src/edwards.rs b/src/edwards.rs index 439d6e6..1e2f31d 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -477,39 +477,16 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsPoint { /// `EdwardsBasepointTable` is approximately 4x faster. fn mul(self, scalar: &'b Scalar) -> EdwardsPoint { // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] { - use backend::avx2::edwards::ExtendedPoint; - let P_avx2 = ExtendedPoint::from(*self); - return EdwardsPoint::from(&P_avx2 * scalar); + #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] + { + use backend::avx2::scalar_mul::variable_base::mul; + mul(self, scalar) } - // Otherwise, proceed as normal: - #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] { - // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] - let lookup_table = LookupTable::::from(self); - - // Setting s = scalar, compute - // - // s = s_0 + s_1*16^1 + ... + s_63*16^63, - // - // with `-8 ≤ s_i < 8` for `0 ≤ i < 63` and `-8 ≤ s_63 ≤ 8`. - let scalar_digits = scalar.to_radix_16(); - - // Compute s*P as - // - // s*P = P*(s_0 + s_1*16^1 + s_2*16^2 + ... + s_63*16^63) - // s*P = P*s_0 + P*s_1*16^1 + P*s_2*16^2 + ... + P*s_63*16^63 - // s*P = P*s_0 + 16*(P*s_1 + 16*(P*s_2 + 16*( ... + P*s_63)...)) - // - // We sum right-to-left. - let mut Q = EdwardsPoint::identity(); - for i in (0..64).rev() { - // Q <-- 16*Q - Q = Q.mul_by_pow_2(4); - // Q <-- Q + P * s_i - Q = (&Q + &lookup_table.select(scalar_digits[i])).to_extended() - } - - Q + // Otherwise, use the serial backend: + #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] + { + use scalar_mul::variable_base::mul; + mul(self, scalar) } } } diff --git a/src/scalar_mul/mod.rs b/src/scalar_mul/mod.rs index bc3ef90..acb5020 100644 --- a/src/scalar_mul/mod.rs +++ b/src/scalar_mul/mod.rs @@ -9,3 +9,5 @@ // - Henry de Valence pub mod window; + +pub mod variable_base; diff --git a/src/scalar_mul/variable_base.rs b/src/scalar_mul/variable_base.rs new file mode 100644 index 0000000..9569bd5 --- /dev/null +++ b/src/scalar_mul/variable_base.rs @@ -0,0 +1,32 @@ +#![allow(non_snake_case)] + +use traits::Identity; +use scalar::Scalar; +use edwards::EdwardsPoint; +use curve_models::ProjectiveNielsPoint; +use scalar_mul::window::LookupTable; + +/// Perform constant-time, variable-base scalar multiplication. +pub(crate) fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint { + // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] + let lookup_table = LookupTable::::from(point); + // Setting s = scalar, compute + // + // s = s_0 + s_1*16^1 + ... + s_63*16^63, + // + // with `-8 ≤ s_i < 8` for `0 ≤ i < 63` and `-8 ≤ s_63 ≤ 8`. + let scalar_digits = scalar.to_radix_16(); + // Compute s*P as + // + // s*P = P*(s_0 + s_1*16^1 + s_2*16^2 + ... + s_63*16^63) + // s*P = P*s_0 + P*s_1*16^1 + P*s_2*16^2 + ... + P*s_63*16^63 + // s*P = P*s_0 + 16*(P*s_1 + 16*(P*s_2 + 16*( ... + P*s_63)...)) + // + // We sum right-to-left. + let mut Q = EdwardsPoint::identity(); + for i in (0..64).rev() { + Q = Q.mul_by_pow_2(4); + Q = (&Q + &lookup_table.select(scalar_digits[i])).to_extended(); + } + Q +} From 2864a422bc2025be78413404a707d19cd66839d1 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 26 Mar 2018 15:32:20 -0700 Subject: [PATCH 04/16] Pull out constant-time straus implementation --- src/backend/avx2/scalar_mul/mod.rs | 2 + src/backend/avx2/scalar_mul/straus.rs | 57 ++++++++++++++++++ src/edwards.rs | 74 ++++-------------------- src/scalar_mul/mod.rs | 2 + src/scalar_mul/straus.rs | 83 +++++++++++++++++++++++++++ src/scalar_mul/vartime_straus.rs | 64 +++++++++++++++++++++ 6 files changed, 219 insertions(+), 63 deletions(-) create mode 100644 src/backend/avx2/scalar_mul/straus.rs create mode 100644 src/scalar_mul/straus.rs create mode 100644 src/scalar_mul/vartime_straus.rs diff --git a/src/backend/avx2/scalar_mul/mod.rs b/src/backend/avx2/scalar_mul/mod.rs index 54f5214..f529730 100644 --- a/src/backend/avx2/scalar_mul/mod.rs +++ b/src/backend/avx2/scalar_mul/mod.rs @@ -9,3 +9,5 @@ // - Henry de Valence pub mod variable_base; + +pub mod straus; diff --git a/src/backend/avx2/scalar_mul/straus.rs b/src/backend/avx2/scalar_mul/straus.rs new file mode 100644 index 0000000..5f76abe --- /dev/null +++ b/src/backend/avx2/scalar_mul/straus.rs @@ -0,0 +1,57 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence +#![allow(non_snake_case)] + +use core::borrow::Borrow; + +use clear_on_drop::ClearOnDrop; + +use traits::Identity; +use scalar::Scalar; +use edwards::EdwardsPoint; +use scalar_mul::window::LookupTable; +use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; + +/// Perform constant-time, variable-base scalar multiplication. +pub(crate) fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint +where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, +{ + // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] + // for each input point P + let lookup_tables: Vec<_> = points + .into_iter() + .map(|point| { + let avx2_point = ExtendedPoint::from(*point.borrow()); + LookupTable::::from(avx2_point) + }) + .collect(); + + let scalar_digits_vec: Vec<_> = scalars + .into_iter() + .map(|s| s.borrow().to_radix_16()) + .collect(); + // Pass ownership to a ClearOnDrop wrapper + let scalar_digits = ClearOnDrop::new(scalar_digits_vec); + + let mut Q = ExtendedPoint::identity(); + for j in (0..64).rev() { + Q = Q.mul_by_pow_2(4); + let it = scalar_digits.iter().zip(lookup_tables.iter()); + for (s_i, lookup_table_i) in it { + // Q = Q + s_{i,j} * P_i + Q = &Q + &lookup_table_i.select(s_i[j]); + } + } + Q.into() +} diff --git a/src/edwards.rs b/src/edwards.rs index 1e2f31d..42802cb 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -547,8 +547,6 @@ impl<'a, 'b> Mul<&'b EdwardsPoint> for &'a Scalar { /// /// assert_eq!(A1.compress(), (-A2).compress()); /// ``` -// XXX later when we do more fancy multiscalar mults, we can delegate -// based on the iter's size hint -- hdevalence #[cfg(any(feature = "alloc", feature = "std"))] pub fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint where I: IntoIterator, @@ -556,70 +554,20 @@ pub fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint J: IntoIterator, J::Item: Borrow, { - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] { - use backend::avx2::edwards as edwards_avx2; + // XXX later when we do more fancy multiscalar mults, we can + // delegate based on the iter's size hint -- hdevalence - edwards_avx2::multiscalar_mul(scalars, points) + // If we built with AVX2, use the AVX2 backend. + #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] + { + use backend::avx2::scalar_mul::straus::multiscalar_mul; + multiscalar_mul(scalars, points) } // Otherwise, proceed as normal: - #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] { - //assert_eq!(scalars.len(), points.len()); - - use clear_on_drop::ClearOnDrop; - - let lookup_tables_vec: Vec<_> = points.into_iter() - .map(|P| LookupTable::::from(P.borrow()) ) - .collect(); - - let lookup_tables = ClearOnDrop::new(lookup_tables_vec); - - // Setting s_i = i-th scalar, compute - // - // s_i = s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63, - // - // with `-8 ≤ s_{i,j} < 8` for `0 ≤ j < 63` and `-8 ≤ s_{i,63} ≤ 8`. - let scalar_digits_vec: Vec<_> = scalars.into_iter() - .map(|c| c.borrow().to_radix_16()) - .collect(); - - // This above puts the scalar digits into a heap-allocated Vec. - // To ensure that these are erased, pass ownership of the Vec into a - // ClearOnDrop wrapper. - let scalar_digits = ClearOnDrop::new(scalar_digits_vec); - - // Compute s_1*P_1 + ... + s_n*P_n: since - // - // s_i*P_i = P_i*(s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63) - // s_i*P_i = P_i*s_{i,0} + P_i*s_{i,1}*16^1 + ... + P_i*s_{i,63}*16^63 - // s_i*P_i = P_i*s_{i,0} + 16*(P_i*s_{i,1} + 16*( ... + 16*P_i*s_{i,63})...) - // - // we have the two-dimensional sum - // - // s_1*P_1 = P_1*s_{1,0} + 16*(P_1*s_{1,1} + 16*( ... + 16*P_1*s_{1,63})...) - // + s_2*P_2 = + P_2*s_{2,0} + 16*(P_2*s_{2,1} + 16*( ... + 16*P_2*s_{2,63})...) - // ... - // + s_n*P_n = + P_n*s_{n,0} + 16*(P_n*s_{n,1} + 16*( ... + 16*P_n*s_{n,63})...) - // - // We sum column-wise top-to-bottom, then right-to-left, - // multiplying by 16 only once per column. - // - // This provides the speedup over doing n independent scalar - // mults: we perform 63 multiplications by 16 instead of 63*n - // multiplications, saving 252*(n-1) doublings. - let mut Q = EdwardsPoint::identity(); - // XXX this impl makes no effort to be cache-aware; maybe it could be improved? - for j in (0..64).rev() { - Q = Q.mul_by_pow_2(4); - let it = scalar_digits.iter().zip(lookup_tables.iter()); - for (s_i, lookup_table_i) in it { - // R_i = s_{i,j} * P_i - let R_i = lookup_table_i.select(s_i[j]); - // Q = Q + R_i - Q = (&Q + &R_i).to_extended(); - } - } - Q + #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] + { + use scalar_mul::straus::multiscalar_mul; + multiscalar_mul(scalars, points) } } diff --git a/src/scalar_mul/mod.rs b/src/scalar_mul/mod.rs index acb5020..b69535d 100644 --- a/src/scalar_mul/mod.rs +++ b/src/scalar_mul/mod.rs @@ -11,3 +11,5 @@ pub mod window; pub mod variable_base; + +pub mod straus; diff --git a/src/scalar_mul/straus.rs b/src/scalar_mul/straus.rs new file mode 100644 index 0000000..e448ee9 --- /dev/null +++ b/src/scalar_mul/straus.rs @@ -0,0 +1,83 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence +#![allow(non_snake_case)] + +use core::borrow::Borrow; + +use clear_on_drop::ClearOnDrop; + +use traits::Identity; +use scalar::Scalar; +use edwards::EdwardsPoint; +use curve_models::ProjectiveNielsPoint; +use scalar_mul::window::LookupTable; + +/// Perform constant-time, variable-base scalar multiplication. +pub(crate) fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint +where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, +{ + // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] + // for each input point P + let lookup_tables: Vec<_> = points + .into_iter() + .map(|point| LookupTable::::from(point.borrow())) + .collect(); + + // Setting s_i = i-th scalar, compute + // + // s_i = s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63, + // + // with `-8 ≤ s_{i,j} < 8` for `0 ≤ j < 63` and `-8 ≤ s_{i,63} ≤ 8`. + // + // This puts the scalar digits into a heap-allocated Vec. + // To ensure that these are erased, pass ownership of the Vec into a + // ClearOnDrop wrapper. + let scalar_digits_vec: Vec<_> = scalars + .into_iter() + .map(|s| s.borrow().to_radix_16()) + .collect(); + let scalar_digits = ClearOnDrop::new(scalar_digits_vec); + + // Compute s_1*P_1 + ... + s_n*P_n: since + // + // s_i*P_i = P_i*(s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63) + // s_i*P_i = P_i*s_{i,0} + P_i*s_{i,1}*16^1 + ... + P_i*s_{i,63}*16^63 + // s_i*P_i = P_i*s_{i,0} + 16*(P_i*s_{i,1} + 16*( ... + 16*P_i*s_{i,63})...) + // + // we have the two-dimensional sum + // + // s_1*P_1 = P_1*s_{1,0} + 16*(P_1*s_{1,1} + 16*( ... + 16*P_1*s_{1,63})...) + // + s_2*P_2 = + P_2*s_{2,0} + 16*(P_2*s_{2,1} + 16*( ... + 16*P_2*s_{2,63})...) + // ... + // + s_n*P_n = + P_n*s_{n,0} + 16*(P_n*s_{n,1} + 16*( ... + 16*P_n*s_{n,63})...) + // + // We sum column-wise top-to-bottom, then right-to-left, + // multiplying by 16 only once per column. + // + // This provides the speedup over doing n independent scalar + // mults: we perform 63 multiplications by 16 instead of 63*n + // multiplications, saving 252*(n-1) doublings. + let mut Q = EdwardsPoint::identity(); + for j in (0..64).rev() { + Q = Q.mul_by_pow_2(4); + let it = scalar_digits.iter().zip(lookup_tables.iter()); + for (s_i, lookup_table_i) in it { + // R_i = s_{i,j} * P_i + let R_i = lookup_table_i.select(s_i[j]); + // Q = Q + R_i + Q = (&Q + &R_i).to_extended(); + } + } + Q +} diff --git a/src/scalar_mul/vartime_straus.rs b/src/scalar_mul/vartime_straus.rs new file mode 100644 index 0000000..890677b --- /dev/null +++ b/src/scalar_mul/vartime_straus.rs @@ -0,0 +1,64 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence +#![allow(non_snake_case)] + +use core::ops::{Add, Sub}; +use core::borrow::Borrow; + +use scalar::Scalar; + +//use super::window::OddLookupTable; +use scalar_mul::window::OddLookupTable; +use traits::{Doubleable, Identity}; + +/// Perform variable-time, variable-base scalar multiplication. +pub(crate) fn multiscalar_mul( + scalars: I, + points: J, +) -> ExtendedPoint +where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + for<'a> OddLookupTable: From<&'a ExtendedPoint>, + for<'a, 'b> &'a ExtendedPoint: Add<&'b CachedPoint, Output = CompletedPoint>, + for<'a, 'b> &'a ExtendedPoint: Sub<&'b CachedPoint, Output = CompletedPoint>, + ExtendedPoint: Identity + From + From, + ProjectivePoint: Identity + Doubleable + From, + CachedPoint: Copy, +{ + let nafs: Vec<_> = scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form()) + .collect(); + let lookup_tables: Vec<_> = points + .into_iter() + .map(|P| OddLookupTable::from(P.borrow())) + .collect(); + + let mut r = ProjectivePoint::identity(); + + for i in (0..255).rev() { + let mut t: CompletedPoint = r.double(); + + for (naf, lookup_table) in nafs.iter().zip(lookup_tables.iter()) { + if naf[i] > 0 { + t = &ExtendedPoint::from(t) + &lookup_table.select(naf[i] as usize); + } else if naf[i] < 0 { + t = &ExtendedPoint::from(t) - &lookup_table.select(-naf[i] as usize); + } + } + + r = ProjectivePoint::from(t); + } + + ExtendedPoint::from(r) +} From 2d99892eabcae205ee88cada2187ac5b5d6a0305 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 26 Mar 2018 15:54:13 -0700 Subject: [PATCH 05/16] Pull out variable-time straus implementation --- src/backend/avx2/edwards.rs | 113 ++++-------------- src/backend/avx2/scalar_mul/mod.rs | 2 + src/backend/avx2/scalar_mul/vartime_straus.rs | 54 +++++++++ src/edwards.rs | 41 ++----- src/scalar_mul/mod.rs | 2 + src/scalar_mul/vartime_straus.rs | 30 ++--- src/scalar_mul/window.rs | 70 ++++++++--- 7 files changed, 156 insertions(+), 156 deletions(-) create mode 100644 src/backend/avx2/scalar_mul/vartime_straus.rs diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 5aad787..0741652 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -144,7 +144,7 @@ impl<'a> Neg for &'a ExtendedPoint { } impl ExtendedPoint { - fn double(&self) -> ExtendedPoint { + pub fn double(&self) -> ExtendedPoint { unsafe { use stdsimd::vendor::_mm256_permute2x128_si256; use stdsimd::vendor::_mm256_permutevar8x32_epi32; @@ -361,6 +361,18 @@ impl<'a, 'b> Sub<&'b ExtendedPoint> for &'a ExtendedPoint { } } +impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint { + type Output = ExtendedPoint; + + /// Implement subtraction by negating the point and adding. + /// + /// Empirically, this seems about the same cost as a custom subtraction impl (maybe because the + /// benefit is cancelled by increased code size?) + fn sub(self, other: &'b CachedPoint) -> ExtendedPoint { + self + &(-other) + } +} + impl From for LookupTable { fn from(P: ExtendedPoint) -> Self { let mut points = [CachedPoint::from(P); 8]; @@ -421,67 +433,18 @@ impl EdwardsBasepointTable { } } -/// Internal multiscalar code. -#[cfg(any(feature = "alloc", feature = "std"))] -pub fn multiscalar_mul(scalars: I, points: J) -> edwards::EdwardsPoint - where I: IntoIterator, - I::Item: Borrow, - J: IntoIterator, - J::Item: Borrow, -{ - //assert_eq!(scalars.len(), points.len()); +use scalar_mul::window::OddLookupTable; - use clear_on_drop::ClearOnDrop; - let lookup_tables_vec: Vec<_> = points.into_iter() - .map(|P| LookupTable::from(ExtendedPoint::from(*P.borrow())) ) - .collect(); - - let lookup_tables = ClearOnDrop::new(lookup_tables_vec); - - // Setting s_i = i-th scalar, compute - // - // s_i = s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63, - // - // with `-8 ≤ s_{i,j} < 8` for `0 ≤ j < 63` and `-8 ≤ s_{i,63} ≤ 8`. - let scalar_digits_vec: Vec<_> = scalars.into_iter() - .map(|c| c.borrow().to_radix_16()) - .collect(); - - // The above puts the scalar digits into a heap-allocated Vec. - // To ensure that these are erased, pass ownership of the Vec into a - // ClearOnDrop wrapper. - let scalar_digits = ClearOnDrop::new(scalar_digits_vec); - - // Compute s_1*P_1 + ... + s_n*P_n: since - // - // s_i*P_i = P_i*(s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63) - // s_i*P_i = P_i*s_{i,0} + P_i*s_{i,1}*16^1 + ... + P_i*s_{i,63}*16^63 - // s_i*P_i = P_i*s_{i,0} + 16*(P_i*s_{i,1} + 16*( ... + 16*P_i*s_{i,63})...) - // - // we have the two-dimensional sum - // - // s_1*P_1 = P_1*s_{1,0} + 16*(P_1*s_{1,1} + 16*( ... + 16*P_1*s_{1,63})...) - // + s_2*P_2 = + P_2*s_{2,0} + 16*(P_2*s_{2,1} + 16*( ... + 16*P_2*s_{2,63})...) - // ... - // + s_n*P_n = + P_n*s_{n,0} + 16*(P_n*s_{n,1} + 16*( ... + 16*P_n*s_{n,63})...) - // - // We sum column-wise top-to-bottom, then right-to-left, - // multiplying by 16 only once per column. - // - // This provides the speedup over doing n independent scalar - // mults: we perform 63 multiplications by 16 instead of 63*n - // multiplications, saving 252*(n-1) doublings. - let mut Q = ExtendedPoint::identity(); - // XXX this algorithm makes no effort to be cache-aware; maybe it could be improved? - for j in (0..64).rev() { - Q = Q.mul_by_pow_2(4); - let it = scalar_digits.iter().zip(lookup_tables.iter()); - for (s_i, lookup_table_i) in it { - // Q = Q + s_{i,j} * P_i - Q = &Q + &lookup_table_i.select(s_i[j]); +impl<'a> From<&'a ExtendedPoint> for OddLookupTable { + fn from(A: &'a ExtendedPoint) -> Self { + let mut Ai = [CachedPoint::from(*A); 8]; + let A2 = A.double(); + for i in 0..7 { + Ai[i + 1] = (&A2 + &Ai[i]).into(); } + // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A] + OddLookupTable(Ai) } - Q.into() } pub mod vartime { @@ -560,38 +523,6 @@ pub mod vartime { Q.into() } - - /// Internal multiscalar function - #[cfg(any(feature = "alloc", feature = "std"))] - pub fn multiscalar_mul(scalars: I, points: J) -> edwards::EdwardsPoint - where I: IntoIterator, - I::Item: Borrow, - J: IntoIterator, - J::Item: Borrow, - { - //assert_eq!(scalars.len(), points.len()); - - let nafs: Vec<_> = scalars.into_iter() - .map(|c| c.borrow().non_adjacent_form()).collect(); - - let odd_multiples: Vec<_> = points.into_iter() - .map(|P| OddMultiples::create((*P.borrow()).into()) ).collect(); - - let mut Q = ExtendedPoint::identity(); - - for i in (0..255).rev() { - Q = Q.double(); - - for (naf, odd_multiple) in nafs.iter().zip(odd_multiples.iter()) { - if naf[i] > 0 { - Q = &Q + &odd_multiple[( naf[i]/2) as usize]; - } else if naf[i] < 0 { - Q = &Q - &odd_multiple[(-naf[i]/2) as usize]; - } - } - } - Q.into() - } } #[cfg(test)] diff --git a/src/backend/avx2/scalar_mul/mod.rs b/src/backend/avx2/scalar_mul/mod.rs index f529730..3c62538 100644 --- a/src/backend/avx2/scalar_mul/mod.rs +++ b/src/backend/avx2/scalar_mul/mod.rs @@ -11,3 +11,5 @@ pub mod variable_base; pub mod straus; + +pub mod vartime_straus; diff --git a/src/backend/avx2/scalar_mul/vartime_straus.rs b/src/backend/avx2/scalar_mul/vartime_straus.rs new file mode 100644 index 0000000..c53e7d4 --- /dev/null +++ b/src/backend/avx2/scalar_mul/vartime_straus.rs @@ -0,0 +1,54 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence +#![allow(non_snake_case)] + +use core::borrow::Borrow; + +use traits::Identity; +use scalar::Scalar; +use edwards::EdwardsPoint; +use scalar_mul::window::OddLookupTable; +use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; + +/// Perform variable-time, variable-base scalar multiplication. +pub(crate) fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint +where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, +{ + let nafs: Vec<_> = scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form()) + .collect(); + let lookup_tables: Vec<_> = points + .into_iter() + .map(|point| { + let avx2_point = ExtendedPoint::from(*point.borrow()); + OddLookupTable::::from(&avx2_point) + }) + .collect(); + + let mut Q = ExtendedPoint::identity(); + + for i in (0..255).rev() { + Q = Q.double(); + + for (naf, lookup_table) in nafs.iter().zip(lookup_tables.iter()) { + if naf[i] > 0 { + Q = &Q + &lookup_table.select(naf[i] as usize); + } else if naf[i] < 0 { + Q = &Q - &lookup_table.select(-naf[i] as usize); + } + } + } + Q.into() +} diff --git a/src/edwards.rs b/src/edwards.rs index 42802cb..0fc0855 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -845,8 +845,6 @@ pub mod vartime { /// /// assert_eq!(A1.compress(), (-A2).compress()); /// ``` - // XXX later when we do more fancy multiscalar mults, we can delegate - // based on the iter's size hint -- hdevalence #[cfg(any(feature = "alloc", feature = "std"))] pub fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint where I: IntoIterator, @@ -854,38 +852,19 @@ pub mod vartime { J: IntoIterator, J::Item: Borrow, { + // XXX later when we do more fancy multiscalar mults, we can delegate + // based on the iter's size hint -- hdevalence // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] { - use backend::avx2::edwards as edwards_avx2; - - edwards_avx2::vartime::multiscalar_mul(scalars, points) + #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] + { + use backend::avx2::scalar_mul::vartime_straus::multiscalar_mul; + multiscalar_mul(scalars, points) } // Otherwise, proceed as normal: - #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] { - //assert_eq!(scalars.len(), points.len()); - - let nafs: Vec<_> = scalars.into_iter() - .map(|c| c.borrow().non_adjacent_form()).collect(); - let odd_multiples: Vec<_> = points.into_iter() - .map(|P| OddMultiples::create(P.borrow())).collect(); - - let mut r = ProjectivePoint::identity(); - - for i in (0..255).rev() { - let mut t = r.double(); - - for (naf, odd_multiple) in nafs.iter().zip(odd_multiples.iter()) { - if naf[i] > 0 { - t = &t.to_extended() + &odd_multiple[( naf[i]/2) as usize]; - } else if naf[i] < 0 { - t = &t.to_extended() - &odd_multiple[(-naf[i]/2) as usize]; - } - } - - r = t.to_projective(); - } - - r.to_extended() + #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] + { + use scalar_mul::vartime_straus::multiscalar_mul; + multiscalar_mul(scalars, points) } } diff --git a/src/scalar_mul/mod.rs b/src/scalar_mul/mod.rs index b69535d..2f295d7 100644 --- a/src/scalar_mul/mod.rs +++ b/src/scalar_mul/mod.rs @@ -13,3 +13,5 @@ pub mod window; pub mod variable_base; pub mod straus; + +pub mod vartime_straus; diff --git a/src/scalar_mul/vartime_straus.rs b/src/scalar_mul/vartime_straus.rs index 890677b..c55ce21 100644 --- a/src/scalar_mul/vartime_straus.rs +++ b/src/scalar_mul/vartime_straus.rs @@ -9,31 +9,21 @@ // - Henry de Valence #![allow(non_snake_case)] -use core::ops::{Add, Sub}; use core::borrow::Borrow; +use traits::Identity; use scalar::Scalar; - -//use super::window::OddLookupTable; +use edwards::EdwardsPoint; +use curve_models::{CompletedPoint, ProjectivePoint, ProjectiveNielsPoint}; use scalar_mul::window::OddLookupTable; -use traits::{Doubleable, Identity}; /// Perform variable-time, variable-base scalar multiplication. -pub(crate) fn multiscalar_mul( - scalars: I, - points: J, -) -> ExtendedPoint +pub(crate) fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint where I: IntoIterator, I::Item: Borrow, J: IntoIterator, - J::Item: Borrow, - for<'a> OddLookupTable: From<&'a ExtendedPoint>, - for<'a, 'b> &'a ExtendedPoint: Add<&'b CachedPoint, Output = CompletedPoint>, - for<'a, 'b> &'a ExtendedPoint: Sub<&'b CachedPoint, Output = CompletedPoint>, - ExtendedPoint: Identity + From + From, - ProjectivePoint: Identity + Doubleable + From, - CachedPoint: Copy, + J::Item: Borrow, { let nafs: Vec<_> = scalars .into_iter() @@ -41,7 +31,7 @@ where .collect(); let lookup_tables: Vec<_> = points .into_iter() - .map(|P| OddLookupTable::from(P.borrow())) + .map(|P| OddLookupTable::::from(P.borrow())) .collect(); let mut r = ProjectivePoint::identity(); @@ -51,14 +41,14 @@ where for (naf, lookup_table) in nafs.iter().zip(lookup_tables.iter()) { if naf[i] > 0 { - t = &ExtendedPoint::from(t) + &lookup_table.select(naf[i] as usize); + t = &t.to_extended() + &lookup_table.select(naf[i] as usize); } else if naf[i] < 0 { - t = &ExtendedPoint::from(t) - &lookup_table.select(-naf[i] as usize); + t = &t.to_extended() - &lookup_table.select(-naf[i] as usize); } } - r = ProjectivePoint::from(t); + r = t.to_projective(); } - ExtendedPoint::from(r) + r.to_extended() } diff --git a/src/scalar_mul/window.rs b/src/scalar_mul/window.rs index ada3706..9b25917 100644 --- a/src/scalar_mul/window.rs +++ b/src/scalar_mul/window.rs @@ -21,6 +21,10 @@ use subtle::Choice; use traits::Identity; +use edwards::EdwardsPoint; +use curve_models::ProjectiveNielsPoint; +use curve_models::AffineNielsPoint; + /// A lookup table of precomputed multiples of a point \\(P\\), used to /// compute \\( xP \\) for \\( -8 \leq x \leq 8 \\). /// @@ -54,22 +58,24 @@ use clear_on_drop::clear::ZeroSafe; unsafe impl ZeroSafe for LookupTable {} impl LookupTable -where T: Identity + ConditionallyAssignable + ConditionallyNegatable +where + T: Identity + ConditionallyAssignable + ConditionallyNegatable, { /// Given \\(-8 \leq x \leq 8\\), return \\(xP\\) in constant time. pub fn select(&self, x: i8) -> T { - debug_assert!(x >= -8); debug_assert!(x <= 8); + debug_assert!(x >= -8); + debug_assert!(x <= 8); // Compute xabs = |x| let xmask = x >> 7; - let xabs = (x + xmask) ^ xmask; + let xabs = (x + xmask) ^ xmask; // Set t = 0 * P = identity let mut t = T::identity(); for j in 1..9 { // Copy `points[j-1] == j*P` onto `t` in constant time if `|x| == j`. let c = (xabs as u8).ct_eq(&(j as u8)); - t.conditional_assign(&self.0[j-1], c); + t.conditional_assign(&self.0[j - 1], c); } // Now t == |x| * P. @@ -93,17 +99,11 @@ impl Debug for LookupTable { } } -use edwards::EdwardsPoint; -use curve_models::ProjectiveNielsPoint; -use curve_models::AffineNielsPoint; - impl<'a> From<&'a EdwardsPoint> for LookupTable { fn from(P: &'a EdwardsPoint) -> Self { let mut points = [P.to_projective_niels(); 8]; for j in 0..7 { - points[j+1] = (P + &points[j]) - .to_extended() - .to_projective_niels(); + points[j + 1] = (P + &points[j]).to_extended().to_projective_niels(); } LookupTable(points) } @@ -114,10 +114,52 @@ impl<'a> From<&'a EdwardsPoint> for LookupTable { let mut points = [P.to_affine_niels(); 8]; // XXX batch inversion would be good if perf mattered here for j in 0..7 { - points[j+1] = (P + &points[j]) - .to_extended() - .to_affine_niels() + points[j + 1] = (P + &points[j]).to_extended().to_affine_niels() } LookupTable(points) } } + +/// Holds odd multiples 1A, 3A, ..., 15A of a point A. +#[derive(Copy, Clone)] +pub(crate) struct OddLookupTable(pub(crate) [T; 8]); + +impl OddLookupTable { + /// Given public, odd \\( x \\) with \\( 0 < x < 2^4 \\), return \\(xA\\). + pub fn select(&self, x: usize) -> T { + debug_assert_eq!(x & 1, 1); + debug_assert!(x < 16); + + self.0[x / 2] + } +} + +impl Debug for OddLookupTable { + fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + write!(f, "OddLookupTable({:?})", self.0) + } +} + +impl<'a> From<&'a EdwardsPoint> for OddLookupTable { + fn from(A: &'a EdwardsPoint) -> Self { + let mut Ai = [A.to_projective_niels(); 8]; + let A2 = A.double(); + for i in 0..7 { + Ai[i + 1] = (&A2 + &Ai[i]).to_extended().to_projective_niels(); + } + // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A] + OddLookupTable(Ai) + } +} + +impl<'a> From<&'a EdwardsPoint> for OddLookupTable { + fn from(A: &'a EdwardsPoint) -> Self { + let mut Ai = [A.to_affine_niels(); 8]; + let A2 = A.double(); + for i in 0..7 { + Ai[i + 1] = (&A2 + &Ai[i]).to_extended().to_affine_niels(); + } + // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A] + OddLookupTable(Ai) + } +} From 0c4e7188a07775a436ef18d859a0c685c8a7a5a0 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 26 Mar 2018 17:02:59 -0700 Subject: [PATCH 06/16] Pull out vartime double-base scalar mul code --- build.rs | 57 +++++---- src/backend/avx2/constants.rs | 119 +++++++++--------- src/backend/avx2/edwards.rs | 78 ------------ src/backend/avx2/scalar_mul/mod.rs | 3 + .../avx2/scalar_mul/vartime_double_base.rs | 61 +++++++++ src/edwards.rs | 89 ++----------- src/scalar_mul/mod.rs | 3 + src/scalar_mul/vartime_double_base.rs | 61 +++++++++ 8 files changed, 231 insertions(+), 240 deletions(-) create mode 100644 src/backend/avx2/scalar_mul/vartime_double_base.rs create mode 100644 src/scalar_mul/vartime_double_base.rs diff --git a/build.rs b/build.rs index 5eb523b..60a9f7a 100644 --- a/build.rs +++ b/build.rs @@ -4,12 +4,12 @@ #![allow(non_snake_case)] #![allow(dead_code)] +extern crate clear_on_drop; extern crate core; -extern crate subtle; -extern crate rand; extern crate digest; extern crate generic_array; -extern crate clear_on_drop; +extern crate rand; +extern crate subtle; use std::env; use std::fs::File; @@ -27,37 +27,39 @@ extern crate serde; extern crate stdsimd; // Macros come first! -#[path="src/macros.rs"] +#[path = "src/macros.rs"] #[macro_use] mod macros; // Public modules -#[path="src/scalar.rs"] +#[path = "src/scalar.rs"] mod scalar; -#[path="src/montgomery.rs"] +#[path = "src/montgomery.rs"] mod montgomery; -#[path="src/edwards.rs"] +#[path = "src/edwards.rs"] mod edwards; -#[path="src/ristretto.rs"] +#[path = "src/ristretto.rs"] mod ristretto; -#[path="src/constants.rs"] +#[path = "src/constants.rs"] mod constants; -#[path="src/traits.rs"] +#[path = "src/traits.rs"] mod traits; // Internal modules -#[path="src/field.rs"] +#[path = "src/field.rs"] mod field; -#[path="src/curve_models/mod.rs"] +#[path = "src/curve_models/mod.rs"] mod curve_models; -#[path="src/backend/mod.rs"] +#[path = "src/backend/mod.rs"] mod backend; -#[path="src/scalar_mul/mod.rs"] +#[path = "src/scalar_mul/mod.rs"] mod scalar_mul; use edwards::EdwardsBasepointTable; +use curve_models::AffineNielsPoint; +use scalar_mul::window::OddLookupTable; fn main() { // Enable the "precomputed_tables" feature in the main build stage @@ -70,7 +72,9 @@ fn main() { // Generate a table of precomputed multiples of the basepoint let table = EdwardsBasepointTable::create(&constants::ED25519_BASEPOINT_POINT); - f.write_all(format!("\n + f.write_all( + format!( + "\n #[cfg(feature=\"radix_51\")] use backend::u64::field::FieldElement64; @@ -82,6 +86,7 @@ use edwards::EdwardsBasepointTable; use curve_models::AffineNielsPoint; use scalar_mul::window::LookupTable; +use scalar_mul::window::OddLookupTable; /// Table containing precomputed multiples of the Ed25519 basepoint \\\\(B = (x, 4/5)\\\\). pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN; @@ -89,18 +94,22 @@ pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TAB /// Inner constant, used to avoid filling the docs with precomputed points. #[doc(hidden)] pub const ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN: EdwardsBasepointTable = {:?}; - \n\n", &table).as_bytes()).unwrap(); +\n\n", + &table + ).as_bytes(), + ).unwrap(); // Now generate AFFINE_ODD_MULTIPLES_OF_BASEPOINT let B = &constants::ED25519_BASEPOINT_POINT; - let B2 = B.double(); - let mut odd_multiples = [B.to_affine_niels(); 8]; - for i in 0..7 { - odd_multiples[i+1] = (&B2 + &odd_multiples[i]).to_extended().to_affine_niels(); - } + let odd_multiples = OddLookupTable::::from(B); - f.write_all(format!("\n + f.write_all( + format!( + "\n /// Odd multiples of the basepoint `[B, 3B, 5B, 7B, 9B, 11B, 13B, 15B]`. -pub(crate) const AFFINE_ODD_MULTIPLES_OF_BASEPOINT: [AffineNielsPoint; 8] = {:?}; - \n\n", &odd_multiples).as_bytes()).unwrap(); +pub(crate) const AFFINE_ODD_MULTIPLES_OF_BASEPOINT: OddLookupTable = {:?}; +\n\n", + &odd_multiples + ).as_bytes(), + ).unwrap(); } diff --git a/src/backend/avx2/constants.rs b/src/backend/avx2/constants.rs index 362b518..3fb77d8 100644 --- a/src/backend/avx2/constants.rs +++ b/src/backend/avx2/constants.rs @@ -12,8 +12,9 @@ use stdsimd::simd::u32x8; +use scalar_mul::window::OddLookupTable; use backend::avx2::field::FieldElement32x4; -use backend::avx2::edwards::ExtendedPoint; +use backend::avx2::edwards::{ExtendedPoint, CachedPoint}; /// The low limbs of (2p, 2p, 2p, 2p), so that /// ```no_run @@ -52,61 +53,61 @@ pub(crate) static P_TIMES_2_MASKED: FieldElement32x4 = FieldElement32x4([ ]); /// Odd multiples of the Ed25519 basepoint: -pub static ODD_MULTIPLES_OF_BASEPOINT: [ExtendedPoint; 8] = [ - ExtendedPoint(FieldElement32x4([ - u32x8::new(52811034, 40265304, 25909283, 26843545, 1, 28827043, 0, 27438313), - u32x8::new(16144682, 13421772, 17082669, 20132659, 0, 39759291, 0, 244362), - u32x8::new(27570973, 26843545, 30858332, 6710886, 0, 8635006, 0, 11264893), - u32x8::new(40966398, 53687091, 8378388, 13421772, 0, 19351346, 0, 13413597), - u32x8::new(20764389, 40265318, 8758491, 26843545, 0, 16611511, 0, 27139452), - ])), - ExtendedPoint(FieldElement32x4([ - u32x8::new(63703867, 19156774, 608100, 2486757, 12685460, 3173753, 21649412, 16313381), - u32x8::new(52397038, 65858675, 26775664, 16661035, 14269998, 9080558, 1059463, 28938752), - u32x8::new( 5461635, 28034025, 23358301, 1245198, 1367765, 20288887, 31111942, 18395221), - u32x8::new( 1886934, 32436996, 681756, 18977693, 8129860, 40112764, 25764567, 11876840), - u32x8::new(63042604, 52399761, 22087481, 29829870, 8565820, 33723612, 28645162, 8502864), - ])), - ExtendedPoint(FieldElement32x4([ - u32x8::new(14879397, 3951036, 9454671, 16606238, 23529732, 44147004, 11890541, 17067526), - u32x8::new(58509479, 57216664, 9671992, 32001147, 60966207, 11801823, 10808378, 15115613), - u32x8::new(54854992, 39210911, 8112050, 1353604, 1337416, 35520540, 32967851, 17786030), - u32x8::new(59007462, 40864509, 26240923, 30403852, 28456403, 21546582, 32732450, 21005910), - u32x8::new(40711675, 22446613, 9664668, 12483629, 26142305, 56254715, 15439904, 214849), - ])), - ExtendedPoint(FieldElement32x4([ - u32x8::new(52231579, 51632644, 173613, 7677257, 26374424, 45994428, 5303371, 1425942), - u32x8::new(38126791, 48854506, 23252518, 30611978, 49977504, 66706952, 1076178, 27100873), - u32x8::new(26349427, 63077566, 20258199, 3884787, 33226507, 2371423, 5787271, 18628170), - u32x8::new(15005754, 22729577, 4978944, 2522289, 1404784, 56367795, 22517039, 29271243), - u32x8::new(22748934, 35977548, 25561257, 31734126, 22775284, 32000077, 927866, 2278697), - ])), - ExtendedPoint(FieldElement32x4([ - u32x8::new(66090281, 61980626, 23780289, 6519561, 62542590, 47174086, 28818882, 15661068), - u32x8::new(17433715, 12931425, 12232056, 7885877, 44179512, 35590146, 32787344, 22631048), - u32x8::new(43729883, 6870635, 15782399, 11810556, 2652935, 31800505, 23683367, 13638649), - u32x8::new(64007953, 40242373, 32810277, 20180235, 20399465, 48133835, 32913956, 19094667), - u32x8::new(56562708, 40269142, 18953105, 9027935, 35700921, 12896915, 14757156, 22773619), - ])), - ExtendedPoint(FieldElement32x4([ - u32x8::new(65129016, 34709402, 25132940, 13788431, 3661652, 16914498, 27409409, 18941039), - u32x8::new(42488074, 49427602, 6177212, 20812339, 41644653, 2977316, 12162542, 5293661), - u32x8::new( 7981168, 12223605, 6239200, 20403609, 20710415, 4828170, 11627702, 4431044), - u32x8::new(65817142, 96824, 25021652, 16364722, 50410869, 24651857, 6979034, 33176209), - u32x8::new(33008344, 8687253, 27859668, 28796356, 30192014, 11975680, 11991047, 27710707), - ])), - ExtendedPoint(FieldElement32x4([ - u32x8::new(14676653, 50945941, 13489249, 31456262, 47726639, 21761847, 3324839, 7843947), - u32x8::new(53352326, 8688989, 12944061, 12994004, 50113821, 37990636, 1537898, 20483689), - u32x8::new(46786852, 15572264, 24004728, 7566233, 32596174, 34437796, 23201722, 3431551), - u32x8::new(49025674, 52497128, 13273618, 10266201, 66795206, 2887684, 30966565, 33449990), - u32x8::new(53210238, 65839385, 15458877, 18409918, 24777464, 25586795, 15335748, 12323382), - ])), - ExtendedPoint(FieldElement32x4([ - u32x8::new(57816016, 23106045, 24948505, 27413507, 32551424, 26145165, 22632568, 27527446), - u32x8::new(53022711, 40974949, 14110533, 30646997, 51399118, 53289754, 32528560, 15822835), - u32x8::new(23810949, 51779690, 17532625, 21326637, 60314333, 43761996, 4852905, 3474945), - u32x8::new(13323962, 10752742, 16431634, 26425049, 24258356, 53260846, 19756601, 19546842), - u32x8::new(17403634, 52199608, 32323720, 5313255, 48522162, 33376516, 31903659, 15291466), - ])), -]; +pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: OddLookupTable = + OddLookupTable([ + CachedPoint(FieldElement32x4([ + u32x8::new(3571425, 10045002, 19036563, 1096096, 243332, 65897020, 0, 28963681), + u32x8::new(30896895, 63055514, 1614915, 5095970, 0, 53791688, 0, 31258312), + u32x8::new(13347627, 40339464, 2236269, 11185503, 0, 22520087, 0, 8659512), + u32x8::new(11125413, 29139905, 32037254, 28360723, 0, 64556417, 0, 9635759), + u32x8::new(33268144, 47262491, 4336918, 15795740, 0, 22027545, 0, 4846528), + ])), + CachedPoint(FieldElement32x4([ + u32x8::new(47099681, 31447946, 29365447, 24740513, 42991046, 18317844, 16051644, 21404226), + u32x8::new(31708133, 28909527, 2366091, 13703791, 469246, 54159622, 2601402, 32988002), + u32x8::new(63432457, 30251794, 15163516, 18491340, 28144087, 35605455, 13682295, 18474872), + u32x8::new(12221607, 4967598, 26061980, 26008006, 20226147, 9726961, 17410, 18051083), + u32x8::new(60569645, 62487085, 11911242, 21920922, 4092105, 38186967, 22431483, 31366585), + ])), + CachedPoint(FieldElement32x4([ + u32x8::new(18147205, 62587998, 2554617, 536692, 11924528, 26674131, 17645433, 24341419), + u32x8::new(11573357, 27579485, 31491870, 29000885, 10800976, 51902791, 28076395, 20464029), + u32x8::new(56031649, 10856669, 11791193, 26769430, 25306956, 5922200, 6630685, 9385098), + u32x8::new(31319348, 23906711, 16290213, 32142166, 61106354, 17181823, 3548308, 12022566), + u32x8::new(5904298, 50218605, 11826440, 5492249, 10379071, 3472255, 172742, 31948344), + ])), + CachedPoint(FieldElement32x4([ + u32x8::new(10625852, 15193821, 22918394, 23676410, 53695416, 54987793, 10067515, 11747680), + u32x8::new(65013325, 1309652, 29616320, 28922974, 60360891, 19621771, 9938982, 30406429), + u32x8::new(54967954, 65931918, 5595602, 25719523, 64909864, 30566415, 15945272, 8495317), + u32x8::new(1167157, 55265018, 11507029, 31641054, 43497904, 2367338, 12937761, 27517066), + u32x8::new(656704, 2544994, 13006713, 480979, 38471594, 62541240, 25353597, 11531760), + ])), + CachedPoint(FieldElement32x4([ + u32x8::new(22176662, 3984313, 27495285, 4110608, 2909584, 30594106, 15677919, 2549183), + u32x8::new(33979105, 62269905, 2071511, 6894756, 53189950, 47232857, 6408191, 6123225), + u32x8::new(32553873, 63948030, 12612401, 3633166, 24054373, 37626618, 14481327, 8520484), + u32x8::new(56552486, 10749438, 12034813, 28811946, 1445640, 36755601, 12104575, 10257833), + u32x8::new(22795808, 48761311, 1136056, 9380768, 1411523, 5341811, 27318329, 9686767)])), + CachedPoint(FieldElement32x4([ + u32x8::new(21157200, 39156966, 20473176, 4934657, 61478183, 45121537, 5429856, 13035023), + u32x8::new(7954529, 58789246, 31440083, 7054221, 38438565, 36856107, 1364112, 14548122), + u32x8::new(26120083, 36321360, 4919997, 31687496, 33757765, 36237559, 15243054, 32163861), + u32x8::new(25878307, 46544824, 19455951, 2414935, 16844726, 56521560, 32680554, 26660660), + u32x8::new(48360220, 43407178, 12187042, 24925816, 7423722, 25746484, 12814654, 17395963), + ])), + CachedPoint(FieldElement32x4([ + u32x8::new(63153652, 32195955, 4087908, 8431689, 30392384, 47203165, 8986649, 9053039), + u32x8::new(63659241, 47988767, 2931872, 19953600, 11747107, 51610101, 20952181, 13364887), + u32x8::new(3659197, 58790649, 5930099, 2605312, 28477896, 580728, 20579735, 2610622), + u32x8::new(41781607, 17161358, 10690531, 24368015, 47027031, 36742339, 5414694, 13156365), + u32x8::new(13237853, 51182423, 8954802, 29006542, 22643989, 56896541, 22830593, 10289708), + ])), + CachedPoint(FieldElement32x4([ + u32x8::new(1401265, 58846825, 30911620, 32239180, 15391552, 15200821, 6339309, 16403588), + u32x8::new(55913797, 29541724, 1664461, 21709410, 38470488, 47097092, 17674945, 32666066), + u32x8::new(22844482, 10797709, 27548106, 31638735, 34500968, 26611503, 19727211, 13160873), + u32x8::new(31485204, 14496164, 13981208, 10276888, 5748808, 35024436, 2740987, 7479021), + u32x8::new(58541207, 14866135, 32344041, 545930, 62661488, 6941250, 27940205, 11976112), + ])), + ]); diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 0741652..caa8322 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -447,84 +447,6 @@ impl<'a> From<&'a ExtendedPoint> for OddLookupTable { } } -pub mod vartime { - //! Variable-time operations on curve points, useful for non-secret data. - use super::*; - - /// Holds odd multiples 1A, 3A, ..., 15A of a point A. - struct OddMultiples([ExtendedPoint; 8]); - - impl OddMultiples { - fn create(A: ExtendedPoint) -> OddMultiples { - // XXX would be great to skip this initialization - let mut Ai = [A; 8]; - let A2 = A.double(); - for i in 0..7 { - Ai[i+1] = &A2 + &Ai[i]; - } - // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A] - OddMultiples(Ai) - } - } - - impl Index for OddMultiples { - type Output = ExtendedPoint; - - fn index(&self, _index: usize) -> &ExtendedPoint { - &(self.0[_index]) - } - } - - /// Given a point `A` and scalars `a` and `b`, compute the point - /// `aA+bB`, where `B` is the Ed25519 basepoint (i.e., `B = (x,4/5)` - /// with x positive). - /// - /// This is the same as calling the iterator-based function, but slightly faster. - pub fn double_scalar_mul_basepoint(a: &Scalar, - A: &edwards::EdwardsPoint, - b: &Scalar) -> edwards::EdwardsPoint { - let a_naf = a.non_adjacent_form(); - let b_naf = b.non_adjacent_form(); - - // Find starting index - let mut i: usize = 255; - for j in (0..255).rev() { - i = j; - if a_naf[i] != 0 || b_naf[i] != 0 { - break; - } - } - - let odd_multiples_of_A = OddMultiples::create((*A).into()); - let odd_multiples_of_B = &avx2::constants::ODD_MULTIPLES_OF_BASEPOINT; - - let mut Q = ExtendedPoint::identity(); - - loop { - Q = Q.double(); - - if a_naf[i] > 0 { - Q = &Q + &odd_multiples_of_A[( a_naf[i]/2) as usize]; - } else if a_naf[i] < 0 { - Q = &Q - &odd_multiples_of_A[(-a_naf[i]/2) as usize]; - } - - if b_naf[i] > 0 { - Q = &Q + &odd_multiples_of_B[( b_naf[i]/2) as usize]; - } else if b_naf[i] < 0 { - Q = &Q - &odd_multiples_of_B[(-b_naf[i]/2) as usize]; - } - - if i == 0 { - break; - } - i -= 1; - } - - Q.into() - } -} - #[cfg(test)] mod test { use super::*; diff --git a/src/backend/avx2/scalar_mul/mod.rs b/src/backend/avx2/scalar_mul/mod.rs index 3c62538..8b1e84e 100644 --- a/src/backend/avx2/scalar_mul/mod.rs +++ b/src/backend/avx2/scalar_mul/mod.rs @@ -10,6 +10,9 @@ pub mod variable_base; +#[cfg(feature="precomputed_tables")] +pub mod vartime_double_base; + pub mod straus; pub mod vartime_straus; diff --git a/src/backend/avx2/scalar_mul/vartime_double_base.rs b/src/backend/avx2/scalar_mul/vartime_double_base.rs new file mode 100644 index 0000000..f7703bb --- /dev/null +++ b/src/backend/avx2/scalar_mul/vartime_double_base.rs @@ -0,0 +1,61 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence +#![allow(non_snake_case)] + +use traits::Identity; +use scalar::Scalar; +use edwards::EdwardsPoint; +use scalar_mul::window::OddLookupTable; +use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; +use backend::avx2::constants::BASEPOINT_ODD_LOOKUP_TABLE; + +/// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. +pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { + let a_naf = a.non_adjacent_form(); + let b_naf = b.non_adjacent_form(); + + // Find starting index + let mut i: usize = 255; + for j in (0..255).rev() { + i = j; + if a_naf[i] != 0 || b_naf[i] != 0 { + break; + } + } + + let avx2_A = ExtendedPoint::from(*A); + let table_A = OddLookupTable::::from(&avx2_A); + let table_B = &BASEPOINT_ODD_LOOKUP_TABLE; + + let mut Q = ExtendedPoint::identity(); + + loop { + Q = Q.double(); + + if a_naf[i] > 0 { + Q = &Q + &table_A.select(a_naf[i] as usize); + } else if a_naf[i] < 0 { + Q = &Q - &table_A.select(-a_naf[i] as usize); + } + + if b_naf[i] > 0 { + Q = &Q + &table_B.select(b_naf[i] as usize); + } else if b_naf[i] < 0 { + Q = &Q - &table_B.select(-b_naf[i] as usize); + } + + if i == 0 { + break; + } + i -= 1; + } + + Q.into() +} diff --git a/src/edwards.rs b/src/edwards.rs index 0fc0855..8487073 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -98,7 +98,6 @@ use core::iter::Iterator; use core::ops::{Add, Sub, Neg}; use core::ops::{AddAssign, SubAssign}; use core::ops::{Mul, MulAssign}; -use core::ops::Index; use core::borrow::Borrow; use subtle::ConditionallyAssignable; @@ -778,30 +777,6 @@ pub mod vartime { //! Variable-time operations on curve points, useful for non-secret data. use super::*; - /// Holds odd multiples 1A, 3A, ..., 15A of a point A. - struct OddMultiples([ProjectiveNielsPoint; 8]); - - impl OddMultiples { - fn create(A: &EdwardsPoint) -> OddMultiples { - let mut Ai = [ProjectiveNielsPoint::identity(); 8]; - let A2 = A.double(); - Ai[0] = A.to_projective_niels(); - for i in 0..7 { - Ai[i+1] = (&A2 + &Ai[i]).to_extended().to_projective_niels(); - } - // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A] - OddMultiples(Ai) - } - } - - impl Index for OddMultiples { - type Output = ProjectiveNielsPoint; - - fn index(&self, _index: usize) -> &ProjectiveNielsPoint { - &(self.0[_index]) - } - } - /// Given an iterator of public scalars and an iterator of public points, compute /// $$ /// Q = c\_1 P\_1 + \cdots + c\_n P\_n. @@ -868,66 +843,22 @@ pub mod vartime { } } - /// Given a point \\(A\\) and scalars \\(a\\) and \\(b\\), compute the point - /// \\(aA+bB\\), where \\(B\\) is the Ed25519 basepoint (i.e., \\(B = (x,4/5)\\) - /// with x positive). + /// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. #[cfg(feature="precomputed_tables")] - pub fn double_scalar_mul_basepoint( - a: &Scalar, - A: &EdwardsPoint, - b: &Scalar, - ) -> EdwardsPoint { + pub fn double_scalar_mul_basepoint(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] { - use backend::avx2::edwards as edwards_avx2; - - edwards_avx2::vartime::double_scalar_mul_basepoint(a, A, b) + #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] + { + use backend::avx2::scalar_mul::vartime_double_base::mul; + mul(a, A, b) } // Otherwise, proceed as normal: - #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] { - let a_naf = a.non_adjacent_form(); - let b_naf = b.non_adjacent_form(); - - // Find starting index - let mut i: usize = 255; - for j in (0..255).rev() { - i = j; - if a_naf[i] != 0 || b_naf[i] != 0 { - break; - } - } - - let odd_multiples_of_A = OddMultiples::create(A); - let odd_multiples_of_B = &constants::AFFINE_ODD_MULTIPLES_OF_BASEPOINT; - - let mut r = ProjectivePoint::identity(); - loop { - let mut t = r.double(); - - if a_naf[i] > 0 { - t = &t.to_extended() + &odd_multiples_of_A[( a_naf[i]/2) as usize]; - } else if a_naf[i] < 0 { - t = &t.to_extended() - &odd_multiples_of_A[(-a_naf[i]/2) as usize]; - } - - if b_naf[i] > 0 { - t = &t.to_extended() + &odd_multiples_of_B[( b_naf[i]/2) as usize]; - } else if b_naf[i] < 0 { - t = &t.to_extended() - &odd_multiples_of_B[(-b_naf[i]/2) as usize]; - } - - r = t.to_projective(); - - if i == 0 { - break; - } - i -= 1; - } - - r.to_extended() + #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] + { + use scalar_mul::vartime_double_base::mul; + mul(a, A, b) } } - } // ------------------------------------------------------------------------ diff --git a/src/scalar_mul/mod.rs b/src/scalar_mul/mod.rs index 2f295d7..7b3cdff 100644 --- a/src/scalar_mul/mod.rs +++ b/src/scalar_mul/mod.rs @@ -12,6 +12,9 @@ pub mod window; pub mod variable_base; +#[cfg(feature="precomputed_tables")] +pub mod vartime_double_base; + pub mod straus; pub mod vartime_straus; diff --git a/src/scalar_mul/vartime_double_base.rs b/src/scalar_mul/vartime_double_base.rs new file mode 100644 index 0000000..60c51e6 --- /dev/null +++ b/src/scalar_mul/vartime_double_base.rs @@ -0,0 +1,61 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence +#![allow(non_snake_case)] + +use constants; +use traits::Identity; +use scalar::Scalar; +use edwards::EdwardsPoint; +use curve_models::{ProjectiveNielsPoint, ProjectivePoint}; +use scalar_mul::window::OddLookupTable; + +/// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. +pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { + let a_naf = a.non_adjacent_form(); + let b_naf = b.non_adjacent_form(); + + // Find starting index + let mut i: usize = 255; + for j in (0..255).rev() { + i = j; + if a_naf[i] != 0 || b_naf[i] != 0 { + break; + } + } + + let table_A = OddLookupTable::::from(A); + let table_B = &constants::AFFINE_ODD_MULTIPLES_OF_BASEPOINT; + + let mut r = ProjectivePoint::identity(); + loop { + let mut t = r.double(); + + if a_naf[i] > 0 { + t = &t.to_extended() + &table_A.select(a_naf[i] as usize); + } else if a_naf[i] < 0 { + t = &t.to_extended() - &table_A.select(-a_naf[i] as usize); + } + + if b_naf[i] > 0 { + t = &t.to_extended() + &table_B.select(b_naf[i] as usize); + } else if b_naf[i] < 0 { + t = &t.to_extended() - &table_B.select(-b_naf[i] as usize); + } + + r = t.to_projective(); + + if i == 0 { + break; + } + i -= 1; + } + + r.to_extended() +} From c73a0fd0d6d0177334d4c37caa51cfdc29ce0d58 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 26 Mar 2018 17:05:54 -0700 Subject: [PATCH 07/16] Remove AVX2 fixed-base code. This was faster than the non-AVX2 code, but the serial code is already so fast that there's no reason not to use it. --- src/backend/avx2/edwards.rs | 50 ------------------------------------- 1 file changed, 50 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index caa8322..9099148 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -383,56 +383,6 @@ impl From for LookupTable { } } -#[derive(Clone)] -pub struct EdwardsBasepointTable(pub [LookupTable; 32]); - -impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsBasepointTable { - type Output = ExtendedPoint; - - fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { - let a = scalar.to_radix_16(); - - let tables = &self.0; - let mut P = ExtendedPoint::identity(); - - for i in (0..64).filter(|x| x % 2 == 1) { - P = &P + &tables[i/2].select(a[i]); - } - - P = P.mul_by_pow_2(4); - - for i in (0..64).filter(|x| x % 2 == 0) { - P = &P + &tables[i/2].select(a[i]); - } - - P - } -} - -impl<'a, 'b> Mul<&'a EdwardsBasepointTable> for &'b Scalar { - type Output = ExtendedPoint; - - /// Given `self` a table of precomputed multiples of the point `B`, compute `B * s`. - fn mul(self, basepoint_table: &'a EdwardsBasepointTable) -> ExtendedPoint { - basepoint_table * &self - } -} - -impl EdwardsBasepointTable { - /// Create a table of precomputed multiples of `basepoint`. - pub fn create(basepoint: &ExtendedPoint) -> EdwardsBasepointTable { - // XXX use init_with - let mut table = EdwardsBasepointTable([LookupTable::default(); 32]); - let mut P = *basepoint; - for i in 0..32 { - // P = (16^2)^i * B - table.0[i] = LookupTable::from(P); - P = P.mul_by_pow_2(8); - } - table - } -} - use scalar_mul::window::OddLookupTable; impl<'a> From<&'a ExtendedPoint> for OddLookupTable { From e8b053b28145bcca6f78d415d11a3215b10cab97 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 26 Mar 2018 17:12:32 -0700 Subject: [PATCH 08/16] Remove AVX2 addition formulas Only the readdition formulas are actually used by scalar multiplication, so there's no reason to implement vectorized addition. --- src/backend/avx2/edwards.rs | 110 +----------------------------------- 1 file changed, 2 insertions(+), 108 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 9099148..6fa72bb 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -132,17 +132,6 @@ impl<'a> Neg for &'a CachedPoint { } } -impl<'a> Neg for &'a ExtendedPoint { - type Output = ExtendedPoint; - - fn neg(self) -> ExtendedPoint { - let mut neg = *self; - // (X Y Z T) -> (-X Y Z -T) - neg.0.negate(A_LANES | D_LANES); - neg - } -} - impl ExtendedPoint { pub fn double(&self) -> ExtendedPoint { unsafe { @@ -287,80 +276,6 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { } } -impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { - type Output = ExtendedPoint; - - /// Uses a slight tweak of the parallel unified formulas of HWCD'08 - fn add(self, other: &'b ExtendedPoint) -> ExtendedPoint { - unsafe { - use stdsimd::vendor::_mm256_permute2x128_si256; - use stdsimd::vendor::_mm256_permutevar8x32_epi32; - use stdsimd::vendor::_mm256_blend_epi32; - - let P: &FieldElement32x4 = &self.0; - let Q: &FieldElement32x4 = &other.0; - - let mut t0 = FieldElement32x4::zero(); - let mut t1 = FieldElement32x4::zero(); - - // set t0 = (X1 Y1 X2 Y2) - for i in 0..5 { - t0.0[i] = _mm256_permute2x128_si256(P.0[i].into(), Q.0[i].into(), 32).into(); - } - - // set t0 = (Y1-X1 Y1+X1 Y2-X2 Y2+X2) = (S0 S1 S2 S3) - t0.diff_sum(0xff); - - // set t1 = (S0 S1 Z1 T1) - // set t0 = (S2 S3 Z2 T2) - for i in 0..5 { - // why does this intrinsic take an i32 for the imm8 ??? - t1.0[i] = _mm256_blend_epi32(t0.0[i].into(), P.0[i].into(), (C_LANES | D_LANES) as i32).into(); - t0.0[i] = _mm256_permute2x128_si256(t0.0[i].into(), Q.0[i].into(), 49).into(); - } - - // set t2 = (S0*S2 S1*S3 Z1*Z2 T1*T2) = (S4 S5 S6 S7) - let mut t2 = &t0 * &t1; - - //// set t2 = (S8 S9 S10 S11) - // set t2 = (121666*S4 121666*S5 2*121666*S6 2*121665*S7) - // = ( S8 S9 S10 -S11) - t2.scale_by_curve_constants(); - - // set t2 = (S8 S9 -S11 S10) - t2.swap_CD(); - - // set t2 = (S9-S8 S9+S8 S10+S11 S10-S11) = (S12 S13 S15 S14) - t2.diff_sum(0xff); - - let c0 = u32x8::new(0,4,2,6,4,0,6,2); // (ABCD) -> (ACCA) - let c1 = u32x8::new(5,1,7,3,5,1,7,3); // (ABCD) -> (DBDB) - - // set t0 = (S12 S15 S15 S12) - // set t1 = (S14 S13 S14 S13) - for i in 0..5 { - t0.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c0); - t1.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c1); - } - - // return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) - ExtendedPoint(&t0 * &t1) - } - } -} - -impl<'a, 'b> Sub<&'b ExtendedPoint> for &'a ExtendedPoint { - type Output = ExtendedPoint; - - /// Implement subtraction by negating the point and adding. - /// - /// Empirically, this seems about the same cost as a custom subtraction impl (maybe because the - /// benefit is cancelled by increased code size?) - fn sub(self, other: &'b ExtendedPoint) -> ExtendedPoint { - self + &(-other) - } -} - impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint { type Output = ExtendedPoint; @@ -466,14 +381,11 @@ mod test { fn addition_test_helper(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) { // Test the serial implementation of the parallel addition formulas let R_serial: edwards::EdwardsPoint = serial_add(P.into(), Q.into()).into(); - // Test the vector implementation of the parallel addition formulas - let R_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) + &ExtendedPoint::from(Q)).into(); - // Test the vector implementation of the parallel subtraction formulas - let S_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) - &ExtendedPoint::from(Q)).into(); // Test the vector implementation of the parallel readdition formulas let cached_Q = CachedPoint::from(ExtendedPoint::from(Q)); - let T_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) + &cached_Q).into(); + let R_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) + &cached_Q).into(); + let S_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) - &cached_Q).into(); println!("Testing point addition:"); println!("P = {:?}", P); @@ -482,32 +394,14 @@ mod test { println!("R = P + Q = {:?}", &P + &Q); println!("R_serial = {:?}", R_serial); println!("R_vector = {:?}", R_vector); - println!("T_vector = {:?}", T_vector); println!("S = P - Q = {:?}", &P - &Q); println!("S_vector = {:?}", S_vector); assert_eq!(R_serial.compress(), (&P + &Q).compress()); assert_eq!(R_vector.compress(), (&P + &Q).compress()); - assert_eq!(T_vector.compress(), (&P + &Q).compress()); assert_eq!(S_vector.compress(), (&P - &Q).compress()); println!("OK!\n"); } - #[test] - fn sub_vs_add_minus() { - let P: ExtendedPoint = edwards::EdwardsPoint::identity().into(); - let Q: ExtendedPoint = edwards::EdwardsPoint::identity().into(); - - let mQ = -&Q; - - println!("sub"); - let R1: edwards::EdwardsPoint = (&P - &Q).into(); - println!("add neg"); - let R2: edwards::EdwardsPoint = (&P + &mQ).into(); - - assert_eq!(R2.compress(), edwards::EdwardsPoint::identity().compress()); - assert_eq!(R1.compress(), edwards::EdwardsPoint::identity().compress()); - } - #[test] fn vector_addition_vs_serial_addition_vs_edwards_extendedpoint() { use constants; From 7ef6a1e6fa0a3a94685c3c216454cb09cd093999 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 26 Mar 2018 17:35:08 -0700 Subject: [PATCH 09/16] Reorganize AVX2 point code --- src/backend/avx2/edwards.rs | 128 +++++++++--------- src/backend/avx2/scalar_mul/straus.rs | 5 +- src/backend/avx2/scalar_mul/variable_base.rs | 4 +- .../avx2/scalar_mul/vartime_double_base.rs | 3 +- src/backend/avx2/scalar_mul/vartime_straus.rs | 5 +- 5 files changed, 68 insertions(+), 77 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 6fa72bb..c9da1ea 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -24,7 +24,7 @@ use subtle::Choice; use edwards; use scalar::Scalar; -use scalar_mul::window::LookupTable; +use scalar_mul::window::{LookupTable, OddLookupTable}; use traits::Identity; @@ -76,62 +76,6 @@ impl Identity for ExtendedPoint { } } -/// A cached point with some precomputed variables used for readdition. -#[derive(Copy, Clone, Debug)] -pub struct CachedPoint(pub(super) FieldElement32x4); - -impl From for CachedPoint { - fn from(P: ExtendedPoint) -> CachedPoint { - let mut x = P.0; - - // x = (S2 S3 Z2 T2) - x.diff_sum(0b00001111); - - // x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2) - x.scale_by_curve_constants(); - - // x = (121666*S2 121666*S3 2*121666*Z2 -2*121665*T2) - x.negate(D_LANES); - - CachedPoint(x) - } -} - -impl Default for CachedPoint { - fn default() -> CachedPoint { - CachedPoint::identity() - } -} - -impl Identity for CachedPoint { - fn identity() -> CachedPoint { - CachedPoint(FieldElement32x4([ - u32x8::new(121647, 121666, 0, 0, 243332, 67108845, 0, 33554431), - u32x8::new(67108864, 0, 33554431, 0, 0, 67108863, 0, 33554431), - u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), - u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), - u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), - ])) - } -} - -impl ConditionallyAssignable for CachedPoint { - fn conditional_assign(&mut self, other: &CachedPoint, choice: Choice) { - self.0.conditional_assign(&other.0, choice); - } -} - -impl<'a> Neg for &'a CachedPoint { - type Output = CachedPoint; - - fn neg(self) -> CachedPoint { - let mut neg = *self; - neg.0.swap_AB(); - neg.0.negate_lazy(D_LANES); - neg - } -} - impl ExtendedPoint { pub fn double(&self) -> ExtendedPoint { unsafe { @@ -236,6 +180,62 @@ impl ExtendedPoint { } } +/// A cached point with some precomputed variables used for readdition. +#[derive(Copy, Clone, Debug)] +pub struct CachedPoint(pub(super) FieldElement32x4); + +impl From for CachedPoint { + fn from(P: ExtendedPoint) -> CachedPoint { + let mut x = P.0; + + // x = (S2 S3 Z2 T2) + x.diff_sum(0b00001111); + + // x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2) + x.scale_by_curve_constants(); + + // x = (121666*S2 121666*S3 2*121666*Z2 -2*121665*T2) + x.negate(D_LANES); + + CachedPoint(x) + } +} + +impl Default for CachedPoint { + fn default() -> CachedPoint { + CachedPoint::identity() + } +} + +impl Identity for CachedPoint { + fn identity() -> CachedPoint { + CachedPoint(FieldElement32x4([ + u32x8::new(121647, 121666, 0, 0, 243332, 67108845, 0, 33554431), + u32x8::new(67108864, 0, 33554431, 0, 0, 67108863, 0, 33554431), + u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), + u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), + u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), + ])) + } +} + +impl ConditionallyAssignable for CachedPoint { + fn conditional_assign(&mut self, other: &CachedPoint, choice: Choice) { + self.0.conditional_assign(&other.0, choice); + } +} + +impl<'a> Neg for &'a CachedPoint { + type Output = CachedPoint; + + fn neg(self) -> CachedPoint { + let mut neg = *self; + neg.0.swap_AB(); + neg.0.negate_lazy(D_LANES); + neg + } +} + impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { type Output = ExtendedPoint; @@ -288,8 +288,9 @@ impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint { } } -impl From for LookupTable { - fn from(P: ExtendedPoint) -> Self { +impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable { + fn from(point: &'a edwards::EdwardsPoint) -> Self { + let P = ExtendedPoint::from(*point); let mut points = [CachedPoint::from(P); 8]; for i in 0..7 { points[i+1] = (&P + &points[i]).into(); @@ -298,11 +299,10 @@ impl From for LookupTable { } } -use scalar_mul::window::OddLookupTable; - -impl<'a> From<&'a ExtendedPoint> for OddLookupTable { - fn from(A: &'a ExtendedPoint) -> Self { - let mut Ai = [CachedPoint::from(*A); 8]; +impl<'a> From<&'a edwards::EdwardsPoint> for OddLookupTable { + fn from(point: &'a edwards::EdwardsPoint) -> Self { + let A = ExtendedPoint::from(*point); + let mut Ai = [CachedPoint::from(A); 8]; let A2 = A.double(); for i in 0..7 { Ai[i + 1] = (&A2 + &Ai[i]).into(); diff --git a/src/backend/avx2/scalar_mul/straus.rs b/src/backend/avx2/scalar_mul/straus.rs index 5f76abe..59e123e 100644 --- a/src/backend/avx2/scalar_mul/straus.rs +++ b/src/backend/avx2/scalar_mul/straus.rs @@ -31,10 +31,7 @@ where // for each input point P let lookup_tables: Vec<_> = points .into_iter() - .map(|point| { - let avx2_point = ExtendedPoint::from(*point.borrow()); - LookupTable::::from(avx2_point) - }) + .map(|point| LookupTable::::from(point.borrow())) .collect(); let scalar_digits_vec: Vec<_> = scalars diff --git a/src/backend/avx2/scalar_mul/variable_base.rs b/src/backend/avx2/scalar_mul/variable_base.rs index e5261fb..6f36bc7 100644 --- a/src/backend/avx2/scalar_mul/variable_base.rs +++ b/src/backend/avx2/scalar_mul/variable_base.rs @@ -8,10 +8,8 @@ use scalar_mul::window::LookupTable; /// Perform constant-time, variable-base scalar multiplication. pub fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint { - // XXX combine these conversions - let avx2_point = ExtendedPoint::from(*point); // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] - let lookup_table = LookupTable::::from(avx2_point); + let lookup_table = LookupTable::::from(point); // Setting s = scalar, compute // // s = s_0 + s_1*16^1 + ... + s_63*16^63, diff --git a/src/backend/avx2/scalar_mul/vartime_double_base.rs b/src/backend/avx2/scalar_mul/vartime_double_base.rs index f7703bb..1ba09f1 100644 --- a/src/backend/avx2/scalar_mul/vartime_double_base.rs +++ b/src/backend/avx2/scalar_mul/vartime_double_base.rs @@ -30,8 +30,7 @@ pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { } } - let avx2_A = ExtendedPoint::from(*A); - let table_A = OddLookupTable::::from(&avx2_A); + let table_A = OddLookupTable::::from(A); let table_B = &BASEPOINT_ODD_LOOKUP_TABLE; let mut Q = ExtendedPoint::identity(); diff --git a/src/backend/avx2/scalar_mul/vartime_straus.rs b/src/backend/avx2/scalar_mul/vartime_straus.rs index c53e7d4..224713d 100644 --- a/src/backend/avx2/scalar_mul/vartime_straus.rs +++ b/src/backend/avx2/scalar_mul/vartime_straus.rs @@ -31,10 +31,7 @@ where .collect(); let lookup_tables: Vec<_> = points .into_iter() - .map(|point| { - let avx2_point = ExtendedPoint::from(*point.borrow()); - OddLookupTable::::from(&avx2_point) - }) + .map(|point| OddLookupTable::::from(point.borrow())) .collect(); let mut Q = ExtendedPoint::identity(); From 4a648df7130ec16e62ca77bd4183694a9cd00fc4 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 26 Mar 2018 17:58:31 -0700 Subject: [PATCH 10/16] Feature-gate multiscalar impls on alloc --- src/backend/avx2/scalar_mul/mod.rs | 2 ++ src/scalar_mul/mod.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/backend/avx2/scalar_mul/mod.rs b/src/backend/avx2/scalar_mul/mod.rs index 8b1e84e..c4c944b 100644 --- a/src/backend/avx2/scalar_mul/mod.rs +++ b/src/backend/avx2/scalar_mul/mod.rs @@ -13,6 +13,8 @@ pub mod variable_base; #[cfg(feature="precomputed_tables")] pub mod vartime_double_base; +#[cfg(any(feature = "alloc", feature = "std"))] pub mod straus; +#[cfg(any(feature = "alloc", feature = "std"))] pub mod vartime_straus; diff --git a/src/scalar_mul/mod.rs b/src/scalar_mul/mod.rs index 7b3cdff..f172788 100644 --- a/src/scalar_mul/mod.rs +++ b/src/scalar_mul/mod.rs @@ -15,6 +15,8 @@ pub mod variable_base; #[cfg(feature="precomputed_tables")] pub mod vartime_double_base; +#[cfg(any(feature = "alloc", feature = "std"))] pub mod straus; +#[cfg(any(feature = "alloc", feature = "std"))] pub mod vartime_straus; From 32817089652f00a32905b0813fec75cdcebd1b9b Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Mon, 2 Apr 2018 21:09:41 +0000 Subject: [PATCH 11/16] Define mul variants for MontgomeryPoints. This results in less changes to the x25519-dalek code to upgrade to the latest version. --- src/montgomery.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/montgomery.rs b/src/montgomery.rs index 8be9ec0..0ebca9a 100644 --- a/src/montgomery.rs +++ b/src/montgomery.rs @@ -229,12 +229,17 @@ fn differential_add_and_double( Q.W = t17; // W_{Q'} = U_D * 4 (W_P U_Q - U_P W_Q)^2 } +define_mul_assign_variants!(LHS = MontgomeryPoint, RHS = Scalar); + +define_mul_variants!(LHS = MontgomeryPoint, RHS = Scalar, Output = MontgomeryPoint); +define_mul_variants!(LHS = Scalar, RHS = MontgomeryPoint, Output = MontgomeryPoint); + /// Multiply this `MontgomeryPoint` by a `Scalar`. -impl Mul for MontgomeryPoint { +impl<'a, 'b> Mul<&'b Scalar> for &'a MontgomeryPoint { type Output = MontgomeryPoint; /// Given `self` \\( = u\_0(P) \\), and a `Scalar` \\(n\\), return \\( u\_0([n]P) \\). - fn mul(self, scalar: Scalar) -> MontgomeryPoint { + fn mul(self, scalar: &'b Scalar) -> MontgomeryPoint { // Algorithm 8 of Costello-Smith 2017 let affine_u = FieldElement::from_bytes(&self.0); let mut x0 = ProjectivePoint::identity(); @@ -256,16 +261,16 @@ impl Mul for MontgomeryPoint { } } -impl MulAssign for MontgomeryPoint { - fn mul_assign(&mut self, scalar: Scalar) { - *self = (*self) * scalar; +impl<'b> MulAssign<&'b Scalar> for MontgomeryPoint { + fn mul_assign(&mut self, scalar: &'b Scalar) { + *self = (self as &MontgomeryPoint) * scalar; } } -impl Mul for Scalar { +impl<'a, 'b> Mul<&'b MontgomeryPoint> for &'a Scalar { type Output = MontgomeryPoint; - fn mul(self, point: MontgomeryPoint) -> MontgomeryPoint { + fn mul(self, point: &'b MontgomeryPoint) -> MontgomeryPoint { point * self } } From b0bda0278cbc49ec69cae1b0c6b8a34ecb3bb050 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 3 Apr 2018 17:58:23 -0700 Subject: [PATCH 12/16] Drop the `stdsimd` crate in favor of `core::{simd, arch}`. This change required some work, because the to-be-stabilized SIMD functions don't allow non-constant `imm8`s. Previously, the `stdsimd` functions had a constifying macro that ensured that the immediates were known. The dalek code used this to build helper functions which would be inlined into different places where the immediates were known. Unfortunately, since constexprs aren't fully supported in Rust yet, this is done by a hidden compiler attribute, and there's no way to propagate "constness". To deal with this, some of the functions are specialized (e.g., `square_and_negate_D` instead of taking a mask), and others use an enum. --- Cargo.toml | 4 +- build.rs | 3 +- src/backend/avx2/constants.rs | 2 +- src/backend/avx2/edwards.rs | 89 ++++++----- src/backend/avx2/field.rs | 279 ++++++++++++++++++---------------- src/lib.rs | 4 +- 6 files changed, 199 insertions(+), 182 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 72eda7f..8387cd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,6 @@ digest = "0.7" generic-array = "0.9" clear_on_drop = "=0.2.3" subtle = { version = "0.6", features = ["generic-impls"], default-features = false } -stdsimd = { version = "0.0.4", optional = true } serde = { version = "1.0", optional = true } rand = { version = "0.4", optional = true } @@ -54,7 +53,6 @@ digest = "0.7" generic-array = "0.9" clear_on_drop = "=0.2.3" subtle = { version = "0.6", features = ["generic-impls"], default-features = false } -stdsimd = { version = "0.0.4", optional = true } serde = { version = "1.0", optional = true } # Allowing rand to be optional during builds causes a build failure when compiling for no_std targets rand = { version = "0.4", optional = false } @@ -70,4 +68,4 @@ radix_51 = [] # Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation. precomputed_tables = [] # experimental avx2 support -avx2_backend = ["nightly", "stdsimd"] +avx2_backend = ["nightly"] diff --git a/build.rs b/build.rs index 60a9f7a..8331688 100644 --- a/build.rs +++ b/build.rs @@ -1,5 +1,6 @@ #![cfg_attr(feature = "nightly", feature(i128_type))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] +#![cfg_attr(all(feature = "nightly", feature = "yolocrypto"), feature(stdsimd))] #![allow(unused_variables)] #![allow(non_snake_case)] #![allow(dead_code)] @@ -23,8 +24,6 @@ use std::path::Path; // For instance, this shouldn't exist here at all, but it does. #[cfg(feature = "serde")] extern crate serde; -#[cfg(feature = "yolocrypto")] -extern crate stdsimd; // Macros come first! #[path = "src/macros.rs"] diff --git a/src/backend/avx2/constants.rs b/src/backend/avx2/constants.rs index 3fb77d8..e0479db 100644 --- a/src/backend/avx2/constants.rs +++ b/src/backend/avx2/constants.rs @@ -10,7 +10,7 @@ //! This module contains constants used by the AVX2 backend. -use stdsimd::simd::u32x8; +use core::simd::u32x8; use scalar_mul::window::OddLookupTable; use backend::avx2::field::FieldElement32x4; diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index c9da1ea..9341ee9 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -17,7 +17,7 @@ use core::convert::From; use core::ops::{Index, Add, Sub, Mul, Neg}; use core::borrow::Borrow; -use stdsimd::simd::{u32x8, i32x8}; +use core::simd::{IntoBits, u32x8, i32x8}; use subtle::ConditionallyAssignable; use subtle::Choice; @@ -28,10 +28,7 @@ use scalar_mul::window::{LookupTable, OddLookupTable}; use traits::Identity; -use backend::avx2::field::FieldElement32x4; - -use backend::avx2::field::{A_LANES, B_LANES, C_LANES, D_LANES, ALL_LANES}; -use backend::avx2::field::D_LANES64; +use backend::avx2::field::{D_LANES, Lanes, FieldElement32x4}; use backend::avx2; @@ -79,10 +76,10 @@ impl Identity for ExtendedPoint { impl ExtendedPoint { pub fn double(&self) -> ExtendedPoint { unsafe { - use stdsimd::vendor::_mm256_permute2x128_si256; - use stdsimd::vendor::_mm256_permutevar8x32_epi32; - use stdsimd::vendor::_mm256_blend_epi32; - use stdsimd::vendor::_mm256_shuffle_epi32; + use core::arch::x86_64::_mm256_permute2x128_si256; + use core::arch::x86_64::_mm256_permutevar8x32_epi32; + use core::arch::x86_64::_mm256_blend_epi32; + use core::arch::x86_64::_mm256_shuffle_epi32; let P = &self.0; @@ -96,18 +93,18 @@ impl ExtendedPoint { // and then adding. // Set t0 = (X1 Y1 X1 Y1) - t0.0[0] = _mm256_permute2x128_si256(P.0[0].into(), P.0[0].into(), 0b0000_0000).into(); - t0.0[1] = _mm256_permute2x128_si256(P.0[1].into(), P.0[1].into(), 0b0000_0000).into(); - t0.0[2] = _mm256_permute2x128_si256(P.0[2].into(), P.0[2].into(), 0b0000_0000).into(); - t0.0[3] = _mm256_permute2x128_si256(P.0[3].into(), P.0[3].into(), 0b0000_0000).into(); - t0.0[4] = _mm256_permute2x128_si256(P.0[4].into(), P.0[4].into(), 0b0000_0000).into(); + t0.0[0] = _mm256_permute2x128_si256(P.0[0].into_bits(), P.0[0].into_bits(), 0b0000_0000).into_bits(); + t0.0[1] = _mm256_permute2x128_si256(P.0[1].into_bits(), P.0[1].into_bits(), 0b0000_0000).into_bits(); + t0.0[2] = _mm256_permute2x128_si256(P.0[2].into_bits(), P.0[2].into_bits(), 0b0000_0000).into_bits(); + t0.0[3] = _mm256_permute2x128_si256(P.0[3].into_bits(), P.0[3].into_bits(), 0b0000_0000).into_bits(); + t0.0[4] = _mm256_permute2x128_si256(P.0[4].into_bits(), P.0[4].into_bits(), 0b0000_0000).into_bits(); // Set t1 = (Y1 X1 Y1 X1) - t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into(), 0b10_11_00_01).into(); - t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into(), 0b10_11_00_01).into(); - t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into(), 0b10_11_00_01).into(); - t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into(), 0b10_11_00_01).into(); - t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into(), 0b10_11_00_01).into(); + t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into_bits(), 0b10_11_00_01).into_bits(); + t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into_bits(), 0b10_11_00_01).into_bits(); + t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into_bits(), 0b10_11_00_01).into_bits(); + t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into_bits(), 0b10_11_00_01).into_bits(); + t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into_bits(), 0b10_11_00_01).into_bits(); // Set t0 = (X1+Y1 X1+Y1 X1+Y1 X1+Y1) t0.0[0] = t0.0[0] + t1.0[0]; @@ -118,19 +115,19 @@ impl ExtendedPoint { // Set t0 = (X1 Y1 Z1 X1+Y1) // why does this intrinsic take an i32 for the imm8 ??? - t0.0[0] = _mm256_blend_epi32(P.0[0].into(), t0.0[0].into(), D_LANES as i32).into(); - t0.0[1] = _mm256_blend_epi32(P.0[1].into(), t0.0[1].into(), D_LANES as i32).into(); - t0.0[2] = _mm256_blend_epi32(P.0[2].into(), t0.0[2].into(), D_LANES as i32).into(); - t0.0[3] = _mm256_blend_epi32(P.0[3].into(), t0.0[3].into(), D_LANES as i32).into(); - t0.0[4] = _mm256_blend_epi32(P.0[4].into(), t0.0[4].into(), D_LANES as i32).into(); + t0.0[0] = _mm256_blend_epi32(P.0[0].into_bits(), t0.0[0].into_bits(), D_LANES as i32).into_bits(); + t0.0[1] = _mm256_blend_epi32(P.0[1].into_bits(), t0.0[1].into_bits(), D_LANES as i32).into_bits(); + t0.0[2] = _mm256_blend_epi32(P.0[2].into_bits(), t0.0[2].into_bits(), D_LANES as i32).into_bits(); + t0.0[3] = _mm256_blend_epi32(P.0[3].into_bits(), t0.0[3].into_bits(), D_LANES as i32).into_bits(); + t0.0[4] = _mm256_blend_epi32(P.0[4].into_bits(), t0.0[4].into_bits(), D_LANES as i32).into_bits(); // Set t1 = t0^2, negating the D values - t1 = t0.square(D_LANES64); + t1 = t0.square_and_negate_D(); // Now t1 = (S1 S2 S3 -S4) - let c0 = u32x8::new(0,0,2,2,0,0,2,2); // (ABCD) -> (AAAA) - let c1 = u32x8::new(1,1,3,3,1,1,3,3); // (ABCD) -> (BBBB) + let c0 = u32x8::new(0,0,2,2,0,0,2,2).into_bits(); // (ABCD) -> (AAAA) + let c1 = u32x8::new(1,1,3,3,1,1,3,3).into_bits(); // (ABCD) -> (BBBB) // See discussion of bounds in the module-level documentation. // @@ -147,24 +144,24 @@ impl ExtendedPoint { // S5 S6 S8 S9 // for i in 0..5 { - let zero = i32x8::splat(0); - let S1 = _mm256_permutevar8x32_epi32(t1.0[i], c0); - let S2 = _mm256_permutevar8x32_epi32(t1.0[i], c1); - let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into(), 0b01010000).into(); + let zero = u32x8::splat(0).into_bits(); + let S1: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c0).into_bits(); + let S2: u32x8 = _mm256_permutevar8x32_epi32(t1.0[i].into_bits(), c1).into_bits(); + let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into_bits(), 0b01010000).into_bits(); // tmp0 = (0 0 2*S3 -S4) - let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into(), t1.0[i].into(), 0b10100000).into(); + let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into_bits(), t1.0[i].into_bits(), 0b10100000).into_bits(); t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1; - t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into(), 0b10100101).into(); - t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into(), zero, 0b10100101).into(); + t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into_bits(), 0b10100101).into_bits(); + t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into_bits(), zero, 0b10100101).into_bits(); } - let c0 = u32x8::new(4,0,6,2,4,0,6,2); // (ABCD) -> (CACA) - let c1 = u32x8::new(5,1,7,3,1,5,3,7); // (ABCD) -> (DBBD) + let c0 = u32x8::new(4,0,6,2,4,0,6,2).into_bits(); // (ABCD) -> (CACA) + let c1 = u32x8::new(5,1,7,3,1,5,3,7).into_bits(); // (ABCD) -> (DBBD) for i in 0..5 { let tmp = t0.0[i]; - t0.0[i] = _mm256_permutevar8x32_epi32(tmp, c0); - t1.0[i] = _mm256_permutevar8x32_epi32(tmp, c1); + t0.0[i] = _mm256_permutevar8x32_epi32(tmp.into_bits(), c0).into_bits(); + t1.0[i] = _mm256_permutevar8x32_epi32(tmp.into_bits(), c1).into_bits(); } ExtendedPoint(&t0 * &t1) @@ -189,13 +186,13 @@ impl From for CachedPoint { let mut x = P.0; // x = (S2 S3 Z2 T2) - x.diff_sum(0b00001111); + x.diff_sum(Lanes::AB); // x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2) x.scale_by_curve_constants(); // x = (121666*S2 121666*S3 2*121666*Z2 -2*121665*T2) - x.negate(D_LANES); + x.negate_D(); CachedPoint(x) } @@ -231,7 +228,7 @@ impl<'a> Neg for &'a CachedPoint { fn neg(self) -> CachedPoint { let mut neg = *self; neg.0.swap_AB(); - neg.0.negate_lazy(D_LANES); + neg.0.negate_D_lazy(); neg } } @@ -242,12 +239,12 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { /// Uses a slight tweak of the parallel unified formulas of HWCD'08 fn add(self, other: &'b CachedPoint) -> ExtendedPoint { unsafe { - use stdsimd::vendor::_mm256_permutevar8x32_epi32; + use core::arch::x86_64::_mm256_permutevar8x32_epi32; let mut tmp = self.0; // tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1) - tmp.diff_sum(A_LANES | B_LANES); + tmp.diff_sum(Lanes::AB); // tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11) tmp = &tmp * &other.0; @@ -256,7 +253,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { tmp.swap_CD(); // tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) - tmp.diff_sum(ALL_LANES); + tmp.diff_sum(Lanes::ALL); let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) @@ -266,8 +263,8 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { let mut t0 = FieldElement32x4::zero(); let mut t1 = FieldElement32x4::zero(); for i in 0..5 { - t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c0); - t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c1); + t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i].into_bits(), c0.into_bits()).into_bits(); + t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i].into_bits(), c1.into_bits()).into_bits(); } // return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 125aa55..2af68b2 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -9,7 +9,6 @@ // - Henry de Valence //! 4-way vectorized 32bit field arithmetic using AVX2. -//! #![allow(bad_style)] @@ -26,13 +25,31 @@ pub const D_LANES64: u8 = 0b11_00_00_00; pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES; use core::ops::Mul; - -use stdsimd::simd::{u32x8, i32x8, u64x4}; +use core::simd::{IntoBits, u32x8, i32x8, u64x4}; use backend::u64::field::FieldElement64; - use backend::avx2::constants::{P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_16_LO, P_TIMES_16_HI}; +#[derive(Copy, Clone)] +pub enum Lanes { + AB, + CD, + ALL, +} + +#[inline(always)] +fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 { + unsafe { + use core::arch::x86_64::_mm256_blend_epi32; + + match control { + Lanes::AB => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (A_LANES | B_LANES) as i32).into_bits(), + Lanes::CD => _mm256_blend_epi32(x.into_bits(), y.into_bits(), (C_LANES | D_LANES) as i32).into_bits(), + Lanes::ALL => _mm256_blend_epi32(x.into_bits(), y.into_bits(), ALL_LANES as i32).into_bits(), + } + } +} + /// A vector of four `FieldElements`, implemented using AVX2. #[derive(Clone, Copy, Debug)] pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]); @@ -50,7 +67,6 @@ impl ConditionallyAssignable for FieldElement32x4 { } } - impl FieldElement32x4 { pub(crate) fn split(&self) -> [FieldElement64; 4] { let mut out = [FieldElement64::zero(); 4]; @@ -108,28 +124,31 @@ impl FieldElement32x4 { return out; } - pub fn negate_lazy(&mut self, mask: u8) { - let mask = mask as i32; + /// Negate the \\(D\\) variable of \\((A,B,C,D)\\). + /// + /// Input limbs must be less than the limbs of \\(2p\\), i.e., freshly reduced. + pub fn negate_D_lazy(&mut self) { unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_2_LO - self.0[0]).into(), mask).into(); - self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_2_HI - self.0[1]).into(), mask).into(); - self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_2_HI - self.0[2]).into(), mask).into(); - self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_2_HI - self.0[3]).into(), mask).into(); - self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_2_HI - self.0[4]).into(), mask).into(); + use core::arch::x86_64::_mm256_blend_epi32; + self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_2_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits(); + self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_2_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits(); + self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_2_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits(); + self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_2_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits(); + self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_2_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits(); } } - /// Negate variables in lanes where mask is set - pub fn negate(&mut self, mask: u8) { - let mask = mask as i32; + /// Negate the \\(D\\) variable of \\((A,B,C,D)\\). + /// + /// Input limbs must be less than the limbs of \\(2p\\), i.e., freshly reduced. + pub fn negate_D(&mut self) { unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_16_LO - self.0[0]).into(), mask).into(); - self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_16_HI - self.0[1]).into(), mask).into(); - self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_16_HI - self.0[2]).into(), mask).into(); - self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_16_HI - self.0[3]).into(), mask).into(); - self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_16_HI - self.0[4]).into(), mask).into(); + use core::arch::x86_64::_mm256_blend_epi32; + self.0[0] = _mm256_blend_epi32(self.0[0].into_bits(), (P_TIMES_16_LO - self.0[0]).into_bits(), D_LANES as i32).into_bits(); + self.0[1] = _mm256_blend_epi32(self.0[1].into_bits(), (P_TIMES_16_HI - self.0[1]).into_bits(), D_LANES as i32).into_bits(); + self.0[2] = _mm256_blend_epi32(self.0[2].into_bits(), (P_TIMES_16_HI - self.0[2]).into_bits(), D_LANES as i32).into_bits(); + self.0[3] = _mm256_blend_epi32(self.0[3].into_bits(), (P_TIMES_16_HI - self.0[3]).into_bits(), D_LANES as i32).into_bits(); + self.0[4] = _mm256_blend_epi32(self.0[4].into_bits(), (P_TIMES_16_HI - self.0[4]).into_bits(), D_LANES as i32).into_bits(); } self.reduce32(); } @@ -137,11 +156,11 @@ impl FieldElement32x4 { /// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)` pub fn swap_AB(&mut self) { unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - use stdsimd::vendor::_mm256_blend_epi32; + use core::arch::x86_64::_mm256_shuffle_epi32; + use core::arch::x86_64::_mm256_blend_epi32; for i in 0..5 { - let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01); - self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b00001111).into(); + let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01); + self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b00001111).into_bits(); } } } @@ -149,55 +168,61 @@ impl FieldElement32x4 { /// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` pub fn swap_CD(&mut self) { unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - use stdsimd::vendor::_mm256_blend_epi32; + use core::arch::x86_64::_mm256_shuffle_epi32; + use core::arch::x86_64::_mm256_blend_epi32; for i in 0..5 { - let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01); - self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b11110000).into(); + let swapped = _mm256_shuffle_epi32(self.0[i].into_bits(), 0b10_11_00_01); + self.0[i] = _mm256_blend_epi32(self.0[i].into_bits(), swapped, 0b11110000).into_bits(); } } } /// Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)` according to `mask`. - pub fn diff_sum(&mut self, mask: u8) { - let mask = mask as i32; + /// + /// This is `#[inline(always)]` because the `mask` parameter should be an immediate. + #[inline(always)] + pub fn diff_sum(&mut self, control: Lanes) { unsafe { - use stdsimd::vendor::{_mm256_shuffle_epi32, _mm256_blend_epi32}; + use core::arch::x86_64::{_mm256_shuffle_epi32, _mm256_blend_epi32}; + + let shuffle = |v: u32x8| -> u32x8 { + _mm256_shuffle_epi32(v.into_bits(), 0b10_11_00_01).into_bits() + }; let x01 = self.0[0]; - let x01_shuf = _mm256_shuffle_epi32(x01.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x01_shuf = shuffle(x01); let v1 = (x01_shuf + P_TIMES_2_LO) - x01; let v2 = x01_shuf + x01; - let diffsum01 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[0] = _mm256_blend_epi32(x01.into(), diffsum01.into(), mask).into(); + let diffsum01 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[0] = blend_lanes(x01, diffsum01, control); let x23 = self.0[1]; - let x23_shuf = _mm256_shuffle_epi32(x23.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x23_shuf = shuffle(x23); let v1 = (x23_shuf + P_TIMES_2_HI) - x23; let v2 = x23_shuf + x23; - let diffsum23 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[1] = _mm256_blend_epi32(x23.into(), diffsum23.into(), mask).into(); + let diffsum23 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[1] = blend_lanes(x23, diffsum23, control); let x45 = self.0[2]; - let x45_shuf = _mm256_shuffle_epi32(x45.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x45_shuf = shuffle(x45); let v1 = (x45_shuf + P_TIMES_2_HI) - x45; let v2 = x45_shuf + x45; - let diffsum45 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[2] = _mm256_blend_epi32(x45.into(), diffsum45.into(), mask).into(); + let diffsum45 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[2] = blend_lanes(x45, diffsum45, control); let x67 = self.0[3]; - let x67_shuf = _mm256_shuffle_epi32(x67.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x67_shuf = shuffle(x67); let v1 = (x67_shuf + P_TIMES_2_HI) - x67; let v2 = x67_shuf + x67; - let diffsum67 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[3] = _mm256_blend_epi32(x67.into(), diffsum67.into(), mask).into(); + let diffsum67 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[3] = blend_lanes(x67, diffsum67, control); let x89 = self.0[4]; - let x89_shuf = _mm256_shuffle_epi32(x89.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let x89_shuf = shuffle(x89); let v1 = (x89_shuf + P_TIMES_2_HI) - x89; let v2 = x89_shuf + x89; - let diffsum89 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); - self.0[4] = _mm256_blend_epi32(x89.into(), diffsum89.into(), mask).into(); + let diffsum89 = _mm256_blend_epi32(v1.into_bits(), v2.into_bits(), 0b10101010).into_bits(); + self.0[4] = blend_lanes(x89, diffsum89, control); } } @@ -208,30 +233,30 @@ impl FieldElement32x4 { pub fn scale_by_curve_constants(&mut self) { let mut b = [u64x4::splat(0); 10]; - let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); + let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); unsafe { - use stdsimd::vendor::_mm256_mul_epu32; + use core::arch::x86_64::_mm256_mul_epu32; let (b0, b1) = unpack_pair(self.0[0]); - b[0] = _mm256_mul_epu32(b0, consts); - b[1] = _mm256_mul_epu32(b1, consts); + b[0] = _mm256_mul_epu32(b0.into_bits(), consts.into_bits()).into_bits(); + b[1] = _mm256_mul_epu32(b1.into_bits(), consts.into_bits()).into_bits(); let (b2, b3) = unpack_pair(self.0[1]); - b[2] = _mm256_mul_epu32(b2, consts); - b[3] = _mm256_mul_epu32(b3, consts); + b[2] = _mm256_mul_epu32(b2.into_bits(), consts.into_bits()).into_bits(); + b[3] = _mm256_mul_epu32(b3.into_bits(), consts.into_bits()).into_bits(); let (b4, b5) = unpack_pair(self.0[2]); - b[4] = _mm256_mul_epu32(b4, consts); - b[5] = _mm256_mul_epu32(b5, consts); + b[4] = _mm256_mul_epu32(b4.into_bits(), consts.into_bits()).into_bits(); + b[5] = _mm256_mul_epu32(b5.into_bits(), consts.into_bits()).into_bits(); let (b6, b7) = unpack_pair(self.0[3]); - b[6] = _mm256_mul_epu32(b6, consts); - b[7] = _mm256_mul_epu32(b7, consts); + b[6] = _mm256_mul_epu32(b6.into_bits(), consts.into_bits()).into_bits(); + b[7] = _mm256_mul_epu32(b7.into_bits(), consts.into_bits()).into_bits(); let (b8, b9) = unpack_pair(self.0[4]); - b[8] = _mm256_mul_epu32(b8, consts); - b[9] = _mm256_mul_epu32(b9, consts); + b[8] = _mm256_mul_epu32(b8.into_bits(), consts.into_bits()).into_bits(); + b[9] = _mm256_mul_epu32(b9.into_bits(), consts.into_bits()).into_bits(); } *self = FieldElement32x4::reduce64(b); @@ -243,24 +268,25 @@ impl FieldElement32x4 { let masks = u32x8::new((1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1, (1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1); + /// Compute the carry-out of each coefficient of v using a left shift. let carry = |v: u32x8| -> u32x8 { unsafe { - use stdsimd::vendor::_mm256_srlv_epi32; - _mm256_srlv_epi32(v.into(), shifts).into() + use core::arch::x86_64::_mm256_srlv_epi32; + _mm256_srlv_epi32(v.into_bits(), shifts.into_bits()).into_bits() } }; let swap_lanes = |v: u32x8| -> u32x8 { unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - _mm256_shuffle_epi32(v.into(), 0b01_00_11_10).into() + use core::arch::x86_64::_mm256_shuffle_epi32; + _mm256_shuffle_epi32(v.into_bits(), 0b01_00_11_10).into_bits() } }; let combine = |v_lo: u32x8, v_hi: u32x8| -> u32x8 { unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - _mm256_blend_epi32(v_lo.into(), v_hi.into(), 0b11_00_11_00).into() + use core::arch::x86_64::_mm256_blend_epi32; + _mm256_blend_epi32(v_lo.into_bits(), v_hi.into_bits(), 0b11_00_11_00).into_bits() } }; @@ -282,11 +308,11 @@ impl FieldElement32x4 { // let c9_19: u32x8; unsafe { - use stdsimd::vendor::_mm256_mul_epu32; - use stdsimd::vendor::_mm256_shuffle_epi32; - let c9_spread: u32x8 = _mm256_shuffle_epi32(c98.into(), 0b11_01_10_00).into(); - let c9_19_spread: u32x8 = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into()).into(); - c9_19 = _mm256_shuffle_epi32(c9_19_spread.into(), 0b11_01_10_00).into(); + use core::arch::x86_64::_mm256_mul_epu32; + use core::arch::x86_64::_mm256_shuffle_epi32; + let c9_spread = _mm256_shuffle_epi32(c98.into_bits(), 0b11_01_10_00); + let c9_19_spread = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into_bits()); + c9_19 = _mm256_shuffle_epi32(c9_19_spread, 0b11_01_10_00).into_bits(); } v[0] = v[0] + c9_19; @@ -331,14 +357,14 @@ impl FieldElement32x4 { let c = z[9] >> 25; z[9] = z[9] & LOW_25_BITS; - let mut c0 = c & LOW_26_BITS; // c0 < 2^26; - let mut c1 = c >> 26; // c1 < 2^(39-26) = 2^13; + let mut c0: u64x4 = c & LOW_26_BITS; // c0 < 2^26; + let mut c1: u64x4 = c >> 26; // c1 < 2^(39-26) = 2^13; unsafe { - use stdsimd::vendor::_mm256_mul_epu32; - let x19 = u32x8::from(u64x4::splat(19)); - c0 = _mm256_mul_epu32(u32x8::from(c0), x19); // c0 < 2^30.25 - c1 = _mm256_mul_epu32(u32x8::from(c1), x19); // c1 < 2^17.25 + use core::arch::x86_64::_mm256_mul_epu32; + let x19 = u64x4::splat(19); + c0 = _mm256_mul_epu32(c0.into_bits(), x19.into_bits()).into_bits(); // c0 < 2^30.25 + c1 = _mm256_mul_epu32(c1.into_bits(), x19.into_bits()).into_bits(); // c1 < 2^17.25 } z[0] = z[0] + c0; // z0 < 2^26 + 2^30.25 < 2^30.33 @@ -348,11 +374,11 @@ impl FieldElement32x4 { // Now repack the [u64x4; 10] into a FieldElement32x4 FieldElement32x4([ - repack_pair(z[0].into(), z[1].into()), - repack_pair(z[2].into(), z[3].into()), - repack_pair(z[4].into(), z[5].into()), - repack_pair(z[6].into(), z[7].into()), - repack_pair(z[8].into(), z[9].into()), + repack_pair(z[0].into_bits(), z[1].into_bits()), + repack_pair(z[2].into_bits(), z[3].into_bits()), + repack_pair(z[4].into_bits(), z[5].into_bits()), + repack_pair(z[6].into_bits(), z[7].into_bits()), + repack_pair(z[8].into_bits(), z[9].into_bits()), ]) } } @@ -363,10 +389,10 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { let b: u32x8; let zero = i32x8::new(0,0,0,0,0,0,0,0); unsafe { - use stdsimd::vendor::_mm256_unpackhi_epi32; - use stdsimd::vendor::_mm256_unpacklo_epi32; - a = _mm256_unpacklo_epi32(src.as_i32x8(), zero).as_u32x8(); - b = _mm256_unpackhi_epi32(src.as_i32x8(), zero).as_u32x8(); + use core::arch::x86_64::_mm256_unpackhi_epi32; + use core::arch::x86_64::_mm256_unpacklo_epi32; + a = _mm256_unpacklo_epi32(src.into_bits(), zero.into_bits()).into_bits(); + b = _mm256_unpackhi_epi32(src.into_bits(), zero.into_bits()).into_bits(); } (a,b) } @@ -374,40 +400,43 @@ pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { #[inline(always)] pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - use stdsimd::vendor::_mm256_blend_epi32; + use core::arch::x86_64::_mm256_shuffle_epi32; + use core::arch::x86_64::_mm256_blend_epi32; // Input: x = (a0, 0, b0, 0, c0, 0, d0) // Input: y = (a1, 0, b1, 0, c1, 0, d1) - let x_shuffled = _mm256_shuffle_epi32(x.into(), 0b11_01_10_00); - let y_shuffled = _mm256_shuffle_epi32(y.into(), 0b10_00_11_01); + let x_shuffled = _mm256_shuffle_epi32(x.into_bits(), 0b11_01_10_00); + let y_shuffled = _mm256_shuffle_epi32(y.into_bits(), 0b10_00_11_01); // x' = (a0, b0, 0, 0, c0, d0, 0, 0) // y' = ( 0, 0, a1, b1, 0, 0, c1, d1) - return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).as_u32x8(); + return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).into_bits(); } } impl FieldElement32x4 { - /// Square this field element, then conditionally negate according to `neg_mask`; for instance, - /// `neg_mask = 0b11_00_00_00` negates the \\( D \\) value. + /// Square this field element, then conditionally negate according + /// to `neg_mask`. This parameter is hardcoded as `neg_mask = + /// D_LANES64` to negate the \\( D \\) value. /// /// # Precondition /// /// Limbs must be bounded by bit-excess \\( b < 2.0 \\). - pub fn square(&self, neg_mask: u8) -> FieldElement32x4 { + pub fn square_and_negate_D(&self) -> FieldElement32x4 { + let neg_mask = D_LANES64; + #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { - use stdsimd::vendor::_mm256_mul_epu32; - unsafe { _mm256_mul_epu32(x,y) } + use core::arch::x86_64::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } } #[inline(always)] fn m_lo(x: u32x8, y: u32x8) -> u32x8 { - use stdsimd::vendor::_mm256_mul_epu32; - unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } + use core::arch::x86_64::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } } let v19 = u32x8::new(19,0,19,0,19,0,19,0); @@ -444,14 +473,6 @@ impl FieldElement32x4 { let mut z8 = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1); let mut z9 = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5); - #[inline(always)] - fn mask_neg(x: u64x4, p: u64x4, mask: u8) -> u64x4 { - unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - _mm256_blend_epi32(x.into(), (p - x).into(), mask as i32).into() - } - } - // The biggest z_i is bounded as z_i < 249*2^(51 + 2*b); // if b < 1.5 we get z_i < 4485585228861014016. // @@ -472,16 +493,23 @@ impl FieldElement32x4 { let even_p37 = u64x4::splat(0x3ffffff << 37); let odd__p37 = u64x4::splat(0x1ffffff << 37); - z0 = mask_neg(z0, low__p37, neg_mask); - z1 = mask_neg(z1, odd__p37, neg_mask); - z2 = mask_neg(z2, even_p37, neg_mask); - z3 = mask_neg(z3, odd__p37, neg_mask); - z4 = mask_neg(z4, even_p37, neg_mask); - z5 = mask_neg(z5, odd__p37, neg_mask); - z6 = mask_neg(z6, even_p37, neg_mask); - z7 = mask_neg(z7, odd__p37, neg_mask); - z8 = mask_neg(z8, even_p37, neg_mask); - z9 = mask_neg(z9, odd__p37, neg_mask); + let negate_D = |x: u64x4, p: u64x4| -> u64x4 { + unsafe { + use core::arch::x86_64::_mm256_blend_epi32; + _mm256_blend_epi32(x.into_bits(), (p - x).into_bits(), D_LANES64 as i32).into_bits() + } + }; + + z0 = negate_D(z0, low__p37); + z1 = negate_D(z1, odd__p37); + z2 = negate_D(z2, even_p37); + z3 = negate_D(z3, odd__p37); + z4 = negate_D(z4, even_p37); + z5 = negate_D(z5, odd__p37); + z6 = negate_D(z6, even_p37); + z7 = negate_D(z7, odd__p37); + z8 = negate_D(z8, even_p37); + z9 = negate_D(z9, odd__p37); FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } @@ -490,17 +518,16 @@ impl FieldElement32x4 { impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { type Output = FieldElement32x4; fn mul(self, _rhs: &'b FieldElement32x4) -> FieldElement32x4 { - #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { - use stdsimd::vendor::_mm256_mul_epu32; - unsafe { _mm256_mul_epu32(x,y) } + use core::arch::x86_64::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } } #[inline(always)] fn m_lo(x: u32x8, y: u32x8) -> u32x8 { - use stdsimd::vendor::_mm256_mul_epu32; - unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } + use core::arch::x86_64::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x.into_bits(),y.into_bits()).into_bits() } } let (x0, x1) = unpack_pair(self.0[0]); @@ -573,7 +600,7 @@ mod test { let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); - vec.diff_sum(0xff); + vec.diff_sum(Lanes::ALL); let result = vec.split(); @@ -583,13 +610,13 @@ mod test { assert_eq!(result[3], &x3 + &x2); let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); - vec.diff_sum(0b01011111); // leave D unchanged + vec.diff_sum(Lanes::AB); // leave C,D unchanged let result = vec.split(); assert_eq!(result[0], &x1 - &x0); assert_eq!(result[1], &x1 + &x0); - assert_eq!(result[2], &x3 - &x2); + assert_eq!(result[2], x2); assert_eq!(result[3], x3); } @@ -602,9 +629,7 @@ mod test { let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); - let neg_mask = 0b11_00_00_00; - - let result = vec.square(neg_mask).split(); + let result = vec.square_and_negate_D().split(); assert_eq!(result[0], &x0 * &x0); assert_eq!(result[1], &x1 * &x1); diff --git a/src/lib.rs b/src/lib.rs index ffe0009..0a8a1a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ #![cfg_attr(feature = "nightly", feature(i128_type))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "nightly", feature(external_doc))] +#![cfg_attr(all(feature = "nightly", feature = "yolocrypto"), feature(stdsimd))] // Refuse to compile if documentation is missing, but only on nightly. // @@ -40,9 +41,6 @@ extern crate alloc; extern crate clear_on_drop; -#[cfg(feature = "yolocrypto")] -extern crate stdsimd; - // The `Digest` trait is implemented using `generic_array`, so we need it // too. Hopefully we can eliminate `generic_array` from `Digest` once const // generics land. From f830268128882c594bfd553068f3fab2d26ce925 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 4 Apr 2018 10:00:18 -0700 Subject: [PATCH 13/16] Drop benchmarks from Travis --- .travis.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 56cad2f..d910557 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,24 +10,10 @@ env: - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='serde' - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='nightly' - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='yolocrypto nightly' - - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly' - - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly' - TEST_COMMAND=build EXTRA_FLAGS=--no-default-features FEATURES='' matrix: exclude: - # We can probably remove this, as we reasonably expect dalek to work on - # stable and beta, but currently we require "test" feature in order to - # run benchmarks, which causes dalek not to build on stable. See - # https://github.com/isislovecruft/curve25519-dalek/pull/38#issuecomment-286027562 - - rust: stable - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly' - - rust: beta - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly' - - rust: stable - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly' - - rust: beta - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly' # Test nightly features, such as radix_51, only on nightly. - rust: stable env: TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='nightly' From 1048100cd06f9adb5035e38e1c5e3fcce20a5083 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 4 Apr 2018 10:16:32 -0700 Subject: [PATCH 14/16] Reduce benchmark runs --- benches/dalek_benchmarks.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs index 81c9056..353f4bf 100644 --- a/benches/dalek_benchmarks.rs +++ b/benches/dalek_benchmarks.rs @@ -13,6 +13,7 @@ extern crate curve25519_dalek; use curve25519_dalek::constants; use curve25519_dalek::scalar::Scalar; +static BATCH_SIZES: [usize; 5] = [1, 2, 4, 8, 16]; static MULTISCALAR_SIZES: [usize; 13] = [1, 2, 4, 8, 16, 32, 64, 128, 256, 384, 512, 768, 1024]; mod edwards_benches { @@ -133,7 +134,7 @@ mod ristretto_benches { .collect(); b.iter(|| RistrettoPoint::double_and_compress_batch(&points)); }, - &MULTISCALAR_SIZES, + &BATCH_SIZES, ); } @@ -186,7 +187,7 @@ mod scalar_benches { Scalar::batch_invert(&mut s); }); }, - &MULTISCALAR_SIZES, + &BATCH_SIZES, ); } From 5f136fbd0cd27e3b83c399c0014f3d0e0b13e284 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 4 Apr 2018 10:25:42 -0700 Subject: [PATCH 15/16] Remove some warnings. Not all of the warnings are removed, since although this code works, it still needs a significant amount of cleanup, editing, and polish. --- src/backend/avx2/edwards.rs | 8 ++------ src/backend/avx2/field.rs | 1 - 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 9341ee9..c0ad9ac 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -14,16 +14,14 @@ #![allow(bad_style)] use core::convert::From; -use core::ops::{Index, Add, Sub, Mul, Neg}; -use core::borrow::Borrow; +use core::ops::{Add, Sub, Neg}; -use core::simd::{IntoBits, u32x8, i32x8}; +use core::simd::{IntoBits, u32x8}; use subtle::ConditionallyAssignable; use subtle::Choice; use edwards; -use scalar::Scalar; use scalar_mul::window::{LookupTable, OddLookupTable}; use traits::Identity; @@ -313,8 +311,6 @@ impl<'a> From<&'a edwards::EdwardsPoint> for OddLookupTable { mod test { use super::*; - use constants; - fn serial_add(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) -> edwards::EdwardsPoint { use backend::u64::field::FieldElement64; diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 2af68b2..96b4baf 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -268,7 +268,6 @@ impl FieldElement32x4 { let masks = u32x8::new((1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1, (1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1); - /// Compute the carry-out of each coefficient of v using a left shift. let carry = |v: u32x8| -> u32x8 { unsafe { use core::arch::x86_64::_mm256_srlv_epi32; From 207fbb640e752589c6a3cbafff018ff4434ae012 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 4 Apr 2018 15:30:12 -0700 Subject: [PATCH 16/16] Bump version to 0.16.1 --- Cargo.toml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8387cd3..9cb39f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "curve25519-dalek" -version = "0.16.0" +version = "0.16.1" authors = ["Isis Lovecruft ", "Henry de Valence "] readme = "README.md" diff --git a/README.md b/README.md index 1ac0bfe..53cfd8b 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ make doc-internal To import `curve25519-dalek`, add the following to the dependencies section of your project's `Cargo.toml`: ```toml -curve25519-dalek = "^0.14" +curve25519-dalek = "^0.16" ``` Then import the crate as: ```rust,no_run