From 20fd561482a592ab22c6954c1bdec00c8cb9fda5 Mon Sep 17 00:00:00 2001 From: Jeff Burdges Date: Wed, 26 Dec 2018 17:09:20 +0100 Subject: [PATCH 01/52] Support passing both borrowed and owned Rngs --- src/ristretto.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ristretto.rs b/src/ristretto.rs index e6375ed..74adefd 100644 --- a/src/ristretto.rs +++ b/src/ristretto.rs @@ -621,7 +621,7 @@ impl RistrettoPoint { /// discrete log of the output point with respect to any other /// point should be unknown. The map is applied twice and the /// results are added, to ensure a uniform distribution. - pub fn random(rng: &mut T) -> Self { + pub fn random(mut rng: T) -> Self { let mut uniform_bytes = [0u8; 64]; rng.fill(&mut uniform_bytes); From d6ca36fa0b376ceffe9588d6b4c4856b4b78a4cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D1=91=D0=BC=20=D0=9F=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=5BArtyom=20Pavlov=5D?= Date: Sat, 5 Jan 2019 14:27:24 +0300 Subject: [PATCH 02/52] replace rand with rand_core+rand_os --- Cargo.toml | 7 ++++--- benches/dalek_benchmarks.rs | 4 ++-- build.rs | 2 +- src/lib.rs | 4 +++- src/montgomery.rs | 2 +- src/ristretto.rs | 14 +++++++------- src/scalar.rs | 12 ++++++------ 7 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 847c5ed..733c62b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ features = ["nightly"] travis-ci = { repository = "dalek-cryptography/curve25519-dalek", branch = "master"} [dev-dependencies] +rand_os = "0.1.0" sha2 = { version = "0.8", default-features = false } bincode = "1" criterion = "0.2" @@ -41,7 +42,7 @@ harness = false # match exactly, since the build.rs uses the crate itself as a library. [dependencies] -rand = { version = "0.6.0", default-features = false } +rand_core = { version = "0.3.0", default-features = false } byteorder = { version = "^1.2.3", default-features = false, features = ["i128"] } digest = { version = "0.8", default-features = false } clear_on_drop = "=0.2.3" @@ -50,7 +51,7 @@ serde = { version = "1.0", optional = true } packed_simd = { version = "0.3.0", features = ["into_bits"], optional = true } [build-dependencies] -rand = { version = "0.6.0", default-features = false } +rand_core = { version = "0.3.0", default-features = false } byteorder = { version = "^1.2.3", default-features = false, features = ["i128"] } digest = { version = "0.8", default-features = false } clear_on_drop = "=0.2.3" @@ -61,7 +62,7 @@ packed_simd = { version = "0.3.0", features = ["into_bits"], optional = true } [features] nightly = ["subtle/nightly", "clear_on_drop/nightly"] default = ["std", "u64_backend"] -std = ["alloc", "subtle/std", "rand/std"] +std = ["alloc", "subtle/std"] alloc = [] yolocrypto = [] diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs index 71792a3..6e51d29 100644 --- a/benches/dalek_benchmarks.rs +++ b/benches/dalek_benchmarks.rs @@ -1,7 +1,7 @@ #![allow(non_snake_case)] -extern crate rand; -use rand::rngs::OsRng; +extern crate rand_os; +use rand_os::OsRng; #[macro_use] extern crate criterion; diff --git a/build.rs b/build.rs index 284bce7..411a5d8 100644 --- a/build.rs +++ b/build.rs @@ -10,7 +10,7 @@ extern crate byteorder; extern crate clear_on_drop; extern crate core; extern crate digest; -extern crate rand; +extern crate rand_core; extern crate subtle; #[cfg(all(feature = "nightly", feature = "avx2_backend"))] diff --git a/src/lib.rs b/src/lib.rs index 4f52c96..a791eda 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,10 +41,12 @@ extern crate std; #[cfg(all(feature = "nightly", feature = "avx2_backend"))] extern crate packed_simd; -extern crate rand; +extern crate rand_core; extern crate clear_on_drop; extern crate byteorder; pub extern crate digest; +#[cfg(all(test, feature = "stage2_build"))] +extern crate rand_os; // Used for traits related to constant-time code. extern crate subtle; diff --git a/src/montgomery.rs b/src/montgomery.rs index da3c9a5..0cd6f8a 100644 --- a/src/montgomery.rs +++ b/src/montgomery.rs @@ -305,7 +305,7 @@ mod test { use super::*; #[cfg(feature = "rand")] - use rand::rngs::OsRng; + use rand_os::OsRng; /// Test Montgomery -> Edwards on the X/Ed25519 basepoint #[test] diff --git a/src/ristretto.rs b/src/ristretto.rs index e6375ed..30cbe82 100644 --- a/src/ristretto.rs +++ b/src/ristretto.rs @@ -164,7 +164,7 @@ use core::ops::{Add, Neg, Sub}; use core::ops::{AddAssign, SubAssign}; use core::ops::{Mul, MulAssign}; -use rand::{CryptoRng, Rng}; +use rand_core::{CryptoRng, RngCore}; use digest::generic_array::typenum::U64; use digest::Digest; @@ -462,8 +462,8 @@ impl RistrettoPoint { /// ``` /// # extern crate curve25519_dalek; /// # use curve25519_dalek::ristretto::RistrettoPoint; - /// extern crate rand; - /// use rand::rngs::OsRng; + /// extern crate rand_os; + /// use rand_os::OsRng; /// /// # // Need fn main() here in comment so the doctest compiles /// # // See https://doc.rust-lang.org/book/documentation.html#documentation-as-tests @@ -609,7 +609,7 @@ impl RistrettoPoint { /// /// # Inputs /// - /// * `rng`: any RNG which implements the `rand::Rng` interface. + /// * `rng`: any RNG which implements the `RngCore + CryptoRng` interface. /// /// # Returns /// @@ -621,9 +621,9 @@ impl RistrettoPoint { /// discrete log of the output point with respect to any other /// point should be unknown. The map is applied twice and the /// results are added, to ensure a uniform distribution. - pub fn random(rng: &mut T) -> Self { + pub fn random(rng: &mut T) -> Self { let mut uniform_bytes = [0u8; 64]; - rng.fill(&mut uniform_bytes); + rng.fill_bytes(&mut uniform_bytes); RistrettoPoint::from_uniform_bytes(&uniform_bytes) } @@ -1014,7 +1014,7 @@ impl Debug for RistrettoPoint { #[cfg(all(test, feature = "stage2_build"))] mod test { #[cfg(feature = "rand")] - use rand::rngs::OsRng; + use rand_os::OsRng; use scalar::Scalar; use constants; diff --git a/src/scalar.rs b/src/scalar.rs index 2e4d256..5c5006e 100644 --- a/src/scalar.rs +++ b/src/scalar.rs @@ -151,7 +151,7 @@ use core::ops::{Sub, SubAssign}; #[allow(unused_imports)] use prelude::*; -use rand::{CryptoRng, Rng}; +use rand_core::{CryptoRng, RngCore}; use digest::generic_array::typenum::U64; use digest::Digest; @@ -507,7 +507,7 @@ impl Scalar { /// /// # Inputs /// - /// * `rng`: any RNG which implements the `rand::CryptoRng` interface. + /// * `rng`: any RNG which implements the `RngCore + CryptoRng` interface. /// /// # Returns /// @@ -516,20 +516,20 @@ impl Scalar { /// # Example /// /// ``` - /// extern crate rand; + /// extern crate rand_os; /// # extern crate curve25519_dalek; /// # /// # fn main() { /// use curve25519_dalek::scalar::Scalar; /// - /// use rand::OsRng; + /// use rand_os::OsRng; /// /// let mut csprng: OsRng = OsRng::new().unwrap(); /// let a: Scalar = Scalar::random(&mut csprng); /// # } - pub fn random(rng: &mut T) -> Self { + pub fn random(rng: &mut T) -> Self { let mut scalar_bytes = [0u8; 64]; - rng.fill(&mut scalar_bytes); + rng.fill_bytes(&mut scalar_bytes); Scalar::from_bytes_mod_order_wide(&scalar_bytes) } From 4e4730303d6a93d6084cbcb862adc3a524e9fbc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D1=80=D1=82=D1=91=D0=BC=20=D0=9F=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=5BArtyom=20Pavlov=5D?= Date: Sat, 5 Jan 2019 14:32:36 +0300 Subject: [PATCH 03/52] enable rand_core/std --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 733c62b..17c478d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,7 +62,7 @@ packed_simd = { version = "0.3.0", features = ["into_bits"], optional = true } [features] nightly = ["subtle/nightly", "clear_on_drop/nightly"] default = ["std", "u64_backend"] -std = ["alloc", "subtle/std"] +std = ["alloc", "subtle/std", "rand_core/std"] alloc = [] yolocrypto = [] From 94cb3e78427c28f220b2283258a98c1b4258cc09 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 8 Nov 2018 21:43:04 -0800 Subject: [PATCH 04/52] Rename field types based on radix. `FieldElement32` -> `FieldElement2625` `FieldElement64` -> `FieldElement51` `Scalar32` -> `Scalar29` `Scalar64` -> `Scalar52` This naming is more accurate and would let us add an ADX backend later. --- build.rs | 4 +- docs/avx2-notes.md | 2 +- src/backend/avx2/constants.rs | 134 ++++++++++++------------- src/backend/avx2/edwards.rs | 24 ++--- src/backend/avx2/field.rs | 182 +++++++++++++++++----------------- src/backend/u32/constants.rs | 94 +++++++++--------- src/backend/u32/field.rs | 114 ++++++++++----------- src/backend/u32/scalar.rs | 122 +++++++++++------------ src/backend/u64/constants.rs | 94 +++++++++--------- src/backend/u64/field.rs | 112 ++++++++++----------- src/backend/u64/scalar.rs | 122 +++++++++++------------ src/constants.rs | 12 +-- src/field.rs | 8 +- src/scalar.rs | 4 +- 14 files changed, 514 insertions(+), 514 deletions(-) diff --git a/build.rs b/build.rs index 284bce7..c6a25eb 100644 --- a/build.rs +++ b/build.rs @@ -81,10 +81,10 @@ fn main() { format!( "\n #[cfg(feature = \"u32_backend\")] -use backend::u32::field::FieldElement32; +use backend::u32::field::FieldElement2625; #[cfg(feature = \"u64_backend\")] -use backend::u64::field::FieldElement64; +use backend::u64::field::FieldElement51; use edwards::EdwardsBasepointTable; diff --git a/docs/avx2-notes.md b/docs/avx2-notes.md index d902650..a382a34 100644 --- a/docs/avx2-notes.md +++ b/docs/avx2-notes.md @@ -444,7 +444,7 @@ There are several directions for future improvement: particularly NEON. The current point arithmetic code is written in terms of field element vectors, which are in turn implemented using platform SIMD vectors. It should be possible to write an alternate - implementation of the `FieldElement32x4` using NEON without changing + implementation of the `FieldElement2625x4` using NEON without changing the point arithmetic. NEON has 128-bit vectors rather than 256-bit vectors, but this may still be worthwhile compared to a serial implementation. diff --git a/src/backend/avx2/constants.rs b/src/backend/avx2/constants.rs index 76a644a..2433497 100644 --- a/src/backend/avx2/constants.rs +++ b/src/backend/avx2/constants.rs @@ -13,11 +13,11 @@ use packed_simd::u32x8; use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; -use backend::avx2::field::FieldElement32x4; +use backend::avx2::field::FieldElement2625x4; use scalar_mul::window::NafLookupTable8; /// The identity element as an `ExtendedPoint`. -pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldElement32x4([ +pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldElement2625x4([ u32x8::new(0, 1, 0, 0, 1, 0, 0, 0), u32x8::splat(0), u32x8::splat(0), @@ -26,7 +26,7 @@ pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldEle ])); /// The identity element as a `CachedPoint`. -pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement32x4([ +pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement2625x4([ u32x8::new(121647, 121666, 0, 0, 243332, 67108845, 0, 33554431), u32x8::new(67108864, 0, 33554431, 0, 0, 67108863, 0, 33554431), u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), @@ -96,7 +96,7 @@ pub(crate) static P_TIMES_16_HI: u32x8 = u32x8::new( /// Odd multiples of the Ed25519 basepoint: pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = NafLookupTable8([ - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 3571425, 10045002, @@ -148,7 +148,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4846528, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 47099681, 31447946, @@ -200,7 +200,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 31366585, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 18147205, 62587998, @@ -252,7 +252,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 31948344, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 10625852, 15193821, @@ -304,7 +304,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 11531760, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 22176662, 3984313, @@ -356,7 +356,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 9686767, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 21157200, 39156966, @@ -408,7 +408,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 17395963, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 63153652, 32195955, @@ -460,7 +460,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10289708, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 1401265, 58846825, @@ -512,7 +512,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 11976112, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 39849808, 44781685, @@ -564,7 +564,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 27313245, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 58237774, 15917425, @@ -616,7 +616,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 22487864, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 12671134, 56419053, @@ -668,7 +668,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 2606889, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 55082775, 45300503, @@ -720,7 +720,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 5956424, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 8211442, 8014184, @@ -772,7 +772,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 1824195, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 59402443, 17056879, @@ -824,7 +824,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 11687259, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 12521903, 26014045, @@ -876,7 +876,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 32973409, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 30654745, 51286025, @@ -928,7 +928,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 32228854, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 49518649, 59119280, @@ -980,7 +980,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 27003505, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 28286608, 10767548, @@ -1032,7 +1032,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 18292949, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 13869851, 31448904, @@ -1084,7 +1084,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 17277037, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 43287109, 27900723, @@ -1136,7 +1136,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 30748046, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 16441817, 36111849, @@ -1188,7 +1188,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10930179, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 28319289, 40734650, @@ -1240,7 +1240,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10938429, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 8373273, 49064494, @@ -1292,7 +1292,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 15812027, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 30741269, 38648744, @@ -1344,7 +1344,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10839820, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 18672548, 57660959, @@ -1396,7 +1396,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 30432268, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 12179834, 41005450, @@ -1448,7 +1448,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 24862543, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 59523541, 62195428, @@ -1500,7 +1500,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 27449522, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 19770733, 26478685, @@ -1552,7 +1552,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 1776722, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 36719806, 20827965, @@ -1604,7 +1604,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 29334408, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 42307254, 57217102, @@ -1656,7 +1656,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 11292096, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 7071115, 46444090, @@ -1708,7 +1708,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 26285185, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 963440, 63742255, @@ -1760,7 +1760,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4928058, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 63655588, 17883670, @@ -1812,7 +1812,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 32462955, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 39158670, 15322548, @@ -1864,7 +1864,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 20307815, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 2863373, 40876242, @@ -1916,7 +1916,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 21388876, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 59276548, 49972346, @@ -1968,7 +1968,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 27257051, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 52951491, 66542164, @@ -2020,7 +2020,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 26001714, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 66783087, 5234346, @@ -2072,7 +2072,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10065424, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 42822326, 57678669, @@ -2124,7 +2124,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4170709, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 26535281, 60238317, @@ -2176,7 +2176,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 33286062, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 54863941, 67016431, @@ -2228,7 +2228,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 347423, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 41391822, 34336880, @@ -2280,7 +2280,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4942942, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 31956192, 59570132, @@ -2332,7 +2332,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 32932252, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 29885826, 51028067, @@ -2384,7 +2384,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 9101885, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 43331297, 18431341, @@ -2436,7 +2436,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 201203, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 20548943, 14334571, @@ -2488,7 +2488,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 25177079, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 41186817, 46681702, @@ -2540,7 +2540,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 7976478, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 46509314, 55327128, @@ -2592,7 +2592,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 22687008, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 15091184, 32550863, @@ -2644,7 +2644,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 542137, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 62776018, 32835413, @@ -2696,7 +2696,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 21024049, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 60835961, 48209103, @@ -2748,7 +2748,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 20924342, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 18135013, 20358426, @@ -2800,7 +2800,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 14572399, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 10785787, 46564798, @@ -2852,7 +2852,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 24110612, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 39907267, 45940262, @@ -2904,7 +2904,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 29853825, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 66303987, 36060363, @@ -2956,7 +2956,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 23261841, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 51218008, 5070126, @@ -3008,7 +3008,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 18326047, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 64176557, 3912400, @@ -3060,7 +3060,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 12655016, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 38372660, 57101970, @@ -3112,7 +3112,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 13421882, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 22917795, 22088359, @@ -3164,7 +3164,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 13175986, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 16185025, 61537525, @@ -3216,7 +3216,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 21409233, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 59411973, 57437124, @@ -3268,7 +3268,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 4211851, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 6757410, 65455566, @@ -3320,7 +3320,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 10524446, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 1497507, 33054449, @@ -3372,7 +3372,7 @@ pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = Naf 29687002, ), ])), - CachedPoint(FieldElement32x4([ + CachedPoint(FieldElement2625x4([ u32x8::new( 35889734, 23047226, diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 18ecfde..e076238 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -45,7 +45,7 @@ use scalar_mul::window::{LookupTable, NafLookupTable5, NafLookupTable8}; use traits::Identity; -use backend::avx2::field::{FieldElement32x4, Lanes, Shuffle}; +use backend::avx2::field::{FieldElement2625x4, Lanes, Shuffle}; use backend::avx2::constants; /// A point on Curve25519, using parallel Edwards formulas for curve @@ -56,11 +56,11 @@ use backend::avx2::constants; /// The coefficients of an `ExtendedPoint` are bounded with /// \\( b < 0.007 \\). #[derive(Copy, Clone, Debug)] -pub struct ExtendedPoint(pub(super) FieldElement32x4); +pub struct ExtendedPoint(pub(super) FieldElement2625x4); impl From for ExtendedPoint { fn from(P: edwards::EdwardsPoint) -> ExtendedPoint { - ExtendedPoint(FieldElement32x4::new(&P.X, &P.Y, &P.Z, &P.T)) + ExtendedPoint(FieldElement2625x4::new(&P.X, &P.Y, &P.Z, &P.T)) } } @@ -78,7 +78,7 @@ impl From for edwards::EdwardsPoint { impl ConditionallySelectable for ExtendedPoint { fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self { - ExtendedPoint(FieldElement32x4::conditional_select(&a.0, &b.0, choice)) + ExtendedPoint(FieldElement2625x4::conditional_select(&a.0, &b.0, choice)) } fn conditional_assign(&mut self, other: &Self, choice: Choice) { @@ -133,7 +133,7 @@ impl ExtendedPoint { // ======================= // S5 S6 S8 S9 - let zero = FieldElement32x4::zero(); + let zero = FieldElement2625x4::zero(); let S_1 = tmp1.shuffle(Shuffle::AAAA); let S_2 = tmp1.shuffle(Shuffle::BBBB); @@ -181,7 +181,7 @@ impl ExtendedPoint { /// As long as the `CachedPoint` is not repeatedly negated, its /// coefficients will be bounded with \\( b < 1.0 \\). #[derive(Copy, Clone, Debug)] -pub struct CachedPoint(pub(super) FieldElement32x4); +pub struct CachedPoint(pub(super) FieldElement2625x4); impl From for CachedPoint { fn from(P: ExtendedPoint) -> CachedPoint { @@ -215,7 +215,7 @@ impl Identity for CachedPoint { impl ConditionallySelectable for CachedPoint { fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self { - CachedPoint(FieldElement32x4::conditional_select(&a.0, &b.0, choice)) + CachedPoint(FieldElement2625x4::conditional_select(&a.0, &b.0, choice)) } fn conditional_assign(&mut self, other: &Self, choice: Choice) { @@ -329,7 +329,7 @@ mod test { use super::*; fn serial_add(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) -> edwards::EdwardsPoint { - use backend::u64::field::FieldElement64; + use backend::u64::field::FieldElement51; let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); let (X2, Y2, Z2, T2) = (Q.X, Q.Y, Q.Z, Q.T); @@ -360,10 +360,10 @@ mod test { print_var!(S7); println!(""); - let S8 = &S4 * &FieldElement64([ 121666,0,0,0,0]); // R5 - let S9 = &S5 * &FieldElement64([ 121666,0,0,0,0]); // R6 - let S10 = &S6 * &FieldElement64([2*121666,0,0,0,0]); // R8 - let S11 = &S7 * &(-&FieldElement64([2*121665,0,0,0,0])); // R7 + let S8 = &S4 * &FieldElement51([ 121666,0,0,0,0]); // R5 + let S9 = &S5 * &FieldElement51([ 121666,0,0,0,0]); // R6 + let S10 = &S6 * &FieldElement51([2*121666,0,0,0,0]); // R8 + let S11 = &S7 * &(-&FieldElement51([2*121665,0,0,0,0])); // R7 print_var!(S8); print_var!(S9); print_var!(S10); diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 27ba558..9364e45 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -11,14 +11,14 @@ //! An implementation of 4-way vectorized 32bit field arithmetic using //! AVX2. //! -//! The `FieldElement32x4` struct provides a vector of four field +//! The `FieldElement2625x4` struct provides a vector of four field //! elements, implemented using AVX2 operations. Its API is designed //! to abstract away the platform-dependent details, so that point //! arithmetic can be implemented only in terms of a vector of field //! elements. //! //! At this level, the API is optimized for speed and not safety. The -//! `FieldElement32x4` does not always perform reductions. The pre- +//! `FieldElement2625x4` does not always perform reductions. The pre- //! and post-conditions on the bounds of the coefficients are //! documented for each method, but it is the caller's responsibility //! to ensure that there are no overflows. @@ -43,7 +43,7 @@ use core::ops::{Add, Mul, Neg}; use packed_simd::{i32x8, u32x8, u64x4, IntoBits}; use backend::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO}; -use backend::u64::field::FieldElement64; +use backend::u64::field::FieldElement51; /// Unpack 32-bit lanes into 64-bit lanes: /// ``` @@ -97,11 +97,11 @@ fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { } /// The `Lanes` enum represents a subset of the lanes `A,B,C,D` of a -/// `FieldElement32x4`. +/// `FieldElement2625x4`. /// /// It's used to specify blend operations without /// having to know details about the data layout of the -/// `FieldElement32x4`. +/// `FieldElement2625x4`. #[derive(Copy, Clone, Debug)] pub enum Lanes { C, @@ -114,7 +114,7 @@ pub enum Lanes { ABCD, } -/// The `Shuffle` enum represents a shuffle of a `FieldElement32x4`. +/// The `Shuffle` enum represents a shuffle of a `FieldElement2625x4`. /// /// The enum variants are named by what they do to a vector \\( /// (A,B,C,D) \\); for instance, `Shuffle::BADC` turns \\( (A, B, C, @@ -135,26 +135,26 @@ pub enum Shuffle { /// A vector of four field elements. /// -/// Each operation on a `FieldElement32x4` has documented effects on +/// Each operation on a `FieldElement2625x4` has documented effects on /// the bounds of the coefficients. This API is designed for speed /// and not safety; it is the caller's responsibility to ensure that /// the post-conditions of one operation are compatible with the /// pre-conditions of the next. #[derive(Clone, Copy, Debug)] -pub struct FieldElement32x4(pub(crate) [u32x8; 5]); +pub struct FieldElement2625x4(pub(crate) [u32x8; 5]); use subtle::Choice; use subtle::ConditionallySelectable; -impl ConditionallySelectable for FieldElement32x4 { +impl ConditionallySelectable for FieldElement2625x4 { fn conditional_select( - a: &FieldElement32x4, - b: &FieldElement32x4, + a: &FieldElement2625x4, + b: &FieldElement2625x4, choice: Choice, - ) -> FieldElement32x4 { + ) -> FieldElement2625x4 { let mask = (-(choice.unwrap_u8() as i32)) as u32; let mask_vec = u32x8::splat(mask); - FieldElement32x4([ + FieldElement2625x4([ a.0[0] ^ (mask_vec & (a.0[0] ^ b.0[0])), a.0[1] ^ (mask_vec & (a.0[1] ^ b.0[1])), a.0[2] ^ (mask_vec & (a.0[2] ^ b.0[2])), @@ -165,7 +165,7 @@ impl ConditionallySelectable for FieldElement32x4 { fn conditional_assign( &mut self, - other: &FieldElement32x4, + other: &FieldElement2625x4, choice: Choice, ) { let mask = (-(choice.unwrap_u8() as i32)) as u32; @@ -178,11 +178,11 @@ impl ConditionallySelectable for FieldElement32x4 { } } -impl FieldElement32x4 { +impl FieldElement2625x4 { /// Split this vector into an array of four (serial) field /// elements. - pub fn split(&self) -> [FieldElement64; 4] { - let mut out = [FieldElement64::zero(); 4]; + pub fn split(&self) -> [FieldElement51; 4] { + let mut out = [FieldElement51::zero(); 4]; for i in 0..5 { let a_2i = self.0[i].extract(0) as u64; // let b_2i = self.0[i].extract(1) as u64; // @@ -208,7 +208,7 @@ impl FieldElement32x4 { /// that when this function is inlined, LLVM is able to lower the /// shuffle using an immediate. #[inline] - pub fn shuffle(&self, control: Shuffle) -> FieldElement32x4 { + pub fn shuffle(&self, control: Shuffle) -> FieldElement2625x4 { #[inline(always)] fn shuffle_lanes(x: u32x8, control: Shuffle) -> u32x8 { unsafe { @@ -233,7 +233,7 @@ impl FieldElement32x4 { } } - FieldElement32x4([ + FieldElement2625x4([ shuffle_lanes(self.0[0], control), shuffle_lanes(self.0[1], control), shuffle_lanes(self.0[2], control), @@ -248,7 +248,7 @@ impl FieldElement32x4 { /// that this function can be inlined and LLVM can lower it to a /// blend instruction using an immediate. #[inline] - pub fn blend(&self, other: FieldElement32x4, control: Lanes) -> FieldElement32x4 { + pub fn blend(&self, other: FieldElement2625x4, control: Lanes) -> FieldElement2625x4 { #[inline(always)] fn blend_lanes(x: u32x8, y: u32x8, control: Lanes) -> u32x8 { unsafe { @@ -310,7 +310,7 @@ impl FieldElement32x4 { } } - FieldElement32x4([ + FieldElement2625x4([ blend_lanes(self.0[0], other.0[0], control), blend_lanes(self.0[1], other.0[1], control), blend_lanes(self.0[2], other.0[2], control), @@ -320,26 +320,26 @@ impl FieldElement32x4 { } /// Construct a vector of zeros. - pub fn zero() -> FieldElement32x4 { - FieldElement32x4([u32x8::splat(0); 5]) + pub fn zero() -> FieldElement2625x4 { + FieldElement2625x4([u32x8::splat(0); 5]) } /// Convenience wrapper around `new(x,x,x,x)`. - pub fn splat(x: &FieldElement64) -> FieldElement32x4 { - FieldElement32x4::new(x, x, x, x) + pub fn splat(x: &FieldElement51) -> FieldElement2625x4 { + FieldElement2625x4::new(x, x, x, x) } - /// Create a `FieldElement32x4` from four `FieldElement64`s. + /// Create a `FieldElement2625x4` from four `FieldElement51`s. /// /// # Postconditions /// - /// The resulting `FieldElement32x4` is bounded with \\( b < 0.0002 \\). + /// The resulting `FieldElement2625x4` is bounded with \\( b < 0.0002 \\). pub fn new( - x0: &FieldElement64, - x1: &FieldElement64, - x2: &FieldElement64, - x3: &FieldElement64, - ) -> FieldElement32x4 { + x0: &FieldElement51, + x1: &FieldElement51, + x2: &FieldElement51, + x3: &FieldElement51, + ) -> FieldElement2625x4 { let mut buf = [u32x8::splat(0); 5]; let low_26_bits = (1 << 26) - 1; for i in 0..5 { @@ -355,10 +355,10 @@ impl FieldElement32x4 { buf[i] = u32x8::new(a_2i, b_2i, a_2i_1, b_2i_1, c_2i, d_2i, c_2i_1, d_2i_1); } - // We don't know that the original `FieldElement64`s were + // We don't know that the original `FieldElement51`s were // fully reduced, so the odd limbs may exceed 2^25. // Reduce them to be sure. - FieldElement32x4(buf).reduce() + FieldElement2625x4(buf).reduce() } /// Given \\((A,B,C,D)\\), compute \\((-A,-B,-C,-D)\\), without @@ -372,11 +372,11 @@ impl FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 1 \\). #[inline] - pub fn negate_lazy(&self) -> FieldElement32x4 { + pub fn negate_lazy(&self) -> FieldElement2625x4 { // The limbs of self are bounded with b < 0.999, while the // smallest limb of 2*p is 67108845 > 2^{26+0.9999}, so // underflows are not possible. - FieldElement32x4([ + FieldElement2625x4([ P_TIMES_2_LO - self.0[0], P_TIMES_2_HI - self.0[1], P_TIMES_2_HI - self.0[2], @@ -395,7 +395,7 @@ impl FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 1.6 \\). #[inline] - pub fn diff_sum(&self) -> FieldElement32x4 { + pub fn diff_sum(&self) -> FieldElement2625x4 { // tmp1 = (B, A, D, C) let tmp1 = self.shuffle(Shuffle::BADC); // tmp2 = (-A, B, -C, D) @@ -410,7 +410,7 @@ impl FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 0.0002 \\). #[inline] - pub fn reduce(&self) -> FieldElement32x4 { + pub fn reduce(&self) -> FieldElement2625x4 { let shifts = i32x8::new(26, 26, 25, 25, 26, 26, 25, 25); let masks = u32x8::new( (1 << 26) - 1, @@ -509,16 +509,16 @@ impl FieldElement32x4 { // c_odd < 2^25 + 2^11.25 < 25.0001 < 2^{25+b} // // where b = 0.0002. - FieldElement32x4(v) + FieldElement2625x4(v) } - /// Given an array of wide coefficients, reduce them to a `FieldElement32x4`. + /// Given an array of wide coefficients, reduce them to a `FieldElement2625x4`. /// /// # Postconditions /// /// The coefficients of the result are bounded with \\( b < 0.007 \\). #[inline] - fn reduce64(mut z: [u64x4; 10]) -> FieldElement32x4 { + fn reduce64(mut z: [u64x4; 10]) -> FieldElement2625x4 { // These aren't const because splat isn't a const fn let LOW_25_BITS: u64x4 = u64x4::splat((1 << 25) - 1); let LOW_26_BITS: u64x4 = u64x4::splat((1 << 26) - 1); @@ -578,7 +578,7 @@ impl FieldElement32x4 { // b = 0 for other z[i]. // // So the packed result is bounded with b = 0.007. - FieldElement32x4([ + FieldElement2625x4([ repack_pair(z[0].into_bits(), z[1].into_bits()), repack_pair(z[2].into_bits(), z[3].into_bits()), repack_pair(z[4].into_bits(), z[5].into_bits()), @@ -596,7 +596,7 @@ impl FieldElement32x4 { /// # Postconditions /// /// The coefficients of the result are bounded with \\( b < 0.007 \\). - pub fn square_and_negate_D(&self) -> FieldElement32x4 { + pub fn square_and_negate_D(&self) -> FieldElement2625x4 { #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { use core::arch::x86_64::_mm256_mul_epu32; @@ -681,12 +681,12 @@ impl FieldElement32x4 { z8 = negate_D(z8, even_p37); z9 = negate_D(z9, odd__p37); - FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) + FieldElement2625x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } } -impl Neg for FieldElement32x4 { - type Output = FieldElement32x4; +impl Neg for FieldElement2625x4 { + type Output = FieldElement2625x4; /// Negate this field element, performing a reduction. /// @@ -701,8 +701,8 @@ impl Neg for FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 0.0002 \\). #[inline] - fn neg(self) -> FieldElement32x4 { - FieldElement32x4([ + fn neg(self) -> FieldElement2625x4 { + FieldElement2625x4([ P_TIMES_16_LO - self.0[0], P_TIMES_16_HI - self.0[1], P_TIMES_16_HI - self.0[2], @@ -712,12 +712,12 @@ impl Neg for FieldElement32x4 { } } -impl Add for FieldElement32x4 { - type Output = FieldElement32x4; - /// Add two `FieldElement32x4`s, without performing a reduction. +impl Add for FieldElement2625x4 { + type Output = FieldElement2625x4; + /// Add two `FieldElement2625x4`s, without performing a reduction. #[inline] - fn add(self, rhs: FieldElement32x4) -> FieldElement32x4 { - FieldElement32x4([ + fn add(self, rhs: FieldElement2625x4) -> FieldElement2625x4 { + FieldElement2625x4([ self.0[0] + rhs.0[0], self.0[1] + rhs.0[1], self.0[2] + rhs.0[2], @@ -727,15 +727,15 @@ impl Add for FieldElement32x4 { } } -impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 { - type Output = FieldElement32x4; +impl Mul<(u32, u32, u32, u32)> for FieldElement2625x4 { + type Output = FieldElement2625x4; /// Perform a multiplication by a vector of small constants. /// /// # Postconditions /// /// The coefficients of the result are bounded with \\( b < 0.007 \\). #[inline] - fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement32x4 { + fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement2625x4 { unsafe { use core::arch::x86_64::_mm256_mul_epu32; @@ -747,7 +747,7 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 { let (b6, b7) = unpack_pair(self.0[3]); let (b8, b9) = unpack_pair(self.0[4]); - FieldElement32x4::reduce64([ + FieldElement2625x4::reduce64([ _mm256_mul_epu32(b0.into_bits(), consts.into_bits()).into_bits(), _mm256_mul_epu32(b1.into_bits(), consts.into_bits()).into_bits(), _mm256_mul_epu32(b2.into_bits(), consts.into_bits()).into_bits(), @@ -763,8 +763,8 @@ impl Mul<(u32, u32, u32, u32)> for FieldElement32x4 { } } -impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { - type Output = FieldElement32x4; +impl<'a, 'b> Mul<&'b FieldElement2625x4> for &'a FieldElement2625x4 { + type Output = FieldElement2625x4; /// Multiply `self` by `rhs`. /// /// # Preconditions @@ -777,7 +777,7 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { /// /// The coefficients of the result are bounded with \\( b < 0.007 \\). /// - fn mul(self, rhs: &'b FieldElement32x4) -> FieldElement32x4 { + fn mul(self, rhs: &'b FieldElement2625x4) -> FieldElement2625x4 { #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { use core::arch::x86_64::_mm256_mul_epu32; @@ -869,7 +869,7 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { // multiplications by 19 into a u32. The tighter bound on b_y // means we could get a tighter bound on the outputs, or a // looser bound on b_x. - FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) + FieldElement2625x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } } @@ -880,25 +880,25 @@ mod test { #[test] fn scale_by_curve_constants() { - let mut x = FieldElement32x4::splat(&FieldElement64::one()); + let mut x = FieldElement2625x4::splat(&FieldElement51::one()); x = x * (121666, 121666, 2*121666, 2*121665); let xs = x.split(); - assert_eq!(xs[0], FieldElement64([121666, 0, 0, 0, 0])); - assert_eq!(xs[1], FieldElement64([121666, 0, 0, 0, 0])); - assert_eq!(xs[2], FieldElement64([2 * 121666, 0, 0, 0, 0])); - assert_eq!(xs[3], FieldElement64([2 * 121665, 0, 0, 0, 0])); + assert_eq!(xs[0], FieldElement51([121666, 0, 0, 0, 0])); + assert_eq!(xs[1], FieldElement51([121666, 0, 0, 0, 0])); + assert_eq!(xs[2], FieldElement51([2 * 121666, 0, 0, 0, 0])); + assert_eq!(xs[3], FieldElement51([2 * 121665, 0, 0, 0, 0])); } #[test] fn diff_sum_vs_serial() { - let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); - let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); - let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); - let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); + let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3).diff_sum(); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3).diff_sum(); let result = vec.split(); @@ -910,12 +910,12 @@ mod test { #[test] fn square_vs_serial() { - let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); - let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); - let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); - let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); + let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3); let result = vec.square_and_negate_D().split(); @@ -927,12 +927,12 @@ mod test { #[test] fn multiply_vs_serial() { - let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); - let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); - let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); - let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); + let x0 = FieldElement51([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement51([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement51([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement51([10300, 10301, 10302, 10303, 10304]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3); let vecprime = vec.clone(); let result = (&vec * &vecprime).split(); @@ -945,12 +945,12 @@ mod test { #[test] fn test_unpack_repack_pair() { - let x0 = FieldElement64([10000 + (10001 << 26), 0, 0, 0, 0]); - let x1 = FieldElement64([10100 + (10101 << 26), 0, 0, 0, 0]); - let x2 = FieldElement64([10200 + (10201 << 26), 0, 0, 0, 0]); - let x3 = FieldElement64([10300 + (10301 << 26), 0, 0, 0, 0]); + let x0 = FieldElement51([10000 + (10001 << 26), 0, 0, 0, 0]); + let x1 = FieldElement51([10100 + (10101 << 26), 0, 0, 0, 0]); + let x2 = FieldElement51([10200 + (10201 << 26), 0, 0, 0, 0]); + let x3 = FieldElement51([10300 + (10301 << 26), 0, 0, 0, 0]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3); let src = vec.0[0]; @@ -969,12 +969,12 @@ mod test { #[test] fn new_split_roundtrips() { - let x0 = FieldElement64::from_bytes(&[0x10; 32]); - let x1 = FieldElement64::from_bytes(&[0x11; 32]); - let x2 = FieldElement64::from_bytes(&[0x12; 32]); - let x3 = FieldElement64::from_bytes(&[0x13; 32]); + let x0 = FieldElement51::from_bytes(&[0x10; 32]); + let x1 = FieldElement51::from_bytes(&[0x11; 32]); + let x2 = FieldElement51::from_bytes(&[0x12; 32]); + let x3 = FieldElement51::from_bytes(&[0x13; 32]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vec = FieldElement2625x4::new(&x0, &x1, &x2, &x3); let splits = vec.split(); diff --git a/src/backend/u32/constants.rs b/src/backend/u32/constants.rs index e0e0525..22def0a 100644 --- a/src/backend/u32/constants.rs +++ b/src/backend/u32/constants.rs @@ -12,48 +12,48 @@ //! and useful field elements like `sqrt(-1)`), as well as //! lookup tables of pre-computed points. -use backend::u32::field::FieldElement32; -use backend::u32::scalar::Scalar32; +use backend::u32::field::FieldElement2625; +use backend::u32::scalar::Scalar29; use edwards::EdwardsPoint; /// Edwards `d` value, equal to `-121665/121666 mod p`. -pub(crate) const EDWARDS_D: FieldElement32 = FieldElement32([ +pub(crate) const EDWARDS_D: FieldElement2625 = FieldElement2625([ 56195235, 13857412, 51736253, 6949390, 114729, 24766616, 60832955, 30306712, 48412415, 21499315, ]); /// Edwards `2*d` value, equal to `2*(-121665/121666) mod p`. -pub(crate) const EDWARDS_D2: FieldElement32 = FieldElement32([ +pub(crate) const EDWARDS_D2: FieldElement2625 = FieldElement2625([ 45281625, 27714825, 36363642, 13898781, 229458, 15978800, 54557047, 27058993, 29715967, 9444199, ]); /// `= sqrt(a*d - 1)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. -pub(crate) const SQRT_AD_MINUS_ONE: FieldElement32 = FieldElement32([ +pub(crate) const SQRT_AD_MINUS_ONE: FieldElement2625 = FieldElement2625([ 24849947, 33400850, 43495378, 6347714, 46036536, 32887293, 41837720, 18186727, 66238516, 14525638, ]); /// `= 1/sqrt(a-d)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. -pub(crate) const INVSQRT_A_MINUS_D: FieldElement32 = FieldElement32([ +pub(crate) const INVSQRT_A_MINUS_D: FieldElement2625 = FieldElement2625([ 6111466, 4156064, 39310137, 12243467, 41204824, 120896, 20826367, 26493656, 6093567, 31568420, ]); /// Precomputed value of one of the square roots of -1 (mod p) -pub(crate) const SQRT_M1: FieldElement32 = FieldElement32([ +pub(crate) const SQRT_M1: FieldElement2625 = FieldElement2625([ 34513072, 25610706, 9377949, 3500415, 12389472, 33281959, 41962654, 31548777, 326685, 11406482, ]); /// `APLUS2_OVER_FOUR` is (A+2)/4. (This is used internally within the Montgomery ladder.) -pub(crate) const APLUS2_OVER_FOUR: FieldElement32 = FieldElement32([ +pub(crate) const APLUS2_OVER_FOUR: FieldElement2625 = FieldElement2625([ 121666, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]); /// `L` is the order of base point, i.e. 2^252 + /// 27742317777372353535851937790883648493 -pub(crate) const L: Scalar32 = Scalar32([ 0x1cf5d3ed, 0x009318d2, 0x1de73596, 0x1df3bd45, +pub(crate) const L: Scalar29 = Scalar29([ 0x1cf5d3ed, 0x009318d2, 0x1de73596, 0x1df3bd45, 0x0000014d, 0x00000000, 0x00000000, 0x00000000, 0x00100000 ]); @@ -61,12 +61,12 @@ pub(crate) const L: Scalar32 = Scalar32([ 0x1cf5d3ed, 0x009318d2, 0x1de73596, 0x pub(crate) const LFACTOR: u32 = 0x12547e1b; /// `R` = R % L where R = 2^261 -pub(crate) const R: Scalar32 = Scalar32([ 0x114df9ed, 0x1a617303, 0x0f7c098c, 0x16793167, +pub(crate) const R: Scalar29 = Scalar29([ 0x114df9ed, 0x1a617303, 0x0f7c098c, 0x16793167, 0x1ffd656e, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x000fffff ]); /// `RR` = (R^2) % L where R = 2^261 -pub(crate) const RR: Scalar32 = Scalar32([ 0x0b5f9d12, 0x1e141b17, 0x158d7f3d, 0x143f3757, +pub(crate) const RR: Scalar29 = Scalar29([ 0x0b5f9d12, 0x1e141b17, 0x158d7f3d, 0x143f3757, 0x1972d781, 0x042feb7c, 0x1ceec73d, 0x1e184d1e, 0x0005046d ]); @@ -76,10 +76,10 @@ pub(crate) const RR: Scalar32 = Scalar32([ 0x0b5f9d12, 0x1e141b17, 0x158d7f3d, 0 /// `ED25519_BASEPOINT_TABLE`, which should be used for scalar /// multiplication (it's much faster). pub const ED25519_BASEPOINT_POINT: EdwardsPoint = EdwardsPoint{ - X: FieldElement32([52811034, 25909283, 16144682, 17082669, 27570973, 30858332, 40966398, 8378388, 20764389, 8758491]), - Y: FieldElement32([40265304, 26843545, 13421772, 20132659, 26843545, 6710886, 53687091, 13421772, 40265318, 26843545]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([28827043, 27438313, 39759291, 244362, 8635006, 11264893, 19351346, 13413597, 16611511, 27139452]), + X: FieldElement2625([52811034, 25909283, 16144682, 17082669, 27570973, 30858332, 40966398, 8378388, 20764389, 8758491]), + Y: FieldElement2625([40265304, 26843545, 13421772, 20132659, 26843545, 6710886, 53687091, 13421772, 40265318, 26843545]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([28827043, 27438313, 39759291, 244362, 8635006, 11264893, 19351346, 13413597, 16611511, 27139452]), }; /// The 8-torsion subgroup \\(\mathcal E [8]\\). @@ -99,51 +99,51 @@ pub const EIGHT_TORSION: [EdwardsPoint; 8] = EIGHT_TORSION_INNER_DOC_HIDDEN; #[doc(hidden)] pub const EIGHT_TORSION_INNER_DOC_HIDDEN: [EdwardsPoint; 8] = [ EdwardsPoint{ - X: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Y: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + X: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Y: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) }, EdwardsPoint{ - X: FieldElement32([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]), - Y: FieldElement32([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251]) + X: FieldElement2625([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]), + Y: FieldElement2625([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251]) }, EdwardsPoint{ - X: FieldElement32([32595773, 7943725, 57730914, 30054016, 54719391, 272472, 25146209, 2005654, 66782178, 22147949]), - Y: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + X: FieldElement2625([32595773, 7943725, 57730914, 30054016, 54719391, 272472, 25146209, 2005654, 66782178, 22147949]), + Y: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) }, EdwardsPoint{ - X: FieldElement32([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]), - Y: FieldElement32([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180]) + X: FieldElement2625([21352778, 5345713, 4660180, 25206575, 24143089, 14568123, 30185756, 21306662, 33579924, 8345318]), + Y: FieldElement2625([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180]) }, EdwardsPoint{ - X: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Y: FieldElement32([67108844, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + X: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Y: FieldElement2625([67108844, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431, 67108863, 33554431]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) }, EdwardsPoint{ - X: FieldElement32([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]), - Y: FieldElement32([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251]) + X: FieldElement2625([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]), + Y: FieldElement2625([60155942, 32288931, 6862340, 26496934, 63071167, 28106709, 31680898, 18229030, 47743011, 1569101]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([41846657, 21581751, 11716001, 27684820, 48915701, 16297738, 20670665, 24995334, 3541542, 28543251]) }, EdwardsPoint{ - X: FieldElement32([34513072, 25610706, 9377949, 3500415, 12389472, 33281959, 41962654, 31548777, 326685, 11406482]), - Y: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + X: FieldElement2625([34513072, 25610706, 9377949, 3500415, 12389472, 33281959, 41962654, 31548777, 326685, 11406482]), + Y: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) }, EdwardsPoint{ - X: FieldElement32([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]), - Y: FieldElement32([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]), - Z: FieldElement32([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), - T: FieldElement32([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180]) + X: FieldElement2625([45756067, 28208718, 62448683, 8347856, 42965774, 18986308, 36923107, 12247769, 33528939, 25209113]), + Y: FieldElement2625([6952903, 1265500, 60246523, 7057497, 4037696, 5447722, 35427965, 15325401, 19365852, 31985330]), + Z: FieldElement2625([1, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + T: FieldElement2625([25262188, 11972680, 55392862, 5869611, 18193162, 17256693, 46438198, 8559097, 63567321, 5011180]) }, ]; diff --git a/src/backend/u32/field.rs b/src/backend/u32/field.rs index 5f2fe1d..8a7ca72 100644 --- a/src/backend/u32/field.rs +++ b/src/backend/u32/field.rs @@ -24,7 +24,7 @@ use core::ops::{Sub, SubAssign}; use subtle::Choice; use subtle::ConditionallySelectable; -/// A `FieldElement32` represents an element of the field +/// A `FieldElement2625` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// /// In the 32-bit implementation, a `FieldElement` is represented in @@ -41,44 +41,44 @@ use subtle::ConditionallySelectable; /// # Note /// /// The `curve25519_dalek::field` module provides a type alias -/// `curve25519_dalek::field::FieldElement` to either `FieldElement64` -/// or `FieldElement32`. +/// `curve25519_dalek::field::FieldElement` to either `FieldElement51` +/// or `FieldElement2625`. /// -/// The backend-specific type `FieldElement32` should not be used +/// The backend-specific type `FieldElement2625` should not be used /// outside of the `curve25519_dalek::field` module. #[derive(Copy, Clone)] -pub struct FieldElement32(pub (crate) [u32; 10]); +pub struct FieldElement2625(pub (crate) [u32; 10]); -impl Debug for FieldElement32 { +impl Debug for FieldElement2625 { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - write!(f, "FieldElement32({:?})", &self.0[..]) + write!(f, "FieldElement2625({:?})", &self.0[..]) } } -impl<'b> AddAssign<&'b FieldElement32> for FieldElement32 { - fn add_assign(&mut self, _rhs: &'b FieldElement32) { +impl<'b> AddAssign<&'b FieldElement2625> for FieldElement2625 { + fn add_assign(&mut self, _rhs: &'b FieldElement2625) { for i in 0..10 { self.0[i] += _rhs.0[i]; } } } -impl<'a, 'b> Add<&'b FieldElement32> for &'a FieldElement32 { - type Output = FieldElement32; - fn add(self, _rhs: &'b FieldElement32) -> FieldElement32 { +impl<'a, 'b> Add<&'b FieldElement2625> for &'a FieldElement2625 { + type Output = FieldElement2625; + fn add(self, _rhs: &'b FieldElement2625) -> FieldElement2625 { let mut output = *self; output += _rhs; output } } -impl<'b> SubAssign<&'b FieldElement32> for FieldElement32 { - fn sub_assign(&mut self, _rhs: &'b FieldElement32) { - // See comment in FieldElement64::Sub +impl<'b> SubAssign<&'b FieldElement2625> for FieldElement2625 { + fn sub_assign(&mut self, _rhs: &'b FieldElement2625) { + // See comment in FieldElement51::Sub // // Compute a - b as ((a + 2^4 * p) - b) to avoid underflow. let b = &_rhs.0; - self.0 = FieldElement32::reduce([ + self.0 = FieldElement2625::reduce([ ((self.0[0] + (0x3ffffed << 4)) - b[0]) as u64, ((self.0[1] + (0x1ffffff << 4)) - b[1]) as u64, ((self.0[2] + (0x3ffffff << 4)) - b[2]) as u64, @@ -93,25 +93,25 @@ impl<'b> SubAssign<&'b FieldElement32> for FieldElement32 { } } -impl<'a, 'b> Sub<&'b FieldElement32> for &'a FieldElement32 { - type Output = FieldElement32; - fn sub(self, _rhs: &'b FieldElement32) -> FieldElement32 { +impl<'a, 'b> Sub<&'b FieldElement2625> for &'a FieldElement2625 { + type Output = FieldElement2625; + fn sub(self, _rhs: &'b FieldElement2625) -> FieldElement2625 { let mut output = *self; output -= _rhs; output } } -impl<'b> MulAssign<&'b FieldElement32> for FieldElement32 { - fn mul_assign(&mut self, _rhs: &'b FieldElement32) { - let result = (self as &FieldElement32) * _rhs; +impl<'b> MulAssign<&'b FieldElement2625> for FieldElement2625 { + fn mul_assign(&mut self, _rhs: &'b FieldElement2625) { + let result = (self as &FieldElement2625) * _rhs; self.0 = result.0; } } -impl<'a, 'b> Mul<&'b FieldElement32> for &'a FieldElement32 { - type Output = FieldElement32; - fn mul(self, _rhs: &'b FieldElement32) -> FieldElement32 { +impl<'a, 'b> Mul<&'b FieldElement2625> for &'a FieldElement2625 { + type Output = FieldElement2625; + fn mul(self, _rhs: &'b FieldElement2625) -> FieldElement2625 { /// Helper function to multiply two 32-bit integers with 64 bits /// of output. #[inline(always)] @@ -206,26 +206,26 @@ impl<'a, 'b> Mul<&'b FieldElement32> for &'a FieldElement32 { // // So z[0] fits into a u64 if 51 + 2*b + lg(249) < 64 // if b < 2.5. - FieldElement32::reduce([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) + FieldElement2625::reduce([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } } -impl<'a> Neg for &'a FieldElement32 { - type Output = FieldElement32; - fn neg(self) -> FieldElement32 { +impl<'a> Neg for &'a FieldElement2625 { + type Output = FieldElement2625; + fn neg(self) -> FieldElement2625 { let mut output = *self; output.negate(); output } } -impl ConditionallySelectable for FieldElement32 { +impl ConditionallySelectable for FieldElement2625 { fn conditional_select( - a: &FieldElement32, - b: &FieldElement32, + a: &FieldElement2625, + b: &FieldElement2625, choice: Choice, - ) -> FieldElement32 { - FieldElement32([ + ) -> FieldElement2625 { + FieldElement2625([ u32::conditional_select(&a.0[0], &b.0[0], choice), u32::conditional_select(&a.0[1], &b.0[1], choice), u32::conditional_select(&a.0[2], &b.0[2], choice), @@ -239,7 +239,7 @@ impl ConditionallySelectable for FieldElement32 { ]) } - fn conditional_assign(&mut self, other: &FieldElement32, choice: Choice) { + fn conditional_assign(&mut self, other: &FieldElement2625, choice: Choice) { self.0[0].conditional_assign(&other.0[0], choice); self.0[1].conditional_assign(&other.0[1], choice); self.0[2].conditional_assign(&other.0[2], choice); @@ -252,7 +252,7 @@ impl ConditionallySelectable for FieldElement32 { self.0[9].conditional_assign(&other.0[9], choice); } - fn conditional_swap(a: &mut FieldElement32, b: &mut FieldElement32, choice: Choice) { + fn conditional_swap(a: &mut FieldElement2625, b: &mut FieldElement2625, choice: Choice) { u32::conditional_swap(&mut a.0[0], &mut b.0[0], choice); u32::conditional_swap(&mut a.0[1], &mut b.0[1], choice); u32::conditional_swap(&mut a.0[2], &mut b.0[2], choice); @@ -266,11 +266,11 @@ impl ConditionallySelectable for FieldElement32 { } } -impl FieldElement32 { +impl FieldElement2625 { /// Invert the sign of this field element pub fn negate(&mut self) { // Compute -b as ((2^4 * p) - b) to avoid underflow. - let neg = FieldElement32::reduce([ + let neg = FieldElement2625::reduce([ ((0x3ffffed << 4) - self.0[0]) as u64, ((0x1ffffff << 4) - self.0[1]) as u64, ((0x3ffffff << 4) - self.0[2]) as u64, @@ -286,25 +286,25 @@ impl FieldElement32 { } /// Construct zero. - pub fn zero() -> FieldElement32 { - FieldElement32([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) + pub fn zero() -> FieldElement2625 { + FieldElement2625([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) } /// Construct one. - pub fn one() -> FieldElement32 { - FieldElement32([ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) + pub fn one() -> FieldElement2625 { + FieldElement2625([ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ]) } /// Construct -1. - pub fn minus_one() -> FieldElement32 { - FieldElement32([ + pub fn minus_one() -> FieldElement2625 { + FieldElement2625([ 0x3ffffec, 0x1ffffff, 0x3ffffff, 0x1ffffff, 0x3ffffff, 0x1ffffff, 0x3ffffff, 0x1ffffff, 0x3ffffff, 0x1ffffff, ]) } /// Given `k > 0`, return `self^(2^k)`. - pub fn pow2k(&self, k: u32) -> FieldElement32 { + pub fn pow2k(&self, k: u32) -> FieldElement2625 { debug_assert!( k > 0 ); let mut z = self.square(); for _ in 1..k { @@ -314,12 +314,12 @@ impl FieldElement32 { } /// Given unreduced coefficients `z[0], ..., z[9]` of any size, - /// carry and reduce them mod p to obtain a `FieldElement32` + /// carry and reduce them mod p to obtain a `FieldElement2625` /// whose coefficients have excess `b < 0.007`. /// /// In other words, each coefficient of the result is bounded by /// either `2^(25 + 0.007)` or `2^(26 + 0.007)`, as appropriate. - fn reduce(mut z: [u64; 10]) -> FieldElement32 { + fn reduce(mut z: [u64; 10]) -> FieldElement2625 { const LOW_25_BITS: u64 = (1 << 25) - 1; const LOW_26_BITS: u64 = (1 << 26) - 1; @@ -361,13 +361,13 @@ impl FieldElement32 { // < 2^25.007 (good enough) // and we're done. - FieldElement32([ + FieldElement2625([ z[0] as u32, z[1] as u32, z[2] as u32, z[3] as u32, z[4] as u32, z[5] as u32, z[6] as u32, z[7] as u32, z[8] as u32, z[9] as u32, ]) } - /// Load a `FieldElement64` from the low 255 bits of a 256-bit + /// Load a `FieldElement51` from the low 255 bits of a 256-bit /// input. /// /// # Warning @@ -378,7 +378,7 @@ impl FieldElement32 { /// encoding of every field element should decode, re-encode to /// the canonical encoding, and check that the input was /// canonical. - pub fn from_bytes(data: &[u8; 32]) -> FieldElement32 { //FeFromBytes + pub fn from_bytes(data: &[u8; 32]) -> FieldElement2625 { //FeFromBytes #[inline] fn load3(b: &[u8]) -> u64 { (b[0] as u64) | ((b[1] as u64) << 8) | ((b[2] as u64) << 16) @@ -402,16 +402,16 @@ impl FieldElement32 { h[8] = load3(&data[26..]) << 4; h[9] = (load3(&data[29..]) & LOW_23_BITS) << 2; - FieldElement32::reduce(h) + FieldElement2625::reduce(h) } - /// Serialize this `FieldElement64` to a 32-byte array. The + /// Serialize this `FieldElement51` to a 32-byte array. The /// encoding is canonical. pub fn to_bytes(&self) -> [u8; 32] { let inp = &self.0; // Reduce the value represented by `in` to the range [0,2*p) - let mut h: [u32; 10] = FieldElement32::reduce([ + let mut h: [u32; 10] = FieldElement2625::reduce([ // XXX this cast is annoying inp[0] as u64, inp[1] as u64, inp[2] as u64, inp[3] as u64, inp[4] as u64, inp[5] as u64, inp[6] as u64, inp[7] as u64, inp[8] as u64, inp[9] as u64, @@ -554,16 +554,16 @@ impl FieldElement32 { } /// Compute `self^2`. - pub fn square(&self) -> FieldElement32 { - FieldElement32::reduce(self.square_inner()) + pub fn square(&self) -> FieldElement2625 { + FieldElement2625::reduce(self.square_inner()) } /// Compute `2*self^2`. - pub fn square2(&self) -> FieldElement32 { + pub fn square2(&self) -> FieldElement2625 { let mut coeffs = self.square_inner(); for i in 0..self.0.len() { coeffs[i] += coeffs[i]; } - FieldElement32::reduce(coeffs) + FieldElement2625::reduce(coeffs) } } diff --git a/src/backend/u32/scalar.rs b/src/backend/u32/scalar.rs index 6c3bb51..8dce8a3 100644 --- a/src/backend/u32/scalar.rs +++ b/src/backend/u32/scalar.rs @@ -15,24 +15,24 @@ use core::ops::{Index, IndexMut}; use constants; -/// The `Scalar32` struct represents an element in ℤ/lℤ as 9 29-bit limbs +/// The `Scalar29` struct represents an element in ℤ/lℤ as 9 29-bit limbs #[derive(Copy,Clone)] -pub struct Scalar32(pub [u32; 9]); +pub struct Scalar29(pub [u32; 9]); -impl Debug for Scalar32 { +impl Debug for Scalar29 { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - write!(f, "Scalar32: {:?}", &self.0[..]) + write!(f, "Scalar29: {:?}", &self.0[..]) } } -impl Index for Scalar32 { +impl Index for Scalar29 { type Output = u32; fn index(&self, _index: usize) -> &u32 { &(self.0[_index]) } } -impl IndexMut for Scalar32 { +impl IndexMut for Scalar29 { fn index_mut(&mut self, _index: usize) -> &mut u32 { &mut (self.0[_index]) } @@ -44,14 +44,14 @@ fn m(x: u32, y: u32) -> u64 { (x as u64) * (y as u64) } -impl Scalar32 { +impl Scalar29 { /// Return the zero scalar. - pub fn zero() -> Scalar32 { - Scalar32([0,0,0,0,0,0,0,0,0]) + pub fn zero() -> Scalar29 { + Scalar29([0,0,0,0,0,0,0,0,0]) } /// Unpack a 32 byte / 256 bit scalar into 9 29-bit limbs. - pub fn from_bytes(bytes: &[u8; 32]) -> Scalar32 { + pub fn from_bytes(bytes: &[u8; 32]) -> Scalar29 { let mut words = [0u32; 8]; for i in 0..8 { for j in 0..4 { @@ -61,7 +61,7 @@ impl Scalar32 { let mask = (1u32 << 29) - 1; let top_mask = (1u32 << 24) - 1; - let mut s = Scalar32::zero(); + let mut s = Scalar29::zero(); s[ 0] = words[0] & mask; s[ 1] = ((words[0] >> 29) | (words[1] << 3)) & mask; @@ -77,7 +77,7 @@ impl Scalar32 { } /// Reduce a 64 byte / 512 bit scalar mod l. - pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar32 { + pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar29 { let mut words = [0u32; 16]; for i in 0..16 { for j in 0..4 { @@ -86,8 +86,8 @@ impl Scalar32 { } let mask = (1u32 << 29) - 1; - let mut lo = Scalar32::zero(); - let mut hi = Scalar32::zero(); + let mut lo = Scalar29::zero(); + let mut hi = Scalar29::zero(); lo[0] = words[ 0] & mask; lo[1] = ((words[ 0] >> 29) | (words[ 1] << 3)) & mask; @@ -108,13 +108,13 @@ impl Scalar32 { hi[7] = ((words[14] >> 16) | (words[15] << 16)) & mask; hi[8] = words[15] >> 13 ; - lo = Scalar32::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo - hi = Scalar32::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R + lo = Scalar29::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo + hi = Scalar29::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R - Scalar32::add(&hi, &lo) // (hi * R) + lo + Scalar29::add(&hi, &lo) // (hi * R) + lo } - /// Pack the limbs of this `Scalar32` into 32 bytes. + /// Pack the limbs of this `Scalar29` into 32 bytes. pub fn to_bytes(&self) -> [u8; 32] { let mut s = [0u8; 32]; @@ -155,8 +155,8 @@ impl Scalar32 { } /// Compute `a + b` (mod l). - pub fn add(a: &Scalar32, b: &Scalar32) -> Scalar32 { - let mut sum = Scalar32::zero(); + pub fn add(a: &Scalar29, b: &Scalar29) -> Scalar29 { + let mut sum = Scalar29::zero(); let mask = (1u32 << 29) - 1; // a + b @@ -167,12 +167,12 @@ impl Scalar32 { } // subtract l if the sum is >= l - Scalar32::sub(&sum, &constants::L) + Scalar29::sub(&sum, &constants::L) } /// Compute `a - b` (mod l). - pub fn sub(a: &Scalar32, b: &Scalar32) -> Scalar32 { - let mut difference = Scalar32::zero(); + pub fn sub(a: &Scalar29, b: &Scalar29) -> Scalar29 { + let mut difference = Scalar29::zero(); let mask = (1u32 << 29) - 1; // a - b @@ -197,7 +197,7 @@ impl Scalar32 { /// /// This is implemented with a one-level refined Karatsuba decomposition #[inline(always)] - pub (crate) fn mul_internal(a: &Scalar32, b: &Scalar32) -> [u64; 17] { + pub (crate) fn mul_internal(a: &Scalar29, b: &Scalar29) -> [u64; 17] { let mut z = [0u64; 17]; z[0] = m(a[0],b[0]); // c00 @@ -254,7 +254,7 @@ impl Scalar32 { /// Compute `a^2`. #[inline(always)] - fn square_internal(a: &Scalar32) -> [u64; 17] { + fn square_internal(a: &Scalar29) -> [u64; 17] { let aa = [ a[0]*2, a[1]*2, @@ -289,7 +289,7 @@ impl Scalar32 { /// Compute `limbs/R` (mod l), where R is the Montgomery modulus 2^261 #[inline(always)] - pub (crate) fn montgomery_reduce(limbs: &[u64; 17]) -> Scalar32 { + pub (crate) fn montgomery_reduce(limbs: &[u64; 17]) -> Scalar29 { #[inline(always)] fn part1(sum: u64) -> (u64, u32) { @@ -329,49 +329,49 @@ impl Scalar32 { let r8 = carry as u32; // result may be >= l, so attempt to subtract l - Scalar32::sub(&Scalar32([r0,r1,r2,r3,r4,r5,r6,r7,r8]), l) + Scalar29::sub(&Scalar29([r0,r1,r2,r3,r4,r5,r6,r7,r8]), l) } /// Compute `a * b` (mod l). #[inline(never)] - pub fn mul(a: &Scalar32, b: &Scalar32) -> Scalar32 { - let ab = Scalar32::montgomery_reduce(&Scalar32::mul_internal(a, b)); - Scalar32::montgomery_reduce(&Scalar32::mul_internal(&ab, &constants::RR)) + pub fn mul(a: &Scalar29, b: &Scalar29) -> Scalar29 { + let ab = Scalar29::montgomery_reduce(&Scalar29::mul_internal(a, b)); + Scalar29::montgomery_reduce(&Scalar29::mul_internal(&ab, &constants::RR)) } /// Compute `a^2` (mod l). #[inline(never)] #[allow(dead_code)] // XXX we don't expose square() via the Scalar API - pub fn square(&self) -> Scalar32 { - let aa = Scalar32::montgomery_reduce(&Scalar32::square_internal(self)); - Scalar32::montgomery_reduce(&Scalar32::mul_internal(&aa, &constants::RR)) + pub fn square(&self) -> Scalar29 { + let aa = Scalar29::montgomery_reduce(&Scalar29::square_internal(self)); + Scalar29::montgomery_reduce(&Scalar29::mul_internal(&aa, &constants::RR)) } /// Compute `(a * b) / R` (mod l), where R is the Montgomery modulus 2^261 #[inline(never)] - pub fn montgomery_mul(a: &Scalar32, b: &Scalar32) -> Scalar32 { - Scalar32::montgomery_reduce(&Scalar32::mul_internal(a, b)) + pub fn montgomery_mul(a: &Scalar29, b: &Scalar29) -> Scalar29 { + Scalar29::montgomery_reduce(&Scalar29::mul_internal(a, b)) } /// Compute `(a^2) / R` (mod l) in Montgomery form, where R is the Montgomery modulus 2^261 #[inline(never)] - pub fn montgomery_square(&self) -> Scalar32 { - Scalar32::montgomery_reduce(&Scalar32::square_internal(self)) + pub fn montgomery_square(&self) -> Scalar29 { + Scalar29::montgomery_reduce(&Scalar29::square_internal(self)) } - /// Puts a Scalar32 in to Montgomery form, i.e. computes `a*R (mod l)` + /// Puts a Scalar29 in to Montgomery form, i.e. computes `a*R (mod l)` #[inline(never)] - pub fn to_montgomery(&self) -> Scalar32 { - Scalar32::montgomery_mul(self, &constants::RR) + pub fn to_montgomery(&self) -> Scalar29 { + Scalar29::montgomery_mul(self, &constants::RR) } - /// Takes a Scalar32 out of Montgomery form, i.e. computes `a/R (mod l)` - pub fn from_montgomery(&self) -> Scalar32 { + /// Takes a Scalar29 out of Montgomery form, i.e. computes `a/R (mod l)` + pub fn from_montgomery(&self) -> Scalar29 { let mut limbs = [0u64; 17]; for i in 0..9 { limbs[i] = self[i] as u64; } - Scalar32::montgomery_reduce(&limbs) + Scalar29::montgomery_reduce(&limbs) } } @@ -385,69 +385,69 @@ mod test { /// x = 2^253-1 = 14474011154664524427946373126085988481658748083205070504932198000989141204991 /// x = 7237005577332262213973186563042994240801631723825162898930247062703686954002 mod l /// x = 5147078182513738803124273553712992179887200054963030844803268920753008712037*R mod l in Montgomery form - pub static X: Scalar32 = Scalar32( + pub static X: Scalar29 = Scalar29( [0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x001fffff]); /// x^2 = 3078544782642840487852506753550082162405942681916160040940637093560259278169 mod l - pub static XX: Scalar32 = Scalar32( + pub static XX: Scalar29 = Scalar29( [0x00217559, 0x000b3401, 0x103ff43b, 0x1462a62c, 0x1d6f9f38, 0x18e7a42f, 0x09a3dcee, 0x008dbe18, 0x0006ce65]); /// x^2 = 2912514428060642753613814151688322857484807845836623976981729207238463947987*R mod l in Montgomery form - pub static XX_MONT: Scalar32 = Scalar32( + pub static XX_MONT: Scalar29 = Scalar29( [0x152b4d2e, 0x0571d53b, 0x1da6d964, 0x188663b6, 0x1d1b5f92, 0x19d50e3f, 0x12306c29, 0x0c6f26fe, 0x00030edb]); /// y = 6145104759870991071742105800796537629880401874866217824609283457819451087098 - pub static Y: Scalar32 = Scalar32( + pub static Y: Scalar29 = Scalar29( [0x1e1458fa, 0x165ba838, 0x1d787b36, 0x0e577f3a, 0x1d2baf06, 0x1d689a19, 0x1fff3047, 0x117704ab, 0x000d9601]); /// x*y = 36752150652102274958925982391442301741 - pub static XY: Scalar32 = Scalar32( + pub static XY: Scalar29 = Scalar29( [0x0ba7632d, 0x017736bb, 0x15c76138, 0x0c69daa1, 0x000001ba, 0x00000000, 0x00000000, 0x00000000, 0x00000000]); /// x*y = 3783114862749659543382438697751927473898937741870308063443170013240655651591*R mod l in Montgomery form - pub static XY_MONT: Scalar32 = Scalar32( + pub static XY_MONT: Scalar29 = Scalar29( [0x077b51e1, 0x1c64e119, 0x02a19ef5, 0x18d2129e, 0x00de0430, 0x045a7bc8, 0x04cfc7c9, 0x1c002681, 0x000bdc1c]); /// a = 2351415481556538453565687241199399922945659411799870114962672658845158063753 - pub static A: Scalar32 = Scalar32( + pub static A: Scalar29 = Scalar29( [0x07b3be89, 0x02291b60, 0x14a99f03, 0x07dc3787, 0x0a782aae, 0x16262525, 0x0cfdb93f, 0x13f5718d, 0x000532da]); /// b = 4885590095775723760407499321843594317911456947580037491039278279440296187236 - pub static B: Scalar32 = Scalar32( + pub static B: Scalar29 = Scalar29( [0x15421564, 0x1e69fd72, 0x093d9692, 0x161785be, 0x1587d69f, 0x09d9dada, 0x130246c0, 0x0c0a8e72, 0x000acd25]); /// a+b = 0 /// a-b = 4702830963113076907131374482398799845891318823599740229925345317690316127506 - pub static AB: Scalar32 = Scalar32( + pub static AB: Scalar29 = Scalar29( [0x0f677d12, 0x045236c0, 0x09533e06, 0x0fb86f0f, 0x14f0555c, 0x0c4c4a4a, 0x19fb727f, 0x07eae31a, 0x000a65b5]); // c = (2^512 - 1) % l = 1627715501170711445284395025044413883736156588369414752970002579683115011840 - pub static C: Scalar32 = Scalar32( + pub static C: Scalar29 = Scalar29( [0x049c0f00, 0x00308f1a, 0x0164d1e9, 0x1c374ed1, 0x1be65d00, 0x19e90bfa, 0x08f73bb1, 0x036f8613, 0x00039941]); #[test] fn mul_max() { - let res = Scalar32::mul(&X, &X); + let res = Scalar29::mul(&X, &X); for i in 0..9 { assert!(res[i] == XX[i]); } @@ -463,7 +463,7 @@ mod test { #[test] fn montgomery_mul_max() { - let res = Scalar32::montgomery_mul(&X, &X); + let res = Scalar29::montgomery_mul(&X, &X); for i in 0..9 { assert!(res[i] == XX_MONT[i]); } @@ -479,7 +479,7 @@ mod test { #[test] fn mul() { - let res = Scalar32::mul(&X, &Y); + let res = Scalar29::mul(&X, &Y); for i in 0..9 { assert!(res[i] == XY[i]); } @@ -487,7 +487,7 @@ mod test { #[test] fn montgomery_mul() { - let res = Scalar32::montgomery_mul(&X, &Y); + let res = Scalar29::montgomery_mul(&X, &Y); for i in 0..9 { assert!(res[i] == XY_MONT[i]); } @@ -495,8 +495,8 @@ mod test { #[test] fn add() { - let res = Scalar32::add(&A, &B); - let zero = Scalar32::zero(); + let res = Scalar29::add(&A, &B); + let zero = Scalar29::zero(); for i in 0..9 { assert!(res[i] == zero[i]); } @@ -504,7 +504,7 @@ mod test { #[test] fn sub() { - let res = Scalar32::sub(&A, &B); + let res = Scalar29::sub(&A, &B); for i in 0..9 { assert!(res[i] == AB[i]); } @@ -513,7 +513,7 @@ mod test { #[test] fn from_bytes_wide() { let bignum = [255u8; 64]; // 2^512 - 1 - let reduced = Scalar32::from_bytes_wide(&bignum); + let reduced = Scalar29::from_bytes_wide(&bignum); for i in 0..9 { assert!(reduced[i] == C[i]); } diff --git a/src/backend/u64/constants.rs b/src/backend/u64/constants.rs index 0ac7fe3..11c86b2 100644 --- a/src/backend/u64/constants.rs +++ b/src/backend/u64/constants.rs @@ -10,43 +10,43 @@ //! This module contains backend-specific constant values, such as the 64-bit limbs of curve constants. -use backend::u64::field::FieldElement64; -use backend::u64::scalar::Scalar64; +use backend::u64::field::FieldElement51; +use backend::u64::scalar::Scalar52; use edwards::EdwardsPoint; /// Edwards `d` value, equal to `-121665/121666 mod p`. -pub(crate) const EDWARDS_D: FieldElement64 = FieldElement64([929955233495203, 466365720129213, 1662059464998953, 2033849074728123, 1442794654840575]); +pub(crate) const EDWARDS_D: FieldElement51 = FieldElement51([929955233495203, 466365720129213, 1662059464998953, 2033849074728123, 1442794654840575]); /// Edwards `2*d` value, equal to `2*(-121665/121666) mod p`. -pub(crate) const EDWARDS_D2: FieldElement64 = FieldElement64([1859910466990425, 932731440258426, 1072319116312658, 1815898335770999, 633789495995903]); +pub(crate) const EDWARDS_D2: FieldElement51 = FieldElement51([1859910466990425, 932731440258426, 1072319116312658, 1815898335770999, 633789495995903]); /// `= sqrt(a*d - 1)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. -pub(crate) const SQRT_AD_MINUS_ONE: FieldElement64 = FieldElement64([ +pub(crate) const SQRT_AD_MINUS_ONE: FieldElement51 = FieldElement51([ 2241493124984347, 425987919032274, 2207028919301688, 1220490630685848, 974799131293748 ]); /// `= 1/sqrt(a-d)`, where `a = -1 (mod p)`, `d` are the Edwards curve parameters. -pub(crate) const INVSQRT_A_MINUS_D: FieldElement64 = FieldElement64([ +pub(crate) const INVSQRT_A_MINUS_D: FieldElement51 = FieldElement51([ 278908739862762, 821645201101625, 8113234426968, 1777959178193151, 2118520810568447 ]); /// Precomputed value of one of the square roots of -1 (mod p) -pub(crate) const SQRT_M1: FieldElement64 = FieldElement64([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]); +pub(crate) const SQRT_M1: FieldElement51 = FieldElement51([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]); /// `APLUS2_OVER_FOUR` is (A+2)/4. (This is used internally within the Montgomery ladder.) -pub(crate) const APLUS2_OVER_FOUR: FieldElement64 = FieldElement64([121666, 0, 0, 0, 0]); +pub(crate) const APLUS2_OVER_FOUR: FieldElement51 = FieldElement51([121666, 0, 0, 0, 0]); /// `L` is the order of base point, i.e. 2^252 + 27742317777372353535851937790883648493 -pub(crate) const L: Scalar64 = Scalar64([ 0x0002631a5cf5d3ed, 0x000dea2f79cd6581, 0x000000000014def9, 0x0000000000000000, 0x0000100000000000 ]); +pub(crate) const L: Scalar52 = Scalar52([ 0x0002631a5cf5d3ed, 0x000dea2f79cd6581, 0x000000000014def9, 0x0000000000000000, 0x0000100000000000 ]); /// `L` * `LFACTOR` = -1 (mod 2^52) pub(crate) const LFACTOR: u64 = 0x51da312547e1b; /// `R` = R % L where R = 2^260 -pub(crate) const R: Scalar64 = Scalar64([ 0x000f48bd6721e6ed, 0x0003bab5ac67e45a, 0x000fffffeb35e51b, 0x000fffffffffffff, 0x00000fffffffffff ]); +pub(crate) const R: Scalar52 = Scalar52([ 0x000f48bd6721e6ed, 0x0003bab5ac67e45a, 0x000fffffeb35e51b, 0x000fffffffffffff, 0x00000fffffffffff ]); /// `RR` = (R^2) % L where R = 2^260 -pub(crate) const RR: Scalar64 = Scalar64([ 0x0009d265e952d13b, 0x000d63c715bea69f, 0x0005be65cb687604, 0x0003dceec73d217f, 0x000009411b7c309a ]); +pub(crate) const RR: Scalar52 = Scalar52([ 0x0009d265e952d13b, 0x000d63c715bea69f, 0x0005be65cb687604, 0x0003dceec73d217f, 0x000009411b7c309a ]); /// The Ed25519 basepoint, as an `EdwardsPoint`. /// @@ -54,10 +54,10 @@ pub(crate) const RR: Scalar64 = Scalar64([ 0x0009d265e952d13b, 0x000d63c715bea69 /// `ED25519_BASEPOINT_TABLE`, which should be used for scalar /// multiplication (it's much faster). pub const ED25519_BASEPOINT_POINT: EdwardsPoint = EdwardsPoint{ - X: FieldElement64([1738742601995546, 1146398526822698, 2070867633025821, 562264141797630, 587772402128613]), - Y: FieldElement64([1801439850948184, 1351079888211148, 450359962737049, 900719925474099, 1801439850948198]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([1841354044333475, 16398895984059, 755974180946558, 900171276175154, 1821297809914039]), + X: FieldElement51([1738742601995546, 1146398526822698, 2070867633025821, 562264141797630, 587772402128613]), + Y: FieldElement51([1801439850948184, 1351079888211148, 450359962737049, 900719925474099, 1801439850948198]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([1841354044333475, 16398895984059, 755974180946558, 900171276175154, 1821297809914039]), }; /// The 8-torsion subgroup \\(\mathcal E [8]\\). @@ -74,58 +74,58 @@ pub const EIGHT_TORSION: [EdwardsPoint; 8] = EIGHT_TORSION_INNER_DOC_HIDDEN; #[doc(hidden)] pub const EIGHT_TORSION_INNER_DOC_HIDDEN: [EdwardsPoint; 8] = [ EdwardsPoint { - X: FieldElement64([0, 0, 0, 0, 0]), - Y: FieldElement64([1, 0, 0, 0, 0]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([0, 0, 0, 0, 0]), + X: FieldElement51([0, 0, 0, 0, 0]), + Y: FieldElement51([1, 0, 0, 0, 0]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([0, 0, 0, 0, 0]), } , EdwardsPoint { - X: FieldElement64([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]), - Y: FieldElement64([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]), + X: FieldElement51([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]), + Y: FieldElement51([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]), } , EdwardsPoint { - X: FieldElement64([533094393274173, 2016890930128738, 18285341111199, 134597186663265, 1486323764102114]), - Y: FieldElement64([0, 0, 0, 0, 0]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([0, 0, 0, 0, 0]), + X: FieldElement51([533094393274173, 2016890930128738, 18285341111199, 134597186663265, 1486323764102114]), + Y: FieldElement51([0, 0, 0, 0, 0]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([0, 0, 0, 0, 0]), } , EdwardsPoint { - X: FieldElement64([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]), - Y: FieldElement64([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]), + X: FieldElement51([358744748052810, 1691584618240980, 977650209285361, 1429865912637724, 560044844278676]), + Y: FieldElement51([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]), } , EdwardsPoint { - X: FieldElement64([0, 0, 0, 0, 0]), - Y: FieldElement64([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([0, 0, 0, 0, 0]), + X: FieldElement51([0, 0, 0, 0, 0]), + Y: FieldElement51([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([0, 0, 0, 0, 0]), } , EdwardsPoint { - X: FieldElement64([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]), - Y: FieldElement64([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]), + X: FieldElement51([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]), + Y: FieldElement51([2166873539340326, 1778179147085316, 1886209374839743, 1223329526802818, 105300633354275]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([1448326834587521, 1857896831960481, 1093722731865333, 1677408490711241, 1915505153018406]), } , EdwardsPoint { - X: FieldElement64([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]), - Y: FieldElement64([0, 0, 0, 0, 0]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([0, 0, 0, 0, 0]), + X: FieldElement51([1718705420411056, 234908883556509, 2233514472574048, 2117202627021982, 765476049583133]), + Y: FieldElement51([0, 0, 0, 0, 0]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([0, 0, 0, 0, 0]), } , EdwardsPoint { - X: FieldElement64([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]), - Y: FieldElement64([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]), - Z: FieldElement64([1, 0, 0, 0, 0]), - T: FieldElement64([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]), + X: FieldElement51([1893055065632419, 560215195444267, 1274149604399886, 821933901047523, 1691754969406571]), + Y: FieldElement51([84926274344903, 473620666599931, 365590438845504, 1028470286882429, 2146499180330972]), + Z: FieldElement51([1, 0, 0, 0, 0]), + T: FieldElement51([803472979097708, 393902981724766, 1158077081819914, 574391322974006, 336294660666841]), } ]; diff --git a/src/backend/u64/field.rs b/src/backend/u64/field.rs index ffe1a48..bdf440c 100644 --- a/src/backend/u64/field.rs +++ b/src/backend/u64/field.rs @@ -20,7 +20,7 @@ use core::ops::{Sub, SubAssign}; use subtle::Choice; use subtle::ConditionallySelectable; -/// A `FieldElement64` represents an element of the field +/// A `FieldElement51` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// /// In the 64-bit implementation, a `FieldElement` is represented in @@ -30,57 +30,57 @@ use subtle::ConditionallySelectable; /// # Note /// /// The `curve25519_dalek::field` module provides a type alias -/// `curve25519_dalek::field::FieldElement` to either `FieldElement64` -/// or `FieldElement32`. +/// `curve25519_dalek::field::FieldElement` to either `FieldElement51` +/// or `FieldElement2625`. /// -/// The backend-specific type `FieldElement64` should not be used +/// The backend-specific type `FieldElement51` should not be used /// outside of the `curve25519_dalek::field` module. #[derive(Copy, Clone)] -pub struct FieldElement64(pub (crate) [u64; 5]); +pub struct FieldElement51(pub (crate) [u64; 5]); -impl Debug for FieldElement64 { +impl Debug for FieldElement51 { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - write!(f, "FieldElement64({:?})", &self.0[..]) + write!(f, "FieldElement51({:?})", &self.0[..]) } } -impl<'b> AddAssign<&'b FieldElement64> for FieldElement64 { - fn add_assign(&mut self, _rhs: &'b FieldElement64) { +impl<'b> AddAssign<&'b FieldElement51> for FieldElement51 { + fn add_assign(&mut self, _rhs: &'b FieldElement51) { for i in 0..5 { self.0[i] += _rhs.0[i]; } } } -impl<'a, 'b> Add<&'b FieldElement64> for &'a FieldElement64 { - type Output = FieldElement64; - fn add(self, _rhs: &'b FieldElement64) -> FieldElement64 { +impl<'a, 'b> Add<&'b FieldElement51> for &'a FieldElement51 { + type Output = FieldElement51; + fn add(self, _rhs: &'b FieldElement51) -> FieldElement51 { let mut output = *self; output += _rhs; output } } -impl<'b> SubAssign<&'b FieldElement64> for FieldElement64 { - fn sub_assign(&mut self, _rhs: &'b FieldElement64) { - let result = (self as &FieldElement64) - _rhs; +impl<'b> SubAssign<&'b FieldElement51> for FieldElement51 { + fn sub_assign(&mut self, _rhs: &'b FieldElement51) { + let result = (self as &FieldElement51) - _rhs; self.0 = result.0; } } -impl<'a, 'b> Sub<&'b FieldElement64> for &'a FieldElement64 { - type Output = FieldElement64; - fn sub(self, _rhs: &'b FieldElement64) -> FieldElement64 { +impl<'a, 'b> Sub<&'b FieldElement51> for &'a FieldElement51 { + type Output = FieldElement51; + fn sub(self, _rhs: &'b FieldElement51) -> FieldElement51 { // To avoid underflow, first add a multiple of p. // Choose 16*p = p << 4 to be larger than 54-bit _rhs. // // If we could statically track the bitlengths of the limbs - // of every FieldElement64, we could choose a multiple of p + // of every FieldElement51, we could choose a multiple of p // just bigger than _rhs and avoid having to do a reduction. // // Since we don't yet have type-level integers to do this, we // have to add an explicit reduction call here. - FieldElement64::reduce([ + FieldElement51::reduce([ (self.0[0] + 36028797018963664u64) - _rhs.0[0], (self.0[1] + 36028797018963952u64) - _rhs.0[1], (self.0[2] + 36028797018963952u64) - _rhs.0[2], @@ -90,16 +90,16 @@ impl<'a, 'b> Sub<&'b FieldElement64> for &'a FieldElement64 { } } -impl<'b> MulAssign<&'b FieldElement64> for FieldElement64 { - fn mul_assign(&mut self, _rhs: &'b FieldElement64) { - let result = (self as &FieldElement64) * _rhs; +impl<'b> MulAssign<&'b FieldElement51> for FieldElement51 { + fn mul_assign(&mut self, _rhs: &'b FieldElement51) { + let result = (self as &FieldElement51) * _rhs; self.0 = result.0; } } -impl<'a, 'b> Mul<&'b FieldElement64> for &'a FieldElement64 { - type Output = FieldElement64; - fn mul(self, _rhs: &'b FieldElement64) -> FieldElement64 { +impl<'a, 'b> Mul<&'b FieldElement51> for &'a FieldElement51 { + type Output = FieldElement51; + fn mul(self, _rhs: &'b FieldElement51) -> FieldElement51 { /// Helper function to multiply two 64-bit integers with 128 /// bits of output. #[inline(always)] @@ -196,26 +196,26 @@ impl<'a, 'b> Mul<&'b FieldElement64> for &'a FieldElement64 { out[0] &= LOW_51_BIT_MASK; // Now out[i] < 2^(51 + epsilon) for all i. - FieldElement64(out) + FieldElement51(out) } } -impl<'a> Neg for &'a FieldElement64 { - type Output = FieldElement64; - fn neg(self) -> FieldElement64 { +impl<'a> Neg for &'a FieldElement51 { + type Output = FieldElement51; + fn neg(self) -> FieldElement51 { let mut output = *self; output.negate(); output } } -impl ConditionallySelectable for FieldElement64 { +impl ConditionallySelectable for FieldElement51 { fn conditional_select( - a: &FieldElement64, - b: &FieldElement64, + a: &FieldElement51, + b: &FieldElement51, choice: Choice, - ) -> FieldElement64 { - FieldElement64([ + ) -> FieldElement51 { + FieldElement51([ u64::conditional_select(&a.0[0], &b.0[0], choice), u64::conditional_select(&a.0[1], &b.0[1], choice), u64::conditional_select(&a.0[2], &b.0[2], choice), @@ -224,7 +224,7 @@ impl ConditionallySelectable for FieldElement64 { ]) } - fn conditional_swap(a: &mut FieldElement64, b: &mut FieldElement64, choice: Choice) { + fn conditional_swap(a: &mut FieldElement51, b: &mut FieldElement51, choice: Choice) { u64::conditional_swap(&mut a.0[0], &mut b.0[0], choice); u64::conditional_swap(&mut a.0[1], &mut b.0[1], choice); u64::conditional_swap(&mut a.0[2], &mut b.0[2], choice); @@ -232,7 +232,7 @@ impl ConditionallySelectable for FieldElement64 { u64::conditional_swap(&mut a.0[4], &mut b.0[4], choice); } - fn conditional_assign(&mut self, other: &FieldElement64, choice: Choice) { + fn conditional_assign(&mut self, other: &FieldElement51, choice: Choice) { self.0[0].conditional_assign(&other.0[0], choice); self.0[1].conditional_assign(&other.0[1], choice); self.0[2].conditional_assign(&other.0[2], choice); @@ -241,11 +241,11 @@ impl ConditionallySelectable for FieldElement64 { } } -impl FieldElement64 { +impl FieldElement51 { /// Invert the sign of this field element pub fn negate(&mut self) { // See commentary in the Sub impl - let neg = FieldElement64::reduce([ + let neg = FieldElement51::reduce([ 36028797018963664u64 - self.0[0], 36028797018963952u64 - self.0[1], 36028797018963952u64 - self.0[2], @@ -256,23 +256,23 @@ impl FieldElement64 { } /// Construct zero. - pub fn zero() -> FieldElement64 { - FieldElement64([ 0, 0, 0, 0, 0 ]) + pub fn zero() -> FieldElement51 { + FieldElement51([ 0, 0, 0, 0, 0 ]) } /// Construct one. - pub fn one() -> FieldElement64 { - FieldElement64([ 1, 0, 0, 0, 0 ]) + pub fn one() -> FieldElement51 { + FieldElement51([ 1, 0, 0, 0, 0 ]) } /// Construct -1. - pub fn minus_one() -> FieldElement64 { - FieldElement64([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]) + pub fn minus_one() -> FieldElement51 { + FieldElement51([2251799813685228, 2251799813685247, 2251799813685247, 2251799813685247, 2251799813685247]) } /// Given 64-bit input limbs, reduce to enforce the bound 2^(51 + epsilon). #[inline(always)] - fn reduce(mut limbs: [u64; 5]) -> FieldElement64 { + fn reduce(mut limbs: [u64; 5]) -> FieldElement51 { const LOW_51_BIT_MASK: u64 = (1u64 << 51) - 1; // Since the input limbs are bounded by 2^64, the biggest @@ -304,10 +304,10 @@ impl FieldElement64 { limbs[3] += c2; limbs[4] += c3; - FieldElement64(limbs) + FieldElement51(limbs) } - /// Load a `FieldElement64` from the low 255 bits of a 256-bit + /// Load a `FieldElement51` from the low 255 bits of a 256-bit /// input. /// /// # Warning @@ -319,7 +319,7 @@ impl FieldElement64 { /// the canonical encoding, and check that the input was /// canonical. /// - pub fn from_bytes(bytes: &[u8; 32]) -> FieldElement64 { + pub fn from_bytes(bytes: &[u8; 32]) -> FieldElement51 { let load8 = |input: &[u8]| -> u64 { (input[0] as u64) | ((input[1] as u64) << 8) @@ -332,7 +332,7 @@ impl FieldElement64 { }; let low_51_bit_mask = (1u64 << 51) - 1; - FieldElement64( + FieldElement51( // load bits [ 0, 64), no shift [ load8(&bytes[ 0..]) & low_51_bit_mask // load bits [ 48,112), shift to [ 51,112) @@ -346,7 +346,7 @@ impl FieldElement64 { ]) } - /// Serialize this `FieldElement64` to a 32-byte array. The + /// Serialize this `FieldElement51` to a 32-byte array. The /// encoding is canonical. pub fn to_bytes(&self) -> [u8; 32] { // Let h = limbs[0] + limbs[1]*2^51 + ... + limbs[4]*2^204. @@ -365,7 +365,7 @@ impl FieldElement64 { // Therefore q can be computed as the carry bit of h + 19. // First, reduce the limbs to ensure h < 2*p. - let mut limbs = FieldElement64::reduce(self.0).0; + let mut limbs = FieldElement51::reduce(self.0).0; let mut q = (limbs[0] + 19) >> 51; q = (limbs[1] + q) >> 51; @@ -433,7 +433,7 @@ impl FieldElement64 { } /// Given `k > 0`, return `self^(2^k)`. - pub fn pow2k(&self, mut k: u32) -> FieldElement64 { + pub fn pow2k(&self, mut k: u32) -> FieldElement51 { debug_assert!( k > 0 ); @@ -535,16 +535,16 @@ impl FieldElement64 { } } - FieldElement64(a) + FieldElement51(a) } /// Returns the square of this field element. - pub fn square(&self) -> FieldElement64 { + pub fn square(&self) -> FieldElement51 { self.pow2k(1) } /// Returns 2 times the square of this field element. - pub fn square2(&self) -> FieldElement64 { + pub fn square2(&self) -> FieldElement51 { let mut square = self.pow2k(1); for i in 0..5 { square.0[i] *= 2; diff --git a/src/backend/u64/scalar.rs b/src/backend/u64/scalar.rs index e14440b..e9031b0 100644 --- a/src/backend/u64/scalar.rs +++ b/src/backend/u64/scalar.rs @@ -16,25 +16,25 @@ use core::ops::{Index, IndexMut}; use constants; -/// The `Scalar64` struct represents an element in +/// The `Scalar52` struct represents an element in /// \\(\mathbb Z / \ell \mathbb Z\\) as 5 \\(52\\)-bit limbs. #[derive(Copy,Clone)] -pub struct Scalar64(pub [u64; 5]); +pub struct Scalar52(pub [u64; 5]); -impl Debug for Scalar64 { +impl Debug for Scalar52 { fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - write!(f, "Scalar64: {:?}", &self.0[..]) + write!(f, "Scalar52: {:?}", &self.0[..]) } } -impl Index for Scalar64 { +impl Index for Scalar52 { type Output = u64; fn index(&self, _index: usize) -> &u64 { &(self.0[_index]) } } -impl IndexMut for Scalar64 { +impl IndexMut for Scalar52 { fn index_mut(&mut self, _index: usize) -> &mut u64 { &mut (self.0[_index]) } @@ -46,14 +46,14 @@ fn m(x: u64, y: u64) -> u128 { (x as u128) * (y as u128) } -impl Scalar64 { +impl Scalar52 { /// Return the zero scalar - pub fn zero() -> Scalar64 { - Scalar64([0,0,0,0,0]) + pub fn zero() -> Scalar52 { + Scalar52([0,0,0,0,0]) } /// Unpack a 32 byte / 256 bit scalar into 5 52-bit limbs. - pub fn from_bytes(bytes: &[u8; 32]) -> Scalar64 { + pub fn from_bytes(bytes: &[u8; 32]) -> Scalar52 { let mut words = [0u64; 4]; for i in 0..4 { for j in 0..8 { @@ -63,7 +63,7 @@ impl Scalar64 { let mask = (1u64 << 52) - 1; let top_mask = (1u64 << 48) - 1; - let mut s = Scalar64::zero(); + let mut s = Scalar52::zero(); s[ 0] = words[0] & mask; s[ 1] = ((words[0] >> 52) | (words[1] << 12)) & mask; @@ -75,7 +75,7 @@ impl Scalar64 { } /// Reduce a 64 byte / 512 bit scalar mod l - pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar64 { + pub fn from_bytes_wide(bytes: &[u8; 64]) -> Scalar52 { let mut words = [0u64; 8]; for i in 0..8 { for j in 0..8 { @@ -84,8 +84,8 @@ impl Scalar64 { } let mask = (1u64 << 52) - 1; - let mut lo = Scalar64::zero(); - let mut hi = Scalar64::zero(); + let mut lo = Scalar52::zero(); + let mut hi = Scalar52::zero(); lo[0] = words[ 0] & mask; lo[1] = ((words[ 0] >> 52) | (words[ 1] << 12)) & mask; @@ -98,13 +98,13 @@ impl Scalar64 { hi[3] = ((words[ 6] >> 32) | (words[ 7] << 32)) & mask; hi[4] = words[ 7] >> 20 ; - lo = Scalar64::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo - hi = Scalar64::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R + lo = Scalar52::montgomery_mul(&lo, &constants::R); // (lo * R) / R = lo + hi = Scalar52::montgomery_mul(&hi, &constants::RR); // (hi * R^2) / R = hi * R - Scalar64::add(&hi, &lo) + Scalar52::add(&hi, &lo) } - /// Pack the limbs of this `Scalar64` into 32 bytes + /// Pack the limbs of this `Scalar52` into 32 bytes pub fn to_bytes(&self) -> [u8; 32] { let mut s = [0u8; 32]; @@ -145,8 +145,8 @@ impl Scalar64 { } /// Compute `a + b` (mod l) - pub fn add(a: &Scalar64, b: &Scalar64) -> Scalar64 { - let mut sum = Scalar64::zero(); + pub fn add(a: &Scalar52, b: &Scalar52) -> Scalar52 { + let mut sum = Scalar52::zero(); let mask = (1u64 << 52) - 1; // a + b @@ -157,12 +157,12 @@ impl Scalar64 { } // subtract l if the sum is >= l - Scalar64::sub(&sum, &constants::L) + Scalar52::sub(&sum, &constants::L) } /// Compute `a - b` (mod l) - pub fn sub(a: &Scalar64, b: &Scalar64) -> Scalar64 { - let mut difference = Scalar64::zero(); + pub fn sub(a: &Scalar52, b: &Scalar52) -> Scalar52 { + let mut difference = Scalar52::zero(); let mask = (1u64 << 52) - 1; // a - b @@ -185,7 +185,7 @@ impl Scalar64 { /// Compute `a * b` #[inline(always)] - pub (crate) fn mul_internal(a: &Scalar64, b: &Scalar64) -> [u128; 9] { + pub (crate) fn mul_internal(a: &Scalar52, b: &Scalar52) -> [u128; 9] { let mut z = [0u128; 9]; z[0] = m(a[0],b[0]); @@ -203,7 +203,7 @@ impl Scalar64 { /// Compute `a^2` #[inline(always)] - fn square_internal(a: &Scalar64) -> [u128; 9] { + fn square_internal(a: &Scalar52) -> [u128; 9] { let aa = [ a[0]*2, a[1]*2, @@ -226,7 +226,7 @@ impl Scalar64 { /// Compute `limbs/R` (mod l), where R is the Montgomery modulus 2^260 #[inline(always)] - pub (crate) fn montgomery_reduce(limbs: &[u128; 9]) -> Scalar64 { + pub (crate) fn montgomery_reduce(limbs: &[u128; 9]) -> Scalar52 { #[inline(always)] fn part1(sum: u128) -> (u128, u64) { @@ -258,50 +258,50 @@ impl Scalar64 { let r4 = carry as u64; // result may be >= l, so attempt to subtract l - Scalar64::sub(&Scalar64([r0,r1,r2,r3,r4]), l) + Scalar52::sub(&Scalar52([r0,r1,r2,r3,r4]), l) } /// Compute `a * b` (mod l) #[inline(never)] - pub fn mul(a: &Scalar64, b: &Scalar64) -> Scalar64 { - let ab = Scalar64::montgomery_reduce(&Scalar64::mul_internal(a, b)); - Scalar64::montgomery_reduce(&Scalar64::mul_internal(&ab, &constants::RR)) + pub fn mul(a: &Scalar52, b: &Scalar52) -> Scalar52 { + let ab = Scalar52::montgomery_reduce(&Scalar52::mul_internal(a, b)); + Scalar52::montgomery_reduce(&Scalar52::mul_internal(&ab, &constants::RR)) } /// Compute `a^2` (mod l) #[inline(never)] #[allow(dead_code)] // XXX we don't expose square() via the Scalar API - pub fn square(&self) -> Scalar64 { - let aa = Scalar64::montgomery_reduce(&Scalar64::square_internal(self)); - Scalar64::montgomery_reduce(&Scalar64::mul_internal(&aa, &constants::RR)) + pub fn square(&self) -> Scalar52 { + let aa = Scalar52::montgomery_reduce(&Scalar52::square_internal(self)); + Scalar52::montgomery_reduce(&Scalar52::mul_internal(&aa, &constants::RR)) } /// Compute `(a * b) / R` (mod l), where R is the Montgomery modulus 2^260 #[inline(never)] - pub fn montgomery_mul(a: &Scalar64, b: &Scalar64) -> Scalar64 { - Scalar64::montgomery_reduce(&Scalar64::mul_internal(a, b)) + pub fn montgomery_mul(a: &Scalar52, b: &Scalar52) -> Scalar52 { + Scalar52::montgomery_reduce(&Scalar52::mul_internal(a, b)) } /// Compute `(a^2) / R` (mod l) in Montgomery form, where R is the Montgomery modulus 2^260 #[inline(never)] - pub fn montgomery_square(&self) -> Scalar64 { - Scalar64::montgomery_reduce(&Scalar64::square_internal(self)) + pub fn montgomery_square(&self) -> Scalar52 { + Scalar52::montgomery_reduce(&Scalar52::square_internal(self)) } - /// Puts a Scalar64 in to Montgomery form, i.e. computes `a*R (mod l)` + /// Puts a Scalar52 in to Montgomery form, i.e. computes `a*R (mod l)` #[inline(never)] - pub fn to_montgomery(&self) -> Scalar64 { - Scalar64::montgomery_mul(self, &constants::RR) + pub fn to_montgomery(&self) -> Scalar52 { + Scalar52::montgomery_mul(self, &constants::RR) } - /// Takes a Scalar64 out of Montgomery form, i.e. computes `a/R (mod l)` + /// Takes a Scalar52 out of Montgomery form, i.e. computes `a/R (mod l)` #[inline(never)] - pub fn from_montgomery(&self) -> Scalar64 { + pub fn from_montgomery(&self) -> Scalar52 { let mut limbs = [0u128; 9]; for i in 0..5 { limbs[i] = self[i] as u128; } - Scalar64::montgomery_reduce(&limbs) + Scalar52::montgomery_reduce(&limbs) } } @@ -316,59 +316,59 @@ mod test { /// x = 14474011154664524427946373126085988481658748083205070504932198000989141204991 /// x = 7237005577332262213973186563042994240801631723825162898930247062703686954002 mod l /// x = 3057150787695215392275360544382990118917283750546154083604586903220563173085*R mod l in Montgomery form - pub static X: Scalar64 = Scalar64( + pub static X: Scalar52 = Scalar52( [0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x00001fffffffffff]); /// x^2 = 3078544782642840487852506753550082162405942681916160040940637093560259278169 mod l - pub static XX: Scalar64 = Scalar64( + pub static XX: Scalar52 = Scalar52( [0x0001668020217559, 0x000531640ffd0ec0, 0x00085fd6f9f38a31, 0x000c268f73bb1cf4, 0x000006ce65046df0]); /// x^2 = 4413052134910308800482070043710297189082115023966588301924965890668401540959*R mod l in Montgomery form - pub static XX_MONT: Scalar64 = Scalar64( + pub static XX_MONT: Scalar52 = Scalar52( [0x000c754eea569a5c, 0x00063b6ed36cb215, 0x0008ffa36bf25886, 0x000e9183614e7543, 0x0000061db6c6f26f]); /// y = 6145104759870991071742105800796537629880401874866217824609283457819451087098 - pub static Y: Scalar64 = Scalar64( + pub static Y: Scalar52 = Scalar52( [0x000b75071e1458fa, 0x000bf9d75e1ecdac, 0x000433d2baf0672b, 0x0005fffcc11fad13, 0x00000d96018bb825]); /// x*y = 36752150652102274958925982391442301741 mod l - pub static XY: Scalar64 = Scalar64( + pub static XY: Scalar52 = Scalar52( [0x000ee6d76ba7632d, 0x000ed50d71d84e02, 0x00000000001ba634, 0x0000000000000000, 0x0000000000000000]); /// x*y = 658448296334113745583381664921721413881518248721417041768778176391714104386*R mod l in Montgomery form - pub static XY_MONT: Scalar64 = Scalar64( + pub static XY_MONT: Scalar52 = Scalar52( [0x0006d52bf200cfd5, 0x00033fb1d7021570, 0x000f201bc07139d8, 0x0001267e3e49169e, 0x000007b839c00268]); /// a = 2351415481556538453565687241199399922945659411799870114962672658845158063753 - pub static A: Scalar64 = Scalar64( + pub static A: Scalar52 = Scalar52( [0x0005236c07b3be89, 0x0001bc3d2a67c0c4, 0x000a4aa782aae3ee, 0x0006b3f6e4fec4c4, 0x00000532da9fab8c]); /// b = 4885590095775723760407499321843594317911456947580037491039278279440296187236 - pub static B: Scalar64 = Scalar64( + pub static B: Scalar52 = Scalar52( [0x000d3fae55421564, 0x000c2df24f65a4bc, 0x0005b5587d69fb0b, 0x00094c091b013b3b, 0x00000acd25605473]); /// a+b = 0 /// a-b = 4702830963113076907131374482398799845891318823599740229925345317690316127506 - pub static AB: Scalar64 = Scalar64( + pub static AB: Scalar52 = Scalar52( [0x000a46d80f677d12, 0x0003787a54cf8188, 0x0004954f0555c7dc, 0x000d67edc9fd8989, 0x00000a65b53f5718]); // c = (2^512 - 1) % l = 1627715501170711445284395025044413883736156588369414752970002579683115011840 - pub static C: Scalar64 = Scalar64( + pub static C: Scalar52 = Scalar52( [0x000611e3449c0f00, 0x000a768859347a40, 0x0007f5be65d00e1b, 0x0009a3dceec73d21, 0x00000399411b7c30]); #[test] fn mul_max() { - let res = Scalar64::mul(&X, &X); + let res = Scalar52::mul(&X, &X); for i in 0..5 { assert!(res[i] == XX[i]); } @@ -384,7 +384,7 @@ mod test { #[test] fn montgomery_mul_max() { - let res = Scalar64::montgomery_mul(&X, &X); + let res = Scalar52::montgomery_mul(&X, &X); for i in 0..5 { assert!(res[i] == XX_MONT[i]); } @@ -400,7 +400,7 @@ mod test { #[test] fn mul() { - let res = Scalar64::mul(&X, &Y); + let res = Scalar52::mul(&X, &Y); for i in 0..5 { assert!(res[i] == XY[i]); } @@ -408,7 +408,7 @@ mod test { #[test] fn montgomery_mul() { - let res = Scalar64::montgomery_mul(&X, &Y); + let res = Scalar52::montgomery_mul(&X, &Y); for i in 0..5 { assert!(res[i] == XY_MONT[i]); } @@ -416,8 +416,8 @@ mod test { #[test] fn add() { - let res = Scalar64::add(&A, &B); - let zero = Scalar64::zero(); + let res = Scalar52::add(&A, &B); + let zero = Scalar52::zero(); for i in 0..5 { assert!(res[i] == zero[i]); } @@ -425,7 +425,7 @@ mod test { #[test] fn sub() { - let res = Scalar64::sub(&A, &B); + let res = Scalar52::sub(&A, &B); for i in 0..5 { assert!(res[i] == AB[i]); } @@ -434,7 +434,7 @@ mod test { #[test] fn from_bytes_wide() { let bignum = [255u8; 64]; // 2^512 - 1 - let reduced = Scalar64::from_bytes_wide(&bignum); + let reduced = Scalar52::from_bytes_wide(&bignum); println!("{:?}", reduced); for i in 0..5 { assert!(reduced[i] == C[i]); diff --git a/src/constants.rs b/src/constants.rs index 51833ac..83d7ec8 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -151,9 +151,9 @@ mod test { #[test] #[cfg(feature = "u32_backend")] fn test_d_vs_ratio() { - use backend::u32::field::FieldElement32; - let a = -&FieldElement32([121665,0,0,0,0,0,0,0,0,0]); - let b = FieldElement32([121666,0,0,0,0,0,0,0,0,0]); + use backend::u32::field::FieldElement2625; + let a = -&FieldElement2625([121665,0,0,0,0,0,0,0,0,0]); + let b = FieldElement2625([121666,0,0,0,0,0,0,0,0,0]); let d = &a * &b.invert(); let d2 = &d + &d; assert_eq!(d, constants::EDWARDS_D); @@ -164,9 +164,9 @@ mod test { #[test] #[cfg(feature = "u64_backend")] fn test_d_vs_ratio() { - use backend::u64::field::FieldElement64; - let a = -&FieldElement64([121665,0,0,0,0]); - let b = FieldElement64([121666,0,0,0,0]); + use backend::u64::field::FieldElement51; + let a = -&FieldElement51([121665,0,0,0,0]); + let b = FieldElement51([121666,0,0,0,0]); let d = &a * &b.invert(); let d2 = &d + &d; assert_eq!(d, constants::EDWARDS_D); diff --git a/src/field.rs b/src/field.rs index 4448695..1587cea 100644 --- a/src/field.rs +++ b/src/field.rs @@ -12,8 +12,8 @@ //! //! The `curve25519_dalek::field` module provides a type alias //! `curve25519_dalek::field::FieldElement` to a field element type -//! defined in the `backend` module; either `FieldElement64` or -//! `FieldElement32`. +//! defined in the `backend` module; either `FieldElement51` or +//! `FieldElement2625`. //! //! Field operations defined in terms of machine //! operations, such as field multiplication or squaring, are defined in @@ -40,7 +40,7 @@ pub use backend::u64::field::*; /// The `FieldElement` type is an alias for one of the platform-specific /// implementations. #[cfg(feature = "u64_backend")] -pub type FieldElement = backend::u64::field::FieldElement64; +pub type FieldElement = backend::u64::field::FieldElement51; #[cfg(feature = "u32_backend")] pub use backend::u32::field::*; @@ -50,7 +50,7 @@ pub use backend::u32::field::*; /// The `FieldElement` type is an alias for one of the platform-specific /// implementations. #[cfg(feature = "u32_backend")] -pub type FieldElement = backend::u32::field::FieldElement32; +pub type FieldElement = backend::u32::field::FieldElement2625; impl Eq for FieldElement {} diff --git a/src/scalar.rs b/src/scalar.rs index 2e4d256..c8d5aa9 100644 --- a/src/scalar.rs +++ b/src/scalar.rs @@ -168,14 +168,14 @@ use constants; /// This is a type alias for one of the scalar types in the `backend` /// module. #[cfg(feature = "u64_backend")] -type UnpackedScalar = backend::u64::scalar::Scalar64; +type UnpackedScalar = backend::u64::scalar::Scalar52; /// An `UnpackedScalar` represents an element of the field GF(l), optimized for speed. /// /// This is a type alias for one of the scalar types in the `backend` /// module. #[cfg(feature = "u32_backend")] -type UnpackedScalar = backend::u32::scalar::Scalar32; +type UnpackedScalar = backend::u32::scalar::Scalar29; /// The `Scalar` struct holds an integer \\(s < 2\^{255} \\) which From f1d2b5182bba57b964de1e63a847c08dd26767f2 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 8 Nov 2018 22:37:35 -0800 Subject: [PATCH 05/52] Restructure source tree into `serial` and `vector` backends. This begins to attempt to restructure the source tree so that the common parts are common and the different parts are different. The backend is now split into two parts: - serial (containing the implementation using serial formulas and mixed-model arithmetic). - vector (containing the implementation using parallel formulas and single-model arithmetic). The serial scalar_mul tree is now under backend::serial::scalar_mul. The avx2 scalar_mul tree is now under backend::avx2::scalar_mul. --- build.rs | 20 +++++----- src/backend/mod.rs | 9 +---- src/{ => backend/serial}/curve_models/mod.rs | 0 src/backend/serial/mod.rs | 39 +++++++++++++++++++ src/{ => backend/serial}/scalar_mul/mod.rs | 2 - src/{ => backend/serial}/scalar_mul/straus.rs | 8 ++-- .../serial}/scalar_mul/variable_base.rs | 4 +- .../serial}/scalar_mul/vartime_double_base.rs | 4 +- src/backend/{ => serial}/u32/constants.rs | 4 +- src/backend/{ => serial}/u32/field.rs | 0 src/backend/{ => serial}/u32/mod.rs | 0 src/backend/{ => serial}/u32/scalar.rs | 0 src/backend/{ => serial}/u64/constants.rs | 4 +- src/backend/{ => serial}/u64/field.rs | 0 src/backend/{ => serial}/u64/mod.rs | 0 src/backend/{ => serial}/u64/scalar.rs | 0 src/backend/{ => vector}/avx2/constants.rs | 6 +-- src/backend/{ => vector}/avx2/edwards.rs | 12 +++--- src/backend/{ => vector}/avx2/field.rs | 4 +- src/backend/{ => vector}/avx2/mod.rs | 1 - src/backend/vector/mod.rs | 32 +++++++++++++++ .../{avx2 => vector}/scalar_mul/mod.rs | 0 .../{avx2 => vector}/scalar_mul/straus.rs | 4 +- .../scalar_mul/variable_base.rs | 8 ++-- .../scalar_mul/vartime_double_base.rs | 10 ++--- src/constants.rs | 8 ++-- src/edwards.rs | 28 ++++++------- src/field.rs | 8 ++-- src/lib.rs | 8 ++-- src/ristretto.rs | 4 +- src/scalar.rs | 4 +- src/{scalar_mul => }/window.rs | 4 +- 32 files changed, 147 insertions(+), 88 deletions(-) rename src/{ => backend/serial}/curve_models/mod.rs (100%) create mode 100644 src/backend/serial/mod.rs rename src/{ => backend/serial}/scalar_mul/mod.rs (97%) rename src/{ => backend/serial}/scalar_mul/straus.rs (96%) rename src/{ => backend/serial}/scalar_mul/variable_base.rs (92%) rename src/{ => backend/serial}/scalar_mul/vartime_double_base.rs (93%) rename src/backend/{ => serial}/u32/constants.rs (98%) rename src/backend/{ => serial}/u32/field.rs (100%) rename src/backend/{ => serial}/u32/mod.rs (100%) rename src/backend/{ => serial}/u32/scalar.rs (100%) rename src/backend/{ => serial}/u64/constants.rs (98%) rename src/backend/{ => serial}/u64/field.rs (100%) rename src/backend/{ => serial}/u64/mod.rs (100%) rename src/backend/{ => serial}/u64/scalar.rs (100%) rename src/backend/{ => vector}/avx2/constants.rs (99%) rename src/backend/{ => vector}/avx2/edwards.rs (98%) rename src/backend/{ => vector}/avx2/field.rs (99%) rename src/backend/{ => vector}/avx2/mod.rs (97%) create mode 100644 src/backend/vector/mod.rs rename src/backend/{avx2 => vector}/scalar_mul/mod.rs (100%) rename src/backend/{avx2 => vector}/scalar_mul/straus.rs (96%) rename src/backend/{avx2 => vector}/scalar_mul/variable_base.rs (91%) rename src/backend/{avx2 => vector}/scalar_mul/vartime_double_base.rs (90%) rename src/{scalar_mul => }/window.rs (98%) diff --git a/build.rs b/build.rs index c6a25eb..38b577a 100644 --- a/build.rs +++ b/build.rs @@ -53,18 +53,16 @@ mod traits; #[path = "src/field.rs"] mod field; -#[path = "src/curve_models/mod.rs"] -mod curve_models; #[path = "src/backend/mod.rs"] mod backend; #[path = "src/prelude.rs"] mod prelude; -#[path = "src/scalar_mul/mod.rs"] -mod scalar_mul; +#[path = "src/window.rs"] +mod window; use edwards::EdwardsBasepointTable; -use curve_models::AffineNielsPoint; -use scalar_mul::window::NafLookupTable8; +use backend::serial::curve_models::AffineNielsPoint; +use window::NafLookupTable8; fn main() { // Enable the "stage2_build" feature in the main build stage @@ -81,17 +79,17 @@ fn main() { format!( "\n #[cfg(feature = \"u32_backend\")] -use backend::u32::field::FieldElement2625; +use backend::serial::u32::field::FieldElement2625; #[cfg(feature = \"u64_backend\")] -use backend::u64::field::FieldElement51; +use backend::serial::u64::field::FieldElement51; use edwards::EdwardsBasepointTable; -use curve_models::AffineNielsPoint; +use backend::serial::curve_models::AffineNielsPoint; -use scalar_mul::window::LookupTable; -use scalar_mul::window::NafLookupTable8; +use window::LookupTable; +use window::NafLookupTable8; /// Table containing precomputed multiples of the Ed25519 basepoint \\\\(B = (x, 4/5)\\\\). pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN; diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 75fa6c0..be537e6 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -27,12 +27,7 @@ compile_error!( please enable one of: u32_backend, u64_backend, avx2_backend" ); -#[cfg(feature = "u32_backend")] -pub mod u32; - -#[cfg(feature = "u64_backend")] -pub mod u64; +pub mod serial; #[cfg(all(feature = "avx2_backend", target_feature = "avx2"))] -pub mod avx2; - +pub mod vector; diff --git a/src/curve_models/mod.rs b/src/backend/serial/curve_models/mod.rs similarity index 100% rename from src/curve_models/mod.rs rename to src/backend/serial/curve_models/mod.rs diff --git a/src/backend/serial/mod.rs b/src/backend/serial/mod.rs new file mode 100644 index 0000000..adce772 --- /dev/null +++ b/src/backend/serial/mod.rs @@ -0,0 +1,39 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +//! Pluggable implementations for different architectures. +//! +//! The naming of the `u32` and `u64` modules is somewhat unfortunate, +//! since these are also the names of primitive types. Since types have +//! a different namespace than modules, this isn't a problem to the +//! compiler, but it could cause confusion. +//! +//! However, it's unlikely that the names of those modules would be +//! brought into scope directly, instead of used as +//! `backend::u32::field` or similar. Unfortunately we can't use +//! `32bit` since identifiers can't start with letters, and the backends +//! do use `u32`/`u64`, so this seems like a least-bad option. + +#[cfg(not(any(feature = "u32_backend", feature = "u64_backend")))] +compile_error!( + "no curve25519-dalek backend cargo feature enabled! \ + please enable one of: u32_backend, u64_backend" +); + +#[cfg(feature = "u32_backend")] +pub mod u32; + +#[cfg(feature = "u64_backend")] +pub mod u64; + +pub mod curve_models; + +pub mod scalar_mul; + diff --git a/src/scalar_mul/mod.rs b/src/backend/serial/scalar_mul/mod.rs similarity index 97% rename from src/scalar_mul/mod.rs rename to src/backend/serial/scalar_mul/mod.rs index 3ebe25f..1421f41 100644 --- a/src/scalar_mul/mod.rs +++ b/src/backend/serial/scalar_mul/mod.rs @@ -16,8 +16,6 @@ //! scalar multiplication implementations, since it only uses one //! curve model. -pub mod window; - pub mod variable_base; #[cfg(feature = "stage2_build")] diff --git a/src/scalar_mul/straus.rs b/src/backend/serial/scalar_mul/straus.rs similarity index 96% rename from src/scalar_mul/straus.rs rename to src/backend/serial/scalar_mul/straus.rs index 0053570..2b7a306 100644 --- a/src/scalar_mul/straus.rs +++ b/src/backend/serial/scalar_mul/straus.rs @@ -115,8 +115,8 @@ impl MultiscalarMul for Straus { { use clear_on_drop::ClearOnDrop; - use curve_models::ProjectiveNielsPoint; - use scalar_mul::window::LookupTable; + use backend::serial::curve_models::ProjectiveNielsPoint; + use window::LookupTable; use traits::Identity; let lookup_tables: Vec<_> = points @@ -167,8 +167,8 @@ impl VartimeMultiscalarMul for Straus { I::Item: Borrow, J: IntoIterator>, { - use curve_models::{CompletedPoint, ProjectiveNielsPoint, ProjectivePoint}; - use scalar_mul::window::NafLookupTable5; + use backend::serial::curve_models::{CompletedPoint, ProjectiveNielsPoint, ProjectivePoint}; + use window::NafLookupTable5; use traits::Identity; let nafs: Vec<_> = scalars diff --git a/src/scalar_mul/variable_base.rs b/src/backend/serial/scalar_mul/variable_base.rs similarity index 92% rename from src/scalar_mul/variable_base.rs rename to src/backend/serial/scalar_mul/variable_base.rs index 9569bd5..588a84e 100644 --- a/src/scalar_mul/variable_base.rs +++ b/src/backend/serial/scalar_mul/variable_base.rs @@ -3,8 +3,8 @@ use traits::Identity; use scalar::Scalar; use edwards::EdwardsPoint; -use curve_models::ProjectiveNielsPoint; -use scalar_mul::window::LookupTable; +use backend::serial::curve_models::ProjectiveNielsPoint; +use window::LookupTable; /// Perform constant-time, variable-base scalar multiplication. pub(crate) fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint { diff --git a/src/scalar_mul/vartime_double_base.rs b/src/backend/serial/scalar_mul/vartime_double_base.rs similarity index 93% rename from src/scalar_mul/vartime_double_base.rs rename to src/backend/serial/scalar_mul/vartime_double_base.rs index 86022d2..d95151f 100644 --- a/src/scalar_mul/vartime_double_base.rs +++ b/src/backend/serial/scalar_mul/vartime_double_base.rs @@ -13,8 +13,8 @@ use constants; use traits::Identity; use scalar::Scalar; use edwards::EdwardsPoint; -use curve_models::{ProjectiveNielsPoint, ProjectivePoint}; -use scalar_mul::window::NafLookupTable5; +use backend::serial::curve_models::{ProjectiveNielsPoint, ProjectivePoint}; +use window::NafLookupTable5; /// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { diff --git a/src/backend/u32/constants.rs b/src/backend/serial/u32/constants.rs similarity index 98% rename from src/backend/u32/constants.rs rename to src/backend/serial/u32/constants.rs index 22def0a..f5102f8 100644 --- a/src/backend/u32/constants.rs +++ b/src/backend/serial/u32/constants.rs @@ -12,8 +12,8 @@ //! and useful field elements like `sqrt(-1)`), as well as //! lookup tables of pre-computed points. -use backend::u32::field::FieldElement2625; -use backend::u32::scalar::Scalar29; +use backend::serial::u32::field::FieldElement2625; +use backend::serial::u32::scalar::Scalar29; use edwards::EdwardsPoint; /// Edwards `d` value, equal to `-121665/121666 mod p`. diff --git a/src/backend/u32/field.rs b/src/backend/serial/u32/field.rs similarity index 100% rename from src/backend/u32/field.rs rename to src/backend/serial/u32/field.rs diff --git a/src/backend/u32/mod.rs b/src/backend/serial/u32/mod.rs similarity index 100% rename from src/backend/u32/mod.rs rename to src/backend/serial/u32/mod.rs diff --git a/src/backend/u32/scalar.rs b/src/backend/serial/u32/scalar.rs similarity index 100% rename from src/backend/u32/scalar.rs rename to src/backend/serial/u32/scalar.rs diff --git a/src/backend/u64/constants.rs b/src/backend/serial/u64/constants.rs similarity index 98% rename from src/backend/u64/constants.rs rename to src/backend/serial/u64/constants.rs index 11c86b2..a5c1f07 100644 --- a/src/backend/u64/constants.rs +++ b/src/backend/serial/u64/constants.rs @@ -10,8 +10,8 @@ //! This module contains backend-specific constant values, such as the 64-bit limbs of curve constants. -use backend::u64::field::FieldElement51; -use backend::u64::scalar::Scalar52; +use backend::serial::u64::field::FieldElement51; +use backend::serial::u64::scalar::Scalar52; use edwards::EdwardsPoint; /// Edwards `d` value, equal to `-121665/121666 mod p`. diff --git a/src/backend/u64/field.rs b/src/backend/serial/u64/field.rs similarity index 100% rename from src/backend/u64/field.rs rename to src/backend/serial/u64/field.rs diff --git a/src/backend/u64/mod.rs b/src/backend/serial/u64/mod.rs similarity index 100% rename from src/backend/u64/mod.rs rename to src/backend/serial/u64/mod.rs diff --git a/src/backend/u64/scalar.rs b/src/backend/serial/u64/scalar.rs similarity index 100% rename from src/backend/u64/scalar.rs rename to src/backend/serial/u64/scalar.rs diff --git a/src/backend/avx2/constants.rs b/src/backend/vector/avx2/constants.rs similarity index 99% rename from src/backend/avx2/constants.rs rename to src/backend/vector/avx2/constants.rs index 2433497..19204e4 100644 --- a/src/backend/avx2/constants.rs +++ b/src/backend/vector/avx2/constants.rs @@ -12,9 +12,9 @@ use packed_simd::u32x8; -use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; -use backend::avx2::field::FieldElement2625x4; -use scalar_mul::window::NafLookupTable8; +use backend::vector::avx2::edwards::{CachedPoint, ExtendedPoint}; +use backend::vector::avx2::field::FieldElement2625x4; +use window::NafLookupTable8; /// The identity element as an `ExtendedPoint`. pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(FieldElement2625x4([ diff --git a/src/backend/avx2/edwards.rs b/src/backend/vector/avx2/edwards.rs similarity index 98% rename from src/backend/avx2/edwards.rs rename to src/backend/vector/avx2/edwards.rs index e076238..675797d 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/vector/avx2/edwards.rs @@ -41,12 +41,12 @@ use subtle::Choice; use subtle::ConditionallySelectable; use edwards; -use scalar_mul::window::{LookupTable, NafLookupTable5, NafLookupTable8}; +use window::{LookupTable, NafLookupTable5, NafLookupTable8}; use traits::Identity; -use backend::avx2::field::{FieldElement2625x4, Lanes, Shuffle}; -use backend::avx2::constants; +use super::constants; +use super::field::{FieldElement2625x4, Lanes, Shuffle}; /// A point on Curve25519, using parallel Edwards formulas for curve /// operations. @@ -190,7 +190,7 @@ impl From for CachedPoint { x = x.blend(x.diff_sum(), Lanes::AB); // x = (X1 - Y1, X2 + Y2, Z2, T2) = (S2 S3 Z2 T2) - x = x * (121666, 121666, 2*121666, 2*121665); + x = x * (121666, 121666, 2 * 121666, 2 * 121665); // x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2) x = x.blend(-x, Lanes::D); @@ -247,7 +247,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { // coefficients grow by one bit. So on input, `self` is // bounded with `b < 0.007` and `other` is bounded with // `b < 1.0`. - + let mut tmp = self.0; tmp = tmp.blend(tmp.diff_sum(), Lanes::AB); @@ -329,7 +329,7 @@ mod test { use super::*; fn serial_add(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) -> edwards::EdwardsPoint { - use backend::u64::field::FieldElement51; + use backend::serial::u64::field::FieldElement51; let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); let (X2, Y2, Z2, T2) = (Q.X, Q.Y, Q.Z, Q.T); diff --git a/src/backend/avx2/field.rs b/src/backend/vector/avx2/field.rs similarity index 99% rename from src/backend/avx2/field.rs rename to src/backend/vector/avx2/field.rs index 9364e45..35cf21c 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/vector/avx2/field.rs @@ -42,8 +42,8 @@ const D_LANES64: u8 = 0b11_00_00_00; use core::ops::{Add, Mul, Neg}; use packed_simd::{i32x8, u32x8, u64x4, IntoBits}; -use backend::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO}; -use backend::u64::field::FieldElement51; +use backend::vector::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_HI, P_TIMES_2_LO}; +use backend::serial::u64::field::FieldElement51; /// Unpack 32-bit lanes into 64-bit lanes: /// ``` diff --git a/src/backend/avx2/mod.rs b/src/backend/vector/avx2/mod.rs similarity index 97% rename from src/backend/avx2/mod.rs rename to src/backend/vector/avx2/mod.rs index 10886af..cdad7ec 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/vector/avx2/mod.rs @@ -27,4 +27,3 @@ pub(crate) mod edwards; pub(crate) mod constants; -pub(crate) mod scalar_mul; diff --git a/src/backend/vector/mod.rs b/src/backend/vector/mod.rs new file mode 100644 index 0000000..7c5609d --- /dev/null +++ b/src/backend/vector/mod.rs @@ -0,0 +1,32 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2018 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +//! Pluggable implementations for different architectures. +//! +//! The naming of the `u32` and `u64` modules is somewhat unfortunate, +//! since these are also the names of primitive types. Since types have +//! a different namespace than modules, this isn't a problem to the +//! compiler, but it could cause confusion. +//! +//! However, it's unlikely that the names of those modules would be +//! brought into scope directly, instead of used as +//! `backend::u32::field` or similar. Unfortunately we can't use +//! `32bit` since identifiers can't start with letters, and the backends +//! do use `u32`/`u64`, so this seems like a least-bad option. + +#[cfg(all(feature = "avx2_backend", target_feature = "avx2"))] +pub mod avx2; +#[cfg(all(feature = "avx2_backend", target_feature = "avx2"))] +pub(crate) use self::avx2::{ + constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint, +}; + +pub mod scalar_mul; + diff --git a/src/backend/avx2/scalar_mul/mod.rs b/src/backend/vector/scalar_mul/mod.rs similarity index 100% rename from src/backend/avx2/scalar_mul/mod.rs rename to src/backend/vector/scalar_mul/mod.rs diff --git a/src/backend/avx2/scalar_mul/straus.rs b/src/backend/vector/scalar_mul/straus.rs similarity index 96% rename from src/backend/avx2/scalar_mul/straus.rs rename to src/backend/vector/scalar_mul/straus.rs index eed9084..9fea5a8 100644 --- a/src/backend/avx2/scalar_mul/straus.rs +++ b/src/backend/vector/scalar_mul/straus.rs @@ -14,10 +14,10 @@ use core::borrow::Borrow; use clear_on_drop::ClearOnDrop; -use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; +use backend::vector::{CachedPoint, ExtendedPoint}; use edwards::EdwardsPoint; use scalar::Scalar; -use scalar_mul::window::{LookupTable, NafLookupTable5}; +use window::{LookupTable, NafLookupTable5}; use traits::{Identity, MultiscalarMul, VartimeMultiscalarMul}; #[allow(unused_imports)] diff --git a/src/backend/avx2/scalar_mul/variable_base.rs b/src/backend/vector/scalar_mul/variable_base.rs similarity index 91% rename from src/backend/avx2/scalar_mul/variable_base.rs rename to src/backend/vector/scalar_mul/variable_base.rs index 6f36bc7..f53c4a0 100644 --- a/src/backend/avx2/scalar_mul/variable_base.rs +++ b/src/backend/vector/scalar_mul/variable_base.rs @@ -1,10 +1,10 @@ #![allow(non_snake_case)] -use traits::Identity; -use scalar::Scalar; +use backend::vector::{CachedPoint, ExtendedPoint}; use edwards::EdwardsPoint; -use backend::avx2::edwards::{ExtendedPoint, CachedPoint}; -use scalar_mul::window::LookupTable; +use scalar::Scalar; +use traits::Identity; +use window::LookupTable; /// Perform constant-time, variable-base scalar multiplication. pub fn mul(point: &EdwardsPoint, scalar: &Scalar) -> EdwardsPoint { diff --git a/src/backend/avx2/scalar_mul/vartime_double_base.rs b/src/backend/vector/scalar_mul/vartime_double_base.rs similarity index 90% rename from src/backend/avx2/scalar_mul/vartime_double_base.rs rename to src/backend/vector/scalar_mul/vartime_double_base.rs index b3d6333..44d92f7 100644 --- a/src/backend/avx2/scalar_mul/vartime_double_base.rs +++ b/src/backend/vector/scalar_mul/vartime_double_base.rs @@ -9,12 +9,12 @@ // - Henry de Valence #![allow(non_snake_case)] -use traits::Identity; -use scalar::Scalar; +use backend::vector::BASEPOINT_ODD_LOOKUP_TABLE; +use backend::vector::{CachedPoint, ExtendedPoint}; use edwards::EdwardsPoint; -use scalar_mul::window::NafLookupTable5; -use backend::avx2::edwards::{CachedPoint, ExtendedPoint}; -use backend::avx2::constants::BASEPOINT_ODD_LOOKUP_TABLE; +use scalar::Scalar; +use traits::Identity; +use window::NafLookupTable5; /// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. pub fn mul(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { diff --git a/src/constants.rs b/src/constants.rs index 83d7ec8..b21ee3a 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -34,9 +34,9 @@ use montgomery::MontgomeryPoint; use scalar::Scalar; #[cfg(feature = "u64_backend")] -pub use backend::u64::constants::*; +pub use backend::serial::u64::constants::*; #[cfg(feature = "u32_backend")] -pub use backend::u32::constants::*; +pub use backend::serial::u32::constants::*; /// The Ed25519 basepoint, in `CompressedEdwardsY` format. /// @@ -151,7 +151,7 @@ mod test { #[test] #[cfg(feature = "u32_backend")] fn test_d_vs_ratio() { - use backend::u32::field::FieldElement2625; + use backend::serial::u32::field::FieldElement2625; let a = -&FieldElement2625([121665,0,0,0,0,0,0,0,0,0]); let b = FieldElement2625([121666,0,0,0,0,0,0,0,0,0]); let d = &a * &b.invert(); @@ -164,7 +164,7 @@ mod test { #[test] #[cfg(feature = "u64_backend")] fn test_d_vs_ratio() { - use backend::u64::field::FieldElement51; + use backend::serial::u64::field::FieldElement51; let a = -&FieldElement51([121665,0,0,0,0]); let b = FieldElement51([121666,0,0,0,0]); let d = &a * &b.invert(); diff --git a/src/edwards.rs b/src/edwards.rs index b37a9da..dbaa1c3 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -112,16 +112,16 @@ use scalar::Scalar; use montgomery::MontgomeryPoint; -use curve_models::ProjectivePoint; -use curve_models::CompletedPoint; -use curve_models::AffineNielsPoint; -use curve_models::ProjectiveNielsPoint; +use backend::serial::curve_models::ProjectivePoint; +use backend::serial::curve_models::CompletedPoint; +use backend::serial::curve_models::AffineNielsPoint; +use backend::serial::curve_models::ProjectiveNielsPoint; + +use window::LookupTable; #[allow(unused_imports)] use prelude::*; -use scalar_mul::window::LookupTable; - use traits::{Identity, IsIdentity}; use traits::ValidityCheck; @@ -575,13 +575,13 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsPoint { // If we built with AVX2, use the AVX2 backend. #[cfg(all(feature="avx2_backend", target_feature="avx2"))] { - use backend::avx2::scalar_mul::variable_base::mul; + use backend::vector::scalar_mul::variable_base::mul; mul(self, scalar) } // Otherwise, use the serial backend: #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] { - use scalar_mul::variable_base::mul; + use backend::serial::scalar_mul::variable_base::mul; mul(self, scalar) } } @@ -636,10 +636,10 @@ impl MultiscalarMul for EdwardsPoint { // If we built with AVX2, use the AVX2 backend. #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::avx2::scalar_mul::straus::Straus; + use backend::vector::scalar_mul::straus::Straus; // Otherwise, proceed as normal: #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use scalar_mul::straus::Straus; + use backend::serial::scalar_mul::straus::Straus; Straus::multiscalar_mul(scalars, points) } @@ -674,10 +674,10 @@ impl VartimeMultiscalarMul for EdwardsPoint { // If we built with AVX2, use the AVX2 backend. #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::avx2::scalar_mul::straus::Straus; + use backend::vector::scalar_mul::straus::Straus; // Otherwise, proceed as normal: #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use scalar_mul::straus::Straus; + use backend::serial::scalar_mul::straus::Straus; Straus::optional_multiscalar_mul(scalars, points) } @@ -689,10 +689,10 @@ impl EdwardsPoint { pub fn vartime_double_scalar_mul_basepoint(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { // If we built with AVX2, use the AVX2 backend. #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::avx2::scalar_mul::vartime_double_base; + use backend::vector::scalar_mul::vartime_double_base; // Otherwise, use the serial backend: #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use scalar_mul::vartime_double_base; + use backend::serial::scalar_mul::vartime_double_base; vartime_double_base::mul(a, A, b) } diff --git a/src/field.rs b/src/field.rs index 1587cea..2ad5d7f 100644 --- a/src/field.rs +++ b/src/field.rs @@ -33,24 +33,24 @@ use constants; use backend; #[cfg(feature = "u64_backend")] -pub use backend::u64::field::*; +pub use backend::serial::u64::field::*; /// A `FieldElement` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// /// The `FieldElement` type is an alias for one of the platform-specific /// implementations. #[cfg(feature = "u64_backend")] -pub type FieldElement = backend::u64::field::FieldElement51; +pub type FieldElement = backend::serial::u64::field::FieldElement51; #[cfg(feature = "u32_backend")] -pub use backend::u32::field::*; +pub use backend::serial::u32::field::*; /// A `FieldElement` represents an element of the field /// \\( \mathbb Z / (2\^{255} - 19)\\). /// /// The `FieldElement` type is an alias for one of the platform-specific /// implementations. #[cfg(feature = "u32_backend")] -pub type FieldElement = backend::u32::field::FieldElement2625; +pub type FieldElement = backend::serial::u32::field::FieldElement2625; impl Eq for FieldElement {} diff --git a/src/lib.rs b/src/lib.rs index 4f52c96..13b7f9c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -90,11 +90,9 @@ pub(crate) mod field; // Arithmetic backends (using u32, u64, etc) live here pub(crate) mod backend; -// Internal curve models which are not part of the public API. -pub(crate) mod curve_models; - // Crate-local prelude (for alloc-dependent features like `Vec`) pub(crate) mod prelude; -// Implementations of scalar mul algorithms live here -pub(crate) mod scalar_mul; +// Generic code for window lookups +pub(crate) mod window; + diff --git a/src/ristretto.rs b/src/ristretto.rs index 234af93..6f2bea7 100644 --- a/src/ristretto.rs +++ b/src/ristretto.rs @@ -185,8 +185,6 @@ use prelude::*; use scalar::Scalar; -use curve_models::CompletedPoint; - use traits::Identity; #[cfg(any(feature = "alloc", feature = "std"))] use traits::{MultiscalarMul, VartimeMultiscalarMul}; @@ -596,6 +594,8 @@ impl RistrettoPoint { let N_t = &(&(&c * &(&r - &one)) * &d_minus_one_sq) - &D; let s_sq = s.square(); + use backend::serial::curve_models::CompletedPoint; + // The conversion from W_i is exactly the conversion from P1xP1. RistrettoPoint(CompletedPoint{ X: &(&s + &s) * &D, diff --git a/src/scalar.rs b/src/scalar.rs index c8d5aa9..3c78bee 100644 --- a/src/scalar.rs +++ b/src/scalar.rs @@ -168,14 +168,14 @@ use constants; /// This is a type alias for one of the scalar types in the `backend` /// module. #[cfg(feature = "u64_backend")] -type UnpackedScalar = backend::u64::scalar::Scalar52; +type UnpackedScalar = backend::serial::u64::scalar::Scalar52; /// An `UnpackedScalar` represents an element of the field GF(l), optimized for speed. /// /// This is a type alias for one of the scalar types in the `backend` /// module. #[cfg(feature = "u32_backend")] -type UnpackedScalar = backend::u32::scalar::Scalar29; +type UnpackedScalar = backend::serial::u32::scalar::Scalar29; /// The `Scalar` struct holds an integer \\(s < 2\^{255} \\) which diff --git a/src/scalar_mul/window.rs b/src/window.rs similarity index 98% rename from src/scalar_mul/window.rs rename to src/window.rs index d5ad2d6..08f2d78 100644 --- a/src/scalar_mul/window.rs +++ b/src/window.rs @@ -22,8 +22,8 @@ use subtle::Choice; use traits::Identity; use edwards::EdwardsPoint; -use curve_models::ProjectiveNielsPoint; -use curve_models::AffineNielsPoint; +use backend::serial::curve_models::ProjectiveNielsPoint; +use backend::serial::curve_models::AffineNielsPoint; /// A lookup table of precomputed multiples of a point \\(P\\), used to /// compute \\( xP \\) for \\( -8 \leq x \leq 8 \\). From 0a97f5fe8c3695b7c429f1fa21d505fe4b3d8db8 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 28 Nov 2018 13:09:35 -0800 Subject: [PATCH 06/52] Don't compile the serial scalar_mul when using the vector one. --- src/backend/serial/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/serial/mod.rs b/src/backend/serial/mod.rs index adce772..b49f851 100644 --- a/src/backend/serial/mod.rs +++ b/src/backend/serial/mod.rs @@ -35,5 +35,9 @@ pub mod u64; pub mod curve_models; +#[cfg(not(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +)))] pub mod scalar_mul; From 9ed2128a100454ed8f8de895fdde43c9e959a39e Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sun, 11 Nov 2018 17:38:28 -0800 Subject: [PATCH 07/52] Add stub code for IFMA intrinsics in Rust. --- Cargo.toml | 1 + build.rs | 4 +++- src/backend/mod.rs | 11 +++++++-- src/backend/vector/ifma/field.rs | 40 ++++++++++++++++++++++++++++++++ src/backend/vector/ifma/mod.rs | 11 +++++++++ src/backend/vector/mod.rs | 3 +++ src/lib.rs | 5 +++- 7 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 src/backend/vector/ifma/field.rs create mode 100644 src/backend/vector/ifma/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 3562f0d..6c7ea30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ u64_backend = [] # The AVX2 backend uses u32x8s with u64x4 products. # It uses the u64 code for serial operations. avx2_backend = ["nightly", "u64_backend", "packed_simd"] +ifma_backend = ["nightly", "u64_backend", "packed_simd", "yolocrypto"] # Signals that we're in the main build stage. This is off by default, # to signal stage 1 of the build, where build.rs loads the library diff --git a/build.rs b/build.rs index 38b577a..e684b37 100644 --- a/build.rs +++ b/build.rs @@ -1,3 +1,5 @@ +#![cfg_attr(feature = "ifma_backend", feature(simd_ffi))] +#![cfg_attr(feature = "ifma_backend", feature(link_llvm_intrinsics))] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![allow(unused_variables)] @@ -13,7 +15,7 @@ extern crate digest; extern crate rand; extern crate subtle; -#[cfg(all(feature = "nightly", feature = "avx2_backend"))] +#[cfg(all(feature = "nightly", any(feature = "avx2_backend", feature = "ifma_backend")))] extern crate packed_simd; use std::env; diff --git a/src/backend/mod.rs b/src/backend/mod.rs index be537e6..82eeb5b 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -21,7 +21,11 @@ //! `32bit` since identifiers can't start with letters, and the backends //! do use `u32`/`u64`, so this seems like a least-bad option. -#[cfg(not(any(feature = "u32_backend", feature = "u64_backend", feature = "avx2_backend")))] +#[cfg(not(any( + feature = "u32_backend", + feature = "u64_backend", + feature = "avx2_backend" +)))] compile_error!( "no curve25519-dalek backend cargo feature enabled! \ please enable one of: u32_backend, u64_backend, avx2_backend" @@ -29,5 +33,8 @@ compile_error!( pub mod serial; -#[cfg(all(feature = "avx2_backend", target_feature = "avx2"))] +#[cfg(any( + all(feature = "ifma_backend", target_feature = "avx512ifma"), + all(feature = "avx2_backend", target_feature = "avx2"), +))] pub mod vector; diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs new file mode 100644 index 0000000..6ff36b7 --- /dev/null +++ b/src/backend/vector/ifma/field.rs @@ -0,0 +1,40 @@ +// -*- mode: rust; coding: utf-8; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2018 Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +use core::ops::{Add, Mul, Neg}; +use packed_simd::{i32x8, u32x8, u64x4, IntoBits}; + +use backend::serial::u64::field::FieldElement51; + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.avx512.vpmadd52l.uq.256"] + fn madd52lo(z: u64x4, x: u64x4, y: u64x4) -> u64x4; + #[link_name = "llvm.x86.avx512.vpmadd52h.uq.256"] + fn madd52hi(z: u64x4, x: u64x4, y: u64x4) -> u64x4; +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn vpmadd52luq() { + let x = u64x4::splat(2); + let y = u64x4::splat(3); + let mut z = u64x4::splat(5); + + z = unsafe { madd52lo(z, x, y) }; + + assert_eq!(z, u64x4::splat(5 + 2*3)); + } + +} + + diff --git a/src/backend/vector/ifma/mod.rs b/src/backend/vector/ifma/mod.rs new file mode 100644 index 0000000..4238412 --- /dev/null +++ b/src/backend/vector/ifma/mod.rs @@ -0,0 +1,11 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2018 Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +pub(crate) mod field; + diff --git a/src/backend/vector/mod.rs b/src/backend/vector/mod.rs index 7c5609d..e7749f2 100644 --- a/src/backend/vector/mod.rs +++ b/src/backend/vector/mod.rs @@ -28,5 +28,8 @@ pub(crate) use self::avx2::{ constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint, }; +#[cfg(all(feature = "ifma_backend", target_feature = "avx512ifma"))] +pub mod ifma; + pub mod scalar_mul; diff --git a/src/lib.rs b/src/lib.rs index 13b7f9c..07a91be 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,9 @@ #![no_std] +#![cfg_attr(feature = "ifma_backend", feature(simd_ffi))] +#![cfg_attr(feature = "ifma_backend", feature(link_llvm_intrinsics))] + #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "nightly", feature(external_doc))] @@ -38,7 +41,7 @@ extern crate alloc; #[macro_use] extern crate std; -#[cfg(all(feature = "nightly", feature = "avx2_backend"))] +#[cfg(all(feature = "nightly", any(feature = "avx2_backend", feature = "ifma_backend")))] extern crate packed_simd; extern crate rand; From 47a164da4dfe732ef23ec9a0f06c8300e9284c3f Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sat, 24 Nov 2018 13:12:20 -0800 Subject: [PATCH 08/52] Replace `avx2_backend` with `simd_backend` (autoselects AVX2/IFMA) --- Cargo.toml | 8 ++--- build.rs | 12 ++++++-- src/backend/mod.rs | 10 +++---- src/backend/vector/mod.rs | 10 ++++--- src/edwards.rs | 61 ++++++++++++++------------------------- src/lib.rs | 24 ++++++++------- 6 files changed, 60 insertions(+), 65 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6c7ea30..fea904c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,10 +69,10 @@ yolocrypto = [] u32_backend = [] # The u64 backend uses u64s with u128 products. u64_backend = [] -# The AVX2 backend uses u32x8s with u64x4 products. -# It uses the u64 code for serial operations. -avx2_backend = ["nightly", "u64_backend", "packed_simd"] -ifma_backend = ["nightly", "u64_backend", "packed_simd", "yolocrypto"] +# The SIMD backend uses parallel formulas, using either AVX2 or AVX512-IFMA. +simd_backend = ["nightly", "u64_backend", "packed_simd"] +# Old name for the SIMD backend, preserved for compatibility +avx2_backend = ["simd_backend"] # Signals that we're in the main build stage. This is off by default, # to signal stage 1 of the build, where build.rs loads the library diff --git a/build.rs b/build.rs index e684b37..f24e47b 100644 --- a/build.rs +++ b/build.rs @@ -1,5 +1,11 @@ -#![cfg_attr(feature = "ifma_backend", feature(simd_ffi))] -#![cfg_attr(feature = "ifma_backend", feature(link_llvm_intrinsics))] +#![cfg_attr( + all(feature = "simd_backend", target_feature = "avx512ifma"), + feature(simd_ffi) +)] +#![cfg_attr( + all(feature = "simd_backend", target_feature = "avx512ifma"), + feature(link_llvm_intrinsics) +)] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![allow(unused_variables)] @@ -15,7 +21,7 @@ extern crate digest; extern crate rand; extern crate subtle; -#[cfg(all(feature = "nightly", any(feature = "avx2_backend", feature = "ifma_backend")))] +#[cfg(all(feature = "nightly", feature = "packed_simd"))] extern crate packed_simd; use std::env; diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 82eeb5b..c088c23 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -24,17 +24,17 @@ #[cfg(not(any( feature = "u32_backend", feature = "u64_backend", - feature = "avx2_backend" + feature = "simd_backend", )))] compile_error!( "no curve25519-dalek backend cargo feature enabled! \ - please enable one of: u32_backend, u64_backend, avx2_backend" + please enable one of: u32_backend, u64_backend, simd_backend" ); pub mod serial; -#[cfg(any( - all(feature = "ifma_backend", target_feature = "avx512ifma"), - all(feature = "avx2_backend", target_feature = "avx2"), +#[cfg(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") ))] pub mod vector; diff --git a/src/backend/vector/mod.rs b/src/backend/vector/mod.rs index e7749f2..5975301 100644 --- a/src/backend/vector/mod.rs +++ b/src/backend/vector/mod.rs @@ -21,15 +21,17 @@ //! `32bit` since identifiers can't start with letters, and the backends //! do use `u32`/`u64`, so this seems like a least-bad option. -#[cfg(all(feature = "avx2_backend", target_feature = "avx2"))] +#[cfg(not(any(target_feature = "avx2", target_feature = "avx512ifma",)))] +compile_error!("simd_backend selected without target_feature=+avx2 or +avx512ifma"); + +#[cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma")))] pub mod avx2; -#[cfg(all(feature = "avx2_backend", target_feature = "avx2"))] +#[cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma")))] pub(crate) use self::avx2::{ constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint, }; -#[cfg(all(feature = "ifma_backend", target_feature = "avx512ifma"))] +#[cfg(all(target_feature = "avx512ifma"))] pub mod ifma; pub mod scalar_mul; - diff --git a/src/edwards.rs b/src/edwards.rs index dbaa1c3..a56822f 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -130,6 +130,17 @@ use traits::MultiscalarMul; #[cfg(any(feature = "alloc", feature = "std"))] use traits::VartimeMultiscalarMul; +#[cfg(not(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +)))] +use backend::serial::scalar_mul; +#[cfg(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +))] +use backend::vector::scalar_mul; + // ------------------------------------------------------------------------ // Compressed points // ------------------------------------------------------------------------ @@ -572,18 +583,7 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsPoint { /// For scalar multiplication of a basepoint, /// `EdwardsBasepointTable` is approximately 4x faster. fn mul(self, scalar: &'b Scalar) -> EdwardsPoint { - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - { - use backend::vector::scalar_mul::variable_base::mul; - mul(self, scalar) - } - // Otherwise, use the serial backend: - #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - { - use backend::serial::scalar_mul::variable_base::mul; - mul(self, scalar) - } + scalar_mul::variable_base::mul(self, scalar) } } @@ -609,7 +609,7 @@ impl<'a, 'b> Mul<&'b EdwardsPoint> for &'a Scalar { #[cfg(feature = "alloc")] impl MultiscalarMul for EdwardsPoint { type Point = EdwardsPoint; - + fn multiscalar_mul(scalars: I, points: J) -> EdwardsPoint where I: IntoIterator, @@ -634,21 +634,14 @@ impl MultiscalarMul for EdwardsPoint { // size-dependent algorithm dispatch, use this as the hint. let _size = s_lo; - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::vector::scalar_mul::straus::Straus; - // Otherwise, proceed as normal: - #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use backend::serial::scalar_mul::straus::Straus; - - Straus::multiscalar_mul(scalars, points) + scalar_mul::straus::Straus::multiscalar_mul(scalars, points) } } #[cfg(feature = "alloc")] impl VartimeMultiscalarMul for EdwardsPoint { type Point = EdwardsPoint; - + fn optional_multiscalar_mul(scalars: I, points: J) -> Option where I: IntoIterator, @@ -672,29 +665,19 @@ impl VartimeMultiscalarMul for EdwardsPoint { // size-dependent algorithm dispatch, use this as the hint. let _size = s_lo; - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::vector::scalar_mul::straus::Straus; - // Otherwise, proceed as normal: - #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use backend::serial::scalar_mul::straus::Straus; - - Straus::optional_multiscalar_mul(scalars, points) + scalar_mul::straus::Straus::optional_multiscalar_mul(scalars, points) } } impl EdwardsPoint { /// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. #[cfg(feature = "stage2_build")] - pub fn vartime_double_scalar_mul_basepoint(a: &Scalar, A: &EdwardsPoint, b: &Scalar) -> EdwardsPoint { - // If we built with AVX2, use the AVX2 backend. - #[cfg(all(feature="avx2_backend", target_feature="avx2"))] - use backend::vector::scalar_mul::vartime_double_base; - // Otherwise, use the serial backend: - #[cfg(not(all(feature="avx2_backend", target_feature="avx2")))] - use backend::serial::scalar_mul::vartime_double_base; - - vartime_double_base::mul(a, A, b) + pub fn vartime_double_scalar_mul_basepoint( + a: &Scalar, + A: &EdwardsPoint, + b: &Scalar, + ) -> EdwardsPoint { + scalar_mul::vartime_double_base::mul(a, A, b) } } diff --git a/src/lib.rs b/src/lib.rs index 07a91be..cd3b583 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,14 +9,18 @@ // - Henry de Valence #![no_std] - -#![cfg_attr(feature = "ifma_backend", feature(simd_ffi))] -#![cfg_attr(feature = "ifma_backend", feature(link_llvm_intrinsics))] - +#![cfg_attr( + all(feature = "simd_backend", target_feature = "avx512ifma"), + feature(simd_ffi) +)] +#![cfg_attr( + all(feature = "simd_backend", target_feature = "avx512ifma"), + feature(link_llvm_intrinsics) +)] +#![cfg_attr(feature = "nightly", feature(test))] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "nightly", feature(external_doc))] - // Refuse to compile if documentation is missing, but only on nightly. // // This means that missing docs will still fail CI, but means we can use @@ -41,21 +45,21 @@ extern crate alloc; #[macro_use] extern crate std; -#[cfg(all(feature = "nightly", any(feature = "avx2_backend", feature = "ifma_backend")))] +#[cfg(all(feature = "nightly", feature = "packed_simd"))] extern crate packed_simd; -extern crate rand; -extern crate clear_on_drop; extern crate byteorder; +extern crate clear_on_drop; pub extern crate digest; +extern crate rand; // Used for traits related to constant-time code. extern crate subtle; -#[cfg(feature = "serde")] -extern crate serde; #[cfg(all(test, feature = "serde"))] extern crate bincode; +#[cfg(feature = "serde")] +extern crate serde; // Internal macros. Must come first! #[macro_use] From e91cc685b1f87ca9661833f10b1d0822f3678ebf Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 29 Nov 2018 10:58:05 -0800 Subject: [PATCH 09/52] Don't generate unused constants in build.rs. --- build.rs | 58 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/build.rs b/build.rs index f24e47b..e9c2a66 100644 --- a/build.rs +++ b/build.rs @@ -44,33 +44,31 @@ mod macros; // Public modules -#[path = "src/scalar.rs"] -mod scalar; -#[path = "src/montgomery.rs"] -mod montgomery; -#[path = "src/edwards.rs"] -mod edwards; -#[path = "src/ristretto.rs"] -mod ristretto; #[path = "src/constants.rs"] mod constants; +#[path = "src/edwards.rs"] +mod edwards; +#[path = "src/montgomery.rs"] +mod montgomery; +#[path = "src/ristretto.rs"] +mod ristretto; +#[path = "src/scalar.rs"] +mod scalar; #[path = "src/traits.rs"] mod traits; // Internal modules -#[path = "src/field.rs"] -mod field; #[path = "src/backend/mod.rs"] mod backend; +#[path = "src/field.rs"] +mod field; #[path = "src/prelude.rs"] mod prelude; #[path = "src/window.rs"] mod window; use edwards::EdwardsBasepointTable; -use backend::serial::curve_models::AffineNielsPoint; -use window::NafLookupTable8; fn main() { // Enable the "stage2_build" feature in the main build stage @@ -97,7 +95,6 @@ use edwards::EdwardsBasepointTable; use backend::serial::curve_models::AffineNielsPoint; use window::LookupTable; -use window::NafLookupTable8; /// Table containing precomputed multiples of the Ed25519 basepoint \\\\(B = (x, 4/5)\\\\). pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN; @@ -107,20 +104,35 @@ pub const ED25519_BASEPOINT_TABLE: EdwardsBasepointTable = ED25519_BASEPOINT_TAB pub const ED25519_BASEPOINT_TABLE_INNER_DOC_HIDDEN: EdwardsBasepointTable = {:?}; \n\n", &table - ).as_bytes(), - ).unwrap(); + ) + .as_bytes(), + ) + .unwrap(); // Now generate AFFINE_ODD_MULTIPLES_OF_BASEPOINT - let B = &constants::ED25519_BASEPOINT_POINT; - let odd_multiples = NafLookupTable8::::from(B); + // if we are going to build the serial scalar_mul backend + #[cfg(not(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") + )))] + { + use backend::serial::curve_models::AffineNielsPoint; + use window::NafLookupTable8; - f.write_all( - format!( - "\n + let B = &constants::ED25519_BASEPOINT_POINT; + let odd_multiples = NafLookupTable8::::from(B); + + f.write_all( + format!( + "\n +use window::NafLookupTable8; /// Odd multiples of the basepoint `[B, 3B, 5B, 7B, 9B, 11B, 13B, 15B, ..., 127B]`. pub(crate) const AFFINE_ODD_MULTIPLES_OF_BASEPOINT: NafLookupTable8 = {:?}; \n\n", - &odd_multiples - ).as_bytes(), - ).unwrap(); + &odd_multiples + ) + .as_bytes(), + ) + .unwrap(); + } } From be77af1cd87fa6f366da606512e8aa778b903d8c Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 29 Nov 2018 12:05:43 -0800 Subject: [PATCH 10/52] Remove unnecessary feature gate. The vector backend doesn't generate constants in build.rs, so this isn't necessary. --- src/backend/vector/scalar_mul/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/vector/scalar_mul/mod.rs b/src/backend/vector/scalar_mul/mod.rs index d3d14e3..9293e3e 100644 --- a/src/backend/vector/scalar_mul/mod.rs +++ b/src/backend/vector/scalar_mul/mod.rs @@ -10,7 +10,6 @@ pub mod variable_base; -#[cfg(feature = "stage2_build")] pub mod vartime_double_base; pub mod straus; From 3233e2eac602404a7dfe388e23f5bd9f12a3e669 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 29 Nov 2018 11:12:01 -0800 Subject: [PATCH 11/52] Fix: cfg_target_feature is stable since 1.27. --- build.rs | 1 - src/lib.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/build.rs b/build.rs index e9c2a66..9d3bae3 100644 --- a/build.rs +++ b/build.rs @@ -7,7 +7,6 @@ feature(link_llvm_intrinsics) )] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] -#![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![allow(unused_variables)] #![allow(non_snake_case)] #![allow(dead_code)] diff --git a/src/lib.rs b/src/lib.rs index cd3b583..2bf5c8a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,7 +19,6 @@ )] #![cfg_attr(feature = "nightly", feature(test))] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] -#![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "nightly", feature(external_doc))] // Refuse to compile if documentation is missing, but only on nightly. // From aa73d7b1bc16147879ae37761c44e95b464a6ac4 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sun, 11 Nov 2018 21:36:55 -0800 Subject: [PATCH 12/52] Add a FieldElement51x4 type. --- src/backend/vector/ifma/field.rs | 115 ++++++++++++++++++++++++++++++- src/backend/vector/ifma/mod.rs | 2 +- src/lib.rs | 1 - 3 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 6ff36b7..d2f59e8 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -20,6 +20,83 @@ extern "C" { fn madd52hi(z: u64x4, x: u64x4, y: u64x4) -> u64x4; } +pub struct FieldElement51x4([u64x4; 5]); + +impl FieldElement51x4 { + pub fn new( + x0: &FieldElement51, + x1: &FieldElement51, + x2: &FieldElement51, + x3: &FieldElement51, + ) -> FieldElement51x4 { + FieldElement51x4([ + u64x4::new(x0.0[0], x1.0[0], x2.0[0], x3.0[0]), + u64x4::new(x0.0[1], x1.0[1], x2.0[1], x3.0[1]), + u64x4::new(x0.0[2], x1.0[2], x2.0[2], x3.0[2]), + u64x4::new(x0.0[3], x1.0[3], x2.0[3], x3.0[3]), + u64x4::new(x0.0[4], x1.0[4], x2.0[4], x3.0[4]), + ]) + .reduce() + } + + pub fn split(&self) -> [FieldElement51; 4] { + let x = &self.0; + [ + FieldElement51([ + x[0].extract(0), + x[1].extract(0), + x[2].extract(0), + x[3].extract(0), + x[4].extract(0), + ]), + FieldElement51([ + x[0].extract(1), + x[1].extract(1), + x[2].extract(1), + x[3].extract(1), + x[4].extract(1), + ]), + FieldElement51([ + x[0].extract(2), + x[1].extract(2), + x[2].extract(2), + x[3].extract(2), + x[4].extract(2), + ]), + FieldElement51([ + x[0].extract(3), + x[1].extract(3), + x[2].extract(3), + x[3].extract(3), + x[4].extract(3), + ]), + ] + } + + #[inline] + pub fn reduce(&self) -> FieldElement51x4 { + let mask = u64x4::splat((1 << 51) - 1); + let r19 = u64x4::splat(19); + + // Compute carryouts in parallel + let c0 = self.0[0] >> 51; + let c1 = self.0[1] >> 51; + let c2 = self.0[2] >> 51; + let c3 = self.0[3] >> 51; + let c4 = self.0[4] >> 51; + + unsafe { + FieldElement51x4([ + madd52lo(self.0[0] & mask, c4, r19), + (self.0[1] & mask) + c0, + (self.0[2] & mask) + c1, + (self.0[3] & mask) + c2, + (self.0[4] & mask) + c3, + ]) + } + } +} + #[cfg(test)] mod test { use super::*; @@ -32,9 +109,41 @@ mod test { z = unsafe { madd52lo(z, x, y) }; - assert_eq!(z, u64x4::splat(5 + 2*3)); + assert_eq!(z, u64x4::splat(5 + 2 * 3)); + } + + #[test] + fn new_split_round_trip_on_reduced_input() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + + let ax4 = FieldElement51x4::new(&a, &a, &a, &a); + let splits = ax4.split(); + + for i in 0..4 { + assert_eq!(a, splits[i]); + } + } + + #[test] + fn new_split_round_trip_on_unreduced_input() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + // ... but now multiply it by 16 without reducing coeffs + let a16 = FieldElement51([ + a.0[0] << 4, + a.0[1] << 4, + a.0[2] << 4, + a.0[3] << 4, + a.0[4] << 4, + ]); + + let a16x4 = FieldElement51x4::new(&a16, &a16, &a16, &a16); + let splits = a16x4.split(); + + for i in 0..4 { + assert_eq!(a16, splits[i]); + } } } - - diff --git a/src/backend/vector/ifma/mod.rs b/src/backend/vector/ifma/mod.rs index 4238412..e066ddd 100644 --- a/src/backend/vector/ifma/mod.rs +++ b/src/backend/vector/ifma/mod.rs @@ -7,5 +7,5 @@ // Authors: // - Henry de Valence -pub(crate) mod field; +pub mod field; diff --git a/src/lib.rs b/src/lib.rs index 2bf5c8a..9dd6608 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -101,4 +101,3 @@ pub(crate) mod prelude; // Generic code for window lookups pub(crate) mod window; - From 70199d6094a015389aff6c860e8a831d3342d012 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 18 Jan 2019 01:04:55 -0800 Subject: [PATCH 13/52] Add impl Mul for FieldElement51x4. --- src/backend/vector/ifma/field.rs | 184 +++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index d2f59e8..ba6939b 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -20,6 +20,7 @@ extern "C" { fn madd52hi(z: u64x4, x: u64x4, y: u64x4) -> u64x4; } +#[derive(Copy, Clone)] pub struct FieldElement51x4([u64x4; 5]); impl FieldElement51x4 { @@ -97,6 +98,146 @@ impl FieldElement51x4 { } } +impl<'a, 'b> Mul<&'b FieldElement51x4> for &'a FieldElement51x4 { + type Output = FieldElement51x4; + fn mul(self, rhs: &'b FieldElement51x4) -> FieldElement51x4 { + unsafe { + // Inputs + let x = &self.0; + let y = &rhs.0; + + // Accumulators for lo-sourced terms + let mut z0lo = u64x4::splat(0); + let mut z1lo = u64x4::splat(0); + let mut z2lo = u64x4::splat(0); + let mut z3lo = u64x4::splat(0); + let mut z4lo = u64x4::splat(0); + let mut z5lo = u64x4::splat(0); + let mut z6lo = u64x4::splat(0); + let mut z7lo = u64x4::splat(0); + let mut z8lo = u64x4::splat(0); + + // Accumulators for hi-sourced terms + // Need to be doubled before adding + let mut z1hi = u64x4::splat(0); + let mut z2hi = u64x4::splat(0); + let mut z3hi = u64x4::splat(0); + let mut z4hi = u64x4::splat(0); + let mut z5hi = u64x4::splat(0); + let mut z6hi = u64x4::splat(0); + let mut z7hi = u64x4::splat(0); + let mut z8hi = u64x4::splat(0); + let mut z9hi = u64x4::splat(0); + + // Wave 0 + z4lo = madd52lo(z4lo, x[4], y[0]); + z5hi = madd52hi(z5hi, x[4], y[0]); + z5lo = madd52lo(z5lo, x[4], y[1]); + z6hi = madd52hi(z6hi, x[4], y[1]); + z6lo = madd52lo(z6lo, x[4], y[2]); + z7hi = madd52hi(z7hi, x[4], y[2]); + z7lo = madd52lo(z7lo, x[4], y[3]); + z8hi = madd52hi(z8hi, x[4], y[3]); + + // Wave 1 + z4lo = madd52lo(z4lo, x[3], y[1]); + z5hi = madd52hi(z5hi, x[3], y[1]); + z5lo = madd52lo(z5lo, x[3], y[2]); + z6hi = madd52hi(z6hi, x[3], y[2]); + z6lo = madd52lo(z6lo, x[3], y[3]); + z7hi = madd52hi(z7hi, x[3], y[3]); + z7lo = madd52lo(z7lo, x[3], y[4]); + z8hi = madd52hi(z8hi, x[3], y[4]); + + // Wave 2 + z8lo = madd52lo(z8lo, x[4], y[4]); + z9hi = madd52hi(z9hi, x[4], y[4]); + z4lo = madd52lo(z4lo, x[2], y[2]); + z5hi = madd52hi(z5hi, x[2], y[2]); + z5lo = madd52lo(z5lo, x[2], y[3]); + z6hi = madd52hi(z6hi, x[2], y[3]); + z6lo = madd52lo(z6lo, x[2], y[4]); + z7hi = madd52hi(z7hi, x[2], y[4]); + + let z8 = z8lo + z8hi + z8hi; + let z9 = z9hi + z9hi; + + // Wave 3 + z3lo = madd52lo(z3lo, x[3], y[0]); + z4hi = madd52hi(z4hi, x[3], y[0]); + z4lo = madd52lo(z4lo, x[1], y[3]); + z5hi = madd52hi(z5hi, x[1], y[3]); + z5lo = madd52lo(z5lo, x[1], y[4]); + z6hi = madd52hi(z6hi, x[1], y[4]); + z2lo = madd52lo(z2lo, x[2], y[0]); + z3hi = madd52hi(z3hi, x[2], y[0]); + + let z6 = z6lo + z6hi + z6hi; + let z7 = z7lo + z7hi + z7hi; + + // Wave 4 + z3lo = madd52lo(z3lo, x[2], y[1]); + z4hi = madd52hi(z4hi, x[2], y[1]); + z4lo = madd52lo(z4lo, x[0], y[4]); + z5hi = madd52hi(z5hi, x[0], y[4]); + z1lo = madd52lo(z1lo, x[1], y[0]); + z2hi = madd52hi(z2hi, x[1], y[0]); + z2lo = madd52lo(z2lo, x[1], y[1]); + z3hi = madd52hi(z3hi, x[1], y[1]); + + let z5 = z5lo + z5hi + z5hi; + + // Wave 5 + z3lo = madd52lo(z3lo, x[1], y[2]); + z4hi = madd52hi(z4hi, x[1], y[2]); + z0lo = madd52lo(z0lo, x[0], y[0]); + z1hi = madd52hi(z1hi, x[0], y[0]); + z1lo = madd52lo(z1lo, x[0], y[1]); + z2lo = madd52lo(z2lo, x[0], y[2]); + z2hi = madd52hi(z2hi, x[0], y[1]); + z3hi = madd52hi(z3hi, x[0], y[2]); + + let r19 = u64x4::splat(19); + let r38 = u64x4::splat(38); + let r1938 = u64x4::splat(19 * 38); + let mut z919hi = u64x4::splat(0); + + // Wave 6 + z3lo = madd52lo(z3lo, x[0], y[3]); + z919hi = madd52hi(z919hi, r19, z9); + z0lo = madd52lo(z0lo, r19, z5); + z4lo = madd52lo(z4lo, r19, z9); + z1lo = madd52lo(z1lo, r19, z6); + z2lo = madd52lo(z2lo, r19, z7); + z1hi = madd52hi(z1hi, r19, z5); + z4hi = madd52hi(z4hi, x[0], y[3]); + + // Wave 7 + z3lo = madd52lo(z3lo, r19, z8); + z2hi = madd52hi(z2hi, r19, z6); + z0lo = madd52lo(z0lo, r38, z919hi); + z4lo = madd52lo(z4lo, r38, z8 >> 52); + z1lo = madd52lo(z1lo, r38, z5 >> 52); + z2lo = madd52lo(z2lo, r38, z6 >> 52); + z3hi = madd52hi(z3hi, r19, z7); + z4hi = madd52hi(z4hi, r19, z8); + + // Wave 8 + z3lo = madd52lo(z3lo, r38, z7 >> 52); + z0lo = madd52lo(z0lo, r1938, z9 >> 52); + + FieldElement51x4([ + z0lo, + z1lo + z1hi + z1hi, + z2lo + z2hi + z2hi, + z3lo + z3hi + z3hi, + z4lo + z4hi + z4hi, + ]) + .reduce() + } + } +} + #[cfg(test)] mod test { use super::*; @@ -146,4 +287,47 @@ mod test { } } + #[test] + fn mul_matches_serial() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + let b = FieldElement51([98098, 87987897, 0, 1, 0]).invert(); + let c = &a * &b; + + let ax4 = FieldElement51x4::new(&a, &a, &a, &a); + let bx4 = FieldElement51x4::new(&b, &b, &b, &b); + let cx4 = &ax4 * &bx4; + + let splits = cx4.split(); + + for i in 0..4 { + assert_eq!(c, splits[i]); + } + } + + #[test] + fn iterated_mul_matches_serial() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + let b = FieldElement51([98098, 87987897, 0, 1, 0]).invert(); + let mut c = &a * &b; + for i in 0..1024 { + c = &a * &c; + c = &b * &c; + } + + let ax4 = FieldElement51x4::new(&a, &a, &a, &a); + let bx4 = FieldElement51x4::new(&b, &b, &b, &b); + let mut cx4 = &ax4 * &bx4; + for i in 0..1024 { + cx4 = &ax4 * &cx4; + cx4 = &bx4 * &cx4; + } + + let splits = cx4.split(); + + for i in 0..4 { + assert_eq!(c, splits[i]); + } + } } From ff4967321412d266ed988ca97942b86cd84d89a6 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 12 Nov 2018 11:58:59 -0800 Subject: [PATCH 14/52] Add impl Mul<(u32, u32, u32, u32)> for FieldElement51x4. --- src/backend/vector/ifma/field.rs | 83 ++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index ba6939b..1b67ea8 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -98,8 +98,67 @@ impl FieldElement51x4 { } } +impl<'a> Mul<(u32, u32, u32, u32)> for &'a FieldElement51x4 { + type Output = FieldElement51x4; + #[inline] + fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement51x4 { + unsafe { + let x = &self.0; + let y = u64x4::new( + scalars.0 as u64, + scalars.1 as u64, + scalars.2 as u64, + scalars.3 as u64, + ); + let mask = u64x4::splat((1 << 51) - 1); + let r19 = u64x4::splat(19); + + let mut z0lo = u64x4::splat(0); + let mut z1lo = u64x4::splat(0); + let mut z2lo = u64x4::splat(0); + let mut z3lo = u64x4::splat(0); + let mut z4lo = u64x4::splat(0); + let mut z1hi = u64x4::splat(0); + let mut z2hi = u64x4::splat(0); + let mut z3hi = u64x4::splat(0); + let mut z4hi = u64x4::splat(0); + let mut z5hi = u64x4::splat(0); + + // Wave 0 + z4hi = madd52hi(z4hi, y, x[3]); + z5hi = madd52hi(z5hi, y, x[4]); + z4lo = madd52lo(z4lo, y, x[4]); + z0lo = madd52lo(z0lo, y, x[0]); + z3lo = madd52lo(z3lo, y, x[3]); + z2lo = madd52lo(z2lo, y, x[2]); + z1lo = madd52lo(z1lo, y, x[1]); + z3hi = madd52hi(z3hi, y, x[2]); + + // Wave 2 + z2hi = madd52hi(z2hi, y, x[1]); + z1hi = madd52hi(z1hi, y, x[0]); + + let z4 = z4hi + z4hi + z4lo; + let c4 = z4 >> 51; + let z5 = z5hi + z5hi + c4; + let z3 = z3hi + z3hi + z3lo; + let z2 = z2hi + z2hi + z2lo; + let z1 = z1hi + z1hi + z1lo; + + FieldElement51x4([ + madd52lo(z0lo & mask, z5, r19), + (z1 & mask) + (z0lo >> 51), + (z2 & mask) + (z1lo >> 51), + (z3 & mask) + (z2lo >> 51), + (z4 & mask) + (z3lo >> 51), + ]) + } + } +} + impl<'a, 'b> Mul<&'b FieldElement51x4> for &'a FieldElement51x4 { type Output = FieldElement51x4; + #[inline] fn mul(self, rhs: &'b FieldElement51x4) -> FieldElement51x4 { unsafe { // Inputs @@ -330,4 +389,28 @@ mod test { assert_eq!(c, splits[i]); } } + + #[test] + fn iterated_u32_mul_matches_serial() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + let b = FieldElement51([121665, 0, 0, 0, 0]); + let mut c = &a * &b; + for i in 0..1024 { + c = &b * &c; + } + + let ax4 = FieldElement51x4::new(&a, &a, &a, &a); + let bx4 = (121665u32, 121665u32, 121665u32, 121665u32); + let mut cx4 = &ax4 * bx4; + for i in 0..1024 { + cx4 = &cx4 * bx4; + } + + let splits = cx4.split(); + + for i in 0..4 { + assert_eq!(c, splits[i]); + } + } } From 14163131f98b7dec85f6e43a57446c6fecb4b240 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 12 Nov 2018 23:32:28 -0800 Subject: [PATCH 15/52] Lift the reduction state of the field elements into the type system. This splits the `FieldElement51x4` type into two types: - `F51x4Reduced` (with reduced limbs) - `F51x4Unreduced` (with unreduced limbs) The reduction is implemented as a `From` impl to convert one type to the other. The output of a multiplication is now a `F51x4Unreduced`. The reason is that the inputs to IFMA operations must be at most 52 bits, so it's not possible to perform an addition of (51+epsilon)-bit values and still be small enough to be used as an input to multiplication. So, it doesn't make sense to perform a reduction at the end of a multiplication, because the reduced values will be fed into an addition or subtraction, which then needs to be re-reduced. --- src/backend/vector/ifma/field.rs | 129 +++++++++++++++++++------------ 1 file changed, 79 insertions(+), 50 deletions(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 1b67ea8..d263eec 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -20,24 +20,28 @@ extern "C" { fn madd52hi(z: u64x4, x: u64x4, y: u64x4) -> u64x4; } -#[derive(Copy, Clone)] -pub struct FieldElement51x4([u64x4; 5]); +/// A vector of four field elements in radix 2^51, with unreduced coefficients. +#[derive(Copy, Clone, Debug)] +pub struct F51x4Unreduced(pub(crate) [u64x4; 5]); -impl FieldElement51x4 { +/// A vector of four field elements in radix 2^51, with reduced coefficients. +#[derive(Copy, Clone, Debug)] +pub struct F51x4Reduced(pub(crate) [u64x4; 5]); + +impl F51x4Unreduced { pub fn new( x0: &FieldElement51, x1: &FieldElement51, x2: &FieldElement51, x3: &FieldElement51, - ) -> FieldElement51x4 { - FieldElement51x4([ + ) -> F51x4Unreduced { + F51x4Unreduced([ u64x4::new(x0.0[0], x1.0[0], x2.0[0], x3.0[0]), u64x4::new(x0.0[1], x1.0[1], x2.0[1], x3.0[1]), u64x4::new(x0.0[2], x1.0[2], x2.0[2], x3.0[2]), u64x4::new(x0.0[3], x1.0[3], x2.0[3], x3.0[3]), u64x4::new(x0.0[4], x1.0[4], x2.0[4], x3.0[4]), ]) - .reduce() } pub fn split(&self) -> [FieldElement51; 4] { @@ -73,35 +77,44 @@ impl FieldElement51x4 { ]), ] } +} +impl From for F51x4Unreduced { #[inline] - pub fn reduce(&self) -> FieldElement51x4 { + fn from(x: F51x4Reduced) -> F51x4Unreduced { + F51x4Unreduced(x.0) + } +} + +impl From for F51x4Reduced { + #[inline] + fn from(x: F51x4Unreduced) -> F51x4Reduced { let mask = u64x4::splat((1 << 51) - 1); let r19 = u64x4::splat(19); // Compute carryouts in parallel - let c0 = self.0[0] >> 51; - let c1 = self.0[1] >> 51; - let c2 = self.0[2] >> 51; - let c3 = self.0[3] >> 51; - let c4 = self.0[4] >> 51; + let c0 = x.0[0] >> 51; + let c1 = x.0[1] >> 51; + let c2 = x.0[2] >> 51; + let c3 = x.0[3] >> 51; + let c4 = x.0[4] >> 51; unsafe { - FieldElement51x4([ - madd52lo(self.0[0] & mask, c4, r19), - (self.0[1] & mask) + c0, - (self.0[2] & mask) + c1, - (self.0[3] & mask) + c2, - (self.0[4] & mask) + c3, + F51x4Reduced([ + madd52lo(x.0[0] & mask, c4, r19), + (x.0[1] & mask) + c0, + (x.0[2] & mask) + c1, + (x.0[3] & mask) + c2, + (x.0[4] & mask) + c3, ]) } } } -impl<'a> Mul<(u32, u32, u32, u32)> for &'a FieldElement51x4 { - type Output = FieldElement51x4; +impl<'a> Mul<(u32, u32, u32, u32)> for &'a F51x4Reduced { + type Output = F51x4Unreduced; #[inline] - fn mul(self, scalars: (u32, u32, u32, u32)) -> FieldElement51x4 { + fn mul(self, scalars: (u32, u32, u32, u32)) -> F51x4Unreduced { unsafe { let x = &self.0; let y = u64x4::new( @@ -137,29 +150,23 @@ impl<'a> Mul<(u32, u32, u32, u32)> for &'a FieldElement51x4 { // Wave 2 z2hi = madd52hi(z2hi, y, x[1]); z1hi = madd52hi(z1hi, y, x[0]); + z0lo = madd52lo(z0lo, z5hi + z5hi, r19); - let z4 = z4hi + z4hi + z4lo; - let c4 = z4 >> 51; - let z5 = z5hi + z5hi + c4; - let z3 = z3hi + z3hi + z3lo; - let z2 = z2hi + z2hi + z2lo; - let z1 = z1hi + z1hi + z1lo; - - FieldElement51x4([ - madd52lo(z0lo & mask, z5, r19), - (z1 & mask) + (z0lo >> 51), - (z2 & mask) + (z1lo >> 51), - (z3 & mask) + (z2lo >> 51), - (z4 & mask) + (z3lo >> 51), + F51x4Unreduced([ + z0lo, + z1hi + z1hi + z1lo, + z2hi + z2hi + z2lo, + z3hi + z3hi + z3lo, + z4hi + z4hi + z4lo, ]) } } } -impl<'a, 'b> Mul<&'b FieldElement51x4> for &'a FieldElement51x4 { - type Output = FieldElement51x4; +impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { + type Output = F51x4Unreduced; #[inline] - fn mul(self, rhs: &'b FieldElement51x4) -> FieldElement51x4 { + fn mul(self, rhs: &'b F51x4Reduced) -> F51x4Unreduced { unsafe { // Inputs let x = &self.0; @@ -285,14 +292,13 @@ impl<'a, 'b> Mul<&'b FieldElement51x4> for &'a FieldElement51x4 { z3lo = madd52lo(z3lo, r38, z7 >> 52); z0lo = madd52lo(z0lo, r1938, z9 >> 52); - FieldElement51x4([ + F51x4Unreduced([ z0lo, z1lo + z1hi + z1hi, z2lo + z2hi + z2hi, z3lo + z3hi + z3hi, z4lo + z4hi + z4hi, ]) - .reduce() } } } @@ -317,7 +323,7 @@ mod test { // Invert a small field element to get a big one let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); - let ax4 = FieldElement51x4::new(&a, &a, &a, &a); + let ax4 = F51x4Unreduced::new(&a, &a, &a, &a); let splits = ax4.split(); for i in 0..4 { @@ -338,7 +344,7 @@ mod test { a.0[4] << 4, ]); - let a16x4 = FieldElement51x4::new(&a16, &a16, &a16, &a16); + let a16x4 = F51x4Unreduced::new(&a16, &a16, &a16, &a16); let splits = a16x4.split(); for i in 0..4 { @@ -346,6 +352,29 @@ mod test { } } + #[test] + fn test_reduction() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + // ... but now multiply it by 128 without reducing coeffs + let abig = FieldElement51([ + a.0[0] << 8, + a.0[1] << 8, + a.0[2] << 8, + a.0[3] << 8, + a.0[4] << 8, + ]); + + let abigx4: F51x4Reduced = F51x4Unreduced::new(&abig, &abig, &abig, &abig).into(); + + let splits = F51x4Unreduced::from(abigx4).split(); + let c = &a * &FieldElement51([128, 0, 0, 0, 0]); + + for i in 0..4 { + assert_eq!(c, splits[i]); + } + } + #[test] fn mul_matches_serial() { // Invert a small field element to get a big one @@ -353,8 +382,8 @@ mod test { let b = FieldElement51([98098, 87987897, 0, 1, 0]).invert(); let c = &a * &b; - let ax4 = FieldElement51x4::new(&a, &a, &a, &a); - let bx4 = FieldElement51x4::new(&b, &b, &b, &b); + let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into(); + let bx4: F51x4Reduced = F51x4Unreduced::new(&b, &b, &b, &b).into(); let cx4 = &ax4 * &bx4; let splits = cx4.split(); @@ -375,12 +404,12 @@ mod test { c = &b * &c; } - let ax4 = FieldElement51x4::new(&a, &a, &a, &a); - let bx4 = FieldElement51x4::new(&b, &b, &b, &b); + let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into(); + let bx4: F51x4Reduced = F51x4Unreduced::new(&b, &b, &b, &b).into(); let mut cx4 = &ax4 * &bx4; for i in 0..1024 { - cx4 = &ax4 * &cx4; - cx4 = &bx4 * &cx4; + cx4 = &ax4 * &F51x4Reduced::from(cx4); + cx4 = &bx4 * &F51x4Reduced::from(cx4); } let splits = cx4.split(); @@ -400,11 +429,11 @@ mod test { c = &b * &c; } - let ax4 = FieldElement51x4::new(&a, &a, &a, &a); + let ax4 = F51x4Unreduced::new(&a, &a, &a, &a); let bx4 = (121665u32, 121665u32, 121665u32, 121665u32); - let mut cx4 = &ax4 * bx4; + let mut cx4 = &F51x4Reduced::from(ax4) * bx4; for i in 0..1024 { - cx4 = &cx4 * bx4; + cx4 = &F51x4Reduced::from(cx4) * bx4; } let splits = cx4.split(); From 607b83903272b1f0007eb0f0a062da5562be843e Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 14 Nov 2018 00:14:57 -0800 Subject: [PATCH 16/52] Change size of limbs in the reduction test. --- src/backend/vector/ifma/field.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index d263eec..5ca3078 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -358,17 +358,17 @@ mod test { let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); // ... but now multiply it by 128 without reducing coeffs let abig = FieldElement51([ - a.0[0] << 8, - a.0[1] << 8, - a.0[2] << 8, - a.0[3] << 8, - a.0[4] << 8, + a.0[0] << 4, + a.0[1] << 4, + a.0[2] << 4, + a.0[3] << 4, + a.0[4] << 4, ]); let abigx4: F51x4Reduced = F51x4Unreduced::new(&abig, &abig, &abig, &abig).into(); let splits = F51x4Unreduced::from(abigx4).split(); - let c = &a * &FieldElement51([128, 0, 0, 0, 0]); + let c = &a * &FieldElement51([(1 << 4), 0, 0, 0, 0]); for i in 0..4 { assert_eq!(c, splits[i]); From c925c763421686cbd350d333d6a748e373c8ff46 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 14 Nov 2018 00:00:03 -0800 Subject: [PATCH 17/52] Change the strategy for reducing the high half of the product. --- src/backend/vector/ifma/field.rs | 39 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 5ca3078..8fdb629 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -185,6 +185,7 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { // Accumulators for hi-sourced terms // Need to be doubled before adding + let mut z0hi = u64x4::splat(0); let mut z1hi = u64x4::splat(0); let mut z2hi = u64x4::splat(0); let mut z3hi = u64x4::splat(0); @@ -263,37 +264,37 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { z2hi = madd52hi(z2hi, x[0], y[1]); z3hi = madd52hi(z3hi, x[0], y[2]); + let mut t0 = u64x4::splat(0); + let mut t1 = u64x4::splat(0); let r19 = u64x4::splat(19); - let r38 = u64x4::splat(38); - let r1938 = u64x4::splat(19 * 38); - let mut z919hi = u64x4::splat(0); + // Wave 6 + t0 = madd52hi(t0, r19, z9); + t1 = madd52lo(t1, r19, z9 >> 52); z3lo = madd52lo(z3lo, x[0], y[3]); - z919hi = madd52hi(z919hi, r19, z9); - z0lo = madd52lo(z0lo, r19, z5); - z4lo = madd52lo(z4lo, r19, z9); - z1lo = madd52lo(z1lo, r19, z6); - z2lo = madd52lo(z2lo, r19, z7); - z1hi = madd52hi(z1hi, r19, z5); z4hi = madd52hi(z4hi, x[0], y[3]); + z1hi = madd52lo(z1hi, r19, z5 >> 52); + z2hi = madd52lo(z2hi, r19, z6 >> 52); + z3hi = madd52lo(z3hi, r19, z7 >> 52); + z0lo = madd52lo(z0lo, r19, z5); // Wave 7 - z3lo = madd52lo(z3lo, r19, z8); - z2hi = madd52hi(z2hi, r19, z6); - z0lo = madd52lo(z0lo, r38, z919hi); - z4lo = madd52lo(z4lo, r38, z8 >> 52); - z1lo = madd52lo(z1lo, r38, z5 >> 52); - z2lo = madd52lo(z2lo, r38, z6 >> 52); - z3hi = madd52hi(z3hi, r19, z7); + z4lo = madd52lo(z4lo, r19, z9); + z1lo = madd52lo(z1lo, r19, z6); + z0hi = madd52lo(z0hi, r19, t0 + t1); z4hi = madd52hi(z4hi, r19, z8); + z2lo = madd52lo(z2lo, r19, z7); + z1hi = madd52hi(z1hi, r19, z5); + z2hi = madd52hi(z2hi, r19, z6); + z3hi = madd52hi(z3hi, r19, z7); // Wave 8 - z3lo = madd52lo(z3lo, r38, z7 >> 52); - z0lo = madd52lo(z0lo, r1938, z9 >> 52); + z3lo = madd52lo(z3lo, r19, z8); + z4hi = madd52lo(z4hi, r19, z8 >> 52); F51x4Unreduced([ - z0lo, + z0lo + z0hi + z0hi, z1lo + z1hi + z1hi, z2lo + z2hi + z2hi, z3lo + z3hi + z3hi, From d90bb779cae5c5a6754b44680c52cef5e11bd9bf Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 14 Nov 2018 12:36:34 -0800 Subject: [PATCH 18/52] Delay all computations using x[0], y[0] Since computation of the 0 term in reduction requires a multiplication with a 4-cycle latency, this ensures that the rest of the computation can start before the 0 term is finished --- src/backend/vector/ifma/field.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 8fdb629..12a72f4 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -197,8 +197,8 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { let mut z9hi = u64x4::splat(0); // Wave 0 - z4lo = madd52lo(z4lo, x[4], y[0]); - z5hi = madd52hi(z5hi, x[4], y[0]); + z4lo = madd52lo(z4lo, x[2], y[2]); + z5hi = madd52hi(z5hi, x[2], y[2]); z5lo = madd52lo(z5lo, x[4], y[1]); z6hi = madd52hi(z6hi, x[4], y[1]); z6lo = madd52lo(z6lo, x[4], y[2]); @@ -219,8 +219,8 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { // Wave 2 z8lo = madd52lo(z8lo, x[4], y[4]); z9hi = madd52hi(z9hi, x[4], y[4]); - z4lo = madd52lo(z4lo, x[2], y[2]); - z5hi = madd52hi(z5hi, x[2], y[2]); + z4lo = madd52lo(z4lo, x[4], y[0]); + z5hi = madd52hi(z5hi, x[4], y[0]); z5lo = madd52lo(z5lo, x[2], y[3]); z6hi = madd52hi(z6hi, x[2], y[3]); z6lo = madd52lo(z6lo, x[2], y[4]); From 5880fced2bb9fa6a31df0da6c72e0fceacc2e02a Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 14 Nov 2018 22:34:19 -0800 Subject: [PATCH 19/52] Add skeleton of shuffle and blend functions. --- src/backend/vector/ifma/field.rs | 118 ++++++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 2 deletions(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 12a72f4..be26a21 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -8,7 +8,7 @@ // - Henry de Valence use core::ops::{Add, Mul, Neg}; -use packed_simd::{i32x8, u32x8, u64x4, IntoBits}; +use packed_simd::{u64x4, FromBits, IntoBits}; use backend::serial::u64::field::FieldElement51; @@ -28,6 +28,38 @@ pub struct F51x4Unreduced(pub(crate) [u64x4; 5]); #[derive(Copy, Clone, Debug)] pub struct F51x4Reduced(pub(crate) [u64x4; 5]); +#[derive(Copy, Clone)] +pub enum Lanes { + AB, +} + +#[derive(Copy, Clone)] +pub enum Shuffle { + AAAA, +} + +#[inline(always)] +fn shuffle_lanes(x: u64x4, control: Shuffle) -> u64x4 { + unsafe { + use core::arch::x86_64::_mm256_permute4x64_epi64 as perm; + + match control { + Shuffle::AAAA => perm(x.into_bits(), 0b00_00_00_00).into_bits(), + } + } +} + +#[inline] +fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 { + unsafe { + use core::arch::x86_64::_mm256_blend_epi32 as blend; + + match control { + Lanes::AB => blend(x.into_bits(), y.into_bits(), 0b00_00_11_11).into_bits(), + } + } +} + impl F51x4Unreduced { pub fn new( x0: &FieldElement51, @@ -77,6 +109,52 @@ impl F51x4Unreduced { ]), ] } + + #[inline] + pub fn shuffle(&self, control: Shuffle) -> F51x4Unreduced { + F51x4Unreduced([ + shuffle_lanes(self.0[0], control), + shuffle_lanes(self.0[1], control), + shuffle_lanes(self.0[2], control), + shuffle_lanes(self.0[3], control), + shuffle_lanes(self.0[4], control), + ]) + } + + #[inline] + pub fn blend(&self, other: &F51x4Unreduced, control: Lanes) -> F51x4Unreduced { + F51x4Unreduced([ + blend_lanes(self.0[0], other.0[0], control), + blend_lanes(self.0[1], other.0[1], control), + blend_lanes(self.0[2], other.0[2], control), + blend_lanes(self.0[3], other.0[3], control), + blend_lanes(self.0[4], other.0[4], control), + ]) + } +} + +impl F51x4Reduced { + #[inline] + pub fn shuffle(&self, control: Shuffle) -> F51x4Reduced { + F51x4Reduced([ + shuffle_lanes(self.0[0], control), + shuffle_lanes(self.0[1], control), + shuffle_lanes(self.0[2], control), + shuffle_lanes(self.0[3], control), + shuffle_lanes(self.0[4], control), + ]) + } + + #[inline] + pub fn blend(&self, other: &F51x4Reduced, control: Lanes) -> F51x4Reduced { + F51x4Reduced([ + blend_lanes(self.0[0], other.0[0], control), + blend_lanes(self.0[1], other.0[1], control), + blend_lanes(self.0[2], other.0[2], control), + blend_lanes(self.0[3], other.0[3], control), + blend_lanes(self.0[4], other.0[4], control), + ]) + } } impl From for F51x4Unreduced { @@ -268,7 +346,6 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { let mut t1 = u64x4::splat(0); let r19 = u64x4::splat(19); - // Wave 6 t0 = madd52hi(t0, r19, z9); t1 = madd52lo(t1, r19, z9 >> 52); @@ -443,4 +520,41 @@ mod test { assert_eq!(c, splits[i]); } } + + #[test] + fn shuffle_AAAA() { + let x0 = FieldElement51::from_bytes(&[0x10; 32]); + let x1 = FieldElement51::from_bytes(&[0x11; 32]); + let x2 = FieldElement51::from_bytes(&[0x12; 32]); + let x3 = FieldElement51::from_bytes(&[0x13; 32]); + + let x = F51x4Unreduced::new(&x0, &x1, &x2, &x3); + + let y = x.shuffle(Shuffle::AAAA); + let splits = y.split(); + + assert_eq!(splits[0], x0); + assert_eq!(splits[1], x0); + assert_eq!(splits[2], x0); + assert_eq!(splits[3], x0); + } + + #[test] + fn blend_AB() { + let x0 = FieldElement51::from_bytes(&[0x10; 32]); + let x1 = FieldElement51::from_bytes(&[0x11; 32]); + let x2 = FieldElement51::from_bytes(&[0x12; 32]); + let x3 = FieldElement51::from_bytes(&[0x13; 32]); + + let x = F51x4Unreduced::new(&x0, &x1, &x2, &x3); + let z = F51x4Unreduced::new(&x3, &x2, &x1, &x0); + + let y = x.blend(&z, Lanes::AB); + let splits = y.split(); + + assert_eq!(splits[0], x3); + assert_eq!(splits[1], x2); + assert_eq!(splits[2], x2); + assert_eq!(splits[3], x3); + } } From 3f21bb3b582717c6a2aabe0c356c4882e279e43a Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 16 Nov 2018 00:13:45 -0800 Subject: [PATCH 20/52] Add skeleton of parallel Edwards formulas. --- src/backend/vector/ifma/edwards.rs | 136 +++++++++++++++++++++++++++++ src/backend/vector/ifma/field.rs | 49 +++++++++++ src/backend/vector/ifma/mod.rs | 2 + 3 files changed, 187 insertions(+) create mode 100644 src/backend/vector/ifma/edwards.rs diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs new file mode 100644 index 0000000..65f7c39 --- /dev/null +++ b/src/backend/vector/ifma/edwards.rs @@ -0,0 +1,136 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2018 Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +use traits::Identity; + +use std::ops::Add; + +use edwards; + +use super::field::{F51x4Reduced, F51x4Unreduced, Lanes, Shuffle}; + +#[derive(Copy, Clone, Debug)] +pub struct ExtendedPoint(pub(super) F51x4Unreduced); + +#[derive(Copy, Clone, Debug)] +pub struct CachedPoint(pub(super) F51x4Reduced); + +impl From for ExtendedPoint { + fn from(P: edwards::EdwardsPoint) -> ExtendedPoint { + ExtendedPoint(F51x4Unreduced::new(&P.X, &P.Y, &P.Z, &P.T)) + } +} + +impl From for edwards::EdwardsPoint { + fn from(P: ExtendedPoint) -> edwards::EdwardsPoint { + let tmp = P.0.split(); + edwards::EdwardsPoint { + X: tmp[0], + Y: tmp[1], + Z: tmp[2], + T: tmp[3], + } + } +} + +impl From for CachedPoint { + fn from(P: ExtendedPoint) -> CachedPoint { + let mut x = P.0; + + x = x.blend(&x.diff_sum(), Lanes::AB); + x = &F51x4Reduced::from(x) * (121666, 121666, 2 * 121666, 2 * 121665); + x = x.blend(&x.negate_lazy(), Lanes::D); + + CachedPoint(F51x4Reduced::from(x)) + } +} + +impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { + type Output = ExtendedPoint; + + /// Add an `ExtendedPoint` and a `CachedPoint`. + fn add(self, other: &'b CachedPoint) -> ExtendedPoint { + let mut tmp = self.0; + + tmp = tmp.blend(&tmp.diff_sum(), Lanes::AB); + // tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1) + + tmp = &F51x4Reduced::from(tmp) * &other.0; + // tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11) + + tmp = tmp.shuffle(Shuffle::ABDC); + // tmp = (S8 S9 S11 S10) + + let tmp = F51x4Reduced::from(tmp.diff_sum()); + // tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) + + let t0 = tmp.shuffle(Shuffle::ADDA); + // t0 = (S12 S15 S15 S12) + let t1 = tmp.shuffle(Shuffle::CBCB); + // t1 = (S14 S13 S14 S13) + + // Return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) + ExtendedPoint(&t0 * &t1) + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn addition_test_helper(P: edwards::EdwardsPoint, Q: edwards::EdwardsPoint) { + // Test the serial implementation of the parallel addition formulas + //let R_serial: edwards::EdwardsPoint = serial_add(P.into(), Q.into()).into(); + + // Test the vector implementation of the parallel readdition formulas + let cached_Q = CachedPoint::from(ExtendedPoint::from(Q)); + let R_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) + &cached_Q).into(); + //et S_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) - &cached_Q).into(); + + println!("Testing point addition:"); + println!("P = {:?}", P); + println!("Q = {:?}", Q); + println!("cached Q = {:?}", cached_Q); + println!("R = P + Q = {:?}", &P + &Q); + //println!("R_serial = {:?}", R_serial); + println!("R_vector = {:?}", R_vector); + //println!("S = P - Q = {:?}", &P - &Q); + //println!("S_vector = {:?}", S_vector); + //assert_eq!(R_serial.compress(), (&P + &Q).compress()); + assert_eq!(R_vector.compress(), (&P + &Q).compress()); + //assert_eq!(S_vector.compress(), (&P - &Q).compress()); + println!("OK!\n"); + } + + #[test] + fn vector_addition_vs_serial_addition_vs_edwards_extendedpoint() { + use constants; + use scalar::Scalar; + + println!("Testing id +- id"); + let P = edwards::EdwardsPoint::identity(); + let Q = edwards::EdwardsPoint::identity(); + addition_test_helper(P, Q); + + println!("Testing id +- B"); + let P = edwards::EdwardsPoint::identity(); + let Q = constants::ED25519_BASEPOINT_POINT; + addition_test_helper(P, Q); + + println!("Testing B +- B"); + let P = constants::ED25519_BASEPOINT_POINT; + let Q = constants::ED25519_BASEPOINT_POINT; + addition_test_helper(P, Q); + + println!("Testing B +- kB"); + let P = constants::ED25519_BASEPOINT_POINT; + let Q = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from(8475983829u64); + addition_test_helper(P, Q); + } +} diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index be26a21..307f807 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -30,12 +30,18 @@ pub struct F51x4Reduced(pub(crate) [u64x4; 5]); #[derive(Copy, Clone)] pub enum Lanes { + D, AB, + AC, } #[derive(Copy, Clone)] pub enum Shuffle { AAAA, + BADC, + ADDA, + CBCB, + ABDC, } #[inline(always)] @@ -45,6 +51,10 @@ fn shuffle_lanes(x: u64x4, control: Shuffle) -> u64x4 { match control { Shuffle::AAAA => perm(x.into_bits(), 0b00_00_00_00).into_bits(), + Shuffle::BADC => perm(x.into_bits(), 0b10_11_00_01).into_bits(), + Shuffle::ADDA => perm(x.into_bits(), 0b00_11_11_00).into_bits(), + Shuffle::CBCB => perm(x.into_bits(), 0b01_10_01_10).into_bits(), + Shuffle::ABDC => perm(x.into_bits(), 0b10_11_01_00).into_bits(), } } } @@ -55,7 +65,9 @@ fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 { use core::arch::x86_64::_mm256_blend_epi32 as blend; match control { + Lanes::D => blend(x.into_bits(), y.into_bits(), 0b11_00_00_00).into_bits(), Lanes::AB => blend(x.into_bits(), y.into_bits(), 0b00_00_11_11).into_bits(), + Lanes::AC => blend(x.into_bits(), y.into_bits(), 0b00_11_00_11).into_bits(), } } } @@ -110,6 +122,29 @@ impl F51x4Unreduced { ] } + #[inline] + pub fn diff_sum(&self) -> F51x4Unreduced { + // tmp1 = (B, A, D, C) + let tmp1 = self.shuffle(Shuffle::BADC); + // tmp2 = (-A, B, -C, D) + let tmp2 = self.blend(&self.negate_lazy(), Lanes::AC); + // (B - A, B + A, D - C, D + C) + tmp1 + tmp2 + } + + #[inline] + pub fn negate_lazy(&self) -> F51x4Unreduced { + let lo = u64x4::splat(36028797018963664u64); + let hi = u64x4::splat(36028797018963952u64); + F51x4Unreduced([ + lo - self.0[0], + hi - self.0[1], + hi - self.0[2], + hi - self.0[3], + hi - self.0[4], + ]) + } + #[inline] pub fn shuffle(&self, control: Shuffle) -> F51x4Unreduced { F51x4Unreduced([ @@ -189,6 +224,20 @@ impl From for F51x4Reduced { } } +impl Add for F51x4Unreduced { + type Output = F51x4Unreduced; + #[inline] + fn add(self, rhs: F51x4Unreduced) -> F51x4Unreduced { + F51x4Unreduced([ + self.0[0] + rhs.0[0], + self.0[1] + rhs.0[1], + self.0[2] + rhs.0[2], + self.0[3] + rhs.0[3], + self.0[4] + rhs.0[4], + ]) + } +} + impl<'a> Mul<(u32, u32, u32, u32)> for &'a F51x4Reduced { type Output = F51x4Unreduced; #[inline] diff --git a/src/backend/vector/ifma/mod.rs b/src/backend/vector/ifma/mod.rs index e066ddd..8a77ed3 100644 --- a/src/backend/vector/ifma/mod.rs +++ b/src/backend/vector/ifma/mod.rs @@ -9,3 +9,5 @@ pub mod field; +pub mod edwards; + From a62571b003e1ee37d797f55446098dd264d58510 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 16 Nov 2018 00:20:51 -0800 Subject: [PATCH 21/52] Reduce field elements during conversion to vector form. The IFMA types have tighter bounds than the u64 types. --- src/backend/vector/ifma/edwards.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index 65f7c39..5339f05 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -29,7 +29,8 @@ impl From for ExtendedPoint { impl From for edwards::EdwardsPoint { fn from(P: ExtendedPoint) -> edwards::EdwardsPoint { - let tmp = P.0.split(); + let reduced = F51x4Reduced::from(P.0); + let tmp = F51x4Unreduced::from(reduced).split(); edwards::EdwardsPoint { X: tmp[0], Y: tmp[1], From c4617b28a9d9e72b0e0fad90a83a3feb634dac21 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 18 Jan 2019 00:48:37 -0800 Subject: [PATCH 22/52] Add squaring implementation. --- src/backend/vector/ifma/field.rs | 148 ++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 307f807..d344514 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -190,6 +190,120 @@ impl F51x4Reduced { blend_lanes(self.0[4], other.0[4], control), ]) } + + #[inline] + pub fn square(&self) -> F51x4Unreduced { + unsafe { + let x = &self.0; + + // Represent values with coeff. 2 + let mut z0_2 = u64x4::splat(0); + let mut z1_2 = u64x4::splat(0); + let mut z2_2 = u64x4::splat(0); + let mut z3_2 = u64x4::splat(0); + let mut z4_2 = u64x4::splat(0); + let mut z5_2 = u64x4::splat(0); + let mut z6_2 = u64x4::splat(0); + let mut z7_2 = u64x4::splat(0); + let mut z9_2 = u64x4::splat(0); + + // Represent values with coeff. 4 + let mut z2_4 = u64x4::splat(0); + let mut z3_4 = u64x4::splat(0); + let mut z4_4 = u64x4::splat(0); + let mut z5_4 = u64x4::splat(0); + let mut z6_4 = u64x4::splat(0); + let mut z7_4 = u64x4::splat(0); + let mut z8_4 = u64x4::splat(0); + + let mut z0_1 = u64x4::splat(0); + z0_1 = madd52lo(z0_1, x[0], x[0]); + + let mut z1_1 = u64x4::splat(0); + z1_2 = madd52lo(z1_2, x[0], x[1]); + z1_2 = madd52hi(z1_2, x[0], x[0]); + + z2_4 = madd52hi(z2_4, x[0], x[1]); + let mut z2_1 = z2_4 << 2; + z2_2 = madd52lo(z2_2, x[0], x[2]); + z2_1 = madd52lo(z2_1, x[1], x[1]); + + z3_4 = madd52hi(z3_4, x[0], x[2]); + let mut z3_1 = z3_4 << 2; + z3_2 = madd52lo(z3_2, x[1], x[2]); + z3_2 = madd52lo(z3_2, x[0], x[3]); + z3_2 = madd52hi(z3_2, x[1], x[1]); + + z4_4 = madd52hi(z4_4, x[1], x[2]); + z4_4 = madd52hi(z4_4, x[0], x[3]); + let mut z4_1 = z4_4 << 2; + z4_2 = madd52lo(z4_2, x[1], x[3]); + z4_2 = madd52lo(z4_2, x[0], x[4]); + z4_1 = madd52lo(z4_1, x[2], x[2]); + + z5_4 = madd52hi(z5_4, x[1], x[3]); + z5_4 = madd52hi(z5_4, x[0], x[4]); + let mut z5_1 = z5_4 << 2; + z5_2 = madd52lo(z5_2, x[2], x[3]); + z5_2 = madd52lo(z5_2, x[1], x[4]); + z5_2 = madd52hi(z5_2, x[2], x[2]); + + z6_4 = madd52hi(z6_4, x[2], x[3]); + z6_4 = madd52hi(z6_4, x[1], x[4]); + let mut z6_1 = z6_4 << 2; + z6_2 = madd52lo(z6_2, x[2], x[4]); + z6_1 = madd52lo(z6_1, x[3], x[3]); + + z7_4 = madd52hi(z7_4, x[2], x[4]); + let mut z7_1 = z7_4 << 2; + z7_2 = madd52lo(z7_2, x[3], x[4]); + z7_2 = madd52hi(z7_2, x[3], x[3]); + + z8_4 = madd52hi(z8_4, x[3], x[4]); + let mut z8_1 = z8_4 << 2; + z8_1 = madd52lo(z8_1, x[4], x[4]); + + let mut z9_1 = u64x4::splat(0); + z9_2 = madd52hi(z9_2, x[4], x[4]); + + z5_1 += z5_2 << 1; + z6_1 += z6_2 << 1; + z7_1 += z7_2 << 1; + z9_1 += z9_2 << 1; + + let mut t0 = u64x4::splat(0); + let mut t1 = u64x4::splat(0); + let r19 = u64x4::splat(19); + + t0 = madd52hi(t0, r19, z9_1); + t1 = madd52lo(t1, r19, z9_1 >> 52); + + z4_2 = madd52lo(z4_2, r19, z8_1 >> 52); + z3_2 = madd52lo(z3_2, r19, z7_1 >> 52); + z2_2 = madd52lo(z2_2, r19, z6_1 >> 52); + z1_2 = madd52lo(z1_2, r19, z5_1 >> 52); + + z0_2 = madd52lo(z0_2, r19, t0 + t1); + z1_2 = madd52hi(z1_2, r19, z5_1); + z2_2 = madd52hi(z2_2, r19, z6_1); + z3_2 = madd52hi(z3_2, r19, z7_1); + z4_2 = madd52hi(z4_2, r19, z8_1); + + z0_1 = madd52lo(z0_1, r19, z5_1); + z1_1 = madd52lo(z1_1, r19, z6_1); + z2_1 = madd52lo(z2_1, r19, z7_1); + z3_1 = madd52lo(z3_1, r19, z8_1); + z4_1 = madd52lo(z4_1, r19, z9_1); + + F51x4Unreduced([ + z0_1 + z0_2 + z0_2, + z1_1 + z1_2 + z1_2, + z2_1 + z2_2 + z2_2, + z3_1 + z3_2 + z3_2, + z4_1 + z4_2 + z4_2, + ]) + } + } } impl From for F51x4Unreduced { @@ -250,7 +364,6 @@ impl<'a> Mul<(u32, u32, u32, u32)> for &'a F51x4Reduced { scalars.2 as u64, scalars.3 as u64, ); - let mask = u64x4::splat((1 << 51) - 1); let r19 = u64x4::splat(19); let mut z0lo = u64x4::splat(0); @@ -546,6 +659,39 @@ mod test { } } + #[test] + fn square_matches_mul() { + // Invert a small field element to get a big one + let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + + let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into(); + let mut cx4 = &ax4 * &ax4; + let mut cx4_sq = ax4.square(); + + let splits = cx4.split(); + let splits_sq = cx4_sq.split(); + + for i in 0..4 { + assert_eq!(splits_sq[i], splits[i]); + } + } + + #[test] + fn iterated_square_matches_serial() { + // Invert a small field element to get a big one + let mut a = FieldElement51([2438, 24, 243, 0, 0]).invert(); + let mut ax4 = F51x4Unreduced::new(&a, &a, &a, &a); + for j in 0..1024 { + a = a.square(); + ax4 = F51x4Reduced::from(ax4).square(); + + let splits = ax4.split(); + for i in 0..4 { + assert_eq!(a, splits[i]); + } + } + } + #[test] fn iterated_u32_mul_matches_serial() { // Invert a small field element to get a big one From 8e38ff2859c10afd14ca38c3a5c9d14d235e7fdb Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 18 Jan 2019 00:53:56 -0800 Subject: [PATCH 23/52] Add Edwards doubling implementation. --- src/backend/vector/ifma/edwards.rs | 76 +++++++++++++++++++++++++++++- src/backend/vector/ifma/field.rs | 32 ++++++++++--- 2 files changed, 100 insertions(+), 8 deletions(-) diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index 5339f05..8e65f2f 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -51,7 +51,50 @@ impl From for CachedPoint { CachedPoint(F51x4Reduced::from(x)) } } - + +impl ExtendedPoint { + pub fn double(&self) -> ExtendedPoint { + // Set tmp0 = (X1 Y1 X1 Y1) + let mut tmp0 = self.0.shuffle(Shuffle::ABAB); + + // Set tmp1 = (Y1 X1 Y1 X1) + let mut tmp1 = tmp0.shuffle(Shuffle::BADC); + + // Set tmp0 = (X1 Y1 Z1 X1+Y1) + tmp0 = self.0.blend(&(tmp0 + tmp1), Lanes::D); + + tmp1 = F51x4Reduced::from(tmp0).square(); + // Now tmp1 = (S1 S2 S3 S4) + + // We want to compute + // + // + | S1 | S1 | S1 | S1 | + // + | S2 | | | S2 | + // + | | | S3 | | + // + | | | S3 | | + // + | |16p |16p |16p | + // - | | S2 | S2 | | + // - | | | | S4 | + // ======================= + // S5 S6 S8 S9 + + let zero = F51x4Unreduced::zero(); + + let S1_S1_S1_S1 = tmp1.shuffle(Shuffle::AAAA); + let S2_S2_S2_S2 = tmp1.shuffle(Shuffle::BBBB); + + let S2_S2_S2_S4 = S2_S2_S2_S2.blend(&tmp1, Lanes::D).negate_lazy(); + + tmp0 = S1_S1_S1_S1 + zero.blend(&(tmp1 + tmp1), Lanes::C); + tmp0 = tmp0 + zero.blend(&S2_S2_S2_S2, Lanes::AD); + tmp0 = tmp0 + zero.blend(&S2_S2_S2_S4, Lanes::BCD); + + let tmp2 = F51x4Reduced::from(tmp0); + + ExtendedPoint(&tmp2.shuffle(Shuffle::DBBD) * &tmp2.shuffle(Shuffle::CACA)) + } +} + impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { type Output = ExtendedPoint; @@ -134,4 +177,35 @@ mod test { let Q = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from(8475983829u64); addition_test_helper(P, Q); } + + fn doubling_test_helper(P: edwards::EdwardsPoint) { + //let R1: edwards::EdwardsPoint = serial_double(P.into()).into(); + let R2: edwards::EdwardsPoint = ExtendedPoint::from(P).double().into(); + println!("Testing point doubling:"); + println!("P = {:?}", P); + //println!("(serial) R1 = {:?}", R1); + println!("(vector) R2 = {:?}", R2); + println!("P + P = {:?}", &P + &P); + //assert_eq!(R1.compress(), (&P + &P).compress()); + assert_eq!(R2.compress(), (&P + &P).compress()); + println!("OK!\n"); + } + + #[test] + fn vector_doubling_vs_serial_doubling_vs_edwards_extendedpoint() { + use constants; + use scalar::Scalar; + + println!("Testing [2]id"); + let P = edwards::EdwardsPoint::identity(); + doubling_test_helper(P); + + println!("Testing [2]B"); + let P = constants::ED25519_BASEPOINT_POINT; + doubling_test_helper(P); + + println!("Testing [2]([k]B)"); + let P = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from(8475983829u64); + doubling_test_helper(P); + } } diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index d344514..40220c5 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -28,20 +28,17 @@ pub struct F51x4Unreduced(pub(crate) [u64x4; 5]); #[derive(Copy, Clone, Debug)] pub struct F51x4Reduced(pub(crate) [u64x4; 5]); -#[derive(Copy, Clone)] -pub enum Lanes { - D, - AB, - AC, -} - #[derive(Copy, Clone)] pub enum Shuffle { AAAA, + BBBB, BADC, ADDA, CBCB, ABDC, + ABAB, + DBBD, + CACA, } #[inline(always)] @@ -51,14 +48,28 @@ fn shuffle_lanes(x: u64x4, control: Shuffle) -> u64x4 { match control { Shuffle::AAAA => perm(x.into_bits(), 0b00_00_00_00).into_bits(), + Shuffle::BBBB => perm(x.into_bits(), 0b01_01_01_01).into_bits(), Shuffle::BADC => perm(x.into_bits(), 0b10_11_00_01).into_bits(), Shuffle::ADDA => perm(x.into_bits(), 0b00_11_11_00).into_bits(), Shuffle::CBCB => perm(x.into_bits(), 0b01_10_01_10).into_bits(), Shuffle::ABDC => perm(x.into_bits(), 0b10_11_01_00).into_bits(), + Shuffle::ABAB => perm(x.into_bits(), 0b01_00_01_00).into_bits(), + Shuffle::DBBD => perm(x.into_bits(), 0b11_01_01_11).into_bits(), + Shuffle::CACA => perm(x.into_bits(), 0b00_10_00_10).into_bits(), } } } +#[derive(Copy, Clone)] +pub enum Lanes { + D, + C, + AB, + AC, + AD, + BCD, +} + #[inline] fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 { unsafe { @@ -66,13 +77,20 @@ fn blend_lanes(x: u64x4, y: u64x4, control: Lanes) -> u64x4 { match control { Lanes::D => blend(x.into_bits(), y.into_bits(), 0b11_00_00_00).into_bits(), + Lanes::C => blend(x.into_bits(), y.into_bits(), 0b00_11_00_00).into_bits(), Lanes::AB => blend(x.into_bits(), y.into_bits(), 0b00_00_11_11).into_bits(), Lanes::AC => blend(x.into_bits(), y.into_bits(), 0b00_11_00_11).into_bits(), + Lanes::AD => blend(x.into_bits(), y.into_bits(), 0b11_00_00_11).into_bits(), + Lanes::BCD => blend(x.into_bits(), y.into_bits(), 0b11_11_11_00).into_bits(), } } } impl F51x4Unreduced { + pub fn zero() -> F51x4Unreduced { + F51x4Unreduced([u64x4::splat(0); 5]) + } + pub fn new( x0: &FieldElement51, x1: &FieldElement51, From 6faaef21dff53e8ec050eb3079ba5d7406963010 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 23 Nov 2018 15:13:52 -0800 Subject: [PATCH 24/52] Add mul_by_pow_2 for IFMA --- src/backend/vector/ifma/edwards.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index 8e65f2f..9c2e7e8 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -93,6 +93,14 @@ impl ExtendedPoint { ExtendedPoint(&tmp2.shuffle(Shuffle::DBBD) * &tmp2.shuffle(Shuffle::CACA)) } + + pub fn mul_by_pow_2(&self, k: u32) -> ExtendedPoint { + let mut tmp: ExtendedPoint = *self; + for _ in 0..k { + tmp = tmp.double(); + } + tmp + } } impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { From 6cec313f16a03f8125148ebdd7873f14267f6cab Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 23 Nov 2018 18:35:08 -0800 Subject: [PATCH 25/52] Add impl of Sub and Neg for Edwards points. --- src/backend/vector/ifma/edwards.rs | 28 +++++++++++++++++++++++----- src/backend/vector/ifma/field.rs | 10 ++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index 9c2e7e8..1a16d42 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -9,7 +9,7 @@ use traits::Identity; -use std::ops::Add; +use std::ops::{Add, Neg, Sub}; use edwards; @@ -132,6 +132,24 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { } } +impl<'a> Neg for &'a CachedPoint { + type Output = CachedPoint; + + fn neg(self) -> CachedPoint { + let swapped = self.0.shuffle(Shuffle::BACD); + CachedPoint(swapped.blend(&(-self.0), Lanes::D)) + } +} + +impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint { + type Output = ExtendedPoint; + + /// Implement subtraction by negating the point and adding. + fn sub(self, other: &'b CachedPoint) -> ExtendedPoint { + self + &(-other) + } +} + #[cfg(test)] mod test { use super::*; @@ -143,7 +161,7 @@ mod test { // Test the vector implementation of the parallel readdition formulas let cached_Q = CachedPoint::from(ExtendedPoint::from(Q)); let R_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) + &cached_Q).into(); - //et S_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) - &cached_Q).into(); + let S_vector: edwards::EdwardsPoint = (&ExtendedPoint::from(P) - &cached_Q).into(); println!("Testing point addition:"); println!("P = {:?}", P); @@ -152,11 +170,11 @@ mod test { println!("R = P + Q = {:?}", &P + &Q); //println!("R_serial = {:?}", R_serial); println!("R_vector = {:?}", R_vector); - //println!("S = P - Q = {:?}", &P - &Q); - //println!("S_vector = {:?}", S_vector); + println!("S = P - Q = {:?}", &P - &Q); + println!("S_vector = {:?}", S_vector); //assert_eq!(R_serial.compress(), (&P + &Q).compress()); assert_eq!(R_vector.compress(), (&P + &Q).compress()); - //assert_eq!(S_vector.compress(), (&P - &Q).compress()); + assert_eq!(S_vector.compress(), (&P - &Q).compress()); println!("OK!\n"); } diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 40220c5..fdc9d61 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -33,6 +33,7 @@ pub enum Shuffle { AAAA, BBBB, BADC, + BACD, ADDA, CBCB, ABDC, @@ -50,6 +51,7 @@ fn shuffle_lanes(x: u64x4, control: Shuffle) -> u64x4 { Shuffle::AAAA => perm(x.into_bits(), 0b00_00_00_00).into_bits(), Shuffle::BBBB => perm(x.into_bits(), 0b01_01_01_01).into_bits(), Shuffle::BADC => perm(x.into_bits(), 0b10_11_00_01).into_bits(), + Shuffle::BACD => perm(x.into_bits(), 0b11_10_00_01).into_bits(), Shuffle::ADDA => perm(x.into_bits(), 0b00_11_11_00).into_bits(), Shuffle::CBCB => perm(x.into_bits(), 0b01_10_01_10).into_bits(), Shuffle::ABDC => perm(x.into_bits(), 0b10_11_01_00).into_bits(), @@ -186,6 +188,14 @@ impl F51x4Unreduced { } } +impl Neg for F51x4Reduced { + type Output = F51x4Reduced; + + fn neg(self) -> F51x4Reduced { + F51x4Unreduced::from(self).negate_lazy().into() + } +} + impl F51x4Reduced { #[inline] pub fn shuffle(&self, control: Shuffle) -> F51x4Reduced { From da6256935542e0a73d6382aa6c0aee5af5bb4b61 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 23 Nov 2018 18:35:32 -0800 Subject: [PATCH 26/52] Add lookup table support. --- src/backend/vector/ifma/edwards.rs | 51 ++++++++++++++++++++++++++++++ src/backend/vector/ifma/field.rs | 33 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index 1a16d42..9d218ca 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -11,7 +11,11 @@ use traits::Identity; use std::ops::{Add, Neg, Sub}; +use subtle::Choice; +use subtle::ConditionallySelectable; + use edwards; +use window::{LookupTable, NafLookupTable5, NafLookupTable8}; use super::field::{F51x4Reduced, F51x4Unreduced, Lanes, Shuffle}; @@ -132,6 +136,16 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { } } +impl ConditionallySelectable for CachedPoint { + fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self { + CachedPoint(F51x4Reduced::conditional_select(&a.0, &b.0, choice)) + } + + fn conditional_assign(&mut self, other: &Self, choice: Choice) { + self.0.conditional_assign(&other.0, choice); + } +} + impl<'a> Neg for &'a CachedPoint { type Output = CachedPoint; @@ -150,6 +164,43 @@ impl<'a, 'b> Sub<&'b CachedPoint> for &'a ExtendedPoint { } } +impl<'a> From<&'a edwards::EdwardsPoint> for LookupTable { + fn from(point: &'a edwards::EdwardsPoint) -> Self { + let P = ExtendedPoint::from(*point); + let mut points = [CachedPoint::from(P); 8]; + for i in 0..7 { + points[i + 1] = (&P + &points[i]).into(); + } + LookupTable(points) + } +} + +impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable5 { + fn from(point: &'a edwards::EdwardsPoint) -> Self { + let A = ExtendedPoint::from(*point); + let mut Ai = [CachedPoint::from(A); 8]; + let A2 = A.double(); + for i in 0..7 { + Ai[i + 1] = (&A2 + &Ai[i]).into(); + } + // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A] + NafLookupTable5(Ai) + } +} + +impl<'a> From<&'a edwards::EdwardsPoint> for NafLookupTable8 { + fn from(point: &'a edwards::EdwardsPoint) -> Self { + let A = ExtendedPoint::from(*point); + let mut Ai = [CachedPoint::from(A); 64]; + let A2 = A.double(); + for i in 0..63 { + Ai[i + 1] = (&A2 + &Ai[i]).into(); + } + // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A, ..., 127A] + NafLookupTable8(Ai) + } +} + #[cfg(test)] mod test { use super::*; diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index fdc9d61..16d3b74 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -196,6 +196,39 @@ impl Neg for F51x4Reduced { } } +use subtle::Choice; +use subtle::ConditionallySelectable; + +impl ConditionallySelectable for F51x4Reduced { + #[inline] + fn conditional_select( + a: &F51x4Reduced, + b: &F51x4Reduced, + choice: Choice, + ) -> F51x4Reduced { + let mask = (-(choice.unwrap_u8() as i64)) as u64; + let mask_vec = u64x4::splat(mask); + F51x4Reduced([ + a.0[0] ^ (mask_vec & (a.0[0] ^ b.0[0])), + a.0[1] ^ (mask_vec & (a.0[1] ^ b.0[1])), + a.0[2] ^ (mask_vec & (a.0[2] ^ b.0[2])), + a.0[3] ^ (mask_vec & (a.0[3] ^ b.0[3])), + a.0[4] ^ (mask_vec & (a.0[4] ^ b.0[4])), + ]) + } + + #[inline] + fn conditional_assign(&mut self, other: &F51x4Reduced, choice: Choice) { + let mask = (-(choice.unwrap_u8() as i64)) as u64; + let mask_vec = u64x4::splat(mask); + self.0[0] ^= mask_vec & (self.0[0] ^ other.0[0]); + self.0[1] ^= mask_vec & (self.0[1] ^ other.0[1]); + self.0[2] ^= mask_vec & (self.0[2] ^ other.0[2]); + self.0[3] ^= mask_vec & (self.0[3] ^ other.0[3]); + self.0[4] ^= mask_vec & (self.0[4] ^ other.0[4]); + } +} + impl F51x4Reduced { #[inline] pub fn shuffle(&self, control: Shuffle) -> F51x4Reduced { From f825426ff4f6e5abaf3e56f2610a141908b77dff Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 18 Jan 2019 01:47:23 -0800 Subject: [PATCH 27/52] Add impl Default, Identity for ExtendedPoint, CachedPoint. --- src/backend/vector/ifma/edwards.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index 9d218ca..6151dda 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -56,6 +56,18 @@ impl From for CachedPoint { } } +impl Default for ExtendedPoint { + fn default() -> ExtendedPoint { + ExtendedPoint::identity() + } +} + +impl Identity for ExtendedPoint { + fn identity() -> ExtendedPoint { + constants::EXTENDEDPOINT_IDENTITY + } +} + impl ExtendedPoint { pub fn double(&self) -> ExtendedPoint { // Set tmp0 = (X1 Y1 X1 Y1) @@ -136,6 +148,18 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { } } +impl Default for CachedPoint { + fn default() -> CachedPoint { + CachedPoint::identity() + } +} + +impl Identity for CachedPoint { + fn identity() -> CachedPoint { + constants::CACHEDPOINT_IDENTITY + } +} + impl ConditionallySelectable for CachedPoint { fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self { CachedPoint(F51x4Reduced::conditional_select(&a.0, &b.0, choice)) From 33aa544cf43ebd8eb81470136a09999e1a1f8268 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 18 Jan 2019 01:47:58 -0800 Subject: [PATCH 28/52] Add IFMA constants file. --- src/backend/vector/ifma/constants.rs | 2062 ++++++++++++++++++++++++++ src/backend/vector/ifma/edwards.rs | 1 + src/backend/vector/ifma/mod.rs | 1 + src/backend/vector/mod.rs | 4 + 4 files changed, 2068 insertions(+) create mode 100644 src/backend/vector/ifma/constants.rs diff --git a/src/backend/vector/ifma/constants.rs b/src/backend/vector/ifma/constants.rs new file mode 100644 index 0000000..257fb54 --- /dev/null +++ b/src/backend/vector/ifma/constants.rs @@ -0,0 +1,2062 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2018 Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +//! This module contains constants used by the IFMA backend. + +use packed_simd::u64x4; + +use window::NafLookupTable8; + +use super::edwards::{CachedPoint, ExtendedPoint}; +use super::field::{F51x4Reduced, F51x4Unreduced}; + +/// The identity element as an `ExtendedPoint`. +pub(crate) static EXTENDEDPOINT_IDENTITY: ExtendedPoint = ExtendedPoint(F51x4Unreduced([ + u64x4::new(0, 1, 1, 0), + u64x4::new(0, 0, 0, 0), + u64x4::new(0, 0, 0, 0), + u64x4::new(0, 0, 0, 0), + u64x4::new(0, 0, 0, 0), +])); + +/// The identity element as a `CachedPoint`. +pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(F51x4Reduced([ + u64x4::new(121647, 121666, 243332, 2251799813685229), + u64x4::new(2251799813685248, 0, 0, 2251799813685247), + u64x4::new(2251799813685247, 0, 0, 2251799813685247), + u64x4::new(2251799813685247, 0, 0, 2251799813685247), + u64x4::new(2251799813685247, 0, 0, 2251799813685247), +])); + +/// Odd multiples of the Ed25519 basepoint: +pub(crate) static BASEPOINT_ODD_LOOKUP_TABLE: NafLookupTable8 = NafLookupTable8([ + CachedPoint(F51x4Reduced([ + u64x4::new(1277522120965857, 73557767439946, 243332, 1943719795065404), + u64x4::new(108375142003455, 341984820733594, 0, 2097709862669256), + u64x4::new(150073485536043, 750646439938056, 0, 581130035634455), + u64x4::new(2149983732744869, 1903255931888577, 0, 646644904824193), + u64x4::new(291045673509296, 1060034214701851, 0, 325245010451737), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1970681836121889, + 1660307753655178, + 1077207637163462, + 1436413309977108, + ), + u64x4::new( + 158785710838757, + 919645875412951, + 174577133496574, + 2213787394009350, + ), + u64x4::new( + 1017606396438281, + 1240932851489554, + 918203302506967, + 1239827708070863, + ), + u64x4::new( + 1748989883612327, + 1745367742532782, + 1168385548387, + 1211387683826673, + ), + u64x4::new( + 799349980018733, + 1471088235739693, + 1505351346057417, + 2104975925096407, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 171437462972293, + 36016853025886, + 1184164975342640, + 1633525003912147, + ), + u64x4::new( + 2113383632509037, + 1946216474924125, + 1884174984466256, + 1373317790955847, + ), + u64x4::new( + 791293623466401, + 1796466048084189, + 444977763198796, + 629823271230872, + ), + u64x4::new( + 1093217720067380, + 2157024270666135, + 238122980108466, + 806820763806847, + ), + u64x4::new( + 793658959468458, + 368578641413741, + 11592529764159, + 2144017075993471, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1538027396670268, + 1588896993892061, + 675619548648376, + 788373514423313, + ), + u64x4::new( + 1987517656073805, + 1940987929951188, + 666993851697339, + 2040540928108427, + ), + u64x4::new( + 375514548584082, + 1726008037083790, + 1070069155000872, + 570111103756303, + ), + u64x4::new( + 772223645372213, + 2123395244967674, + 868238486911408, + 1846639042240362, + ), + u64x4::new( + 872865734460736, + 32277956842850, + 1701451131455402, + 773883376061880, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1845177363882902, + 275858237213625, + 1052127336883600, + 171072805852218, + ), + u64x4::new( + 139016783952609, + 462699304987089, + 430046471494974, + 410922720999257, + ), + u64x4::new( + 846403935976337, + 243817706931454, + 971825428236901, + 571800039596794, + ), + u64x4::new( + 807642685434918, + 1933536976438782, + 812324278898440, + 688391556487313, + ), + u64x4::new( + 76239450396192, + 629532732688863, + 1833302026979779, + 650067934544499, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1373931604989264, + 331159264656614, + 364391529321767, + 874765630865409, + ), + u64x4::new( + 2109908262150241, + 473400816504190, + 91544045127333, + 976307977609515, + ), + u64x4::new( + 330175435673491, + 2126511895885904, + 1022944071588421, + 2158480209801463, + ), + u64x4::new( + 1305666795527971, + 162063591028664, + 2193154870675382, + 1789166662611800, + ), + u64x4::new( + 817858592500508, + 1672743239440202, + 859976879916778, + 1167423340862516, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 274334925170164, + 565841102587251, + 603083835949120, + 607539210240861, + ), + u64x4::new( + 196754662972649, + 1339063476699167, + 1406077076979491, + 896902435668469, + ), + u64x4::new( + 397962210956733, + 174839587476217, + 1381082665748936, + 175195877334136, + ), + u64x4::new( + 717429432748391, + 1635309821746318, + 363374010274647, + 882908746261699, + ), + u64x4::new( + 600946602802781, + 1946596133370711, + 1532135183320341, + 690530671668253, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2074443704000945, + 2163534804938345, + 425423840926528, + 1100826171404853, + ), + u64x4::new( + 111700142796101, + 1456893872751964, + 1186145518682968, + 2192182627706116, + ), + u64x4::new( + 1848722121856066, + 2123239575044749, + 1323870754599272, + 883211262889775, + ), + u64x4::new( + 938263017712916, + 689670293631396, + 183944529557576, + 501908638166580, + ), + u64x4::new( + 2170571907220631, + 36636756989655, + 1875035480138608, + 803703278398018, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1053429956874064, + 1636640618139765, + 1556890827801070, + 2142720579528828, + ), + u64x4::new( + 1814240918422814, + 692326274601777, + 1054896561802157, + 2025454041705534, + ), + u64x4::new( + 2109495823888757, + 1287497869997176, + 194170063200096, + 621116840113213, + ), + u64x4::new( + 2156505873679998, + 2197064359737385, + 1312887672223536, + 369862818895912, + ), + u64x4::new( + 977381163563657, + 1878897311974033, + 2144566861359744, + 1832960882773351, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1266492498289486, + 1301524759372145, + 324789537938521, + 442710471023019, + ), + u64x4::new( + 1232722320001345, + 1191193089162455, + 176474006074813, + 2158950213252857, + ), + u64x4::new( + 1901782191467749, + 494791441598902, + 1820415815322129, + 854954583485223, + ), + u64x4::new( + 1511383667649702, + 792536415032464, + 2027741263854728, + 1727944381044738, + ), + u64x4::new( + 606355788891204, + 1670687521471220, + 582824350365415, + 1509135066079912, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1079942762813598, + 2015830004785901, + 479916361323351, + 1907956590950158, + ), + u64x4::new( + 2053400302939156, + 1319799126867070, + 19493088767391, + 1908755581402373, + ), + u64x4::new( + 2235858054780980, + 885832711204321, + 810332865560178, + 103174191215441, + ), + u64x4::new( + 1843466881032833, + 355511728384038, + 693846715794114, + 186545012724117, + ), + u64x4::new( + 1661758432892509, + 1491022339899281, + 698941123765263, + 174945407208560, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1075933251927831, + 400263885306647, + 1308157532880528, + 347933379126665, + ), + u64x4::new( + 673811632329433, + 1584860147186478, + 271778891257244, + 498194055154207, + ), + u64x4::new( + 703783427747558, + 1051624728592032, + 1371463103351544, + 230351033002960, + ), + u64x4::new( + 860729466483372, + 421647596766583, + 1520613871336707, + 635298775280054, + ), + u64x4::new( + 1168352891728845, + 1691216293752089, + 1799491997061519, + 399728882318504, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 420156727446514, + 1483649215777128, + 165508610199900, + 1918121104840431, + ), + u64x4::new( + 2129902293682427, + 730952770435213, + 2184527544565390, + 1939880362232986, + ), + u64x4::new( + 1771978364905086, + 510975579746524, + 927564335219142, + 177574146260558, + ), + u64x4::new( + 2164104536437514, + 1532598873799015, + 406875369182421, + 1367005937406517, + ), + u64x4::new( + 35073200082587, + 1981124717036219, + 1854087014063833, + 122419694385217, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1963785875777739, + 411497142699119, + 1974557512687408, + 1268304422747183, + ), + u64x4::new( + 762752575978150, + 1443822019541748, + 1331556159904338, + 377726798263780, + ), + u64x4::new( + 825953972847841, + 353487068141356, + 1955697322427207, + 2048226560172078, + ), + u64x4::new( + 1482378558684434, + 657691905625918, + 923870001994493, + 1694132799397736, + ), + u64x4::new( + 1643904759603122, + 170495566698285, + 1218312703413378, + 784318735038131, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 939230507241903, + 2238763473105245, + 1827325199528162, + 1153939339775538, + ), + u64x4::new( + 38544505283339, + 258889431497015, + 351721979677947, + 1357907379592829, + ), + u64x4::new( + 1393974676373341, + 1131355528938676, + 473104915298872, + 978783482501776, + ), + u64x4::new( + 2131516168980501, + 2113911780991092, + 1477027502354261, + 542884524860340, + ), + u64x4::new( + 1029606261349423, + 64226378557628, + 1669131167474348, + 2212808057234874, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1423176501543193, + 163313632579593, + 2220495688893001, + 2220041045291870, + ), + u64x4::new( + 1111834224023697, + 1026815658023689, + 1404605100939775, + 1412149108248227, + ), + u64x4::new( + 1542537854906076, + 1270288391129127, + 991419278941933, + 1824939809581980, + ), + u64x4::new( + 1142003215657891, + 525980550896367, + 1508270666157963, + 917719462309053, + ), + u64x4::new( + 400851268057105, + 1620818232405188, + 1251478578139510, + 2162841805361886, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2125383272208441, + 1368790097335984, + 11813369275978, + 639513785921674, + ), + u64x4::new( + 2200806265616284, + 1041996387620216, + 1275149397833084, + 1723371028064068, + ), + u64x4::new( + 603720163891275, + 2135593511176153, + 2049641644431548, + 1198460677818310, + ), + u64x4::new( + 1862491879401621, + 2008116580769441, + 626566325260235, + 1058308304975798, + ), + u64x4::new( + 628557314314858, + 1075323332046522, + 1631772244117095, + 1812174547405683, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1222773123817104, + 363276129291452, + 796237592807883, + 1914425291893078, + ), + u64x4::new( + 1721259057429088, + 734941709009373, + 1553365830564638, + 1492120931079419, + ), + u64x4::new( + 1009354843273686, + 293884504384873, + 1050281954944357, + 134132942667344, + ), + u64x4::new( + 23119363298711, + 1694754778833445, + 1725925193393496, + 1738396998222001, + ), + u64x4::new( + 1753692057254667, + 118428526447110, + 840961387840295, + 1227619055408558, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1004186117579547, + 508771992330056, + 1426571663072421, + 2238524171903259, + ), + u64x4::new( + 744764613007812, + 398885442368825, + 2047459490294949, + 2141797621077959, + ), + u64x4::new( + 4556204156489, + 1708213022802363, + 1071381560923933, + 393474529142567, + ), + u64x4::new( + 350116198213005, + 945907227204695, + 168267474358731, + 1801504420122711, + ), + u64x4::new( + 728788674520360, + 1262722049156121, + 455259596607008, + 1159442365834489, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2226818917892677, + 185673745808179, + 2240952219732549, + 324137961621908, + ), + u64x4::new( + 1659527641857410, + 973964060249383, + 1349692151487730, + 1172743533370593, + ), + u64x4::new( + 310591478467746, + 2123977244137170, + 774562885265820, + 430035546191685, + ), + u64x4::new( + 2150863173197992, + 2101978317708856, + 193592648406011, + 1375328504508580, + ), + u64x4::new( + 1946235834250479, + 121741431658675, + 1004342690620100, + 2063466488599450, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 463079632200153, + 40415275714025, + 545935352782679, + 1458043501600908, + ), + u64x4::new( + 783771976559993, + 880839641726471, + 1782028201271831, + 41664413404590, + ), + u64x4::new( + 985129151724159, + 187728621410000, + 16620051933318, + 378011085567733, + ), + u64x4::new( + 1820372198168638, + 905710046480679, + 1912961774249737, + 1868135861067161, + ), + u64x4::new( + 474460473983187, + 1455684425673661, + 652771171116843, + 733511920760779, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1088886980746809, + 1660218575261626, + 527921875040240, + 915086639857889, + ), + u64x4::new( + 1814735788528175, + 1586698876186367, + 2040856637532862, + 405684812785624, + ), + u64x4::new( + 658578559700999, + 1751442070931114, + 1293623371490094, + 715026719042518, + ), + u64x4::new( + 382156225644820, + 897982285504960, + 577673183555858, + 1158728558309719, + ), + u64x4::new( + 1865791902475663, + 124491617513788, + 758484125168765, + 734065580770143, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 330985690350617, + 2214424721795630, + 973374650780848, + 1507267060932964, + ), + u64x4::new( + 1733823971011290, + 1730742552292995, + 669018866977489, + 604527664126146, + ), + u64x4::new( + 1082092498645474, + 1029182053935309, + 756799947765834, + 1764720030308351, + ), + u64x4::new( + 969912105693756, + 38116887248276, + 2148030115687613, + 995140534653865, + ), + u64x4::new( + 2154373397460354, + 298128883464656, + 479587543632539, + 1061127201140779, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 843064865526549, + 2019481782959016, + 1873125524281672, + 2013330239022371, + ), + u64x4::new( + 1192932403815186, + 1818108671859220, + 1247005102016258, + 1210577394628058, + ), + u64x4::new( + 132359273326717, + 795492788299178, + 1235924489372816, + 891705064411550, + ), + u64x4::new( + 1425833709104858, + 152114045731085, + 991347902581315, + 1387773338707683, + ), + u64x4::new( + 48024203807922, + 157005564892977, + 1474053161953744, + 727448023498345, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1076621484026788, + 1309917234320927, + 1786998180233659, + 1595497085944737, + ), + u64x4::new( + 1737334672694726, + 2038133716999447, + 1929061192400917, + 620544235219084, + ), + u64x4::new( + 1550527313469747, + 329096759623509, + 1585214659209474, + 693419841748324, + ), + u64x4::new( + 1450010875912315, + 2085047082180569, + 757421110771886, + 389367139787400, + ), + u64x4::new( + 781339490566117, + 132941783448971, + 258650459725225, + 2042274962585613, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 859638991542650, + 2249840007426442, + 1138753070862357, + 793751342318913, + ), + u64x4::new( + 2133476133447306, + 1027010646129239, + 436851910892865, + 866949948830344, + ), + u64x4::new( + 1936003572431223, + 531513680252193, + 1929877059408416, + 830585477662503, + ), + u64x4::new( + 1460760405777960, + 686673748420916, + 275475330051554, + 1581792376993692, + ), + u64x4::new( + 894482039456784, + 1801274480988632, + 16407898635278, + 1668497039215206, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 258585746227669, + 936490904651492, + 1826793887434108, + 1201219990633823, + ), + u64x4::new( + 979462791643635, + 461762372210187, + 218708929991480, + 1378150755760178, + ), + u64x4::new( + 642542170229970, + 787135445552820, + 371168855880557, + 182642566486693, + ), + u64x4::new( + 1152277399721904, + 1726910452705576, + 1452393215705343, + 2117799581546845, + ), + u64x4::new( + 1211265143925330, + 14373046151823, + 1745528818271507, + 1842106288572078, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 635154614562157, + 1956763034454109, + 509123035953043, + 445727657534780, + ), + u64x4::new( + 2072765509783252, + 1282639891593570, + 1075086397362049, + 722996110178195, + ), + u64x4::new( + 1385572918825603, + 1190035835509576, + 218317841176013, + 1047865370756924, + ), + u64x4::new( + 473991569426488, + 1910588123704592, + 1338270051770806, + 401676861680875, + ), + u64x4::new( + 992455353618436, + 126422733426929, + 1955248037756399, + 119233843022643, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1555272991526078, + 2214378187116349, + 366893798097444, + 1401502118355702, + ), + u64x4::new( + 1157229521930713, + 2144787187506262, + 1681597469697840, + 847499096518697, + ), + u64x4::new( + 1872802655800758, + 1027119609820793, + 1137278714788290, + 1664750301179485, + ), + u64x4::new( + 1091289858897030, + 910126419483563, + 1101920147235731, + 597083075893952, + ), + u64x4::new( + 1711011533670315, + 185206680336278, + 1620960612579784, + 1968598849170880, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 73077300235958, + 257216723095630, + 466947267713785, + 847105214181598, + ), + u64x4::new( + 1322905631406309, + 407458059314731, + 230045063190376, + 923800751267786, + ), + u64x4::new( + 1146027205000415, + 1541328763727623, + 768510249199119, + 1630223587589059, + ), + u64x4::new( + 1930368769879433, + 1376145403022159, + 1898149855343131, + 1709421930518180, + ), + u64x4::new( + 633944191571764, + 58314960742839, + 2050971151574988, + 757799756090059, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 361576929158539, + 1035682890165818, + 160945739362874, + 266975208626222, + ), + u64x4::new( + 1635371797076046, + 2106722851965197, + 451585919077206, + 6692426667180, + ), + u64x4::new( + 175820543533852, + 2057511393764025, + 1531846543720469, + 1648320903946519, + ), + u64x4::new( + 947461770620940, + 1107335044817620, + 1725565474111216, + 2182263619949220, + ), + u64x4::new( + 726444888601221, + 1379664085279206, + 1517215633290417, + 1763968936542507, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 686545355846512, + 1712283265573167, + 1743509592736302, + 1653906616429153, + ), + u64x4::new( + 985108805667149, + 2244347650874753, + 1304749057936860, + 321846134330589, + ), + u64x4::new( + 296321076156886, + 1717929256240029, + 450933772486425, + 2015536856431605, + ), + u64x4::new( + 1690393512821866, + 646913049470189, + 2198650647576397, + 1230646705710442, + ), + u64x4::new( + 601961913448442, + 878806578800541, + 620497587492381, + 330716414244629, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 631510982676132, + 1755753187697174, + 1596201246674299, + 2197888384902121, + ), + u64x4::new( + 626957678275745, + 1447583371478595, + 1375375216702128, + 1443613232818823, + ), + u64x4::new( + 1962997804660501, + 1051744123184519, + 1002558639300437, + 1237313314603385, + ), + u64x4::new( + 2118828335274995, + 226398203764759, + 889099617161107, + 1620967117678504, + ), + u64x4::new( + 227261019362935, + 2046897556746842, + 591524060355369, + 2178552047369691, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1375403119051662, + 222313965014452, + 539873444241395, + 213198095917915, + ), + u64x4::new( + 1436952871599114, + 1229749762725246, + 1174441562267670, + 265367077740349, + ), + u64x4::new( + 11107426165917, + 985954476039181, + 1147329112365579, + 1133931640328107, + ), + u64x4::new( + 585235055006843, + 699515259687482, + 299559608721134, + 2134819767146767, + ), + u64x4::new( + 1376401105588528, + 391412107507860, + 302743651807545, + 1362834426455518, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1802940904616205, + 1615132760193234, + 869321663313735, + 666494072545310, + ), + u64x4::new( + 1452849320020701, + 1472716813676364, + 472862999490802, + 359937983286145, + ), + u64x4::new( + 1221198323133843, + 491718521756528, + 1387135774113906, + 793779904904008, + ), + u64x4::new( + 1032129287829151, + 30730741946697, + 217603185195068, + 2118169309744162, + ), + u64x4::new( + 225899335574721, + 1767553399797342, + 881082465669982, + 1435383196392870, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1127093564374276, + 2245188499702906, + 1250041622887441, + 2179324911668149, + ), + u64x4::new( + 908019210866875, + 1879900391060964, + 1355047706206597, + 647218945377302, + ), + u64x4::new( + 1616265604422592, + 2134336781521657, + 1157711219915601, + 1227494173135033, + ), + u64x4::new( + 136450294813355, + 1984543542455033, + 1199486053011083, + 33687889941331, + ), + u64x4::new( + 1053447012707371, + 68239344331930, + 537448158443925, + 1829189783369646, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 996806463322563, + 2043104667851348, + 1110361398300309, + 1218740346887957, + ), + u64x4::new( + 399141907016839, + 1307691109658227, + 532535384961264, + 896201194398872, + ), + u64x4::new( + 111705272106160, + 1790972382466021, + 1159338112559144, + 303544352897203, + ), + u64x4::new( + 1036600573322969, + 1457119922663674, + 334117653665514, + 460023361701263, + ), + u64x4::new( + 1363773215189933, + 1915594049343802, + 1661249423378694, + 1744945551969247, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 3093919631215, + 574886478077610, + 1704446919728971, + 250093147254210, + ), + u64x4::new( + 1387413348737796, + 360142717826981, + 2116185073015983, + 474541388374100, + ), + u64x4::new( + 1632539630892580, + 1332404016215719, + 2145297637794728, + 1289783723173504, + ), + u64x4::new( + 1030244179060173, + 579782698595797, + 1062365251139982, + 677149839815546, + ), + u64x4::new( + 6671539419876, + 1426937459653775, + 406942403696343, + 675479224223817, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 271984148441782, + 1708099625818957, + 1499011822959235, + 516808451044836, + ), + u64x4::new( + 1124847751346323, + 2038336022958449, + 1721698491022600, + 705944403212572, + ), + u64x4::new( + 85459783780275, + 1715213099986669, + 1728445509034791, + 730657630359717, + ), + u64x4::new( + 1185034652652387, + 755472578204310, + 476118360897817, + 1800434542785310, + ), + u64x4::new( + 1815589628676941, + 491778500674079, + 1547664984392513, + 279891608681267, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2036337168672113, + 1730787524684269, + 639134121311693, + 698060925015524, + ), + u64x4::new( + 315211075189491, + 1329055848835358, + 688621136402134, + 1271193060119448, + ), + u64x4::new( + 1697984374314012, + 459330773536457, + 305481314707918, + 61676911066002, + ), + u64x4::new( + 2166631826859191, + 2105217187401781, + 937587962768434, + 357397435365683, + ), + u64x4::new( + 1206757093145471, + 1287847622009294, + 1951336140421622, + 2233789834777410, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 82144190081093, + 1568417433687791, + 907555979158442, + 2037855062523867, + ), + u64x4::new( + 1225315484058853, + 315317868015613, + 1765025920288384, + 175223259828436, + ), + u64x4::new( + 1215010304871271, + 662713408454950, + 429517658575616, + 991062684008811, + ), + u64x4::new( + 993837615254894, + 1485561584889450, + 2001836754226476, + 1915943063896801, + ), + u64x4::new( + 818895101625673, + 1342479472068804, + 1380235330010671, + 23315169761453, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1500726307559118, + 956166860173424, + 512663951564436, + 1940180717699824, + ), + u64x4::new( + 1789521472720825, + 779456898652427, + 2035063615853504, + 863582140589407, + ), + u64x4::new( + 634508890793787, + 1748041666732214, + 259642099961634, + 1294936839797812, + ), + u64x4::new( + 2183334898697038, + 2197242820694806, + 2217225409073703, + 992633998226449, + ), + u64x4::new( + 2197077498155916, + 1562008797791883, + 1395088759904208, + 331715244679294, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 186854731652320, + 284389440026580, + 1252175415119400, + 1025377410100223, + ), + u64x4::new( + 1578732129417607, + 898645497852382, + 2237766074482974, + 1939197790303592, + ), + u64x4::new( + 1438830390640145, + 1682452015845597, + 1108441197232223, + 1984134492898664, + ), + u64x4::new( + 282668727301669, + 1609018289552856, + 390363439795705, + 1138459124667912, + ), + u64x4::new( + 18889015928490, + 532489638086725, + 324621535996080, + 2210046082697453, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2041327051605378, + 2244037852176483, + 2116336876147147, + 9616672544864, + ), + u64x4::new( + 969847387559191, + 1059119127679639, + 1764630094670633, + 364568045311834, + ), + u64x4::new( + 505938893153679, + 2075421412172902, + 326984153045666, + 1959549727324704, + ), + u64x4::new( + 1088715617911260, + 13917085151028, + 950568481355929, + 23687195265771, + ), + u64x4::new( + 1798284568673198, + 808382292203333, + 2214698741961545, + 610817203275867, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1731488929623777, + 1158815615106413, + 1491090861948525, + 1428384712900962, + ), + u64x4::new( + 722237139522457, + 1514290328911535, + 1366197913116230, + 1519472657321210, + ), + u64x4::new( + 246028966932273, + 1888239319448405, + 423720022211163, + 455243905681470, + ), + u64x4::new( + 738323403716001, + 1758018973481179, + 1180718299482318, + 1008495946606708, + ), + u64x4::new( + 334959381596119, + 1704599537529481, + 2172191232106896, + 13502508918495, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 273393076768079, + 427388720298603, + 1071733376018227, + 1715429388968611, + ), + u64x4::new( + 751776629892313, + 1965239102856011, + 541955408230119, + 831043488876080, + ), + u64x4::new( + 643718536393104, + 390543998404644, + 2176730661486279, + 499459234889079, + ), + u64x4::new( + 1482404333915009, + 865527293526285, + 507957951411713, + 216456252558825, + ), + u64x4::new( + 2210281256300231, + 1519357818277551, + 1257866936775246, + 1689605217672864, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2135395168187905, + 2214400157568614, + 2032983817870823, + 1124945109072647, + ), + u64x4::new( + 1602820011758145, + 906675633903289, + 782700735390986, + 2067218823525601, + ), + u64x4::new( + 786785748926382, + 1433583123655616, + 905839404290873, + 2249680349963778, + ), + u64x4::new( + 1940824582370584, + 1610961256326291, + 285307858781375, + 1755588655461194, + ), + u64x4::new( + 233682812055333, + 2146114223476434, + 41132209533476, + 535292431776371, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 600257696476418, + 18449221564824, + 1422209458591138, + 239571584769716, + ), + u64x4::new( + 2056372917056980, + 1155290566623531, + 1252473955568148, + 1276690716882081, + ), + u64x4::new( + 246974369025311, + 658117221519903, + 2000380937898441, + 1351183273924850, + ), + u64x4::new( + 1803747363753112, + 1736801515030186, + 2025633577199091, + 603378480769167, + ), + u64x4::new( + 57348749438551, + 1893551220299655, + 657926732731806, + 1522499384853705, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 591809128842736, + 284860517232591, + 27436696863545, + 886306697195798, + ), + u64x4::new( + 2113192175751749, + 1405882509906423, + 561316282804847, + 835573846576266, + ), + u64x4::new( + 94407289485409, + 1781534171669004, + 2098782516531528, + 598529921520053, + ), + u64x4::new( + 1860137004504786, + 2197323407480349, + 1516772733981532, + 961740253777086, + ), + u64x4::new( + 1484139612868217, + 1593557644636881, + 838834937143441, + 36382198263380, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1165898865828562, + 1153420815042389, + 1068625028915785, + 1945927229911090, + ), + u64x4::new( + 843454394017146, + 571029655293754, + 386282254545998, + 1804608237584150, + ), + u64x4::new( + 370552451091100, + 1279105656351124, + 1864742949668631, + 2093071521726981, + ), + u64x4::new( + 1872542389052198, + 1679083953574330, + 349872262454465, + 1470311090717925, + ), + u64x4::new( + 685345654160323, + 319718985807814, + 1359932285384164, + 1410900103316331, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 2083666668832889, + 314624387816655, + 1496694646480345, + 1946728950459189, + ), + u64x4::new( + 1579153761571203, + 508771185291380, + 1002249659402007, + 551517831173801, + ), + u64x4::new( + 2132371471626150, + 1988122278556533, + 1552195130653890, + 1327637750292755, + ), + u64x4::new( + 118937099181527, + 382610380973142, + 634951529106471, + 382740054041699, + ), + u64x4::new( + 801287519643470, + 87822941589258, + 1908825350108451, + 1404208826499115, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 330347226380261, + 672119116965146, + 1761510370768005, + 1959200302484704, + ), + u64x4::new( + 1631876583009250, + 1684917718484264, + 1027256947805920, + 2174612545251129, + ), + u64x4::new( + 636668855699872, + 625187713984839, + 265886954766790, + 167898557908504, + ), + u64x4::new( + 1210974548180860, + 2051308710365526, + 907620584086428, + 1081788677970850, + ), + u64x4::new( + 621792955460854, + 1450945504745382, + 1666728650687828, + 977937146451674, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 24725936182267, + 2226765032752574, + 2036560083102883, + 2002351185719584, + ), + u64x4::new( + 1620080779405308, + 1493220053370419, + 2245691691038916, + 1152182628629603, + ), + u64x4::new( + 317928527147500, + 1855194218440212, + 979380281964169, + 861442286685289, + ), + u64x4::new( + 393308472784625, + 486143087279967, + 1234071346236405, + 777748237119399, + ), + u64x4::new( + 43850412814718, + 1497656407486446, + 744128331046695, + 1618035787321792, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1670169946550211, + 1230951698726438, + 806586940221293, + 23159779184607, + ), + u64x4::new( + 634011340979302, + 764182085034744, + 731065727766955, + 1737985776442180, + ), + u64x4::new( + 240492712141842, + 73976435954441, + 162810587166835, + 697230894340912, + ), + u64x4::new( + 1299745598348388, + 1359436039694544, + 1856609816731554, + 25228008461513, + ), + u64x4::new( + 2180690501932381, + 2161211192848458, + 87069466793408, + 2003456332883860, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1106932458043379, + 1675181364231371, + 1681785724775243, + 131824742557210, + ), + u64x4::new( + 1671649414647169, + 1827849994880670, + 1097958057111899, + 701956891169434, + ), + u64x4::new( + 2095539283710881, + 591029812888096, + 1699571518315654, + 1297589045812566, + ), + u64x4::new( + 1345612272298537, + 2166754730876055, + 2047982622154948, + 1785222806258129, + ), + u64x4::new( + 2181915268829890, + 1895697064378670, + 1288412327355885, + 1561075738281368, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 741330264098392, + 357073519729966, + 1603572339180975, + 433572083688575, + ), + u64x4::new( + 699685108971208, + 1719650727634959, + 1941668009419214, + 870374958347891, + ), + u64x4::new( + 385971389331537, + 11655507719711, + 94814615497633, + 515572102810609, + ), + u64x4::new( + 1396688200590426, + 1518748475144123, + 162386454324368, + 2083303971579002, + ), + u64x4::new( + 1511688632419263, + 251584258592336, + 545345887993880, + 1229840230314160, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1298668855706029, + 2017860934939344, + 2224150456036391, + 1925926576297971, + ), + u64x4::new( + 259522963883544, + 1312469129541229, + 1647530465049600, + 1113737129047154, + ), + u64x4::new( + 733193298663145, + 2115712816303403, + 897628702762311, + 116440277571901, + ), + u64x4::new( + 1998719395229750, + 1662774553684237, + 194395608126452, + 98796702872301, + ), + u64x4::new( + 2226158244229144, + 91961728239158, + 526869903032152, + 849263805316773, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 472779569333556, + 854477760843410, + 2070906720349401, + 734613359834689, + ), + u64x4::new( + 1771897100487404, + 1604024196006064, + 319699348925383, + 437152129592623, + ), + u64x4::new( + 627618365135361, + 1768642666037955, + 588564169143939, + 35295037750744, + ), + u64x4::new( + 220241884231278, + 319104161410840, + 1048165719448798, + 1583931089774347, + ), + u64x4::new( + 166479451884333, + 1623611819962804, + 59990366193679, + 900727256046987, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 1944687327687331, + 1328410791053991, + 2083980670913902, + 609396833380574, + ), + u64x4::new( + 1907563845734496, + 1385619047697883, + 869817384774457, + 106642388505109, + ), + u64x4::new( + 1006516581737154, + 1561918369633937, + 1921172883211450, + 2216650451558824, + ), + u64x4::new( + 1780506017391778, + 233064930371847, + 1332962603425752, + 1380075261612354, + ), + u64x4::new( + 1907624789747741, + 1310065402098523, + 1838275780706825, + 884225500782782, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 198729830692545, + 100156148743413, + 2140568641558859, + 2220606475942394, + ), + u64x4::new( + 1108788217903741, + 1706330932366163, + 2050449866410661, + 684907598542847, + ), + u64x4::new( + 1101958322366646, + 659427843062405, + 253899933868173, + 896574852821269, + ), + u64x4::new( + 1157052140740658, + 440541103447032, + 2173354981480949, + 604768603561932, + ), + u64x4::new( + 961238337866054, + 830849154351308, + 1643852412409441, + 1436749321770368, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 784870637473285, + 1180234052037572, + 2086951602998715, + 419328169540373, + ), + u64x4::new( + 1966862397394559, + 788036164772123, + 2024355635709481, + 1471696676696146, + ), + u64x4::new( + 1468884300957205, + 1408016588131185, + 2229595828577885, + 240413942963547, + ), + u64x4::new( + 1481791691942441, + 970648959691160, + 1635500996148197, + 2236917233261585, + ), + u64x4::new( + 31660820731028, + 801794768903647, + 1069092619607344, + 282652554845923, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 911659428682786, + 762502588057038, + 1311399152500807, + 1966922911783311, + ), + u64x4::new( + 1229849228728540, + 258161307933217, + 2140796867375541, + 1569345075547911, + ), + u64x4::new( + 1487354676143742, + 1818317546165791, + 811033554173350, + 1768788663337616, + ), + u64x4::new( + 450017165913234, + 962535873747168, + 2099104262993585, + 503030952485785, + ), + u64x4::new( + 1259958681304518, + 479589250923541, + 1503904042161640, + 706283657294305, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 794562643024291, + 198670993088241, + 1678984629358943, + 273399517554618, + ), + u64x4::new( + 188458991574433, + 1389872130156447, + 1461868931574746, + 795140878721432, + ), + u64x4::new( + 624046647169653, + 630363741191019, + 911018499983500, + 1410140563046579, + ), + u64x4::new( + 1675056174405076, + 632544713589250, + 795454163559811, + 1535271563341780, + ), + u64x4::new( + 25504547444781, + 812510098987855, + 51290042016232, + 1992260991700127, + ), + ])), + CachedPoint(F51x4Reduced([ + u64x4::new( + 269968325452358, + 470932785179706, + 1684444304834150, + 1027482126748243, + ), + u64x4::new( + 457941065342419, + 2117377568137882, + 1209423706730905, + 2192403099717071, + ), + u64x4::new( + 1899046404863678, + 1359500336071762, + 1492389156724726, + 1455627081827750, + ), + u64x4::new( + 2016101061876546, + 1967000012916571, + 582539481696050, + 1197538178790094, + ), + u64x4::new( + 639684852217504, + 1799941252757449, + 1470016556327743, + 846111828965901, + ), + ])), +]); diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index 6151dda..429f3a7 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -17,6 +17,7 @@ use subtle::ConditionallySelectable; use edwards; use window::{LookupTable, NafLookupTable5, NafLookupTable8}; +use super::constants; use super::field::{F51x4Reduced, F51x4Unreduced, Lanes, Shuffle}; #[derive(Copy, Clone, Debug)] diff --git a/src/backend/vector/ifma/mod.rs b/src/backend/vector/ifma/mod.rs index 8a77ed3..106118a 100644 --- a/src/backend/vector/ifma/mod.rs +++ b/src/backend/vector/ifma/mod.rs @@ -11,3 +11,4 @@ pub mod field; pub mod edwards; +pub mod constants; diff --git a/src/backend/vector/mod.rs b/src/backend/vector/mod.rs index 5975301..2718ae9 100644 --- a/src/backend/vector/mod.rs +++ b/src/backend/vector/mod.rs @@ -33,5 +33,9 @@ pub(crate) use self::avx2::{ #[cfg(all(target_feature = "avx512ifma"))] pub mod ifma; +#[cfg(all(target_feature = "avx512ifma"))] +pub(crate) use self::ifma::{ + constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint, +}; pub mod scalar_mul; From 06766bfdead69149db1dda660b4ca8d0e0224270 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 28 Nov 2018 12:47:25 -0800 Subject: [PATCH 29/52] Delay shuffles that cross the "Great Wall of Intel". --- src/backend/vector/ifma/edwards.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index 429f3a7..beefc71 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -71,14 +71,14 @@ impl Identity for ExtendedPoint { impl ExtendedPoint { pub fn double(&self) -> ExtendedPoint { - // Set tmp0 = (X1 Y1 X1 Y1) - let mut tmp0 = self.0.shuffle(Shuffle::ABAB); + // (Y1 X1 T1 Z1) -- uses vpshufd (1c latency @ 1/c) + let mut tmp0 = self.0.shuffle(Shuffle::BADC); - // Set tmp1 = (Y1 X1 Y1 X1) - let mut tmp1 = tmp0.shuffle(Shuffle::BADC); + // (X1+Y1 X1+Y1 X1+Y1 X1+Y1) -- can use vpinserti128 + let mut tmp1 = (self.0 + tmp0).shuffle(Shuffle::ABAB); - // Set tmp0 = (X1 Y1 Z1 X1+Y1) - tmp0 = self.0.blend(&(tmp0 + tmp1), Lanes::D); + // (X1 Y1 Z1 X1+Y1) + tmp0 = self.0.blend(&tmp1, Lanes::D); tmp1 = F51x4Reduced::from(tmp0).square(); // Now tmp1 = (S1 S2 S3 S4) From a84f9eafe4e67224b6b022a8c966715911e9d0a3 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 28 Nov 2018 13:19:15 -0800 Subject: [PATCH 30/52] Allow non_snake_case in the IFMA field and edwards modules. --- src/backend/vector/ifma/edwards.rs | 2 ++ src/backend/vector/ifma/field.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/backend/vector/ifma/edwards.rs b/src/backend/vector/ifma/edwards.rs index beefc71..cc2b241 100644 --- a/src/backend/vector/ifma/edwards.rs +++ b/src/backend/vector/ifma/edwards.rs @@ -7,6 +7,8 @@ // Authors: // - Henry de Valence +#![allow(non_snake_case)] + use traits::Identity; use std::ops::{Add, Neg, Sub}; diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 16d3b74..9169f4a 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -7,6 +7,8 @@ // Authors: // - Henry de Valence +#![allow(non_snake_case)] + use core::ops::{Add, Mul, Neg}; use packed_simd::{u64x4, FromBits, IntoBits}; From fa427082577e686cbdaefd0aec0c2cce8a764987 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 29 Nov 2018 10:50:17 -0800 Subject: [PATCH 31/52] Change naming scheme for multiplication accumulators. --- src/backend/vector/ifma/field.rs | 248 ++++++++++++++++--------------- 1 file changed, 126 insertions(+), 122 deletions(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 9169f4a..5b38e3a 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -429,38 +429,38 @@ impl<'a> Mul<(u32, u32, u32, u32)> for &'a F51x4Reduced { ); let r19 = u64x4::splat(19); - let mut z0lo = u64x4::splat(0); - let mut z1lo = u64x4::splat(0); - let mut z2lo = u64x4::splat(0); - let mut z3lo = u64x4::splat(0); - let mut z4lo = u64x4::splat(0); - let mut z1hi = u64x4::splat(0); - let mut z2hi = u64x4::splat(0); - let mut z3hi = u64x4::splat(0); - let mut z4hi = u64x4::splat(0); - let mut z5hi = u64x4::splat(0); + let mut z0_1 = u64x4::splat(0); + let mut z1_1 = u64x4::splat(0); + let mut z2_1 = u64x4::splat(0); + let mut z3_1 = u64x4::splat(0); + let mut z4_1 = u64x4::splat(0); + let mut z1_2 = u64x4::splat(0); + let mut z2_2 = u64x4::splat(0); + let mut z3_2 = u64x4::splat(0); + let mut z4_2 = u64x4::splat(0); + let mut z5_2 = u64x4::splat(0); // Wave 0 - z4hi = madd52hi(z4hi, y, x[3]); - z5hi = madd52hi(z5hi, y, x[4]); - z4lo = madd52lo(z4lo, y, x[4]); - z0lo = madd52lo(z0lo, y, x[0]); - z3lo = madd52lo(z3lo, y, x[3]); - z2lo = madd52lo(z2lo, y, x[2]); - z1lo = madd52lo(z1lo, y, x[1]); - z3hi = madd52hi(z3hi, y, x[2]); + z4_2 = madd52hi(z4_2, y, x[3]); + z5_2 = madd52hi(z5_2, y, x[4]); + z4_1 = madd52lo(z4_1, y, x[4]); + z0_1 = madd52lo(z0_1, y, x[0]); + z3_1 = madd52lo(z3_1, y, x[3]); + z2_1 = madd52lo(z2_1, y, x[2]); + z1_1 = madd52lo(z1_1, y, x[1]); + z3_2 = madd52hi(z3_2, y, x[2]); // Wave 2 - z2hi = madd52hi(z2hi, y, x[1]); - z1hi = madd52hi(z1hi, y, x[0]); - z0lo = madd52lo(z0lo, z5hi + z5hi, r19); + z2_2 = madd52hi(z2_2, y, x[1]); + z1_2 = madd52hi(z1_2, y, x[0]); + z0_1 = madd52lo(z0_1, z5_2 + z5_2, r19); F51x4Unreduced([ - z0lo, - z1hi + z1hi + z1lo, - z2hi + z2hi + z2lo, - z3hi + z3hi + z3lo, - z4hi + z4hi + z4lo, + z0_1, + z1_1 + z1_2 + z1_2, + z2_1 + z2_2 + z2_2, + z3_1 + z3_2 + z3_2, + z4_1 + z4_2 + z4_2, ]) } } @@ -475,97 +475,101 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { let x = &self.0; let y = &rhs.0; - // Accumulators for lo-sourced terms - let mut z0lo = u64x4::splat(0); - let mut z1lo = u64x4::splat(0); - let mut z2lo = u64x4::splat(0); - let mut z3lo = u64x4::splat(0); - let mut z4lo = u64x4::splat(0); - let mut z5lo = u64x4::splat(0); - let mut z6lo = u64x4::splat(0); - let mut z7lo = u64x4::splat(0); - let mut z8lo = u64x4::splat(0); + // Accumulators for terms with coeff 1 + let mut z0_1 = u64x4::splat(0); + let mut z1_1 = u64x4::splat(0); + let mut z2_1 = u64x4::splat(0); + let mut z3_1 = u64x4::splat(0); + let mut z4_1 = u64x4::splat(0); + let mut z5_1 = u64x4::splat(0); + let mut z6_1 = u64x4::splat(0); + let mut z7_1 = u64x4::splat(0); + let mut z8_1 = u64x4::splat(0); - // Accumulators for hi-sourced terms - // Need to be doubled before adding - let mut z0hi = u64x4::splat(0); - let mut z1hi = u64x4::splat(0); - let mut z2hi = u64x4::splat(0); - let mut z3hi = u64x4::splat(0); - let mut z4hi = u64x4::splat(0); - let mut z5hi = u64x4::splat(0); - let mut z6hi = u64x4::splat(0); - let mut z7hi = u64x4::splat(0); - let mut z8hi = u64x4::splat(0); - let mut z9hi = u64x4::splat(0); + // Accumulators for terms with coeff 2 + let mut z0_2 = u64x4::splat(0); + let mut z1_2 = u64x4::splat(0); + let mut z2_2 = u64x4::splat(0); + let mut z3_2 = u64x4::splat(0); + let mut z4_2 = u64x4::splat(0); + let mut z5_2 = u64x4::splat(0); + let mut z6_2 = u64x4::splat(0); + let mut z7_2 = u64x4::splat(0); + let mut z8_2 = u64x4::splat(0); + let mut z9_2 = u64x4::splat(0); + + // LLVM doesn't seem to do much work reordering IFMA + // instructions, so try to organize them into "waves" of 8 + // independent operations (4c latency, 0.5 c throughput + // means 8 in flight) // Wave 0 - z4lo = madd52lo(z4lo, x[2], y[2]); - z5hi = madd52hi(z5hi, x[2], y[2]); - z5lo = madd52lo(z5lo, x[4], y[1]); - z6hi = madd52hi(z6hi, x[4], y[1]); - z6lo = madd52lo(z6lo, x[4], y[2]); - z7hi = madd52hi(z7hi, x[4], y[2]); - z7lo = madd52lo(z7lo, x[4], y[3]); - z8hi = madd52hi(z8hi, x[4], y[3]); + z4_1 = madd52lo(z4_1, x[2], y[2]); + z5_2 = madd52hi(z5_2, x[2], y[2]); + z5_1 = madd52lo(z5_1, x[4], y[1]); + z6_2 = madd52hi(z6_2, x[4], y[1]); + z6_1 = madd52lo(z6_1, x[4], y[2]); + z7_2 = madd52hi(z7_2, x[4], y[2]); + z7_1 = madd52lo(z7_1, x[4], y[3]); + z8_2 = madd52hi(z8_2, x[4], y[3]); // Wave 1 - z4lo = madd52lo(z4lo, x[3], y[1]); - z5hi = madd52hi(z5hi, x[3], y[1]); - z5lo = madd52lo(z5lo, x[3], y[2]); - z6hi = madd52hi(z6hi, x[3], y[2]); - z6lo = madd52lo(z6lo, x[3], y[3]); - z7hi = madd52hi(z7hi, x[3], y[3]); - z7lo = madd52lo(z7lo, x[3], y[4]); - z8hi = madd52hi(z8hi, x[3], y[4]); + z4_1 = madd52lo(z4_1, x[3], y[1]); + z5_2 = madd52hi(z5_2, x[3], y[1]); + z5_1 = madd52lo(z5_1, x[3], y[2]); + z6_2 = madd52hi(z6_2, x[3], y[2]); + z6_1 = madd52lo(z6_1, x[3], y[3]); + z7_2 = madd52hi(z7_2, x[3], y[3]); + z7_1 = madd52lo(z7_1, x[3], y[4]); + z8_2 = madd52hi(z8_2, x[3], y[4]); // Wave 2 - z8lo = madd52lo(z8lo, x[4], y[4]); - z9hi = madd52hi(z9hi, x[4], y[4]); - z4lo = madd52lo(z4lo, x[4], y[0]); - z5hi = madd52hi(z5hi, x[4], y[0]); - z5lo = madd52lo(z5lo, x[2], y[3]); - z6hi = madd52hi(z6hi, x[2], y[3]); - z6lo = madd52lo(z6lo, x[2], y[4]); - z7hi = madd52hi(z7hi, x[2], y[4]); + z8_1 = madd52lo(z8_1, x[4], y[4]); + z9_2 = madd52hi(z9_2, x[4], y[4]); + z4_1 = madd52lo(z4_1, x[4], y[0]); + z5_2 = madd52hi(z5_2, x[4], y[0]); + z5_1 = madd52lo(z5_1, x[2], y[3]); + z6_2 = madd52hi(z6_2, x[2], y[3]); + z6_1 = madd52lo(z6_1, x[2], y[4]); + z7_2 = madd52hi(z7_2, x[2], y[4]); - let z8 = z8lo + z8hi + z8hi; - let z9 = z9hi + z9hi; + let z8 = z8_1 + z8_2 + z8_2; + let z9 = z9_2 + z9_2; // Wave 3 - z3lo = madd52lo(z3lo, x[3], y[0]); - z4hi = madd52hi(z4hi, x[3], y[0]); - z4lo = madd52lo(z4lo, x[1], y[3]); - z5hi = madd52hi(z5hi, x[1], y[3]); - z5lo = madd52lo(z5lo, x[1], y[4]); - z6hi = madd52hi(z6hi, x[1], y[4]); - z2lo = madd52lo(z2lo, x[2], y[0]); - z3hi = madd52hi(z3hi, x[2], y[0]); + z3_1 = madd52lo(z3_1, x[3], y[0]); + z4_2 = madd52hi(z4_2, x[3], y[0]); + z4_1 = madd52lo(z4_1, x[1], y[3]); + z5_2 = madd52hi(z5_2, x[1], y[3]); + z5_1 = madd52lo(z5_1, x[1], y[4]); + z6_2 = madd52hi(z6_2, x[1], y[4]); + z2_1 = madd52lo(z2_1, x[2], y[0]); + z3_2 = madd52hi(z3_2, x[2], y[0]); - let z6 = z6lo + z6hi + z6hi; - let z7 = z7lo + z7hi + z7hi; + let z6 = z6_1 + z6_2 + z6_2; + let z7 = z7_1 + z7_2 + z7_2; // Wave 4 - z3lo = madd52lo(z3lo, x[2], y[1]); - z4hi = madd52hi(z4hi, x[2], y[1]); - z4lo = madd52lo(z4lo, x[0], y[4]); - z5hi = madd52hi(z5hi, x[0], y[4]); - z1lo = madd52lo(z1lo, x[1], y[0]); - z2hi = madd52hi(z2hi, x[1], y[0]); - z2lo = madd52lo(z2lo, x[1], y[1]); - z3hi = madd52hi(z3hi, x[1], y[1]); + z3_1 = madd52lo(z3_1, x[2], y[1]); + z4_2 = madd52hi(z4_2, x[2], y[1]); + z4_1 = madd52lo(z4_1, x[0], y[4]); + z5_2 = madd52hi(z5_2, x[0], y[4]); + z1_1 = madd52lo(z1_1, x[1], y[0]); + z2_2 = madd52hi(z2_2, x[1], y[0]); + z2_1 = madd52lo(z2_1, x[1], y[1]); + z3_2 = madd52hi(z3_2, x[1], y[1]); - let z5 = z5lo + z5hi + z5hi; + let z5 = z5_1 + z5_2 + z5_2; // Wave 5 - z3lo = madd52lo(z3lo, x[1], y[2]); - z4hi = madd52hi(z4hi, x[1], y[2]); - z0lo = madd52lo(z0lo, x[0], y[0]); - z1hi = madd52hi(z1hi, x[0], y[0]); - z1lo = madd52lo(z1lo, x[0], y[1]); - z2lo = madd52lo(z2lo, x[0], y[2]); - z2hi = madd52hi(z2hi, x[0], y[1]); - z3hi = madd52hi(z3hi, x[0], y[2]); + z3_1 = madd52lo(z3_1, x[1], y[2]); + z4_2 = madd52hi(z4_2, x[1], y[2]); + z0_1 = madd52lo(z0_1, x[0], y[0]); + z1_2 = madd52hi(z1_2, x[0], y[0]); + z1_1 = madd52lo(z1_1, x[0], y[1]); + z2_1 = madd52lo(z2_1, x[0], y[2]); + z2_2 = madd52hi(z2_2, x[0], y[1]); + z3_2 = madd52hi(z3_2, x[0], y[2]); let mut t0 = u64x4::splat(0); let mut t1 = u64x4::splat(0); @@ -574,33 +578,33 @@ impl<'a, 'b> Mul<&'b F51x4Reduced> for &'a F51x4Reduced { // Wave 6 t0 = madd52hi(t0, r19, z9); t1 = madd52lo(t1, r19, z9 >> 52); - z3lo = madd52lo(z3lo, x[0], y[3]); - z4hi = madd52hi(z4hi, x[0], y[3]); - z1hi = madd52lo(z1hi, r19, z5 >> 52); - z2hi = madd52lo(z2hi, r19, z6 >> 52); - z3hi = madd52lo(z3hi, r19, z7 >> 52); - z0lo = madd52lo(z0lo, r19, z5); + z3_1 = madd52lo(z3_1, x[0], y[3]); + z4_2 = madd52hi(z4_2, x[0], y[3]); + z1_2 = madd52lo(z1_2, r19, z5 >> 52); + z2_2 = madd52lo(z2_2, r19, z6 >> 52); + z3_2 = madd52lo(z3_2, r19, z7 >> 52); + z0_1 = madd52lo(z0_1, r19, z5); // Wave 7 - z4lo = madd52lo(z4lo, r19, z9); - z1lo = madd52lo(z1lo, r19, z6); - z0hi = madd52lo(z0hi, r19, t0 + t1); - z4hi = madd52hi(z4hi, r19, z8); - z2lo = madd52lo(z2lo, r19, z7); - z1hi = madd52hi(z1hi, r19, z5); - z2hi = madd52hi(z2hi, r19, z6); - z3hi = madd52hi(z3hi, r19, z7); + z4_1 = madd52lo(z4_1, r19, z9); + z1_1 = madd52lo(z1_1, r19, z6); + z0_2 = madd52lo(z0_2, r19, t0 + t1); + z4_2 = madd52hi(z4_2, r19, z8); + z2_1 = madd52lo(z2_1, r19, z7); + z1_2 = madd52hi(z1_2, r19, z5); + z2_2 = madd52hi(z2_2, r19, z6); + z3_2 = madd52hi(z3_2, r19, z7); // Wave 8 - z3lo = madd52lo(z3lo, r19, z8); - z4hi = madd52lo(z4hi, r19, z8 >> 52); + z3_1 = madd52lo(z3_1, r19, z8); + z4_2 = madd52lo(z4_2, r19, z8 >> 52); F51x4Unreduced([ - z0lo + z0hi + z0hi, - z1lo + z1hi + z1hi, - z2lo + z2hi + z2hi, - z3lo + z3hi + z3hi, - z4lo + z4hi + z4hi, + z0_1 + z0_2 + z0_2, + z1_1 + z1_2 + z1_2, + z2_1 + z2_2 + z2_2, + z3_1 + z3_2 + z3_2, + z4_1 + z4_2 + z4_2, ]) } } From 5700add507f5a20c86f0034fa592fc9da11da5b7 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 29 Nov 2018 12:08:06 -0800 Subject: [PATCH 32/52] Clean up warnings. --- src/backend/vector/ifma/field.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/vector/ifma/field.rs b/src/backend/vector/ifma/field.rs index 5b38e3a..d220a5a 100644 --- a/src/backend/vector/ifma/field.rs +++ b/src/backend/vector/ifma/field.rs @@ -10,7 +10,7 @@ #![allow(non_snake_case)] use core::ops::{Add, Mul, Neg}; -use packed_simd::{u64x4, FromBits, IntoBits}; +use packed_simd::{u64x4, IntoBits}; use backend::serial::u64::field::FieldElement51; @@ -706,7 +706,7 @@ mod test { let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); let b = FieldElement51([98098, 87987897, 0, 1, 0]).invert(); let mut c = &a * &b; - for i in 0..1024 { + for _i in 0..1024 { c = &a * &c; c = &b * &c; } @@ -714,7 +714,7 @@ mod test { let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into(); let bx4: F51x4Reduced = F51x4Unreduced::new(&b, &b, &b, &b).into(); let mut cx4 = &ax4 * &bx4; - for i in 0..1024 { + for _i in 0..1024 { cx4 = &ax4 * &F51x4Reduced::from(cx4); cx4 = &bx4 * &F51x4Reduced::from(cx4); } @@ -732,8 +732,8 @@ mod test { let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); let ax4: F51x4Reduced = F51x4Unreduced::new(&a, &a, &a, &a).into(); - let mut cx4 = &ax4 * &ax4; - let mut cx4_sq = ax4.square(); + let cx4 = &ax4 * &ax4; + let cx4_sq = ax4.square(); let splits = cx4.split(); let splits_sq = cx4_sq.split(); @@ -748,7 +748,7 @@ mod test { // Invert a small field element to get a big one let mut a = FieldElement51([2438, 24, 243, 0, 0]).invert(); let mut ax4 = F51x4Unreduced::new(&a, &a, &a, &a); - for j in 0..1024 { + for _j in 0..1024 { a = a.square(); ax4 = F51x4Reduced::from(ax4).square(); @@ -765,14 +765,14 @@ mod test { let a = FieldElement51([2438, 24, 243, 0, 0]).invert(); let b = FieldElement51([121665, 0, 0, 0, 0]); let mut c = &a * &b; - for i in 0..1024 { + for _i in 0..1024 { c = &b * &c; } let ax4 = F51x4Unreduced::new(&a, &a, &a, &a); let bx4 = (121665u32, 121665u32, 121665u32, 121665u32); let mut cx4 = &F51x4Reduced::from(ax4) * bx4; - for i in 0..1024 { + for _i in 0..1024 { cx4 = &F51x4Reduced::from(cx4) * bx4; } From dc8431429f679dd7ff5be6b5a3a4dab61aa0a033 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 30 Nov 2018 13:18:55 -0800 Subject: [PATCH 33/52] Refactor documentation to use doc_cfg. --- build.rs | 1 + docs/avx2-notes.md | 323 +------------------------------------ docs/parallel-formulas.md | 329 ++++++++++++++++++++++++++++++++++++++ src/backend/mod.rs | 46 ++++-- src/backend/serial/mod.rs | 22 +-- src/backend/vector/mod.rs | 33 ++-- src/lib.rs | 12 +- 7 files changed, 401 insertions(+), 365 deletions(-) create mode 100644 docs/parallel-formulas.md diff --git a/build.rs b/build.rs index 9d3bae3..631515c 100644 --- a/build.rs +++ b/build.rs @@ -7,6 +7,7 @@ feature(link_llvm_intrinsics) )] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] +#![cfg_attr(feature = "nightly", feature(doc_cfg))] #![allow(unused_variables)] #![allow(non_snake_case)] #![allow(dead_code)] diff --git a/docs/avx2-notes.md b/docs/avx2-notes.md index a382a34..87992b3 100644 --- a/docs/avx2-notes.md +++ b/docs/avx2-notes.md @@ -1,218 +1,4 @@ -A vectorized implementation of group operations on the twisted Edwards -form of Curve25519, using a modification of the 4-way parallel -formulas of Hisil, Wong, Carter, and Dawson. - -# Overview - -The 2008 paper [_Twisted Edwards Curves Revisited_][hwcd08] by Hisil, -Wong, Carter, and Dawson (HWCD) introduced the “extended coordinates” -and mixed-model representations which are used by most Edwards curve -implementations. - -However, they also describe 4-way parallel formulas for point addition -and doubling: a unified addition algorithm taking an effective -\\(2\mathbf M + 1\mathbf D\\), a doubling algorithm taking an -effective \\(1\mathbf M + 1\mathbf S\\), and a dedicated (i.e., for -distinct points) addition algorithm taking an effective \\(2 \mathbf M -\\). They compare these formulas with a 2-way parallel variant of the -Montgomery ladder. - -Unlike their serial formulas, which are used widely, their parallel -formulas do not seem to have been implemented in software before. The -2-way parallel Montgomery ladder was used in 2015 by Tung Chou's -`sandy2x` implementation. Curiously, however, although the [`sandy2x` -paper][sandy2x] also implements Edwards arithmetic, and cites HWCD08, -it doesn't mention their parallel Edwards formulas. -A 2015 paper by Hernández and López describes an AVX2 implementation -of X25519. Neither the paper nor the code are publicly available, but -it apparently gives only a [slight speedup][avx2trac], suggesting that -it uses a 4-way parallel Montgomery ladder rather than parallel -Edwards formulas. - -The reason may be that HWCD08 describe their formulas as operating on -four independent processors, which would make a software -implementation impractical: all of the operations are too low-latency -to effectively synchronize. But a closer inspection reveals that the -(more expensive) multiplication and squaring steps are uniform, while -the instruction divergence occurs in the (much cheaper) addition and -subtraction steps. This means that a SIMD implementation can perform -the expensive steps uniformly, and handle divergence in the -inexpensive steps using masking. - -These notes describe modifications to the original parallel formulas -to allow a SIMD implementation, and this module contains an -implementation of the modified formulas using 256-bit AVX2 vector -operations. - -# Parallel formulas in HWCD'08 - -The doubling formula is presented in the HWCD paper as follows: - -| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | -|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| | idle | idle | idle | \\( R\_1 \gets X\_1 + Y\_1 \\) | -| \\(1\mathbf S\\) | \\( R\_2 \gets X\_1\^2 \\) | \\( R\_3 \gets Y\_1\^2 \\) | \\( R\_4 \gets Z\_1\^2 \\) | \\( R\_5 \gets R\_1\^2 \\) | -| | \\( R\_6 \gets R\_2 + R\_3 \\) | \\( R\_7 \gets R\_2 - R\_3 \\) | \\( R\_4 \gets 2 R\_4 \\) | idle | -| | idle | \\( R\_1 \gets R\_4 + R\_7 \\) | idle | \\( R\_2 \gets R\_6 - R\_5 \\) | -| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_6 R\_7 \\) | \\( T\_3 \gets R\_2 R\_6 \\) | \\( Z\_3 \gets R\_1 R\_7 \\) | - -and the unified addition algorithm is presented as follows: - -| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | -|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| | \\( R\_1 \gets Y\_1 - X\_1 \\) | \\( R\_2 \gets Y\_2 - X\_2 \\) | \\( R\_3 \gets Y\_1 + X\_1 \\) | \\( R\_4 \gets Y\_2 + X\_2 \\) | -| \\(1\mathbf M\\) | \\( R\_5 \gets R\_1 R\_2 \\) | \\( R\_6 \gets R\_3 R\_4 \\) | \\( R\_7 \gets T\_1 T\_2 \\) | \\( R\_8 \gets Z\_1 Z\_2 \\) | -| \\(1\mathbf D\\) | idle | idle | \\( R\_7 \gets k R\_7 \\) | \\( R\_8 \gets 2 R\_8 \\) | -| | \\( R\_1 \gets R\_6 - R\_5 \\) | \\( R\_2 \gets R\_8 - R\_7 \\) | \\( R\_3 \gets R\_8 + R\_7 \\) | \\( R\_4 \gets R\_6 + R\_5 \\) | -| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_3 R\_4 \\) | \\( T\_3 \gets R\_1 R\_4 \\) | \\( Z\_3 \gets R\_2 R\_3 \\) | - -Here \\(\mathbf M\\) and \\(\mathbf S\\) represent the cost of -multiplication and squaring of generic field elements, \\(\mathbf D\\) -represents the cost of multiplication by a curve constant (in this -case \\( k = 2d \\)). - -Notice that the \\(1\mathbf M\\) and \\(1\mathbf S\\) steps are -uniform. The non-uniform steps are all inexpensive additions or -subtractions, with the exception of the multiplication by the curve -constant \\(k = 2d\\): -$$ -R\_7 \gets 2 d R\_7. -$$ - -HWCD suggest parallelising this step by breaking \\(k = 2d\\) into four -parts as \\(k = k_0 + 2\^n k_1 + 2\^{2n} k_2 + 2\^{3n} k_3 \\) and -computing \\(k_i R_7 \\) in parallel. This is quite awkward, but if -the curve constant is a ratio \\( d = d\_1/d\_2 \\), then projective -coordinates allow us to instead compute -$$ -(R\_5, R\_6, R\_7, R\_8) \gets (d\_2 R\_5, d\_2 R\_6, 2d\_1 R\_7, d\_2 R\_8). -$$ -This can be performed as a uniform multiplication by a vector of -constants, and if \\(d\_1, d\_2\\) are small, it is relatively -inexpensive. (This trick was suggested by Mike Hamburg). -In the Curve25519 case, we have -$$ -d = \frac{d\_1}{d\_2} = \frac{-121665}{121666}; -$$ -Since \\(2 \cdot 121666 < 2\^{18}\\), all the constants above fit (up -to sign) in 32 bits, so this can be done in parallel as four -multiplications by small constants \\( (121666, 121666, 2\cdot 121665, -2\cdot 121666) \\), followed by a negation to compute \\( - 2\cdot 121665\\). - -# Modified parallel formulas - -Using the modifications sketched above, we can write SIMD-friendly -versions of the parallel formulas as follows. To avoid confusion with -the original formulas, temporary variables are named \\(S\\) instead -of \\(R\\) and are in static single-assignment form. - -## Addition - -To add points -\\(P_1 = (X_1 : Y_1 : Z_1 : T_1) \\) -and -\\(P_2 = (X_2 : Y_2 : Z_2 : T_2 ) \\), -we compute -$$ -\begin{aligned} -(S\_0 &&,&& S\_1 &&,&& S\_2 &&,&& S\_3 ) -&\gets -(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Y\_2 - X\_2&&,&& Y\_2 + X\_2) -\\\\ -(S\_4 &&,&& S\_5 &&,&& S\_6 &&,&& S\_7 ) -&\gets -(S\_0 \cdot S\_2&&,&& S\_1 \cdot S\_3&&,&& Z\_1 \cdot Z\_2&&,&& T\_1 \cdot T\_2) -\\\\ -(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} ) -&\gets -(d\_2 \cdot S\_4 &&,&& d\_2 \cdot S\_5 &&,&& 2 d\_2 \cdot S\_6 &&,&& 2 d\_1 \cdot S\_7 ) -\\\\ -(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15}) -&\gets -(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11}) -\\\\ -(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3) -&\gets -(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13}) -\end{aligned} -$$ -to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). -This costs \\( 2\mathbf M + 1 \mathbf D\\). - -## Readdition - -If the point \\( P_2 = (X\_2 : Y\_2 : Z\_2 : T\_2) \\) is fixed, we -can cache the multiplication of the curve constants by computing -$$ -\begin{aligned} -(S\_2' &&,&& S\_3' &&,&& Z\_2' &&,&& T\_2' ) -&\gets -(d\_2 \cdot (Y\_2 - X\_2)&&,&& d\_2 \cdot (Y\_1 + X\_1)&&,&& 2d\_2 \cdot Z\_2 &&,&& 2d\_1 \cdot T\_2). -\end{aligned} -$$ -This costs \\( 1\mathbf D\\); with \\( (S\_2', S\_3', Z\_2', T\_2')\\) -in hand, the addition formulas above become -$$ -\begin{aligned} -(S\_0 &&,&& S\_1 &&,&& Z\_1 &&,&& T\_1 ) -&\gets -(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Z\_1 &&,&& T\_1) -\\\\ -(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} ) -&\gets -(S\_0 \cdot S\_2' &&,&& S\_1 \cdot S\_3'&&,&& Z\_1 \cdot Z\_2' &&,&& T\_1 \cdot T\_2') -\\\\ -(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15}) -&\gets -(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11}) -\\\\ -(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3) -&\gets -(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13}) -\end{aligned} -$$ -which costs only \\( 2\mathbf M \\). This precomputation is -essentially similar to the precomputation that HWCD suggest for their -serial formulas. Because the cost of precomputation and then -readdition is the same as addition, it's sufficient to only -implement caching and readdition. - -## Doubling - -The non-uniform portions of the (re)addition formulas have a fairly -regular structure. Unfortunately, this is not the case for the -doubling formulas, which are much less nice. - -To double a point \\( P = (X\_1 : Y\_1 : Z\_1 : T\_1) \\), we compute -$$ -\begin{aligned} -(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& S\_0) -&\gets -(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& X\_1 + Y\_1) -\\\\ -(S\_1 &&,&& S\_2 &&,&& S\_3 &&,&& S\_4 ) -&\gets -(X\_1\^2 &&,&& Y\_1\^2&&,&& Z\_1\^2 &&,&& S\_0\^2) -\\\\ -(S\_5 &&,&& S\_6 &&,&& S\_8 &&,&& S\_9 ) -&\gets -(S\_1 + S\_2 &&,&& S\_1 - S\_2 &&,&& S\_1 + 2S\_3 - S\_2 &&,&& S\_1 + S\_2 - S\_4) -\\\\ -(X\_3 &&,&& Y\_3 &&,&& Z\_3 &&,&& T\_3 ) -&\gets -(S\_8 \cdot S\_9 &&,&& S\_5 \cdot S\_6 &&,&& S\_8 \cdot S\_6 &&,&& S\_5 \cdot S\_9) -\end{aligned} -$$ -to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = [2]P\_1 \\). - -The intermediate step between the squaring and multiplication requires -a long chain of additions, but with some care and finesse, -described below, it is possible (in our case) to arrange this -computation without requiring an intermediate reduction. - -However, it does mean that the doubling formulas have proportionately -more vectorization overhead than the (re)addition formulas. The -effects of this are discussed in the comparison section below. +An AVX2 implementation of the vectorized point operation strategy. # Field element representation @@ -266,9 +52,7 @@ the form Since this breaks cleanly into two 128-bit lanes, it may be possible to adapt it to 128-bit vector instructions such as NEON without too -much difficulty. Going the other direction, to extend this to AVX512, -we could either run two point operations in parallel in lower and upper -halves of the registers, or use 2-way parallelism within a field operation. +much difficulty. # Avoiding Overflow in Doubling @@ -354,106 +138,3 @@ $$ whose right-hand sides are all bounded with \\( b < 1.75 \\) and whose left-hand sides are all bounded with \\( b < 2.5 \\), so that we can avoid any intermediate reductions. - -# Comparison to non-vectorized formulas - -In theory, the parallel Edwards formulas seem to allow a \\(4\\)-way -speedup from parallelism. However, an actual vectorized -implementation has several slowdowns that cut into this speedup. - -First, the parallel formulas can only use a \\( 32 \times 32 -\rightarrow 64 \\)-bit integer multiplier, so the speedup from -vectorization must overcome the disadvantage of losing the \\( 64 -\times 64 \rightarrow 128\\)-bit (serial) integer multiplier. The -effect of this slowdown is microarchitecture-dependent, since it -requires accounting for the total number of multiplications and -additions and their relative costs. In the future, it will probably -be possible to avoid this slowdown by using the `IFMA52` instructions, -whose parallelism is perfectly suited to these formulas. - -Second, the parallel doubling formulas incur both a theoretical and -practical slowdown. The parallel formulas described above work on the -\\( \mathbb P\^3 \\) “extended” coordinates. The \\( \mathbb P\^2 \\) -model introduced earlier by [Bernstein, Birkner, Joye, Lange, and -Peters][bbjlp08] allows slightly faster doublings, so HWCD suggest -mixing coordinate systems while performing scalar multiplication -(attributing the idea to [a 1998 paper][cmo98] by Cohen, Miyagi, and -Ono). The \\( T \\) coordinate is not required for doublings, so when -doublings are followed by doublings, its computation can be skipped. -More details on this approach and the different coordinate systems can -be found in the [`curve_models` module documentation][curve_models]. - -Unfortunately, this optimization is not compatible with the parallel -formulas, which cannot save time by skipping a single variable, so the -parallel doubling formulas do slightly more work when counting the -total number of field multiplications and squarings. - -In addition, the parallel doubling formulas have a less regular -pattern of additions and subtractions than the parallel addition -formulas, so the vectorization overhead is proportionately greater. -Both the parallel addition and parallel doubling formulas also require -some shuffling to rearrange data within the vectors, which places more -pressure on the shuffle unit than is desirable. - -This means that the speedup from using a vectorized implementation of -parallel Edwards formulas is likely to be greatest in applications -that do fewer doublings and more additions (like a large multiscalar -multiplication) rather than applications that do fewer additions and -more doublings (like a double-base scalar multiplication). - -Third, current Intel CPUs perform thermal throttling when using wide -vector instructions. A detailed description can be found in §15.26 of -[the Intel Optimization Manual][intel], but using wide vector -instructions prevents the core from operating at higher frequencies. -The core can return to the higher-frequency state after 2 -milliseconds, but this timer is reset every time high-power -instructions are used. - -Any speedup from vectorization therefore has to be weighed against a -slowdown for the next few million instructions. For a mixed workload, -where point operations are interspersed with other tasks, this can -reduce overall performance. This implementation is therefore probably -not suitable for basic applications, like signatures, but is -worthwhile for complex applications, like zero-knowledge proofs, which -do sustained work. - -For this reason, the AVX2 backend is not enabled by default, but can -be selected using the `avx2_backend` feature. - -# Future work - -There are several directions for future improvement: - -* Using the vectorized field arithmetic code to parallelize across - point operations rather than within a single point operation. This - is less flexible, but would give a speedup both from allowing use of - the faster mixed-model arithmetic and from reducing shuffle - pressure. One approach in this direction would be to implement - batched scalar-point operations using vectors of points (AoSoA - layout). This less generally useful but would give a speedup for - Bulletproofs. - -* Extending the implementation to use the full width of AVX512, either - handling the extra parallelism internally to a single point - operation (by using a 2-way parallel implementation of field - arithmetic instead of a wordsliced one), or externally, - parallelizing across point operations. Internal parallelism would - be preferable but might require too much shuffle pressure. - -* Generalizing the implementation to non-AVX2 instructions, - particularly NEON. The current point arithmetic code is written in - terms of field element vectors, which are in turn implemented using - platform SIMD vectors. It should be possible to write an alternate - implementation of the `FieldElement2625x4` using NEON without changing - the point arithmetic. NEON has 128-bit vectors rather than 256-bit - vectors, but this may still be worthwhile compared to a serial - implementation. - - -[sandy2x]: https://eprint.iacr.org/2015/943.pdf -[avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28 -[hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf -[curve_models]: https://doc-internal.dalek.rs/curve25519_dalek/curve_models/index.html -[bbjlp08]: https://eprint.iacr.org/2008/013 -[cmo98]: https://link.springer.com/content/pdf/10.1007%2F3-540-49649-1_6.pdf -[intel]: https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf \ No newline at end of file diff --git a/docs/parallel-formulas.md b/docs/parallel-formulas.md new file mode 100644 index 0000000..a356740 --- /dev/null +++ b/docs/parallel-formulas.md @@ -0,0 +1,329 @@ +Vectorized implementations of field and point operations, using a +modification of the 4-way parallel formulas of Hisil, Wong, Carter, +and Dawson. + +# Overview + +The 2008 paper [_Twisted Edwards Curves Revisited_][hwcd08] by Hisil, +Wong, Carter, and Dawson (HWCD) introduced the “extended coordinates” +and mixed-model representations which are used by most Edwards curve +implementations. + +However, they also describe 4-way parallel formulas for point addition +and doubling: a unified addition algorithm taking an effective +\\(2\mathbf M + 1\mathbf D\\), a doubling algorithm taking an +effective \\(1\mathbf M + 1\mathbf S\\), and a dedicated (i.e., for +distinct points) addition algorithm taking an effective \\(2 \mathbf M +\\). They compare these formulas with a 2-way parallel variant of the +Montgomery ladder. + +Unlike their serial formulas, which are used widely, their parallel +formulas do not seem to have been implemented in software before. The +2-way parallel Montgomery ladder was used in 2015 by Tung Chou's +`sandy2x` implementation. Curiously, however, although the [`sandy2x` +paper][sandy2x] also implements Edwards arithmetic, and cites HWCD08, +it doesn't mention their parallel Edwards formulas. +A 2015 paper by Hernández and López describes an AVX2 implementation +of X25519. Neither the paper nor the code are publicly available, but +it apparently gives only a [slight speedup][avx2trac], suggesting that +it uses a 4-way parallel Montgomery ladder rather than parallel +Edwards formulas. + +The reason may be that HWCD08 describe their formulas as operating on +four independent processors, which would make a software +implementation impractical: all of the operations are too low-latency +to effectively synchronize. But a closer inspection reveals that the +(more expensive) multiplication and squaring steps are uniform, while +the instruction divergence occurs in the (much cheaper) addition and +subtraction steps. This means that a SIMD implementation can perform +the expensive steps uniformly, and handle divergence in the +inexpensive steps using masking. + +These notes describe modifications to the original parallel formulas +to allow a SIMD implementation, and this module contains +implementations of the modified formulas targeting either AVX2 or +AVX512-IFMA. + +# Parallel formulas in HWCD'08 + +The doubling formula is presented in the HWCD paper as follows: + +| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | +|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| | idle | idle | idle | \\( R\_1 \gets X\_1 + Y\_1 \\) | +| \\(1\mathbf S\\) | \\( R\_2 \gets X\_1\^2 \\) | \\( R\_3 \gets Y\_1\^2 \\) | \\( R\_4 \gets Z\_1\^2 \\) | \\( R\_5 \gets R\_1\^2 \\) | +| | \\( R\_6 \gets R\_2 + R\_3 \\) | \\( R\_7 \gets R\_2 - R\_3 \\) | \\( R\_4 \gets 2 R\_4 \\) | idle | +| | idle | \\( R\_1 \gets R\_4 + R\_7 \\) | idle | \\( R\_2 \gets R\_6 - R\_5 \\) | +| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_6 R\_7 \\) | \\( T\_3 \gets R\_2 R\_6 \\) | \\( Z\_3 \gets R\_1 R\_7 \\) | + +and the unified addition algorithm is presented as follows: + +| Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | +|------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| | \\( R\_1 \gets Y\_1 - X\_1 \\) | \\( R\_2 \gets Y\_2 - X\_2 \\) | \\( R\_3 \gets Y\_1 + X\_1 \\) | \\( R\_4 \gets Y\_2 + X\_2 \\) | +| \\(1\mathbf M\\) | \\( R\_5 \gets R\_1 R\_2 \\) | \\( R\_6 \gets R\_3 R\_4 \\) | \\( R\_7 \gets T\_1 T\_2 \\) | \\( R\_8 \gets Z\_1 Z\_2 \\) | +| \\(1\mathbf D\\) | idle | idle | \\( R\_7 \gets k R\_7 \\) | \\( R\_8 \gets 2 R\_8 \\) | +| | \\( R\_1 \gets R\_6 - R\_5 \\) | \\( R\_2 \gets R\_8 - R\_7 \\) | \\( R\_3 \gets R\_8 + R\_7 \\) | \\( R\_4 \gets R\_6 + R\_5 \\) | +| \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_3 R\_4 \\) | \\( T\_3 \gets R\_1 R\_4 \\) | \\( Z\_3 \gets R\_2 R\_3 \\) | + +Here \\(\mathbf M\\) and \\(\mathbf S\\) represent the cost of +multiplication and squaring of generic field elements, \\(\mathbf D\\) +represents the cost of multiplication by a curve constant (in this +case \\( k = 2d \\)). + +Notice that the \\(1\mathbf M\\) and \\(1\mathbf S\\) steps are +uniform. The non-uniform steps are all inexpensive additions or +subtractions, with the exception of the multiplication by the curve +constant \\(k = 2d\\): +$$ +R\_7 \gets 2 d R\_7. +$$ + +HWCD suggest parallelising this step by breaking \\(k = 2d\\) into four +parts as \\(k = k_0 + 2\^n k_1 + 2\^{2n} k_2 + 2\^{3n} k_3 \\) and +computing \\(k_i R_7 \\) in parallel. This is quite awkward, but if +the curve constant is a ratio \\( d = d\_1/d\_2 \\), then projective +coordinates allow us to instead compute +$$ +(R\_5, R\_6, R\_7, R\_8) \gets (d\_2 R\_5, d\_2 R\_6, 2d\_1 R\_7, d\_2 R\_8). +$$ +This can be performed as a uniform multiplication by a vector of +constants, and if \\(d\_1, d\_2\\) are small, it is relatively +inexpensive. (This trick was suggested by Mike Hamburg). +In the Curve25519 case, we have +$$ +d = \frac{d\_1}{d\_2} = \frac{-121665}{121666}; +$$ +Since \\(2 \cdot 121666 < 2\^{18}\\), all the constants above fit (up +to sign) in 32 bits, so this can be done in parallel as four +multiplications by small constants \\( (121666, 121666, 2\cdot 121665, +2\cdot 121666) \\), followed by a negation to compute \\( - 2\cdot 121665\\). + +# Modified parallel formulas + +Using the modifications sketched above, we can write SIMD-friendly +versions of the parallel formulas as follows. To avoid confusion with +the original formulas, temporary variables are named \\(S\\) instead +of \\(R\\) and are in static single-assignment form. + +## Addition + +To add points +\\(P_1 = (X_1 : Y_1 : Z_1 : T_1) \\) +and +\\(P_2 = (X_2 : Y_2 : Z_2 : T_2 ) \\), +we compute +$$ +\begin{aligned} +(S\_0 &&,&& S\_1 &&,&& S\_2 &&,&& S\_3 ) +&\gets +(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Y\_2 - X\_2&&,&& Y\_2 + X\_2) +\\\\ +(S\_4 &&,&& S\_5 &&,&& S\_6 &&,&& S\_7 ) +&\gets +(S\_0 \cdot S\_2&&,&& S\_1 \cdot S\_3&&,&& Z\_1 \cdot Z\_2&&,&& T\_1 \cdot T\_2) +\\\\ +(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} ) +&\gets +(d\_2 \cdot S\_4 &&,&& d\_2 \cdot S\_5 &&,&& 2 d\_2 \cdot S\_6 &&,&& 2 d\_1 \cdot S\_7 ) +\\\\ +(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15}) +&\gets +(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11}) +\\\\ +(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3) +&\gets +(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13}) +\end{aligned} +$$ +to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). +This costs \\( 2\mathbf M + 1 \mathbf D\\). + +## Readdition + +If the point \\( P_2 = (X\_2 : Y\_2 : Z\_2 : T\_2) \\) is fixed, we +can cache the multiplication of the curve constants by computing +$$ +\begin{aligned} +(S\_2' &&,&& S\_3' &&,&& Z\_2' &&,&& T\_2' ) +&\gets +(d\_2 \cdot (Y\_2 - X\_2)&&,&& d\_2 \cdot (Y\_1 + X\_1)&&,&& 2d\_2 \cdot Z\_2 &&,&& 2d\_1 \cdot T\_2). +\end{aligned} +$$ +This costs \\( 1\mathbf D\\); with \\( (S\_2', S\_3', Z\_2', T\_2')\\) +in hand, the addition formulas above become +$$ +\begin{aligned} +(S\_0 &&,&& S\_1 &&,&& Z\_1 &&,&& T\_1 ) +&\gets +(Y\_1 - X\_1&&,&& Y\_1 + X\_1&&,&& Z\_1 &&,&& T\_1) +\\\\ +(S\_8 &&,&& S\_9 &&,&& S\_{10} &&,&& S\_{11} ) +&\gets +(S\_0 \cdot S\_2' &&,&& S\_1 \cdot S\_3'&&,&& Z\_1 \cdot Z\_2' &&,&& T\_1 \cdot T\_2') +\\\\ +(S\_{12} &&,&& S\_{13} &&,&& S\_{14} &&,&& S\_{15}) +&\gets +(S\_9 - S\_8&&,&& S\_9 + S\_8&&,&& S\_{10} - S\_{11}&&,&& S\_{10} + S\_{11}) +\\\\ +(X\_3&&,&& Y\_3&&,&& Z\_3&&,&& T\_3) +&\gets +(S\_{12} \cdot S\_{14}&&,&& S\_{15} \cdot S\_{13}&&,&& S\_{15} \cdot S\_{14}&&,&& S\_{12} \cdot S\_{13}) +\end{aligned} +$$ +which costs only \\( 2\mathbf M \\). This precomputation is +essentially similar to the precomputation that HWCD suggest for their +serial formulas. Because the cost of precomputation and then +readdition is the same as addition, it's sufficient to only +implement caching and readdition. + +## Doubling + +The non-uniform portions of the (re)addition formulas have a fairly +regular structure. Unfortunately, this is not the case for the +doubling formulas, which are much less nice. + +To double a point \\( P = (X\_1 : Y\_1 : Z\_1 : T\_1) \\), we compute +$$ +\begin{aligned} +(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& S\_0) +&\gets +(X\_1 &&,&& Y\_1 &&,&& Z\_1 &&,&& X\_1 + Y\_1) +\\\\ +(S\_1 &&,&& S\_2 &&,&& S\_3 &&,&& S\_4 ) +&\gets +(X\_1\^2 &&,&& Y\_1\^2&&,&& Z\_1\^2 &&,&& S\_0\^2) +\\\\ +(S\_5 &&,&& S\_6 &&,&& S\_8 &&,&& S\_9 ) +&\gets +(S\_1 + S\_2 &&,&& S\_1 - S\_2 &&,&& S\_1 + 2S\_3 - S\_2 &&,&& S\_1 + S\_2 - S\_4) +\\\\ +(X\_3 &&,&& Y\_3 &&,&& Z\_3 &&,&& T\_3 ) +&\gets +(S\_8 \cdot S\_9 &&,&& S\_5 \cdot S\_6 &&,&& S\_8 \cdot S\_6 &&,&& S\_5 \cdot S\_9) +\end{aligned} +$$ +to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = [2]P\_1 \\). + +The intermediate step between the squaring and multiplication requires +a long chain of additions. For the IFMA-based implementation, this is not a problem; for the AVX2-based implementation, it is, but with some care and finesse, it's possible to arrange the computation without requiring an intermediate reduction. + +# Implementation + +These formulas aren't specific to a particular representation of field +element vectors, whose optimum choice is determined by the details of +the instruction set. However, it's not possible to perfectly separate +the implementation of the field element vectors from the +implementation of the point operations. Instead, the [`avx2`] and +[`ifma`] backends provide `ExtendedPoint` and `CachedPoint` types, and +the [`scalar_mul`] code uses one of the backend types by a type alias. + +# Comparison to non-vectorized formulas + +In theory, the parallel Edwards formulas seem to allow a \\(4\\)-way +speedup from parallelism. However, an actual vectorized +implementation has several slowdowns that cut into this speedup. + +First, the parallel formulas can only use the available vector +multiplier. For AVX2, this is a \\( 32 \times 32 \rightarrow 64 +\\)-bit integer multiplier, so the speedup from vectorization must +overcome the disadvantage of losing the \\( 64 \times 64 \rightarrow +128\\)-bit (serial) integer multiplier. The effect of this slowdown +is microarchitecture-dependent, since it requires accounting for the +total number of multiplications and additions and their relative +costs. IFMA allows using a \\( 52 \times 52 \rightarrow 104 \\)-bit +multiplier, but the high and low halves need to be computed +separately, and the reduction requires extra work because it's not +possible to pre-multiply by \\(19\\). + +Second, the parallel doubling formulas incur both a theoretical and +practical slowdown. The parallel formulas described above work on the +\\( \mathbb P\^3 \\) “extended” coordinates. The \\( \mathbb P\^2 \\) +model introduced earlier by [Bernstein, Birkner, Joye, Lange, and +Peters][bbjlp08] allows slightly faster doublings, so HWCD suggest +mixing coordinate systems while performing scalar multiplication +(attributing the idea to [a 1998 paper][cmo98] by Cohen, Miyagi, and +Ono). The \\( T \\) coordinate is not required for doublings, so when +doublings are followed by doublings, its computation can be skipped. +More details on this approach and the different coordinate systems can +be found in the [`curve_models` module documentation][curve_models]. + +Unfortunately, this optimization is not compatible with the parallel +formulas, which cannot save time by skipping a single variable, so the +parallel doubling formulas do slightly more work when counting the +total number of field multiplications and squarings. + +In addition, the parallel doubling formulas have a less regular +pattern of additions and subtractions than the parallel addition +formulas, so the vectorization overhead is proportionately greater. +Both the parallel addition and parallel doubling formulas also require +some shuffling to rearrange data within the vectors, which places more +pressure on the shuffle unit than is desirable. + +This means that the speedup from using a vectorized implementation of +parallel Edwards formulas is likely to be greatest in applications +that do fewer doublings and more additions (like a large multiscalar +multiplication) rather than applications that do fewer additions and +more doublings (like a double-base scalar multiplication). + +Third, Amdahl's law says that the speedup is limited to the portion +which can be parallelized. Normally, the field multiplications +dominate the cost of point operations, but with the IFMA backend, the +multiplications are so fast that the non-parallel additions end up as +a significant portion of the total time. + +Fourth, current Intel CPUs perform thermal throttling when using wide +vector instructions. A detailed description can be found in §15.26 of +[the Intel Optimization Manual][intel], but using wide vector +instructions prevents the core from operating at higher frequencies. +The core can return to the higher-frequency state after 2 +milliseconds, but this timer is reset every time high-power +instructions are used. + +Any speedup from vectorization therefore has to be weighed against a +slowdown for the next few million instructions. For a mixed workload, +where point operations are interspersed with other tasks, this can +reduce overall performance. This implementation is therefore probably +not suitable for basic applications, like signatures, but is +worthwhile for complex applications, like zero-knowledge proofs, which +do sustained work. + +# Future work + +There are several directions for future improvement: + +* Using the vectorized field arithmetic code to parallelize across + point operations rather than within a single point operation. This + is less flexible, but would give a speedup both from allowing use of + the faster mixed-model arithmetic and from reducing shuffle + pressure. One approach in this direction would be to implement + batched scalar-point operations using vectors of points (AoSoA + layout). This less generally useful but would give a speedup for + Bulletproofs. + +* Extending the IFMA implementation to use the full width of AVX512, + either handling the extra parallelism internally to a single point + operation (by using a 2-way parallel implementation of field + arithmetic instead of a wordsliced one), or externally, + parallelizing across point operations. Internal parallelism would + be preferable but might require too much shuffle pressure. For now, + the only available CPU which runs IFMA operations executes them at + 256-bits wide anyways, so this isn't yet important. + +* Generalizing the implementation to non-AVX2 instructions, + particularly NEON. The current point arithmetic code is written in + terms of field element vectors, which are in turn implemented using + platform SIMD vectors. It should be possible to write an alternate + implementation of the `FieldElement2625x4` using NEON without changing + the point arithmetic. NEON has 128-bit vectors rather than 256-bit + vectors, but this may still be worthwhile compared to a serial + implementation. + + +[sandy2x]: https://eprint.iacr.org/2015/943.pdf +[avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28 +[hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf +[curve_models]: https://doc-internal.dalek.rs/curve25519_dalek/curve_models/index.html +[bbjlp08]: https://eprint.iacr.org/2008/013 +[cmo98]: https://link.springer.com/content/pdf/10.1007%2F3-540-49649-1_6.pdf +[intel]: https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf \ No newline at end of file diff --git a/src/backend/mod.rs b/src/backend/mod.rs index c088c23..aad3871 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -10,16 +10,28 @@ //! Pluggable implementations for different architectures. //! -//! The naming of the `u32` and `u64` modules is somewhat unfortunate, -//! since these are also the names of primitive types. Since types have -//! a different namespace than modules, this isn't a problem to the -//! compiler, but it could cause confusion. +//! The backend code is split into two parts: a serial backend, +//! and a vector backend. //! -//! However, it's unlikely that the names of those modules would be -//! brought into scope directly, instead of used as -//! `backend::u32::field` or similar. Unfortunately we can't use -//! `32bit` since identifiers can't start with letters, and the backends -//! do use `u32`/`u64`, so this seems like a least-bad option. +//! The [`serial`] backend contains 32- and 64-bit implementations of +//! field arithmetic and scalar arithmetic, as well as implementations +//! of point operations using the mixed-model strategy (passing +//! between different curve models depending on the operation). +//! +//! The [`vector`] backend contains implementations of vectorized +//! field arithmetic, used to implement point operations using a novel +//! implementation strategy derived from parallel formulas of Hisil, +//! Wong, Carter, and Dawson. +//! +//! Because the two strategies give rise to different curve models, +//! it's not possible to reuse exactly the same scalar multiplication +//! code (or to write it generically), so both serial and vector +//! backends contain matching implementations of scalar multiplication +//! algorithms. These are intended to be selected by a `#[cfg]`-based +//! type alias. +//! +//! The [`vector`] backend is selected by the `simd_backend` cargo +//! feature; it uses the [`serial`] backend for non-vectorized operations. #[cfg(not(any( feature = "u32_backend", @@ -33,8 +45,18 @@ compile_error!( pub mod serial; -#[cfg(all( - feature = "simd_backend", - any(target_feature = "avx2", target_feature = "avx512ifma") +#[cfg(any( + all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") + ), + rustdoc ))] +#[cfg_attr( + feature = "nightly", + doc(cfg(any(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") + )))) +)] pub mod vector; diff --git a/src/backend/serial/mod.rs b/src/backend/serial/mod.rs index b49f851..f7476e8 100644 --- a/src/backend/serial/mod.rs +++ b/src/backend/serial/mod.rs @@ -8,18 +8,19 @@ // - Isis Agora Lovecruft // - Henry de Valence -//! Pluggable implementations for different architectures. +//! Serial implementations of field, scalar, point arithmetic. //! -//! The naming of the `u32` and `u64` modules is somewhat unfortunate, -//! since these are also the names of primitive types. Since types have -//! a different namespace than modules, this isn't a problem to the -//! compiler, but it could cause confusion. +//! When the vector backend is disabled, the crate uses the +//! mixed-model strategy for implementing point operations and scalar +//! multiplication; see the [`curve_models`](self::curve_models) and +//! [`scalar_mul`](self::scalar_mul) documentation for more +//! information. //! -//! However, it's unlikely that the names of those modules would be -//! brought into scope directly, instead of used as -//! `backend::u32::field` or similar. Unfortunately we can't use -//! `32bit` since identifiers can't start with letters, and the backends -//! do use `u32`/`u64`, so this seems like a least-bad option. +//! When the vector backend is enabled, the field and scalar +//! implementations are still used for non-vectorized operations. +//! +//! Note: at this time the `u32` and `u64` backends cannot be built +//! together. #[cfg(not(any(feature = "u32_backend", feature = "u64_backend")))] compile_error!( @@ -40,4 +41,3 @@ pub mod curve_models; any(target_feature = "avx2", target_feature = "avx512ifma") )))] pub mod scalar_mul; - diff --git a/src/backend/vector/mod.rs b/src/backend/vector/mod.rs index 2718ae9..d034e6c 100644 --- a/src/backend/vector/mod.rs +++ b/src/backend/vector/mod.rs @@ -8,32 +8,35 @@ // - Isis Agora Lovecruft // - Henry de Valence -//! Pluggable implementations for different architectures. -//! -//! The naming of the `u32` and `u64` modules is somewhat unfortunate, -//! since these are also the names of primitive types. Since types have -//! a different namespace than modules, this isn't a problem to the -//! compiler, but it could cause confusion. -//! -//! However, it's unlikely that the names of those modules would be -//! brought into scope directly, instead of used as -//! `backend::u32::field` or similar. Unfortunately we can't use -//! `32bit` since identifiers can't start with letters, and the backends -//! do use `u32`/`u64`, so this seems like a least-bad option. +// Conditionally include the notes if: +// - we're on nightly (so we can include docs at all) +// - we're in stage 2 of the build. +// The latter point prevents a really silly and annoying problem, +// where the location of ".." is different depending on whether we're +// building the crate for real, or whether we're in build.rs +// generating the lookup tables (in which case we're relative to the +// location of build.rs, not lib.rs, so the markdown file appears +// missing). +#![cfg_attr( + all(feature = "nightly", feature = "stage2_build"), + doc(include = "../docs/parallel-formulas.md") +)] #[cfg(not(any(target_feature = "avx2", target_feature = "avx512ifma",)))] compile_error!("simd_backend selected without target_feature=+avx2 or +avx512ifma"); -#[cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma")))] +#[cfg(any(all(target_feature = "avx2", not(target_feature = "avx512ifma")), rustdoc))] +#[doc(cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma"))))] pub mod avx2; #[cfg(all(target_feature = "avx2", not(target_feature = "avx512ifma")))] pub(crate) use self::avx2::{ constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint, }; -#[cfg(all(target_feature = "avx512ifma"))] +#[cfg(any(target_feature = "avx512ifma", rustdoc))] +#[doc(cfg(target_feature = "avx512ifma"))] pub mod ifma; -#[cfg(all(target_feature = "avx512ifma"))] +#[cfg(target_feature = "avx512ifma")] pub(crate) use self::ifma::{ constants::BASEPOINT_ODD_LOOKUP_TABLE, edwards::CachedPoint, edwards::ExtendedPoint, }; diff --git a/src/lib.rs b/src/lib.rs index 9dd6608..85ad392 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,16 +10,16 @@ #![no_std] #![cfg_attr( - all(feature = "simd_backend", target_feature = "avx512ifma"), - feature(simd_ffi) -)] -#![cfg_attr( - all(feature = "simd_backend", target_feature = "avx512ifma"), - feature(link_llvm_intrinsics) + any( + all(feature = "simd_backend", target_feature = "avx512ifma"), + all(rustdoc, feature = "nightly") + ), + feature(simd_ffi, link_llvm_intrinsics) )] #![cfg_attr(feature = "nightly", feature(test))] #![cfg_attr(all(feature = "alloc", not(feature = "std")), feature(alloc))] #![cfg_attr(feature = "nightly", feature(external_doc))] +#![cfg_attr(feature = "nightly", feature(doc_cfg))] // Refuse to compile if documentation is missing, but only on nightly. // // This means that missing docs will still fail CI, but means we can use From 9ddbf1ba837ee346b50f507954338e9ef46183cf Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sat, 1 Dec 2018 11:07:08 -0800 Subject: [PATCH 34/52] Add notes on the IFMA implementation. --- docs/ifma-notes.md | 580 +++++++++++++++++++++++++++++++++ docs/parallel-formulas.md | 20 +- src/backend/vector/avx2/mod.rs | 4 +- src/backend/vector/ifma/mod.rs | 5 + 4 files changed, 599 insertions(+), 10 deletions(-) create mode 100644 docs/ifma-notes.md diff --git a/docs/ifma-notes.md b/docs/ifma-notes.md new file mode 100644 index 0000000..c6fd3b3 --- /dev/null +++ b/docs/ifma-notes.md @@ -0,0 +1,580 @@ +An AVX512-IFMA implementation of the vectorized point operation +strategy. + +# IFMA instructions + +AVX512-IFMA is an extension to AVX-512 consisting of two instructions: + +* `vpmadd52luq`: packed multiply of unsigned 52-bit integers and add + the low 52 product bits to 64-bit accumulators; +* `vpmadd52huq`: packed multiply of unsigned 52-bit integers and add + the high 52 product bits to 64-bit accumulators; + +These operate on 64-bit lanes of their source vectors, taking the low +52 bits of each lane of each source vector, computing the 104-bit +products of each pair, and then adding either the high or low 52 bits +of the 104-bit products to the 64-bit lanes of the destination vector. +The multiplication is performed internally by reusing circuitry for +floating-point arithmetic. Although these instructions are part of +AVX512, the AVX512VL (vector length) extension (present whenever IFMA +is) allows using them with 512, 256, or 128-bit operands. + +This provides a major advantage to vectorized integer operations: +previously, vector operations could only use a \\(32 \times 32 +\rightarrow 64\\)-bit multiplier, while serial code could use a +\\(64\times 64 \rightarrow 128\\)-bit multiplier. + +## IFMA for big-integer multiplications + +A detailed example of the intended use of the IFMA instructions can be +found in a 2016 paper by Gueron and Krasnov, [_Accelerating Big +Integer Arithmetic Using Intel IFMA Extensions_][2016_gueron_krasnov]. +The basic idea is that multiplication of large integers (such as 1024, +2048, or more bits) can be performed as follows. + +First, convert a “packed” 64-bit representation +\\[ +\begin{aligned} +x &= x'_0 + x'_1 2^{64} + x'_2 2^{128} + \cdots \\\\ +y &= y'_0 + y'_1 2^{64} + y'_2 2^{128} + \cdots +\end{aligned} +\\] +into a “redundant” 52-bit representation +\\[ +\begin{aligned} +x &= x_0 + x_1 2^{52} + x_2 2^{104} + \cdots \\\\ +y &= y_0 + y_1 2^{52} + y_2 2^{104} + \cdots +\end{aligned} +\\] +with each \\(x_i, y_j\\) in a 64-bit lane. + +Writing the product as \\(z = z_0 + z_1 2^{52} + z_2 2^{104} + \cdots\\), +the “schoolbook” multiplication strategy gives +\\[ +\begin{aligned} +&z_0 &&=& x_0 & y_0 & & & & & & & & \\\\ +&z_1 &&=& x_1 & y_0 &+ x_0 & y_1 & & & & & & \\\\ +&z_2 &&=& x_2 & y_0 &+ x_1 & y_1 &+ x_0 & y_2 & & & & \\\\ +&z_3 &&=& x_3 & y_0 &+ x_2 & y_1 &+ x_1 & y_2 &+ x_0 & y_3 & & \\\\ +&z_4 &&=& \vdots\\;&\\;\vdots &+ x_3 & y_1 &+ x_2 & y_2 &+ x_1 & y_3 &+ \cdots& \\\\ +&z_5 &&=& & & \vdots\\;&\\;\vdots &+ x_3 & y_2 &+ x_2 & y_3 &+ \cdots& \\\\ +&z_6 &&=& & & & & \vdots\\;&\\;\vdots &+ x_3 & y_3 &+ \cdots& \\\\ +&z_7 &&=& & & & & & & \vdots\\;&\\;\vdots &+ \cdots& \\\\ +&\vdots&&=& & & & & & & & & \ddots& \\\\ +\end{aligned} +\\] +Notice that the product coefficient \\(z_k\\), representing the value +\\(z_k 2^{52k}\\), is the sum of all product terms +\\( +(x_i 2^{52 i}) (y_j 2^{52 j}) +\\) +with \\(k = i + j\\). +Write the IFMA operators \\(\mathrm{lo}(a,b)\\), denoting the low +\\(52\\) bits of \\(ab\\), and +\\(\mathrm{hi}(a,b)\\), denoting the high \\(52\\) bits of +\\(ab\\). +Now we can rewrite the product terms as +\\[ +\begin{aligned} +(x_i 2^{52 i}) (y_j 2^{52 j}) +&= +2^{52 (i+j)}( +\mathrm{lo}(x_i, y_j) + +\mathrm{hi}(x_i, y_j) 2^{52} +) +\\\\ +&= +\mathrm{lo}(x_i, y_j) 2^{52 (i+j)} + +\mathrm{hi}(x_i, y_j) 2^{52 (i+j+1)}. +\end{aligned} +\\] +This means that the low half of \\(x_i y_j\\) can be accumulated onto +the product limb \\(z_{i+j}\\) and the high half can be directly +accumulated onto the next-higher product limb \\(z_{i+j+1}\\) with no +additional operations. This allows rewriting the schoolbook +multiplication into the form +\\[ +\begin{aligned} +&z_0 &&=& \mathrm{lo}(x_0,&y_0) & & & & & & & & & & \\\\ +&z_1 &&=& \mathrm{lo}(x_1,&y_0) &+\mathrm{hi}(x_0,&y_0) &+\mathrm{lo}(x_0,&y_1) & & & & & & \\\\ +&z_2 &&=& \mathrm{lo}(x_2,&y_0) &+\mathrm{hi}(x_1,&y_0) &+\mathrm{lo}(x_1,&y_1) &+\mathrm{hi}(x_0,&y_1) &+\mathrm{lo}(x_0,&y_2) & & \\\\ +&z_3 &&=& \mathrm{lo}(x_3,&y_0) &+\mathrm{hi}(x_2,&y_0) &+\mathrm{lo}(x_2,&y_1) &+\mathrm{hi}(x_1,&y_1) &+\mathrm{lo}(x_1,&y_2) &+ \cdots& \\\\ +&z_4 &&=& \vdots\\;&\\;\vdots &+\mathrm{hi}(x_3,&y_0) &+\mathrm{lo}(x_3,&y_1) &+\mathrm{hi}(x_2,&y_1) &+\mathrm{lo}(x_2,&y_2) &+ \cdots& \\\\ +&z_5 &&=& & & \vdots\\;&\\;\vdots & \vdots\\;&\\;\vdots &+\mathrm{hi}(x_3,&y_1) &+\mathrm{lo}(x_3,&y_2) &+ \cdots& \\\\ +&z_6 &&=& & & & & & & \vdots\\;&\\;\vdots & \vdots\\;&\\;\vdots &+ \cdots& \\\\ +&\vdots&&=& & & & & & & & & & & \ddots& \\\\ +\end{aligned} +\\] +Gueron and Krasnov implement multiplication by constructing vectors +out of the columns of this diagram, so that the source operands for +the IFMA instructions are of the form \\((x_0, x_1, x_2, \ldots)\\) +and \\((y_i, y_i, y_i, \ldots)\\). +After performing the multiplication, +the product terms \\(z_i\\) are then repacked into a 64-bit representation. + +## An alternative strategy + +The strategy described above is aimed at big-integer multiplications, +such as 1024, 2048, or 4096 bits, which would be used for applications +like RSA. However, elliptic curve cryptography uses much smaller field +sizes, such as 256 or 384 bits, so a different strategy is needed. + +The parallel Edwards formulas provide parallelism at the level of the +formulas for curve operations. This means that instead of scanning +through the terms of the source operands and parallelizing *within* a +field element (as described above), we can arrange the computation in +product-scanning form and parallelize *across* field elements (as +described below). + +The parallel Edwards +formulas provide 4-way parallelism, so they can be implemented using +256-bit vectors using a single 64-bit lane for each element, or using +512-bit vectors using two 64-bit lanes. +The only available CPU supporting IFMA (the +i3-8121U) executes 512-bit IFMA instructions at half rate compared to +256-bit instructions, so for now there's no throughput advantage to +using 512-bit IFMA instructions, and this implementation uses 256-bit +vectors. + +To extend this to 512-bit vectors, it's only only necessary to achieve +2-way parallelism, and it's possible (with a small amount of overhead) +to create a hybrid strategy that operates entirely within 128-bit +lanes. This means that cross-lane operations can use the faster +`vpshufd` (1c latency) instead of a general shuffle instruction (3c +latency). + +# Choice of radix + +The inputs to IFMA instructions are 52 bits wide, so the radix \\(r\\) +used to represent a multiprecision integer must be \\( r \leq 52 \\). +The obvious choice is the "native" radix \\(r = 52\\). + +As described above, this choice +has the advantage that for \\(x_i, y_j \in [0,2^{52})\\), the product term +\\[ +\begin{aligned} +(x_i 2^{52 i}) (y_j 2^{52 j}) +&= +2^{52 (i+j)}( +\mathrm{lo}(x_i, y_j) + +\mathrm{hi}(x_i, y_j) 2^{52} +) +\\\\ +&= +\mathrm{lo}(x_i, y_j) 2^{52 (i+j)} + +\mathrm{hi}(x_i, y_j) 2^{52 (i+j+1)}, +\end{aligned} +\\] +so that the low and high halves of the product can be directly accumulated +onto the product limbs. +In contrast, when using a smaller radix \\(r = 52 - k\\), +the product term has the form +\\[ +\begin{aligned} +(x_i 2^{r i}) (y_j 2^{r j}) +&= +2^{r (i+j)}( +\mathrm{lo}(x_i, y_j) + +\mathrm{hi}(x_i, y_j) 2^{52} +) +\\\\ +&= +\mathrm{lo}(x_i, y_j) 2^{r (i+j)} + +( +\mathrm{hi}(x_i, y_j) 2^k +) +2^{r (i+j+1)}. +\end{aligned} +\\] +What's happening is that the product \\(x_i y_j\\) of size \\(2r\\) +bits is split not at \\(r\\) but at \\(52\\), so \\(k\\) product bits +are placed into the low half instead of the high half. This means +that the high half of the product cannot be directly accumulated onto +\\(z_{i+j+1}\\), but must first be multiplied by \\(2^k\\) (i.e., left +shifted by \\(k\\)). In addition, the low half of the product is +\\(52\\) bits large instead of \\(r\\) bits. + +## Handling offset product terms + +[Drucker and Gueron][2018_drucker_gueron] analyze the choice of radix +in the context of big-integer squaring, outlining three ways to handle +the offset product terms, before concluding that all of them are +suboptimal: + +1. Shift the results after accumulation; +2. Shift the input operands before multiplication; +3. Split the MAC operation, accumulating into a zeroed register, + shifting the result, and then adding. + +The first option is rejected because it could double-shift some +previously accumulated terms, the second doesn't work because the +inputs could become larger than \\(52\\) bits, and the third requires +additional instructions to handle the shifting and adding. + +Based on an analysis of total number of instructions, they suggest an +addition to the instruction set, which they call `FMSA` (fused +multiply-shift-add). This would shift the result according to an 8-bit +immediate value before accumulating it into the destination register. + +However, this change to the instruction set doesn't seem to be +necessary. Instead, the product terms can be grouped according to +their coefficients, accumulated together, then shifted once before +adding them to the final sum. This uses an extra register, shift, and +add, but only once per product term (accumulation target), not once +per source term (as in the Drucker-Gueron paper). + +Moreover, because IFMA instructions execute only on two ports +(presumably 0 and 1), while adds and shifts can execute on three ports +(0, 1, and 5), the adds and shifts can execute independently of the +IFMA operations, as long as there is not too much pressure on port 5. +This means that, although the total number of instructions increases, +the shifts and adds do not necessarily increase the execution time, as +long as throughput is limited by IFMA operations. + +Finally, because IFMA instructions have 4 cycle latency and 0.5/1 +cycle throughput (for 256/512 bit vectors), maximizing IFMA throughput +requires either 8 (for 256) or 4 (for 512) independent operations. So +accumulating groups of terms independently before adding them at the +end may be necessary anyways, in order to prevent long chains of +dependent instructions. + +## Advantages of a smaller radix + +Using a smaller radix has other advantages. Although radix \\(52\\) +is an unsaturated representation from the point of view of the +\\(64\\)-bit accumulators (because up to 4096 product terms can be +accumulated without carries), it's a saturated representation from the +point of view of the multiplier (since \\(52\\)-bit values are the +maximum input size). + +Because the inputs to a multiplication must have all of their limbs +bounded by \\(2^{52}\\), limbs in excess of \\(2^{52}\\) must be +reduced before they can be used as an input. The +[Gueron-Krasnov][2016_gueron_krasnov] paper suggests normalizing +values using a standard, sequential carry chain: for each limb, add +the carryin from reducing the previous limb, compute the carryout and +reduce the current limb, then move to the next limb. + +However, when using a smaller radix, such as \\(51\\), each limb can +store a carry bit and still be used as the input to a multiplication. +This means that the inputs do not need to be normalized, and instead +of using a sequential carry chain, we can compute all carryouts in +parallel, reduce all limbs in parallel, and then add the carryins in +parallel (possibly growing the limb values by one bit). + +Because the output of this partial reduction is an acceptable +multiplication input, we can "close the loop" using partial reductions +and never have to normalize to a canonical representation through the +entire computation, in contrast to the Gueron-Krasnov approach, which +converts back to a packed representation after every operation. (This +idea seems to trace back to at least as early as [this 1999 +paper][1999_walter]). + +Using \\(r = 51\\) is enough to keep a carry bit in each limb and +avoid normalizations. What about an even smaller radix? One reason +to choose a smaller radix would be to align the limb boundaries with +an inline reduction (for instance, choosing \\(r = 43\\) for the +Mersenne field \\(p = 2^{127} - 1\\)), but for \\(p = 2^{255 - 19}\\), +\\(r = 51 = 255/5\\) is the natural choice. + +# Multiplication + +The inputs to a multiplication are two field elements +\\[ +\begin{aligned} +x &= x_0 + x_1 2^{51} + x_2 2^{102} + x_3 2^{153} + x_4 2^{204} \\\\ +y &= y_0 + y_1 2^{51} + y_2 2^{102} + y_3 2^{153} + y_4 2^{204}, +\end{aligned} +\\] +with limbs in range \\([0,2^{52})\\). + +Writing the product terms as +\\[ +\begin{aligned} +z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\ + &+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459}, +\end{aligned} +\\] +a schoolbook multiplication in product scanning form takes the form +\\[ +\begin{aligned} +z_0 &= x_0 y_0 \\\\ +z_1 &= x_1 y_0 + x_0 y_1 \\\\ +z_2 &= x_2 y_0 + x_1 y_1 + x_0 y_2 \\\\ +z_3 &= x_3 y_0 + x_2 y_1 + x_1 y_2 + x_0 y_3 \\\\ +z_4 &= x_4 y_0 + x_3 y_1 + x_2 y_2 + x_1 y_3 + x_0 y_4 \\\\ +z_5 &= x_4 y_1 + x_3 y_2 + x_2 y_3 + x_1 y_4 \\\\ +z_6 &= x_4 y_2 + x_3 y_3 + x_2 y_4 \\\\ +z_7 &= x_4 y_3 + x_3 y_4 \\\\ +z_8 &= x_4 y_4 \\\\ +z_9 &= 0 \\\\ +\end{aligned} +\\] +Each term \\(x_i y_j\\) can be written in terms of IFMA operations as +\\[ +x_i y_j = \mathrm{lo}(x_i,y_j) + 2\mathrm{hi}(x_i,y_j)2^{51}. +\\] +Substituting this equation into the schoolbook multiplication, then +moving terms to eliminate the \\(2^{51}\\) factors gives +\\[ +\begin{aligned} +z_0 &= \mathrm{lo}(x_0, y_0) \\\\ + &+ \qquad 0 \\\\ +z_1 &= \mathrm{lo}(x_1, y_0) + \mathrm{lo}(x_0, y_1) \\\\ + &+ \qquad 2( \mathrm{hi}(x_0, y_0) )\\\\ +z_2 &= \mathrm{lo}(x_2, y_0) + \mathrm{lo}(x_1, y_1) + \mathrm{lo}(x_0, y_2) \\\\ + &+ \qquad 2( \mathrm{hi}(x_1, y_0) + \mathrm{hi}(x_0, y_1) )\\\\ +z_3 &= \mathrm{lo}(x_3, y_0) + \mathrm{lo}(x_2, y_1) + \mathrm{lo}(x_1, y_2) + \mathrm{lo}(x_0, y_3) \\\\ + &+ \qquad 2( \mathrm{hi}(x_2, y_0) + \mathrm{hi}(x_1, y_1) + \mathrm{hi}(x_0, y_2) )\\\\ +z_4 &= \mathrm{lo}(x_4, y_0) + \mathrm{lo}(x_3, y_1) + \mathrm{lo}(x_2, y_2) + \mathrm{lo}(x_1, y_3) + \mathrm{lo}(x_0, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_3, y_0) + \mathrm{hi}(x_2, y_1) + \mathrm{hi}(x_1, y_2) + \mathrm{hi}(x_0, y_3) )\\\\ +z_5 &= \mathrm{lo}(x_4, y_1) + \mathrm{lo}(x_3, y_2) + \mathrm{lo}(x_2, y_3) + \mathrm{lo}(x_1, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_0) + \mathrm{hi}(x_3, y_1) + \mathrm{hi}(x_2, y_2) + \mathrm{hi}(x_1, y_3) + \mathrm{hi}(x_0, y_4) )\\\\ +z_6 &= \mathrm{lo}(x_4, y_2) + \mathrm{lo}(x_3, y_3) + \mathrm{lo}(x_2, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_1) + \mathrm{hi}(x_3, y_2) + \mathrm{hi}(x_2, y_3) + \mathrm{hi}(x_1, y_4) )\\\\ +z_7 &= \mathrm{lo}(x_4, y_3) + \mathrm{lo}(x_3, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_2) + \mathrm{hi}(x_3, y_3) + \mathrm{hi}(x_2, y_4) )\\\\ +z_8 &= \mathrm{lo}(x_4, y_4) \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_3) + \mathrm{hi}(x_3, y_4) )\\\\ +z_9 &= 0 \\\\ + &+ \qquad 2( \mathrm{hi}(x_4, y_4) )\\\\ +\end{aligned} +\\] +As noted above, our strategy will be to multiply and accumulate the +terms with coefficient \\(2\\) separately from those with coefficient +\\(1\\), before combining them at the end. This can alternately be +thought of as accumulating product terms into a *doubly-redundant* +representation, with two limbs for each digit, before collapsing +the doubly-redundant representation by shifts and adds. + +This computation requires 25 `vpmadd52luq` and 25 `vpmadd52huq` +operations. For 256-bit vectors, IFMA operations execute on an +i3-8121U with latency 4 cycles, throughput 0.5 cycles, so executing 50 +instructions requires 25 cycles' worth of throughput. Accumulating +terms with coefficient \\(1\\) and \\(2\\) seperately means that the +longest dependency chain has length 5, so the critical path has length +20 cycles and the bottleneck is throughput. + +# Reduction modulo \\(p\\) + +The next question is how to handle the reduction modulo \\(p\\). +Because \\(p = 2^{255} - 19\\), \\(2^{255} = 19 \pmod p\\), so we can +alternately write +\\[ +\begin{aligned} +z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\ + &+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459} +\end{aligned} +\\] +as +\\[ +\begin{aligned} +z &= (z_0 + 19z_5) + (z_1 + 19z_6) 2^{51} + (z_2 + 19z_7) 2^{102} + (z_3 + 19z_8) 2^{153} + (z_4 + 19z_9) 2^{204}. +\end{aligned} +\\] +When using a \\(64 \times 64 \rightarrow 128\\)-bit multiplier, this +can be handled (as in [Ed25519][ed25519_paper]) by premultiplying +source terms by \\(19\\). Since \\(\lg(19) < 4.25\\), this increases +their size by less than \\(4.25\\) bits, and the rest of the +multiplication can be shown to work out. + +Here, we have at most \\(1\\) bit of headroom. In order to allow +premultiplication, we would need to use radix \\(2^{47}\\), which +would require six limbs instead of five. Instead, we compute the high +terms \\(z_5, \ldots, z_9\\), each using two chains of IFMA +operations, then multiply by \\(19\\) and combine with the lower terms +\\(z_0, \ldots, z_4\\). There are two ways to perform the +multiplication by \\(19\\): using more IFMA operations, or using the +`vpmullq` instruction, which computes the low \\(64\\) bits of a \\(64 +\times 64\\)-bit product. However, `vpmullq` has 15c/1.5c +latency/throughput, in contrast to the 4c/0.5c latency/throughput of +IFMA operations, so it seems like a worse choice. + +The high terms \\(z_5, \ldots, z_9\\) are sums of \\(52\\)-bit terms, +so they are larger than \\(52\\) bits. Write these terms in radix \\(52\\) as +\\[ +z_{5+i} = z_{5+i}' + z_{5+i}'' 2^{52}, \qquad z_{5+i}' < 2^{52}. +\\] +Then the contribution of \\(z_{5+i}\\), taken modulo \\(p\\), is +\\[ +\begin{aligned} +z_{5+i} 2^{255} 2^{51 i} +&= +19 (z_{5+i}' + z_{5+i}'' 2^{52}) 2^{51 i} +\\\\ +&= +19 z_{5+i}' 2^{51 i} + 2 \cdot 19 z_{5+i}'' 2^{51 (i+1)} +\\\\ +\end{aligned} +\\] +The products \\(19 z_{5+i}', 19 z_{5+i}''\\) can be written in terms of IFMA operations as +\\[ +\begin{aligned} +19 z_{5+i}' &= \mathrm{lo}(19, z_{5+i}') + 2 \mathrm{hi}(19, z_{5+i}') 2^{51}, \\\\ +19 z_{5+i}'' &= \mathrm{lo}(19, z_{5+i}'') + 2 \mathrm{hi}(19, z_{5+i}'') 2^{51}. \\\\ +\end{aligned} +\\] +Because \\(z_{5+i} < 2^{64}\\), \\(z_{5+i}'' < 2^{12} \\), so \\(19 +z_{5+i}'' < 2^{17} < 2^{52} \\) and \\(\mathrm{hi}(19, z_{5+i}'') = 0\\). +Because IFMA operations ignore the high bits of their source +operands, we do not need to compute \\(z\_{5+i}'\\) explicitly: +the high bits will be ignored. +Combining these observations, we can write +\\[ +\begin{aligned} +z_{5+i} 2^{255} 2^{51 i} +&= +19 z_{5+i}' 2^{51 i} + 2 \cdot 19 z_{5+i}'' 2^{51 (i+1)} +\\\\ +&= +\mathrm{lo}(19, z_{5+i}) 2^{51 i} +\+ 2 \mathrm{hi}(19, z_{5+i}) 2^{51 (i+1)} +\+ 2 \mathrm{lo}(19, z_{5+i}/2^{52}) 2^{51 (i+1)}. +\end{aligned} +\\] + +For \\(i = 0,1,2,3\\), this allows reducing \\(z_{5+i}\\) onto +\\(z_{i}, z_{i+1}\\), and if the low terms are computed using a +doubly-redundant representation, no additional shifts are needed to +handle the \\(2\\) coefficients. For \\(i = 4\\), there's a +complication: the contribution becomes +\\[ +\begin{aligned} +z_{9} 2^{255} 2^{204} +&= +\mathrm{lo}(19, z_{9}) 2^{204} +\+ 2 \mathrm{hi}(19, z_{9}) 2^{255} +\+ 2 \mathrm{lo}(19, z_{9}/2^{52}) 2^{255} +\\\\ +&= +\mathrm{lo}(19, z_{9}) 2^{204} +\+ 2 \mathrm{hi}(19, z_{9}) 19 +\+ 2 \mathrm{lo}(19, z_{9}/2^{52}) 19 +\\\\ +&= +\mathrm{lo}(19, z_{9}) 2^{204} +\+ 2 +\mathrm{lo}(19, \mathrm{hi}(19, z_{9}) + \mathrm{lo}(19, z_{9}/2^{52})). +\\\\ +\end{aligned} +\\] + +It would be possible to cut the number of multiplications from 3 to 2 +by carrying the high part of each \\(z_i\\) onto \\(z_{i+1}\\). This +would eliminate 5 multiplications, clearing 2.5 cycles of port +pressure, at the cost of 5 additions, adding 1.66 cycles of port +pressure. But doing this would create a dependency between terms +(e.g., \\(z_{5}\\) must be computed before the reduction of +\\(z_{6}\\) can begin), whereas with the approach above, all +contributions to all terms are computed independently, to maximize ILP +and flexibility for the processor to schedule instructions. + +This strategy performs 16 IFMA operations, adding two IFMA operations +to each of the \\(2\\)-coefficient terms and one to each of the +\\(1\\)-coefficient terms. Considering the multiplication and +reduction together, we use 66 IFMA operations, requiring 33 cycles' +throughput, while the longest chain of IFMA operations is in the +reduction of \\(z_5\\) onto \\(z_1\\), of length 7 (so 28 cycles, plus +2 cycles to combine the two parts of \\(z_5\\), and the bottleneck is +again throughput. + +Once this is done, we have computed the product terms +\\[ +z = z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204}, +\\] +without reducing the \\(z_i\\) to fit in \\(52\\) bits. Because the +overall flow of operations alternates multiplications and additions or +subtractions, we would have to perform a reduction after an addition +but before the next multiplication anyways, so there's no benefit to +fully reducing the limbs at the end of a multiplication. Instead, we +leave them unreduced, and track the reduction state using the type +system to ensure that unreduced limbs are not accidentally used as an +input to a multiplication. + +# Squaring + +Squaring operates similarly to multiplication, but with the +possibility to combine identical terms. +As before, we write the input as +\\[ +\begin{aligned} +x &= x_0 + x_1 2^{51} + x_2 2^{102} + x_3 2^{153} + x_4 2^{204} +\end{aligned} +\\] +with limbs in range \\([0,2^{52})\\). +Writing the product terms as +\\[ +\begin{aligned} +z &= z_0 + z_1 2^{51} + z_2 2^{102} + z_3 2^{153} + z_4 2^{204} \\\\ + &+ z_5 2^{255} + z_6 2^{306} + z_7 2^{357} + z_8 2^{408} + z_9 2^{459}, +\end{aligned} +\\] +a schoolbook squaring in product scanning form takes the form +\\[ +\begin{aligned} +z_0 &= x_0 x_0 \\\\ +z_1 &= 2 x_1 x_0 \\\\ +z_2 &= 2 x_2 x_0 + x_1 x_1 \\\\ +z_3 &= 2 x_3 x_0 + 2 x_2 x_1 \\\\ +z_4 &= 2 x_4 x_0 + 2 x_3 x_1 + x_2 x_2 \\\\ +z_5 &= 2 x_4 x_1 + 2 x_3 x_2 \\\\ +z_6 &= 2 x_4 x_2 + x_3 x_3 \\\\ +z_7 &= 2 x_4 x_3 \\\\ +z_8 &= x_4 x_4 \\\\ +z_9 &= 0 \\\\ +\end{aligned} +\\] +As before, we write \\(x_i x_j\\) as +\\[ +x_i x_j = \mathrm{lo}(x_i,x_j) + 2\mathrm{hi}(x_i,x_j)2^{51}, +\\] +and substitute to obtain +\\[ +\begin{aligned} +z_0 &= \mathrm{lo}(x_0, x_0) + 0 \\\\ +z_1 &= 2 \mathrm{lo}(x_1, x_0) + 2 \mathrm{hi}(x_0, x_0) \\\\ +z_2 &= 2 \mathrm{lo}(x_2, x_0) + \mathrm{lo}(x_1, x_1) + 4 \mathrm{hi}(x_1, x_0) \\\\ +z_3 &= 2 \mathrm{lo}(x_3, x_0) + 2 \mathrm{lo}(x_2, x_1) + 4 \mathrm{hi}(x_2, x_0) + 2 \mathrm{hi}(x_1, x_1) \\\\ +z_4 &= 2 \mathrm{lo}(x_4, x_0) + 2 \mathrm{lo}(x_3, x_1) + \mathrm{lo}(x_2, x_2) + 4 \mathrm{hi}(x_3, x_0) + 4 \mathrm{hi}(x_2, x_1) \\\\ +z_5 &= 2 \mathrm{lo}(x_4, x_1) + 2 \mathrm{lo}(x_3, x_2) + 4 \mathrm{hi}(x_4, x_0) + 4 \mathrm{hi}(x_3, x_1) + 2 \mathrm{hi}(x_2, x_2) \\\\ +z_6 &= 2 \mathrm{lo}(x_4, x_2) + \mathrm{lo}(x_3, x_3) + 4 \mathrm{hi}(x_4, x_1) + 4 \mathrm{hi}(x_3, x_2) \\\\ +z_7 &= 2 \mathrm{lo}(x_4, x_3) + 4 \mathrm{hi}(x_4, x_2) + 2 \mathrm{hi}(x_3, x_3) \\\\ +z_8 &= \mathrm{lo}(x_4, x_4) + 4 \mathrm{hi}(x_4, x_3) \\\\ +z_9 &= 0 + 2 \mathrm{hi}(x_4, x_4) \\\\ +\end{aligned} +\\] +To implement these, we group terms by their coefficient, computing +those with coefficient \\(2\\) on set of IFMA chains, and on another +set of chains, we begin with coefficient-\\(4\\) terms, then shift +left before continuing with the coefficient-\\(1\\) terms. +The reduction strategy is the same as for multiplication. + +# Future improvements + +LLVM won't use blend operations on [256-bit vectors yet][llvm_blend], +so there's a bunch of blend instructions that could be omitted. + +Although the multiplications and squarings are much faster, there's no +speedup to the additions and subtractions, so there are diminishing +returns. In fact, the complications in the doubling formulas mean +that doubling is actually slower than readdition. This also suggests +that moving to 512-bit vectors won't be much help for a strategy aimed +at parallelism within a group operation, so to extract performance +gains from 512-bit vectors it will probably be necessary to create a +parallel-friendly multiscalar multiplication algorithm. This could +also help with reducing shuffle pressure. + +The squaring implementation could probably be optimized, but without +`perf` support on Cannonlake it's difficult to make actual +measurements. + +Another improvement would be to implement vectorized square root +computations, which would allow creating an iterator adaptor for point +decompression that bunched decompression operations and executed them +in parallel. This would accelerate batch verification. + +[2016_gueron_krasnov]: https://ieeexplore.ieee.org/document/7563269 +[2018_drucker_gueron]: https://eprint.iacr.org/2018/335 +[1999_walter]: https://pdfs.semanticscholar.org/0e6a/3e8f30b63b556679f5dff2cbfdfe9523f4fa.pdf +[ed25519_paper]: https://ed25519.cr.yp.to/ed25519-20110926.pdf +[llvm_blend]: https://bugs.llvm.org/show_bug.cgi?id=38343 diff --git a/docs/parallel-formulas.md b/docs/parallel-formulas.md index a356740..22f59cd 100644 --- a/docs/parallel-formulas.md +++ b/docs/parallel-formulas.md @@ -2,6 +2,10 @@ Vectorized implementations of field and point operations, using a modification of the 4-way parallel formulas of Hisil, Wong, Carter, and Dawson. +These notes explain the parallel formulas and our strategy for using +them with SIMD operations. There are two backend implementations: one +using AVX2, and the other using AVX512-IFMA. + # Overview The 2008 paper [_Twisted Edwards Curves Revisited_][hwcd08] by Hisil, @@ -310,13 +314,13 @@ There are several directions for future improvement: the only available CPU which runs IFMA operations executes them at 256-bits wide anyways, so this isn't yet important. -* Generalizing the implementation to non-AVX2 instructions, - particularly NEON. The current point arithmetic code is written in - terms of field element vectors, which are in turn implemented using - platform SIMD vectors. It should be possible to write an alternate - implementation of the `FieldElement2625x4` using NEON without changing - the point arithmetic. NEON has 128-bit vectors rather than 256-bit - vectors, but this may still be worthwhile compared to a serial +* Generalizing the implementation to NEON instructions. The current + point arithmetic code is written in terms of field element vectors, + which are in turn implemented using platform SIMD vectors. It + should be possible to write an alternate implementation of the + `FieldElement2625x4` using NEON without changing the point + arithmetic. NEON has 128-bit vectors rather than 256-bit vectors, + but this may still be worthwhile compared to a serial implementation. @@ -326,4 +330,4 @@ There are several directions for future improvement: [curve_models]: https://doc-internal.dalek.rs/curve25519_dalek/curve_models/index.html [bbjlp08]: https://eprint.iacr.org/2008/013 [cmo98]: https://link.springer.com/content/pdf/10.1007%2F3-540-49649-1_6.pdf -[intel]: https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf \ No newline at end of file +[intel]: https://software.intel.com/sites/default/files/managed/9e/bc/64-ia-32-architectures-optimization-manual.pdf diff --git a/src/backend/vector/avx2/mod.rs b/src/backend/vector/avx2/mod.rs index cdad7ec..0b30085 100644 --- a/src/backend/vector/avx2/mod.rs +++ b/src/backend/vector/avx2/mod.rs @@ -18,7 +18,8 @@ // location of build.rs, not lib.rs, so the markdown file appears // missing). #![cfg_attr( - all(feature = "nightly", feature = "stage2_build"), doc(include = "../docs/avx2-notes.md") + all(feature = "nightly", feature = "stage2_build"), + doc(include = "../docs/avx2-notes.md") )] pub(crate) mod field; @@ -26,4 +27,3 @@ pub(crate) mod field; pub(crate) mod edwards; pub(crate) mod constants; - diff --git a/src/backend/vector/ifma/mod.rs b/src/backend/vector/ifma/mod.rs index 106118a..996f941 100644 --- a/src/backend/vector/ifma/mod.rs +++ b/src/backend/vector/ifma/mod.rs @@ -7,6 +7,11 @@ // Authors: // - Henry de Valence +#![cfg_attr( + all(feature = "nightly", feature = "stage2_build"), + doc(include = "../docs/ifma-notes.md") +)] + pub mod field; pub mod edwards; From e3ca8961fce5f3b0afbbbee84371983d8b0bebb6 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 17 Jan 2019 19:49:41 -0800 Subject: [PATCH 35/52] Fix rustdoc feature flags to compile on stable. --- src/backend/mod.rs | 2 +- src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index aad3871..33d5bc1 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -50,7 +50,7 @@ pub mod serial; feature = "simd_backend", any(target_feature = "avx2", target_feature = "avx512ifma") ), - rustdoc + all(feature = "nightly", rustdoc) ))] #[cfg_attr( feature = "nightly", diff --git a/src/lib.rs b/src/lib.rs index 85ad392..7fef688 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,7 +12,7 @@ #![cfg_attr( any( all(feature = "simd_backend", target_feature = "avx512ifma"), - all(rustdoc, feature = "nightly") + all(feature = "nightly", rustdoc) ), feature(simd_ffi, link_llvm_intrinsics) )] From 4270d4ff5f3b1f405aad955cb48f1fcda660cd9d Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Tue, 22 Jan 2019 01:01:08 +0000 Subject: [PATCH 36/52] Fix two typos in FieldElement::invsqrt documentation. --- src/field.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/field.rs b/src/field.rs index 2ad5d7f..f3c3320 100644 --- a/src/field.rs +++ b/src/field.rs @@ -255,7 +255,7 @@ impl FieldElement { (was_nonzero_square, r) } - /// Attempt to compute `1/sqrt(self)` in constant time. + /// Attempt to compute `sqrt(1/self)` in constant time. /// /// Convenience wrapper around `sqrt_ratio_i`. /// @@ -265,7 +265,7 @@ impl FieldElement { /// /// - `(Choice(1), +sqrt(1/self)) ` if `self` is a nonzero square; /// - `(Choice(0), zero) ` if `self` is zero; - /// - `(Choice(0), +sqrt(i*u/v)) ` if `self` is a nonzero nonsquare; + /// - `(Choice(0), +sqrt(i/self)) ` if `self` is a nonzero nonsquare; /// pub fn invsqrt(&self) -> (Choice, FieldElement) { FieldElement::sqrt_ratio_i(&FieldElement::one(), self) From 0d23260a75949372a6dd40cd27a1329f828966ab Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Thu, 24 Jan 2019 21:35:43 +0000 Subject: [PATCH 37/52] I found a weird pub, not the kind you get cidre in. --- src/field.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/field.rs b/src/field.rs index 2ad5d7f..70445bf 100644 --- a/src/field.rs +++ b/src/field.rs @@ -280,7 +280,7 @@ mod test { /// Random element a of GF(2^255-19), from Sage /// a = 1070314506888354081329385823235218444233221\ /// 2228051251926706380353716438957572 - pub static A_BYTES: [u8; 32] = + static A_BYTES: [u8; 32] = [ 0x04, 0xfe, 0xdf, 0x98, 0xa7, 0xfa, 0x0a, 0x68, 0x84, 0x92, 0xbd, 0x59, 0x08, 0x07, 0xa7, 0x03, 0x9e, 0xd1, 0xf6, 0xf2, 0xe1, 0xd9, 0xe2, 0xa4, From 98e713ef91947070b9224a5c8a59dc49d841089f Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 3 May 2018 15:31:08 -0700 Subject: [PATCH 38/52] Add a trait for multiscalar multiplication with precomputation. --- src/traits.rs | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/src/traits.rs b/src/traits.rs index 8db963a..630e510 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -224,6 +224,166 @@ pub trait VartimeMultiscalarMul { } } +/// A trait for constant-time multiscalar multiplication with precomputation. +/// +/// A general multiscalar multiplication with precomputation can be written as +/// $$ +/// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, +/// $$ +/// where the \\(B_i\\) are *static* points, for which precomputation +/// is possible, and the \\(A_j\\) are *dynamic* points, for which +/// precomputation is not possible. +pub trait PrecomputedMultiscalarMul: Sized { + /// The type of point to be multiplied, e.g., `RistrettoPoint`. + type Point; + + /// Given the static points \\( B_i \\), perform precomputation + /// and return the precomputation data. + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow; + + /// Given `static_scalars`, an iterator of (possibly secret) + /// scalars \\(b_i\\), `dynamic_scalars`, an iterator of (possibly + /// secret) scalars \\(a_i\\), and `dynamic_points`, an iterator + /// of points \\(A_i\\), compute + /// $$ + /// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// The trait bound aims for maximum flexibility: the inputs must be + /// convertable to iterators (`I: IntoIter`), and the iterator's items + /// must be `Borrow` (or `Borrow`), to allow + /// iterators returning either `Scalar`s or `&Scalar`s. + fn mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow; + + /// Given `static_scalars`, an iterator of (possibly secret) + /// scalars \\(b_i\\), compute + /// $$ + /// Q = b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// The trait bound aims for maximum flexibility: the input must + /// be convertable to iterators (`I: IntoIter`), and the + /// iterator's items must be `Borrow`, to allow iterators + /// returning either `Scalar`s or `&Scalar`s. + fn multiscalar_mul(&self, static_scalars: I) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + { + use core::iter; + + Self::mixed_multiscalar_mul( + self, + static_scalars, + iter::empty::(), + iter::empty::(), + ) + } +} + +/// A trait for variable-time multiscalar multiplication with precomputation. +/// +/// A general multiscalar multiplication with precomputation can be written as +/// $$ +/// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, +/// $$ +/// where the \\(B_i\\) are *static* points, for which precomputation +/// is possible, and the \\(A_j\\) are *dynamic* points, for which +/// precomputation is not possible. +pub trait VartimePrecomputedMultiscalarMul: Sized { + /// The type of point to be multiplied, e.g., `RistrettoPoint`. + type Point; + + /// Given the static points \\( B_i \\), perform precomputation + /// and return the precomputation data. + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow; + + /// Given `static_scalars`, an iterator of public scalars + /// \\(b_i\\), `dynamic_scalars`, an iterator of public scalars + /// \\(a_i\\), and `dynamic_points`, an iterator of points + /// \\(A_i\\), compute + /// $$ + /// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// The trait bound aims for maximum flexibility: the inputs must be + /// convertable to iterators (`I: IntoIter`), and the iterator's items + /// must be `Borrow` (or `Borrow`), to allow + /// iterators returning either `Scalar`s or `&Scalar`s. + fn vartime_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow; + + /// Given `static_scalars`, an iterator of public scalars + /// \\(b_i\\), compute + /// $$ + /// Q = b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// The trait bound aims for maximum flexibility: the input must + /// be convertable to iterators (`I: IntoIter`), and the + /// iterator's items must be `Borrow`, to allow iterators + /// returning either `Scalar`s or `&Scalar`s. + fn vartime_multiscalar_mul(&self, static_scalars: I) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + { + use core::iter; + + Self::vartime_mixed_multiscalar_mul( + self, + static_scalars, + iter::empty::(), + iter::empty::(), + ) + } +} + // ------------------------------------------------------------------------ // Private Traits // ------------------------------------------------------------------------ From 00675b4c560e1264608a80b505ef1eb0072a524f Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 24 Jan 2019 13:29:28 -0800 Subject: [PATCH 39/52] Add benchmarks for precomputed multiscalar multiplication. --- benches/dalek_benchmarks.rs | 150 ++++++++++++++++++++++++++++++------ 1 file changed, 128 insertions(+), 22 deletions(-) diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs index 71792a3..011e6a6 100644 --- a/benches/dalek_benchmarks.rs +++ b/benches/dalek_benchmarks.rs @@ -25,9 +25,7 @@ mod edwards_benches { fn compress(c: &mut Criterion) { let B = &constants::ED25519_BASEPOINT_POINT; - c.bench_function("EdwardsPoint compression", move |b| { - b.iter(|| B.compress()) - }); + c.bench_function("EdwardsPoint compression", move |b| b.iter(|| B.compress())); } fn decompress(c: &mut Criterion) { @@ -63,16 +61,40 @@ mod edwards_benches { }); } + criterion_group!{ + name = edwards_benches; + config = Criterion::default(); + targets = + compress, + decompress, + consttime_fixed_base_scalar_mul, + consttime_variable_base_scalar_mul, + vartime_double_base_scalar_mul, + } +} + +mod multiscalar_benches { + use super::*; + use curve25519_dalek::edwards; + use curve25519_dalek::edwards::EdwardsPoint; + use curve25519_dalek::traits::MultiscalarMul; + use curve25519_dalek::traits::VartimeMultiscalarMul; + + fn construct(n: usize) -> (Vec, Vec) { + let mut rng = OsRng::new().unwrap(); + let scalars: Vec = (0..n).map(|_| Scalar::random(&mut rng)).collect(); + let points: Vec = scalars + .iter() + .map(|s| s * &constants::ED25519_BASEPOINT_TABLE) + .collect(); + (scalars, points) + } + fn consttime_multiscalar_mul(c: &mut Criterion) { c.bench_function_over_inputs( "Constant-time variable-base multiscalar multiplication", |b, &&size| { - let mut rng = OsRng::new().unwrap(); - let scalars: Vec = (0..size).map(|_| Scalar::random(&mut rng)).collect(); - let points: Vec = scalars - .iter() - .map(|s| s * &constants::ED25519_BASEPOINT_TABLE) - .collect(); + let (scalars, points) = construct(size); b.iter(|| EdwardsPoint::multiscalar_mul(&scalars, &points)); }, &MULTISCALAR_SIZES, @@ -83,29 +105,112 @@ mod edwards_benches { c.bench_function_over_inputs( "Variable-time variable-base multiscalar multiplication", |b, &&size| { - let mut rng = OsRng::new().unwrap(); - let scalars: Vec = (0..size).map(|_| Scalar::random(&mut rng)).collect(); - let points: Vec = scalars - .iter() - .map(|s| s * &constants::ED25519_BASEPOINT_TABLE) - .collect(); + let (scalars, points) = construct(size); b.iter(|| EdwardsPoint::vartime_multiscalar_mul(&scalars, &points)); }, &MULTISCALAR_SIZES, ); } + fn precomputed_ct_straus_helper(c: &mut Criterion, dynamic_fraction: f64) { + let label = format!( + "Constant-time mixed-base Straus ({:.2}pct dyn)", + 100.0*dynamic_fraction, + ); + c.bench_function_over_inputs( + &label, + move |b, &&total_size| { + let dynamic_size = ((total_size as f64) * dynamic_fraction) as usize; + let static_size = total_size - dynamic_size; + + let (static_scalars, static_points) = construct(static_size); + let (dynamic_scalars, dynamic_points) = construct(dynamic_size); + + use curve25519_dalek::edwards::PrecomputedStraus; + use curve25519_dalek::traits::PrecomputedMultiscalarMul; + + let precomp = PrecomputedStraus::new(&static_points); + + b.iter(|| { + precomp.mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ) + }); + }, + &MULTISCALAR_SIZES, + ); + } + + fn precomputed_vt_straus_helper(c: &mut Criterion, dynamic_fraction: f64) { + let label = format!( + "Variable-time mixed-base Straus ({:.2}pct dyn)", + 100.0*dynamic_fraction, + ); + c.bench_function_over_inputs( + &label, + move |b, &&total_size| { + let dynamic_size = ((total_size as f64) * dynamic_fraction) as usize; + let static_size = total_size - dynamic_size; + + let (static_scalars, static_points) = construct(static_size); + let (dynamic_scalars, dynamic_points) = construct(dynamic_size); + + use curve25519_dalek::edwards::VartimePrecomputedStraus; + use curve25519_dalek::traits::VartimePrecomputedMultiscalarMul; + + let precomp = VartimePrecomputedStraus::new(&static_points); + + b.iter(|| { + precomp.vartime_mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ) + }); + }, + &MULTISCALAR_SIZES, + ); + } + + fn precomputed_ct_straus_00_pct_dynamic(c: &mut Criterion) { + precomputed_ct_straus_helper(c, 0.0); + } + + fn precomputed_ct_straus_20_pct_dynamic(c: &mut Criterion) { + precomputed_ct_straus_helper(c, 0.2); + } + + fn precomputed_ct_straus_50_pct_dynamic(c: &mut Criterion) { + precomputed_ct_straus_helper(c, 0.5); + } + + fn precomputed_vt_straus_00_pct_dynamic(c: &mut Criterion) { + precomputed_vt_straus_helper(c, 0.0); + } + + fn precomputed_vt_straus_20_pct_dynamic(c: &mut Criterion) { + precomputed_vt_straus_helper(c, 0.2); + } + + fn precomputed_vt_straus_50_pct_dynamic(c: &mut Criterion) { + precomputed_vt_straus_helper(c, 0.5); + } + criterion_group!{ - name = edwards_benches; - config = Criterion::default(); + name = multiscalar_benches; + // Lower the sample size to run the benchmarks faster + config = Criterion::default().sample_size(15); targets = - compress, - decompress, - consttime_fixed_base_scalar_mul, - consttime_variable_base_scalar_mul, - vartime_double_base_scalar_mul, consttime_multiscalar_mul, vartime_multiscalar_mul, + precomputed_ct_straus_00_pct_dynamic, + precomputed_ct_straus_20_pct_dynamic, + precomputed_ct_straus_50_pct_dynamic, + precomputed_vt_straus_00_pct_dynamic, + precomputed_vt_straus_20_pct_dynamic, + precomputed_vt_straus_50_pct_dynamic, } } @@ -208,4 +313,5 @@ criterion_main!( montgomery_benches::montgomery_benches, ristretto_benches::ristretto_benches, edwards_benches::edwards_benches, + multiscalar_benches::multiscalar_benches, ); From 5daff6607975ca86f562a1f2ffb5f2ae51e392c0 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 11 Feb 2019 07:25:16 -0800 Subject: [PATCH 40/52] Move cfgs into outer module. --- src/backend/serial/scalar_mul/mod.rs | 1 + src/backend/serial/scalar_mul/straus.rs | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/backend/serial/scalar_mul/mod.rs b/src/backend/serial/scalar_mul/mod.rs index 1421f41..2aba024 100644 --- a/src/backend/serial/scalar_mul/mod.rs +++ b/src/backend/serial/scalar_mul/mod.rs @@ -21,4 +21,5 @@ pub mod variable_base; #[cfg(feature = "stage2_build")] pub mod vartime_double_base; +#[cfg(feature = "alloc")] pub mod straus; diff --git a/src/backend/serial/scalar_mul/straus.rs b/src/backend/serial/scalar_mul/straus.rs index 2b7a306..4053ea3 100644 --- a/src/backend/serial/scalar_mul/straus.rs +++ b/src/backend/serial/scalar_mul/straus.rs @@ -12,16 +12,11 @@ #![allow(non_snake_case)] -#[cfg(any(feature = "alloc", feature = "std"))] use core::borrow::Borrow; -#[cfg(any(feature = "alloc", feature = "std"))] use edwards::EdwardsPoint; -#[cfg(any(feature = "alloc", feature = "std"))] use scalar::Scalar; -#[cfg(any(feature = "alloc", feature = "std"))] use traits::MultiscalarMul; -#[cfg(any(feature = "alloc", feature = "std"))] use traits::VartimeMultiscalarMul; #[allow(unused_imports)] @@ -48,10 +43,8 @@ use prelude::*; /// /// [solution]: https://www.jstor.org/stable/2310929 /// [problem]: https://www.jstor.org/stable/2312273 -#[cfg(any(feature = "alloc", feature = "std"))] pub struct Straus {} -#[cfg(feature = "alloc")] impl MultiscalarMul for Straus { type Point = EdwardsPoint; @@ -148,7 +141,6 @@ impl MultiscalarMul for Straus { } } -#[cfg(feature = "alloc")] impl VartimeMultiscalarMul for Straus { type Point = EdwardsPoint; From c6acdfd5e2b74e6b2d29cc320f1c8defe979f624 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 12 Feb 2019 11:18:03 -0800 Subject: [PATCH 41/52] Add serial implementation of precomputation. --- benches/dalek_benchmarks.rs | 10 +- src/backend/serial/scalar_mul/mod.rs | 3 + .../serial/scalar_mul/precomputed_straus.rs | 202 ++++++++++++++++++ src/edwards.rs | 176 ++++++++++++++- 4 files changed, 380 insertions(+), 11 deletions(-) create mode 100644 src/backend/serial/scalar_mul/precomputed_straus.rs diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs index 011e6a6..118a809 100644 --- a/benches/dalek_benchmarks.rs +++ b/benches/dalek_benchmarks.rs @@ -20,8 +20,6 @@ mod edwards_benches { use super::*; use curve25519_dalek::edwards; use curve25519_dalek::edwards::EdwardsPoint; - use curve25519_dalek::traits::MultiscalarMul; - use curve25519_dalek::traits::VartimeMultiscalarMul; fn compress(c: &mut Criterion) { let B = &constants::ED25519_BASEPOINT_POINT; @@ -126,10 +124,10 @@ mod multiscalar_benches { let (static_scalars, static_points) = construct(static_size); let (dynamic_scalars, dynamic_points) = construct(dynamic_size); - use curve25519_dalek::edwards::PrecomputedStraus; + use curve25519_dalek::edwards::EdwardsPrecomputation; use curve25519_dalek::traits::PrecomputedMultiscalarMul; - let precomp = PrecomputedStraus::new(&static_points); + let precomp = EdwardsPrecomputation::new(&static_points); b.iter(|| { precomp.mixed_multiscalar_mul( @@ -157,10 +155,10 @@ mod multiscalar_benches { let (static_scalars, static_points) = construct(static_size); let (dynamic_scalars, dynamic_points) = construct(dynamic_size); - use curve25519_dalek::edwards::VartimePrecomputedStraus; + use curve25519_dalek::edwards::VartimeEdwardsPrecomputation; use curve25519_dalek::traits::VartimePrecomputedMultiscalarMul; - let precomp = VartimePrecomputedStraus::new(&static_points); + let precomp = VartimeEdwardsPrecomputation::new(&static_points); b.iter(|| { precomp.vartime_mixed_multiscalar_mul( diff --git a/src/backend/serial/scalar_mul/mod.rs b/src/backend/serial/scalar_mul/mod.rs index 2aba024..bec874b 100644 --- a/src/backend/serial/scalar_mul/mod.rs +++ b/src/backend/serial/scalar_mul/mod.rs @@ -23,3 +23,6 @@ pub mod vartime_double_base; #[cfg(feature = "alloc")] pub mod straus; + +#[cfg(feature = "alloc")] +pub mod precomputed_straus; diff --git a/src/backend/serial/scalar_mul/precomputed_straus.rs b/src/backend/serial/scalar_mul/precomputed_straus.rs new file mode 100644 index 0000000..989d92a --- /dev/null +++ b/src/backend/serial/scalar_mul/precomputed_straus.rs @@ -0,0 +1,202 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2019 Henry de Valence. +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +//! Precomputation for Straus's method. + +#![allow(non_snake_case)] + +use core::borrow::Borrow; + +use clear_on_drop::ClearOnDrop; + +use backend::serial::curve_models::{ + AffineNielsPoint, CompletedPoint, ProjectiveNielsPoint, ProjectivePoint, +}; +use edwards::EdwardsPoint; +use scalar::Scalar; +use traits::Identity; +use traits::{PrecomputedMultiscalarMul, VartimePrecomputedMultiscalarMul}; +use window::{LookupTable, NafLookupTable5, NafLookupTable8}; + +#[allow(unused_imports)] +use prelude::*; + +pub struct PrecomputedStraus { + static_lookup_tables: Vec>, +} + +impl PrecomputedMultiscalarMul for PrecomputedStraus { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + PrecomputedStraus { + static_lookup_tables: static_points + .into_iter() + .map(|point| LookupTable::::from(point.borrow())) + .collect(), + } + } + + fn mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + let mut static_scalars = static_scalars.into_iter(); + let mut dynamic_scalars = dynamic_scalars.into_iter(); + let mut dynamic_points = dynamic_points.into_iter(); + + // Check that the input lengths are consistent with each other: + let (ss_lo, ss_hi) = static_scalars.by_ref().size_hint(); + let (ds_lo, ds_hi) = dynamic_scalars.by_ref().size_hint(); + let (dp_lo, dp_hi) = dynamic_points.by_ref().size_hint(); + + // Static points match static scalars + let sp = self.static_lookup_tables.len(); + assert_eq!(ss_lo, sp); + assert_eq!(ss_hi, Some(sp)); + + // Dynamic points match dynamic scalars + assert_eq!(ds_lo, dp_lo); + assert_eq!(ds_hi, Some(ds_lo)); + assert_eq!(ds_hi, dp_hi); + let dp = dp_lo; + + // This does two allocs for the scalar digits instead of + // putting them in a contiguous array, which makes handling + // the two kinds of lookup tables slightly easier. + // Use a ClearOnDrop wrapper. + + let static_scalar_digits_vec: Vec<_> = + static_scalars.map(|s| s.borrow().to_radix_16()).collect(); + let static_scalar_digits = ClearOnDrop::new(static_scalar_digits_vec); + + let dynamic_scalar_digits_vec: Vec<_> = + dynamic_scalars.map(|s| s.borrow().to_radix_16()).collect(); + let dynamic_scalar_digits = ClearOnDrop::new(dynamic_scalar_digits_vec); + + // Build lookup tables for dynamic points + let dynamic_lookup_tables: Vec<_> = dynamic_points + .map(|point| LookupTable::::from(point.borrow())) + .collect(); + + let mut R = EdwardsPoint::identity(); + for j in (0..64).rev() { + R = R.mul_by_pow_2(4); + for i in 0..dp { + let t_ij = dynamic_scalar_digits[i][j]; + R = (&R + &dynamic_lookup_tables[i].select(t_ij)).to_extended(); + } + for i in 0..sp { + let s_ij = static_scalar_digits[i][j]; + R = (&R + &self.static_lookup_tables[i].select(s_ij)).to_extended(); + } + } + + R + } +} + +pub struct VartimePrecomputedStraus { + static_lookup_tables: Vec>, +} + +impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self { + static_lookup_tables: static_points + .into_iter() + .map(|P| NafLookupTable8::::from(P.borrow())) + .collect(), + } + } + + fn vartime_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + let static_nafs = static_scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form(5)) + .collect::>(); + let dynamic_nafs: Vec<_> = dynamic_scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form(5)) + .collect::>(); + + let dynamic_lookup_tables = dynamic_points + .into_iter() + .map(|P| NafLookupTable5::::from(P.borrow())) + .collect::>(); + + let sp = self.static_lookup_tables.len(); + let dp = dynamic_lookup_tables.len(); + assert_eq!(sp, static_nafs.len()); + assert_eq!(dp, dynamic_nafs.len()); + + // We could save some doublings by looking for the highest + // nonzero NAF coefficient, but since we might have a lot of + // them to search, it's not clear it's worthwhile to check. + let mut S = ProjectivePoint::identity(); + for j in (0..255).rev() { + let mut R: CompletedPoint = S.double(); + + for i in 0..dp { + let t_ij = dynamic_nafs[i][j]; + if t_ij > 0 { + R = &R.to_extended() + &dynamic_lookup_tables[i].select(t_ij as usize); + } else if t_ij < 0 { + R = &R.to_extended() - &dynamic_lookup_tables[i].select(-t_ij as usize); + } + } + + for i in 0..sp { + let t_ij = static_nafs[i][j]; + if t_ij > 0 { + R = &R.to_extended() + &self.static_lookup_tables[i].select(t_ij as usize); + } else if t_ij < 0 { + R = &R.to_extended() - &self.static_lookup_tables[i].select(-t_ij as usize); + } + } + + S = R.to_projective(); + } + + S.to_extended() + } +} diff --git a/src/edwards.rs b/src/edwards.rs index 102a5ea..fa7029e 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -112,23 +112,23 @@ use scalar::Scalar; use montgomery::MontgomeryPoint; -use backend::serial::curve_models::ProjectivePoint; -use backend::serial::curve_models::CompletedPoint; use backend::serial::curve_models::AffineNielsPoint; +use backend::serial::curve_models::CompletedPoint; use backend::serial::curve_models::ProjectiveNielsPoint; +use backend::serial::curve_models::ProjectivePoint; use window::LookupTable; #[allow(unused_imports)] use prelude::*; -use traits::{Identity, IsIdentity}; use traits::ValidityCheck; +use traits::{Identity, IsIdentity}; #[cfg(any(feature = "alloc", feature = "std"))] -use traits::MultiscalarMul; +use traits::{MultiscalarMul, PrecomputedMultiscalarMul}; #[cfg(any(feature = "alloc", feature = "std"))] -use traits::VartimeMultiscalarMul; +use traits::{VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul}; #[cfg(not(all( feature = "simd_backend", @@ -673,6 +673,84 @@ impl VartimeMultiscalarMul for EdwardsPoint { } } +/// Precomputation for multiscalar multiplication with `EdwardsPoint`s. +// This wraps the inner implementation in a facade type so that we can +// decouple stability of the inner type from the stability of the +// outer type. +#[cfg(all(feature = "alloc", feature = "yolocrypto"))] +pub struct EdwardsPrecomputation(scalar_mul::precomputed_straus::PrecomputedStraus); + +/// Precomputation for variable-time multiscalar multiplication with `EdwardsPoint`s. +// This wraps the inner implementation in a facade type so that we can +// decouple stability of the inner type from the stability of the +// outer type. +#[cfg(all(feature = "alloc", feature = "yolocrypto"))] +pub struct VartimeEdwardsPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus); + +#[cfg(all(feature = "alloc", feature = "yolocrypto"))] +impl PrecomputedMultiscalarMul for EdwardsPrecomputation { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self(scalar_mul::precomputed_straus::PrecomputedStraus::new( + static_points, + )) + } + + fn mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + self.0 + .mixed_multiscalar_mul(static_scalars, dynamic_scalars, dynamic_points) + } +} + +#[cfg(all(feature = "alloc", feature = "yolocrypto"))] +impl VartimePrecomputedMultiscalarMul for VartimeEdwardsPrecomputation { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self(scalar_mul::precomputed_straus::VartimePrecomputedStraus::new(static_points)) + } + + fn vartime_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + self.0 + .vartime_mixed_multiscalar_mul(static_scalars, dynamic_scalars, dynamic_points) + } +} + impl EdwardsPoint { /// Compute \\(aA + bB\\) in variable time, where \\(B\\) is the Ed25519 basepoint. #[cfg(feature = "stage2_build")] @@ -1203,6 +1281,94 @@ mod test { assert!(P1.compress().to_bytes() == P2.compress().to_bytes()); } + #[test] + #[cfg(feature = "yolocrypto")] + fn precomputed_vs_nonprecomputed_multiscalar() { + let mut rng = rand::thread_rng(); + + let B = &::constants::ED25519_BASEPOINT_TABLE; + + let static_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let dynamic_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let check_scalar: Scalar = static_scalars + .iter() + .chain(dynamic_scalars.iter()) + .map(|s| s * s) + .sum(); + + let static_points = static_scalars.iter().map(|s| s * B).collect::>(); + let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::>(); + + let precomputation = EdwardsPrecomputation::new(static_points.iter()); + + let P = precomputation.mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ); + + use traits::MultiscalarMul; + let Q = EdwardsPoint::multiscalar_mul( + static_scalars.iter().chain(dynamic_scalars.iter()), + static_points.iter().chain(dynamic_points.iter()), + ); + + let R = &check_scalar * B; + + assert_eq!(P.compress(), R.compress()); + assert_eq!(Q.compress(), R.compress()); + } + + #[test] + #[cfg(feature = "yolocrypto")] + fn vartime_precomputed_vs_nonprecomputed_multiscalar() { + let mut rng = rand::thread_rng(); + + let B = &::constants::ED25519_BASEPOINT_TABLE; + + let static_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let dynamic_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let check_scalar: Scalar = static_scalars + .iter() + .chain(dynamic_scalars.iter()) + .map(|s| s * s) + .sum(); + + let static_points = static_scalars.iter().map(|s| s * B).collect::>(); + let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::>(); + + let precomputation = VartimeEdwardsPrecomputation::new(static_points.iter()); + + let P = precomputation.vartime_mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ); + + use traits::VartimeMultiscalarMul; + let Q = EdwardsPoint::vartime_multiscalar_mul( + static_scalars.iter().chain(dynamic_scalars.iter()), + static_points.iter().chain(dynamic_points.iter()), + ); + + let R = &check_scalar * B; + + assert_eq!(P.compress(), R.compress()); + assert_eq!(Q.compress(), R.compress()); + } + mod vartime { use super::super::*; use super::{A_SCALAR, B_SCALAR, A_TIMES_BASEPOINT, DOUBLE_SCALAR_MULT_RESULT}; From 8b0ad2b03dd9f44994d955fb6d9eedeaf8af8465 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 12 Feb 2019 11:36:34 -0800 Subject: [PATCH 42/52] Add vector implementation of precomputation. --- src/backend/vector/scalar_mul/mod.rs | 4 + .../vector/scalar_mul/precomputed_straus.rs | 198 ++++++++++++++++++ src/backend/vector/scalar_mul/straus.rs | 2 - 3 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 src/backend/vector/scalar_mul/precomputed_straus.rs diff --git a/src/backend/vector/scalar_mul/mod.rs b/src/backend/vector/scalar_mul/mod.rs index 9293e3e..5c8734d 100644 --- a/src/backend/vector/scalar_mul/mod.rs +++ b/src/backend/vector/scalar_mul/mod.rs @@ -12,4 +12,8 @@ pub mod variable_base; pub mod vartime_double_base; +#[cfg(feature = "alloc")] pub mod straus; + +#[cfg(feature = "alloc")] +pub mod precomputed_straus; diff --git a/src/backend/vector/scalar_mul/precomputed_straus.rs b/src/backend/vector/scalar_mul/precomputed_straus.rs new file mode 100644 index 0000000..dc3fc3f --- /dev/null +++ b/src/backend/vector/scalar_mul/precomputed_straus.rs @@ -0,0 +1,198 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2019 Henry de Valence. +// See LICENSE for licensing information. +// +// Authors: +// - Henry de Valence + +//! Precomputation for Straus's method. + +#![allow(non_snake_case)] + +use core::borrow::Borrow; + +use clear_on_drop::ClearOnDrop; + +use backend::vector::{CachedPoint, ExtendedPoint}; +use edwards::EdwardsPoint; +use scalar::Scalar; +use traits::Identity; +use traits::{PrecomputedMultiscalarMul, VartimePrecomputedMultiscalarMul}; +use window::{LookupTable, NafLookupTable5, NafLookupTable8}; + +#[allow(unused_imports)] +use prelude::*; + +pub struct PrecomputedStraus { + static_lookup_tables: Vec>, +} + +impl PrecomputedMultiscalarMul for PrecomputedStraus { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + PrecomputedStraus { + static_lookup_tables: static_points + .into_iter() + .map(|point| LookupTable::::from(point.borrow())) + .collect(), + } + } + + fn mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + let mut static_scalars = static_scalars.into_iter(); + let mut dynamic_scalars = dynamic_scalars.into_iter(); + let mut dynamic_points = dynamic_points.into_iter(); + + // Check that the input lengths are consistent with each other: + let (ss_lo, ss_hi) = static_scalars.by_ref().size_hint(); + let (ds_lo, ds_hi) = dynamic_scalars.by_ref().size_hint(); + let (dp_lo, dp_hi) = dynamic_points.by_ref().size_hint(); + + // Static points match static scalars + let sp = self.static_lookup_tables.len(); + assert_eq!(ss_lo, sp); + assert_eq!(ss_hi, Some(sp)); + + // Dynamic points match dynamic scalars + assert_eq!(ds_lo, dp_lo); + assert_eq!(ds_hi, Some(ds_lo)); + assert_eq!(ds_hi, dp_hi); + let dp = dp_lo; + + // This does two allocs for the scalar digits instead of + // putting them in a contiguous array, which makes handling + // the two kinds of lookup tables slightly easier. + // Use a ClearOnDrop wrapper. + + let static_scalar_digits_vec: Vec<_> = + static_scalars.map(|s| s.borrow().to_radix_16()).collect(); + let static_scalar_digits = ClearOnDrop::new(static_scalar_digits_vec); + + let dynamic_scalar_digits_vec: Vec<_> = + dynamic_scalars.map(|s| s.borrow().to_radix_16()).collect(); + let dynamic_scalar_digits = ClearOnDrop::new(dynamic_scalar_digits_vec); + + // Build lookup tables for dynamic points + let dynamic_lookup_tables: Vec<_> = dynamic_points + .map(|point| LookupTable::::from(point.borrow())) + .collect(); + + let mut R = ExtendedPoint::identity(); + for j in (0..64).rev() { + R = R.mul_by_pow_2(4); + for i in 0..dp { + let t_ij = dynamic_scalar_digits[i][j]; + R = &R + &dynamic_lookup_tables[i].select(t_ij); + } + for i in 0..sp { + let s_ij = static_scalar_digits[i][j]; + R = &R + &self.static_lookup_tables[i].select(s_ij); + } + } + + R.into() + } +} + +pub struct VartimePrecomputedStraus { + static_lookup_tables: Vec>, +} + +impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { + type Point = EdwardsPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self { + static_lookup_tables: static_points + .into_iter() + .map(|P| NafLookupTable8::::from(P.borrow())) + .collect(), + } + } + + fn vartime_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + let static_nafs = static_scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form(5)) + .collect::>(); + let dynamic_nafs: Vec<_> = dynamic_scalars + .into_iter() + .map(|c| c.borrow().non_adjacent_form(5)) + .collect::>(); + + let dynamic_lookup_tables = dynamic_points + .into_iter() + .map(|P| NafLookupTable5::::from(P.borrow())) + .collect::>(); + + let sp = self.static_lookup_tables.len(); + let dp = dynamic_lookup_tables.len(); + assert_eq!(sp, static_nafs.len()); + assert_eq!(dp, dynamic_nafs.len()); + + // We could save some doublings by looking for the highest + // nonzero NAF coefficient, but since we might have a lot of + // them to search, it's not clear it's worthwhile to check. + let mut R = ExtendedPoint::identity(); + for j in (0..255).rev() { + R = R.double(); + + for i in 0..dp { + let t_ij = dynamic_nafs[i][j]; + if t_ij > 0 { + R = &R + &dynamic_lookup_tables[i].select(t_ij as usize); + } else if t_ij < 0 { + R = &R - &dynamic_lookup_tables[i].select(-t_ij as usize); + } + } + + for i in 0..sp { + let t_ij = static_nafs[i][j]; + if t_ij > 0 { + R = &R + &self.static_lookup_tables[i].select(t_ij as usize); + } else if t_ij < 0 { + R = &R - &self.static_lookup_tables[i].select(-t_ij as usize); + } + } + } + + R.into() + } +} diff --git a/src/backend/vector/scalar_mul/straus.rs b/src/backend/vector/scalar_mul/straus.rs index 9fea5a8..506693d 100644 --- a/src/backend/vector/scalar_mul/straus.rs +++ b/src/backend/vector/scalar_mul/straus.rs @@ -33,7 +33,6 @@ use prelude::*; /// point representation on the fly. pub struct Straus {} -#[cfg(feature = "alloc")] impl MultiscalarMul for Straus { type Point = EdwardsPoint; @@ -71,7 +70,6 @@ impl MultiscalarMul for Straus { } } -#[cfg(feature = "alloc")] impl VartimeMultiscalarMul for Straus { type Point = EdwardsPoint; From e693d7f020fd4c7732f646ddb5693d805f8ae3c5 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 12 Feb 2019 11:38:39 -0800 Subject: [PATCH 43/52] fixup AVX2 ascii blocks so they don't run as doctests --- src/backend/vector/avx2/constants.rs | 8 ++++---- src/backend/vector/avx2/field.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/vector/avx2/constants.rs b/src/backend/vector/avx2/constants.rs index 19204e4..614286c 100644 --- a/src/backend/vector/avx2/constants.rs +++ b/src/backend/vector/avx2/constants.rs @@ -35,7 +35,7 @@ pub(crate) static CACHEDPOINT_IDENTITY: CachedPoint = CachedPoint(FieldElement26 ])); /// The low limbs of (2p, 2p, 2p, 2p), so that -/// ```no_run +/// ```ascii,no_run /// (2p, 2p, 2p, 2p) = [P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI] /// ``` pub(crate) static P_TIMES_2_LO: u32x8 = u32x8::new( @@ -50,7 +50,7 @@ pub(crate) static P_TIMES_2_LO: u32x8 = u32x8::new( ); /// The high limbs of (2p, 2p, 2p, 2p), so that -/// ```no_run +/// ```ascii,no_run /// (2p, 2p, 2p, 2p) = [P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI] /// ``` pub(crate) static P_TIMES_2_HI: u32x8 = u32x8::new( @@ -65,7 +65,7 @@ pub(crate) static P_TIMES_2_HI: u32x8 = u32x8::new( ); /// The low limbs of (16p, 16p, 16p, 16p), so that -/// ```no_run +/// ```ascii,no_run /// (16p, 16p, 16p, 16p) = [P_TIMES_16_LO, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI] /// ``` pub(crate) static P_TIMES_16_LO: u32x8 = u32x8::new( @@ -80,7 +80,7 @@ pub(crate) static P_TIMES_16_LO: u32x8 = u32x8::new( ); /// The high limbs of (16p, 16p, 16p, 16p), so that -/// ```no_run +/// ```ascii,no_run /// (16p, 16p, 16p, 16p) = [P_TIMES_16_LO, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI] /// ``` pub(crate) static P_TIMES_16_HI: u32x8 = u32x8::new( diff --git a/src/backend/vector/avx2/field.rs b/src/backend/vector/avx2/field.rs index 35cf21c..e7ee916 100644 --- a/src/backend/vector/avx2/field.rs +++ b/src/backend/vector/avx2/field.rs @@ -46,11 +46,11 @@ use backend::vector::avx2::constants::{P_TIMES_16_HI, P_TIMES_16_LO, P_TIMES_2_H use backend::serial::u64::field::FieldElement51; /// Unpack 32-bit lanes into 64-bit lanes: -/// ``` +/// ```ascii,no_run /// (a0, b0, a1, b1, c0, d0, c1, d1) /// ``` /// into -/// ``` +/// ```ascii,no_run /// (a0, 0, b0, 0, c0, 0, d0, 0) /// (a1, 0, b1, 0, c1, 0, d1, 0) /// ``` @@ -69,12 +69,12 @@ fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { } /// Repack 64-bit lanes into 32-bit lanes: -/// ``` +/// ```ascii,no_run /// (a0, 0, b0, 0, c0, 0, d0, 0) /// (a1, 0, b1, 0, c1, 0, d1, 0) /// ``` /// into -/// ``` +/// ```ascii,no_run /// (a0, b0, a1, b1, c0, d0, c1, d1) /// ``` #[inline(always)] From 727ba862924e702be95db7eb6ce59e8cb71b2df4 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 12 Feb 2019 12:28:24 -0800 Subject: [PATCH 44/52] Rework the vartime precomputation trait to be Option-al --- .../serial/scalar_mul/precomputed_straus.rs | 19 +-- .../vector/scalar_mul/precomputed_straus.rs | 19 +-- src/edwards.rs | 9 +- src/traits.rs | 124 +++++++++++++----- 4 files changed, 117 insertions(+), 54 deletions(-) diff --git a/src/backend/serial/scalar_mul/precomputed_straus.rs b/src/backend/serial/scalar_mul/precomputed_straus.rs index 989d92a..ceeb8b9 100644 --- a/src/backend/serial/scalar_mul/precomputed_straus.rs +++ b/src/backend/serial/scalar_mul/precomputed_straus.rs @@ -136,19 +136,18 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { } } - fn vartime_mixed_multiscalar_mul( + fn optional_mixed_multiscalar_mul( &self, static_scalars: I, dynamic_scalars: J, dynamic_points: K, - ) -> Self::Point + ) -> Option where I: IntoIterator, I::Item: Borrow, J: IntoIterator, J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow, + K: IntoIterator>, { let static_nafs = static_scalars .into_iter() @@ -159,10 +158,14 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { .map(|c| c.borrow().non_adjacent_form(5)) .collect::>(); - let dynamic_lookup_tables = dynamic_points + let dynamic_lookup_tables = match dynamic_points .into_iter() - .map(|P| NafLookupTable5::::from(P.borrow())) - .collect::>(); + .map(|P_opt| P_opt.map(|P| NafLookupTable5::::from(&P))) + .collect::>>() + { + Some(x) => x, + None => return None, + }; let sp = self.static_lookup_tables.len(); let dp = dynamic_lookup_tables.len(); @@ -197,6 +200,6 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { S = R.to_projective(); } - S.to_extended() + Some(S.to_extended()) } } diff --git a/src/backend/vector/scalar_mul/precomputed_straus.rs b/src/backend/vector/scalar_mul/precomputed_straus.rs index dc3fc3f..2ddfa24 100644 --- a/src/backend/vector/scalar_mul/precomputed_straus.rs +++ b/src/backend/vector/scalar_mul/precomputed_straus.rs @@ -134,19 +134,18 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { } } - fn vartime_mixed_multiscalar_mul( + fn optional_mixed_multiscalar_mul( &self, static_scalars: I, dynamic_scalars: J, dynamic_points: K, - ) -> Self::Point + ) -> Option where I: IntoIterator, I::Item: Borrow, J: IntoIterator, J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow, + K: IntoIterator>, { let static_nafs = static_scalars .into_iter() @@ -157,10 +156,14 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { .map(|c| c.borrow().non_adjacent_form(5)) .collect::>(); - let dynamic_lookup_tables = dynamic_points + let dynamic_lookup_tables = match dynamic_points .into_iter() - .map(|P| NafLookupTable5::::from(P.borrow())) - .collect::>(); + .map(|P_opt| P_opt.map(|P| NafLookupTable5::::from(&P))) + .collect::>>() + { + Some(x) => x, + None => return None, + }; let sp = self.static_lookup_tables.len(); let dp = dynamic_lookup_tables.len(); @@ -193,6 +196,6 @@ impl VartimePrecomputedMultiscalarMul for VartimePrecomputedStraus { } } - R.into() + Some(R.into()) } } diff --git a/src/edwards.rs b/src/edwards.rs index fa7029e..04c6565 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -732,22 +732,21 @@ impl VartimePrecomputedMultiscalarMul for VartimeEdwardsPrecomputation { Self(scalar_mul::precomputed_straus::VartimePrecomputedStraus::new(static_points)) } - fn vartime_mixed_multiscalar_mul( + fn optional_mixed_multiscalar_mul( &self, static_scalars: I, dynamic_scalars: J, dynamic_points: K, - ) -> Self::Point + ) -> Option where I: IntoIterator, I::Item: Borrow, J: IntoIterator, J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow, + K: IntoIterator>, { self.0 - .vartime_mixed_multiscalar_mul(static_scalars, dynamic_scalars, dynamic_points) + .optional_mixed_multiscalar_mul(static_scalars, dynamic_scalars, dynamic_points) } } diff --git a/src/traits.rs b/src/traits.rs index 630e510..c1166f5 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -219,8 +219,9 @@ pub trait VartimeMultiscalarMul { { Self::optional_multiscalar_mul( scalars, - points.into_iter().map(|P| Some(P.borrow().clone())) - ).unwrap() + points.into_iter().map(|P| Some(P.borrow().clone())), + ) + .unwrap() } } @@ -313,9 +314,26 @@ pub trait PrecomputedMultiscalarMul: Sized { /// where the \\(B_i\\) are *static* points, for which precomputation /// is possible, and the \\(A_j\\) are *dynamic* points, for which /// precomputation is not possible. +/// +/// This trait has three methods for performing this computation: +/// +/// * [`vartime_multiscalar_mul`], which handles the special case +/// where \\(n = 0\\) and there are no dynamic points; +/// +/// * [`vartime_mixed_multiscalar_mul`], which takes the dynamic +/// points as already-validated `Point`s and is infallible; +/// +/// * [`optional_mixed_multiscalar_mul`], which takes the dynamic +/// points as `Option`s and returns an `Option`, +/// allowing decompression to be composed into the input iterators. +/// +/// All methods require that the lengths of the input iterators be +/// known and matching, as if they were `ExactSizeIterator`s. (It +/// does not require `ExactSizeIterator` only because that trait is +/// broken). pub trait VartimePrecomputedMultiscalarMul: Sized { /// The type of point to be multiplied, e.g., `RistrettoPoint`. - type Point; + type Point: Clone; /// Given the static points \\( B_i \\), perform precomputation /// and return the precomputation data. @@ -324,36 +342,6 @@ pub trait VartimePrecomputedMultiscalarMul: Sized { I: IntoIterator, I::Item: Borrow; - /// Given `static_scalars`, an iterator of public scalars - /// \\(b_i\\), `dynamic_scalars`, an iterator of public scalars - /// \\(a_i\\), and `dynamic_points`, an iterator of points - /// \\(A_i\\), compute - /// $$ - /// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, - /// $$ - /// where the \\(B_j\\) are the points that were supplied to `new`. - /// - /// It is an error to call this function with iterators of - /// inconsistent lengths. - /// - /// The trait bound aims for maximum flexibility: the inputs must be - /// convertable to iterators (`I: IntoIter`), and the iterator's items - /// must be `Borrow` (or `Borrow`), to allow - /// iterators returning either `Scalar`s or `&Scalar`s. - fn vartime_mixed_multiscalar_mul( - &self, - static_scalars: I, - dynamic_scalars: J, - dynamic_points: K, - ) -> Self::Point - where - I: IntoIterator, - I::Item: Borrow, - J: IntoIterator, - J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow; - /// Given `static_scalars`, an iterator of public scalars /// \\(b_i\\), compute /// $$ @@ -382,6 +370,76 @@ pub trait VartimePrecomputedMultiscalarMul: Sized { iter::empty::(), ) } + + /// Given `static_scalars`, an iterator of public scalars + /// \\(b_i\\), `dynamic_scalars`, an iterator of public scalars + /// \\(a_i\\), and `dynamic_points`, an iterator of points + /// \\(A_i\\), compute + /// $$ + /// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// The trait bound aims for maximum flexibility: the inputs must be + /// convertable to iterators (`I: IntoIter`), and the iterator's items + /// must be `Borrow` (or `Borrow`), to allow + /// iterators returning either `Scalar`s or `&Scalar`s. + fn vartime_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + Self::optional_mixed_multiscalar_mul( + self, + static_scalars, + dynamic_scalars, + dynamic_points.into_iter().map(|P| Some(P.borrow().clone())), + ) + .unwrap() + } + + /// Given `static_scalars`, an iterator of public scalars + /// \\(b_i\\), `dynamic_scalars`, an iterator of public scalars + /// \\(a_i\\), and `dynamic_points`, an iterator of points + /// \\(A_i\\), compute + /// $$ + /// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, + /// $$ + /// where the \\(B_j\\) are the points that were supplied to `new`. + /// + /// If any of the dynamic points were `None`, return `None`. + /// + /// It is an error to call this function with iterators of + /// inconsistent lengths. + /// + /// This function is particularly useful when verifying statements + /// involving compressed points. Accepting `Option` allows + /// inlining point decompression into the multiscalar call, + /// avoiding the need for temporary buffers. + fn optional_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Option + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator>; } // ------------------------------------------------------------------------ From 8adcfb7fa341a02b029b12cf8d1789ebbbaf1e3d Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 12 Feb 2019 12:35:01 -0800 Subject: [PATCH 45/52] Simplify length checking. --- .../serial/scalar_mul/precomputed_straus.rs | 38 +++++++------------ .../vector/scalar_mul/precomputed_straus.rs | 38 +++++++------------ 2 files changed, 28 insertions(+), 48 deletions(-) diff --git a/src/backend/serial/scalar_mul/precomputed_straus.rs b/src/backend/serial/scalar_mul/precomputed_straus.rs index ceeb8b9..905b168 100644 --- a/src/backend/serial/scalar_mul/precomputed_straus.rs +++ b/src/backend/serial/scalar_mul/precomputed_straus.rs @@ -61,44 +61,34 @@ impl PrecomputedMultiscalarMul for PrecomputedStraus { K: IntoIterator, K::Item: Borrow, { - let mut static_scalars = static_scalars.into_iter(); - let mut dynamic_scalars = dynamic_scalars.into_iter(); - let mut dynamic_points = dynamic_points.into_iter(); - - // Check that the input lengths are consistent with each other: - let (ss_lo, ss_hi) = static_scalars.by_ref().size_hint(); - let (ds_lo, ds_hi) = dynamic_scalars.by_ref().size_hint(); - let (dp_lo, dp_hi) = dynamic_points.by_ref().size_hint(); - - // Static points match static scalars - let sp = self.static_lookup_tables.len(); - assert_eq!(ss_lo, sp); - assert_eq!(ss_hi, Some(sp)); - - // Dynamic points match dynamic scalars - assert_eq!(ds_lo, dp_lo); - assert_eq!(ds_hi, Some(ds_lo)); - assert_eq!(ds_hi, dp_hi); - let dp = dp_lo; - // This does two allocs for the scalar digits instead of // putting them in a contiguous array, which makes handling // the two kinds of lookup tables slightly easier. // Use a ClearOnDrop wrapper. - let static_scalar_digits_vec: Vec<_> = - static_scalars.map(|s| s.borrow().to_radix_16()).collect(); + let static_scalar_digits_vec: Vec<_> = static_scalars + .into_iter() + .map(|s| s.borrow().to_radix_16()) + .collect(); let static_scalar_digits = ClearOnDrop::new(static_scalar_digits_vec); - let dynamic_scalar_digits_vec: Vec<_> = - dynamic_scalars.map(|s| s.borrow().to_radix_16()).collect(); + let dynamic_scalar_digits_vec: Vec<_> = dynamic_scalars + .into_iter() + .map(|s| s.borrow().to_radix_16()) + .collect(); let dynamic_scalar_digits = ClearOnDrop::new(dynamic_scalar_digits_vec); // Build lookup tables for dynamic points let dynamic_lookup_tables: Vec<_> = dynamic_points + .into_iter() .map(|point| LookupTable::::from(point.borrow())) .collect(); + let sp = self.static_lookup_tables.len(); + let dp = dynamic_lookup_tables.len(); + assert_eq!(sp, static_scalar_digits.len()); + assert_eq!(dp, dynamic_scalar_digits.len()); + let mut R = EdwardsPoint::identity(); for j in (0..64).rev() { R = R.mul_by_pow_2(4); diff --git a/src/backend/vector/scalar_mul/precomputed_straus.rs b/src/backend/vector/scalar_mul/precomputed_straus.rs index 2ddfa24..9ca767f 100644 --- a/src/backend/vector/scalar_mul/precomputed_straus.rs +++ b/src/backend/vector/scalar_mul/precomputed_straus.rs @@ -59,44 +59,34 @@ impl PrecomputedMultiscalarMul for PrecomputedStraus { K: IntoIterator, K::Item: Borrow, { - let mut static_scalars = static_scalars.into_iter(); - let mut dynamic_scalars = dynamic_scalars.into_iter(); - let mut dynamic_points = dynamic_points.into_iter(); - - // Check that the input lengths are consistent with each other: - let (ss_lo, ss_hi) = static_scalars.by_ref().size_hint(); - let (ds_lo, ds_hi) = dynamic_scalars.by_ref().size_hint(); - let (dp_lo, dp_hi) = dynamic_points.by_ref().size_hint(); - - // Static points match static scalars - let sp = self.static_lookup_tables.len(); - assert_eq!(ss_lo, sp); - assert_eq!(ss_hi, Some(sp)); - - // Dynamic points match dynamic scalars - assert_eq!(ds_lo, dp_lo); - assert_eq!(ds_hi, Some(ds_lo)); - assert_eq!(ds_hi, dp_hi); - let dp = dp_lo; - // This does two allocs for the scalar digits instead of // putting them in a contiguous array, which makes handling // the two kinds of lookup tables slightly easier. // Use a ClearOnDrop wrapper. - let static_scalar_digits_vec: Vec<_> = - static_scalars.map(|s| s.borrow().to_radix_16()).collect(); + let static_scalar_digits_vec: Vec<_> = static_scalars + .into_iter() + .map(|s| s.borrow().to_radix_16()) + .collect(); let static_scalar_digits = ClearOnDrop::new(static_scalar_digits_vec); - let dynamic_scalar_digits_vec: Vec<_> = - dynamic_scalars.map(|s| s.borrow().to_radix_16()).collect(); + let dynamic_scalar_digits_vec: Vec<_> = dynamic_scalars + .into_iter() + .map(|s| s.borrow().to_radix_16()) + .collect(); let dynamic_scalar_digits = ClearOnDrop::new(dynamic_scalar_digits_vec); // Build lookup tables for dynamic points let dynamic_lookup_tables: Vec<_> = dynamic_points + .into_iter() .map(|point| LookupTable::::from(point.borrow())) .collect(); + let sp = self.static_lookup_tables.len(); + let dp = dynamic_lookup_tables.len(); + assert_eq!(sp, static_scalar_digits.len()); + assert_eq!(dp, dynamic_scalar_digits.len()); + let mut R = ExtendedPoint::identity(); for j in (0..64).rev() { R = R.mul_by_pow_2(4); From 27daa5215e7baacf934b2b80ebd4a6c50eb71cf8 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 12 Feb 2019 12:45:56 -0800 Subject: [PATCH 46/52] Add Ristretto precomputation facade. --- src/edwards.rs | 10 +-- src/ristretto.rs | 194 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 195 insertions(+), 9 deletions(-) diff --git a/src/edwards.rs b/src/edwards.rs index 04c6565..3ab3e60 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -677,17 +677,17 @@ impl VartimeMultiscalarMul for EdwardsPoint { // This wraps the inner implementation in a facade type so that we can // decouple stability of the inner type from the stability of the // outer type. -#[cfg(all(feature = "alloc", feature = "yolocrypto"))] +#[cfg(feature = "alloc")] pub struct EdwardsPrecomputation(scalar_mul::precomputed_straus::PrecomputedStraus); /// Precomputation for variable-time multiscalar multiplication with `EdwardsPoint`s. // This wraps the inner implementation in a facade type so that we can // decouple stability of the inner type from the stability of the // outer type. -#[cfg(all(feature = "alloc", feature = "yolocrypto"))] +#[cfg(feature = "alloc")] pub struct VartimeEdwardsPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus); -#[cfg(all(feature = "alloc", feature = "yolocrypto"))] +#[cfg(feature = "alloc")] impl PrecomputedMultiscalarMul for EdwardsPrecomputation { type Point = EdwardsPoint; @@ -720,7 +720,7 @@ impl PrecomputedMultiscalarMul for EdwardsPrecomputation { } } -#[cfg(all(feature = "alloc", feature = "yolocrypto"))] +#[cfg(feature = "alloc")] impl VartimePrecomputedMultiscalarMul for VartimeEdwardsPrecomputation { type Point = EdwardsPoint; @@ -1281,7 +1281,6 @@ mod test { } #[test] - #[cfg(feature = "yolocrypto")] fn precomputed_vs_nonprecomputed_multiscalar() { let mut rng = rand::thread_rng(); @@ -1325,7 +1324,6 @@ mod test { } #[test] - #[cfg(feature = "yolocrypto")] fn vartime_precomputed_vs_nonprecomputed_multiscalar() { let mut rng = rand::thread_rng(); diff --git a/src/ristretto.rs b/src/ristretto.rs index 2daabaf..34e3c33 100644 --- a/src/ristretto.rs +++ b/src/ristretto.rs @@ -187,7 +187,21 @@ use scalar::Scalar; use traits::Identity; #[cfg(any(feature = "alloc", feature = "std"))] -use traits::{MultiscalarMul, VartimeMultiscalarMul}; +use traits::{ + MultiscalarMul, PrecomputedMultiscalarMul, VartimeMultiscalarMul, + VartimePrecomputedMultiscalarMul, +}; + +#[cfg(not(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +)))] +use backend::serial::scalar_mul; +#[cfg(all( + feature = "simd_backend", + any(target_feature = "avx2", target_feature = "avx512ifma") +))] +use backend::vector::scalar_mul; // ------------------------------------------------------------------------ // Compressed points @@ -891,8 +905,96 @@ impl VartimeMultiscalarMul for RistrettoPoint { { let extended_points = points.into_iter().map(|opt_P| opt_P.map(|P| P.borrow().0)); - EdwardsPoint::optional_multiscalar_mul(scalars, extended_points) - .map(|P| RistrettoPoint(P)) + EdwardsPoint::optional_multiscalar_mul(scalars, extended_points).map(|P| RistrettoPoint(P)) + } +} + +/// Precomputation for multiscalar multiplication with `RistrettoPoint`s. +// This wraps the inner implementation in a facade type so that we can +// decouple stability of the inner type from the stability of the +// outer type. +#[cfg(feature = "alloc")] +pub struct RistrettoPrecomputation(scalar_mul::precomputed_straus::PrecomputedStraus); + +/// Precomputation for variable-time multiscalar multiplication with `RistrettoPoint`s. +// This wraps the inner implementation in a facade type so that we can +// decouple stability of the inner type from the stability of the +// outer type. +#[cfg(feature = "alloc")] +pub struct VartimeRistrettoPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus); + +#[cfg(feature = "alloc")] +impl PrecomputedMultiscalarMul for RistrettoPrecomputation { + type Point = RistrettoPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self(scalar_mul::precomputed_straus::PrecomputedStraus::new( + static_points.into_iter().map(|P| P.borrow().0), + )) + } + + fn mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Self::Point + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator, + K::Item: Borrow, + { + RistrettoPoint(self.0.mixed_multiscalar_mul( + static_scalars, + dynamic_scalars, + dynamic_points.into_iter().map(|P| P.borrow().0), + )) + } +} + +#[cfg(feature = "alloc")] +impl VartimePrecomputedMultiscalarMul for VartimeRistrettoPrecomputation { + type Point = RistrettoPoint; + + fn new(static_points: I) -> Self + where + I: IntoIterator, + I::Item: Borrow, + { + Self( + scalar_mul::precomputed_straus::VartimePrecomputedStraus::new( + static_points.into_iter().map(|P| P.borrow().0), + ), + ) + } + + fn optional_mixed_multiscalar_mul( + &self, + static_scalars: I, + dynamic_scalars: J, + dynamic_points: K, + ) -> Option + where + I: IntoIterator, + I::Item: Borrow, + J: IntoIterator, + J::Item: Borrow, + K: IntoIterator>, + { + self.0 + .optional_mixed_multiscalar_mul( + static_scalars, + dynamic_scalars, + dynamic_points.into_iter().map(|P_opt| P_opt.map(|P| P.0)), + ) + .map(|P_ed| RistrettoPoint(P_ed)) } } @@ -1263,4 +1365,90 @@ mod test { P.compress(); } } + + #[test] + fn precomputed_vs_nonprecomputed_multiscalar() { + let mut rng = rand::thread_rng(); + + let B = &::constants::RISTRETTO_BASEPOINT_TABLE; + + let static_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let dynamic_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let check_scalar: Scalar = static_scalars + .iter() + .chain(dynamic_scalars.iter()) + .map(|s| s * s) + .sum(); + + let static_points = static_scalars.iter().map(|s| s * B).collect::>(); + let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::>(); + + let precomputation = RistrettoPrecomputation::new(static_points.iter()); + + let P = precomputation.mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ); + + use traits::MultiscalarMul; + let Q = RistrettoPoint::multiscalar_mul( + static_scalars.iter().chain(dynamic_scalars.iter()), + static_points.iter().chain(dynamic_points.iter()), + ); + + let R = &check_scalar * B; + + assert_eq!(P.compress(), R.compress()); + assert_eq!(Q.compress(), R.compress()); + } + + #[test] + fn vartime_precomputed_vs_nonprecomputed_multiscalar() { + let mut rng = rand::thread_rng(); + + let B = &::constants::RISTRETTO_BASEPOINT_TABLE; + + let static_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let dynamic_scalars = (0..128) + .map(|_| Scalar::random(&mut rng)) + .collect::>(); + + let check_scalar: Scalar = static_scalars + .iter() + .chain(dynamic_scalars.iter()) + .map(|s| s * s) + .sum(); + + let static_points = static_scalars.iter().map(|s| s * B).collect::>(); + let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::>(); + + let precomputation = VartimeRistrettoPrecomputation::new(static_points.iter()); + + let P = precomputation.vartime_mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ); + + use traits::VartimeMultiscalarMul; + let Q = RistrettoPoint::vartime_multiscalar_mul( + static_scalars.iter().chain(dynamic_scalars.iter()), + static_points.iter().chain(dynamic_points.iter()), + ); + + let R = &check_scalar * B; + + assert_eq!(P.compress(), R.compress()); + assert_eq!(Q.compress(), R.compress()); + } } From 96df60895f0bc4f390fc50141327e012a248455b Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 12 Feb 2019 19:30:18 -0800 Subject: [PATCH 47/52] Fix comment on Montgomery conversion. Closes #213. --- src/edwards.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/edwards.rs b/src/edwards.rs index 102a5ea..3b65e25 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -450,6 +450,10 @@ impl EdwardsPoint { /// Convert this `EdwardsPoint` on the Edwards model to the /// corresponding `MontgomeryPoint` on the Montgomery model. /// + /// This function has one exceptional case; the identity point of + /// the Edwards curve is sent to the 2-torsion point \\((0,0)\\) + /// on the Montgomery curve. + /// /// Note that this is a one-way conversion, since the Montgomery /// model does not retain sign information. pub fn to_montgomery(&self) -> MontgomeryPoint { @@ -457,7 +461,7 @@ impl EdwardsPoint { // // The denominator is zero only when y=1, the identity point of // the Edwards curve. Since 0.invert() = 0, in this case we - // compute u = 0, the identity point of the Montgomery line. + // compute the 2-torsion point (0,0). let U = &self.Z + &self.Y; let W = &self.Z - &self.Y; let u = &U * &W.invert(); From 092ff52cb0927696ccda4100203446c2374e1d3a Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 13 Feb 2019 11:10:47 -0800 Subject: [PATCH 48/52] Remove constant-time multiscalar precomputation. This doesn't (yet) give any speedup over the non-precomputed multiscalar multiplication, and it's not clear that it's a good idea to commit to supporting it in the future. Removing it means that it's not committed-to as part of the public API, but the source is still there in the tree if we want to revisit it later. --- benches/dalek_benchmarks.rs | 46 ---------- .../serial/scalar_mul/precomputed_straus.rs | 85 +---------------- .../vector/scalar_mul/precomputed_straus.rs | 84 +---------------- src/edwards.rs | 85 +---------------- src/ristretto.rs | 91 +------------------ src/traits.rs | 80 ---------------- 6 files changed, 6 insertions(+), 465 deletions(-) diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs index 118a809..e2e3f5b 100644 --- a/benches/dalek_benchmarks.rs +++ b/benches/dalek_benchmarks.rs @@ -110,37 +110,6 @@ mod multiscalar_benches { ); } - fn precomputed_ct_straus_helper(c: &mut Criterion, dynamic_fraction: f64) { - let label = format!( - "Constant-time mixed-base Straus ({:.2}pct dyn)", - 100.0*dynamic_fraction, - ); - c.bench_function_over_inputs( - &label, - move |b, &&total_size| { - let dynamic_size = ((total_size as f64) * dynamic_fraction) as usize; - let static_size = total_size - dynamic_size; - - let (static_scalars, static_points) = construct(static_size); - let (dynamic_scalars, dynamic_points) = construct(dynamic_size); - - use curve25519_dalek::edwards::EdwardsPrecomputation; - use curve25519_dalek::traits::PrecomputedMultiscalarMul; - - let precomp = EdwardsPrecomputation::new(&static_points); - - b.iter(|| { - precomp.mixed_multiscalar_mul( - &static_scalars, - &dynamic_scalars, - &dynamic_points, - ) - }); - }, - &MULTISCALAR_SIZES, - ); - } - fn precomputed_vt_straus_helper(c: &mut Criterion, dynamic_fraction: f64) { let label = format!( "Variable-time mixed-base Straus ({:.2}pct dyn)", @@ -172,18 +141,6 @@ mod multiscalar_benches { ); } - fn precomputed_ct_straus_00_pct_dynamic(c: &mut Criterion) { - precomputed_ct_straus_helper(c, 0.0); - } - - fn precomputed_ct_straus_20_pct_dynamic(c: &mut Criterion) { - precomputed_ct_straus_helper(c, 0.2); - } - - fn precomputed_ct_straus_50_pct_dynamic(c: &mut Criterion) { - precomputed_ct_straus_helper(c, 0.5); - } - fn precomputed_vt_straus_00_pct_dynamic(c: &mut Criterion) { precomputed_vt_straus_helper(c, 0.0); } @@ -203,9 +160,6 @@ mod multiscalar_benches { targets = consttime_multiscalar_mul, vartime_multiscalar_mul, - precomputed_ct_straus_00_pct_dynamic, - precomputed_ct_straus_20_pct_dynamic, - precomputed_ct_straus_50_pct_dynamic, precomputed_vt_straus_00_pct_dynamic, precomputed_vt_straus_20_pct_dynamic, precomputed_vt_straus_50_pct_dynamic, diff --git a/src/backend/serial/scalar_mul/precomputed_straus.rs b/src/backend/serial/scalar_mul/precomputed_straus.rs index 905b168..4019b14 100644 --- a/src/backend/serial/scalar_mul/precomputed_straus.rs +++ b/src/backend/serial/scalar_mul/precomputed_straus.rs @@ -13,99 +13,18 @@ use core::borrow::Borrow; -use clear_on_drop::ClearOnDrop; - use backend::serial::curve_models::{ AffineNielsPoint, CompletedPoint, ProjectiveNielsPoint, ProjectivePoint, }; use edwards::EdwardsPoint; use scalar::Scalar; use traits::Identity; -use traits::{PrecomputedMultiscalarMul, VartimePrecomputedMultiscalarMul}; -use window::{LookupTable, NafLookupTable5, NafLookupTable8}; +use traits::VartimePrecomputedMultiscalarMul; +use window::{NafLookupTable5, NafLookupTable8}; #[allow(unused_imports)] use prelude::*; -pub struct PrecomputedStraus { - static_lookup_tables: Vec>, -} - -impl PrecomputedMultiscalarMul for PrecomputedStraus { - type Point = EdwardsPoint; - - fn new(static_points: I) -> Self - where - I: IntoIterator, - I::Item: Borrow, - { - PrecomputedStraus { - static_lookup_tables: static_points - .into_iter() - .map(|point| LookupTable::::from(point.borrow())) - .collect(), - } - } - - fn mixed_multiscalar_mul( - &self, - static_scalars: I, - dynamic_scalars: J, - dynamic_points: K, - ) -> Self::Point - where - I: IntoIterator, - I::Item: Borrow, - J: IntoIterator, - J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow, - { - // This does two allocs for the scalar digits instead of - // putting them in a contiguous array, which makes handling - // the two kinds of lookup tables slightly easier. - // Use a ClearOnDrop wrapper. - - let static_scalar_digits_vec: Vec<_> = static_scalars - .into_iter() - .map(|s| s.borrow().to_radix_16()) - .collect(); - let static_scalar_digits = ClearOnDrop::new(static_scalar_digits_vec); - - let dynamic_scalar_digits_vec: Vec<_> = dynamic_scalars - .into_iter() - .map(|s| s.borrow().to_radix_16()) - .collect(); - let dynamic_scalar_digits = ClearOnDrop::new(dynamic_scalar_digits_vec); - - // Build lookup tables for dynamic points - let dynamic_lookup_tables: Vec<_> = dynamic_points - .into_iter() - .map(|point| LookupTable::::from(point.borrow())) - .collect(); - - let sp = self.static_lookup_tables.len(); - let dp = dynamic_lookup_tables.len(); - assert_eq!(sp, static_scalar_digits.len()); - assert_eq!(dp, dynamic_scalar_digits.len()); - - let mut R = EdwardsPoint::identity(); - for j in (0..64).rev() { - R = R.mul_by_pow_2(4); - for i in 0..dp { - let t_ij = dynamic_scalar_digits[i][j]; - R = (&R + &dynamic_lookup_tables[i].select(t_ij)).to_extended(); - } - for i in 0..sp { - let s_ij = static_scalar_digits[i][j]; - R = (&R + &self.static_lookup_tables[i].select(s_ij)).to_extended(); - } - } - - R - } -} - pub struct VartimePrecomputedStraus { static_lookup_tables: Vec>, } diff --git a/src/backend/vector/scalar_mul/precomputed_straus.rs b/src/backend/vector/scalar_mul/precomputed_straus.rs index 9ca767f..49d1be4 100644 --- a/src/backend/vector/scalar_mul/precomputed_straus.rs +++ b/src/backend/vector/scalar_mul/precomputed_straus.rs @@ -13,96 +13,16 @@ use core::borrow::Borrow; -use clear_on_drop::ClearOnDrop; - use backend::vector::{CachedPoint, ExtendedPoint}; use edwards::EdwardsPoint; use scalar::Scalar; use traits::Identity; -use traits::{PrecomputedMultiscalarMul, VartimePrecomputedMultiscalarMul}; -use window::{LookupTable, NafLookupTable5, NafLookupTable8}; +use traits::VartimePrecomputedMultiscalarMul; +use window::{NafLookupTable5, NafLookupTable8}; #[allow(unused_imports)] use prelude::*; -pub struct PrecomputedStraus { - static_lookup_tables: Vec>, -} - -impl PrecomputedMultiscalarMul for PrecomputedStraus { - type Point = EdwardsPoint; - - fn new(static_points: I) -> Self - where - I: IntoIterator, - I::Item: Borrow, - { - PrecomputedStraus { - static_lookup_tables: static_points - .into_iter() - .map(|point| LookupTable::::from(point.borrow())) - .collect(), - } - } - - fn mixed_multiscalar_mul( - &self, - static_scalars: I, - dynamic_scalars: J, - dynamic_points: K, - ) -> Self::Point - where - I: IntoIterator, - I::Item: Borrow, - J: IntoIterator, - J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow, - { - // This does two allocs for the scalar digits instead of - // putting them in a contiguous array, which makes handling - // the two kinds of lookup tables slightly easier. - // Use a ClearOnDrop wrapper. - - let static_scalar_digits_vec: Vec<_> = static_scalars - .into_iter() - .map(|s| s.borrow().to_radix_16()) - .collect(); - let static_scalar_digits = ClearOnDrop::new(static_scalar_digits_vec); - - let dynamic_scalar_digits_vec: Vec<_> = dynamic_scalars - .into_iter() - .map(|s| s.borrow().to_radix_16()) - .collect(); - let dynamic_scalar_digits = ClearOnDrop::new(dynamic_scalar_digits_vec); - - // Build lookup tables for dynamic points - let dynamic_lookup_tables: Vec<_> = dynamic_points - .into_iter() - .map(|point| LookupTable::::from(point.borrow())) - .collect(); - - let sp = self.static_lookup_tables.len(); - let dp = dynamic_lookup_tables.len(); - assert_eq!(sp, static_scalar_digits.len()); - assert_eq!(dp, dynamic_scalar_digits.len()); - - let mut R = ExtendedPoint::identity(); - for j in (0..64).rev() { - R = R.mul_by_pow_2(4); - for i in 0..dp { - let t_ij = dynamic_scalar_digits[i][j]; - R = &R + &dynamic_lookup_tables[i].select(t_ij); - } - for i in 0..sp { - let s_ij = static_scalar_digits[i][j]; - R = &R + &self.static_lookup_tables[i].select(s_ij); - } - } - - R.into() - } -} pub struct VartimePrecomputedStraus { static_lookup_tables: Vec>, diff --git a/src/edwards.rs b/src/edwards.rs index 3ab3e60..06f53ea 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -126,7 +126,7 @@ use traits::ValidityCheck; use traits::{Identity, IsIdentity}; #[cfg(any(feature = "alloc", feature = "std"))] -use traits::{MultiscalarMul, PrecomputedMultiscalarMul}; +use traits::MultiscalarMul; #[cfg(any(feature = "alloc", feature = "std"))] use traits::{VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul}; @@ -673,13 +673,6 @@ impl VartimeMultiscalarMul for EdwardsPoint { } } -/// Precomputation for multiscalar multiplication with `EdwardsPoint`s. -// This wraps the inner implementation in a facade type so that we can -// decouple stability of the inner type from the stability of the -// outer type. -#[cfg(feature = "alloc")] -pub struct EdwardsPrecomputation(scalar_mul::precomputed_straus::PrecomputedStraus); - /// Precomputation for variable-time multiscalar multiplication with `EdwardsPoint`s. // This wraps the inner implementation in a facade type so that we can // decouple stability of the inner type from the stability of the @@ -687,39 +680,6 @@ pub struct EdwardsPrecomputation(scalar_mul::precomputed_straus::PrecomputedStra #[cfg(feature = "alloc")] pub struct VartimeEdwardsPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus); -#[cfg(feature = "alloc")] -impl PrecomputedMultiscalarMul for EdwardsPrecomputation { - type Point = EdwardsPoint; - - fn new(static_points: I) -> Self - where - I: IntoIterator, - I::Item: Borrow, - { - Self(scalar_mul::precomputed_straus::PrecomputedStraus::new( - static_points, - )) - } - - fn mixed_multiscalar_mul( - &self, - static_scalars: I, - dynamic_scalars: J, - dynamic_points: K, - ) -> Self::Point - where - I: IntoIterator, - I::Item: Borrow, - J: IntoIterator, - J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow, - { - self.0 - .mixed_multiscalar_mul(static_scalars, dynamic_scalars, dynamic_points) - } -} - #[cfg(feature = "alloc")] impl VartimePrecomputedMultiscalarMul for VartimeEdwardsPrecomputation { type Point = EdwardsPoint; @@ -1280,49 +1240,6 @@ mod test { assert!(P1.compress().to_bytes() == P2.compress().to_bytes()); } - #[test] - fn precomputed_vs_nonprecomputed_multiscalar() { - let mut rng = rand::thread_rng(); - - let B = &::constants::ED25519_BASEPOINT_TABLE; - - let static_scalars = (0..128) - .map(|_| Scalar::random(&mut rng)) - .collect::>(); - - let dynamic_scalars = (0..128) - .map(|_| Scalar::random(&mut rng)) - .collect::>(); - - let check_scalar: Scalar = static_scalars - .iter() - .chain(dynamic_scalars.iter()) - .map(|s| s * s) - .sum(); - - let static_points = static_scalars.iter().map(|s| s * B).collect::>(); - let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::>(); - - let precomputation = EdwardsPrecomputation::new(static_points.iter()); - - let P = precomputation.mixed_multiscalar_mul( - &static_scalars, - &dynamic_scalars, - &dynamic_points, - ); - - use traits::MultiscalarMul; - let Q = EdwardsPoint::multiscalar_mul( - static_scalars.iter().chain(dynamic_scalars.iter()), - static_points.iter().chain(dynamic_points.iter()), - ); - - let R = &check_scalar * B; - - assert_eq!(P.compress(), R.compress()); - assert_eq!(Q.compress(), R.compress()); - } - #[test] fn vartime_precomputed_vs_nonprecomputed_multiscalar() { let mut rng = rand::thread_rng(); diff --git a/src/ristretto.rs b/src/ristretto.rs index 34e3c33..3851949 100644 --- a/src/ristretto.rs +++ b/src/ristretto.rs @@ -187,10 +187,7 @@ use scalar::Scalar; use traits::Identity; #[cfg(any(feature = "alloc", feature = "std"))] -use traits::{ - MultiscalarMul, PrecomputedMultiscalarMul, VartimeMultiscalarMul, - VartimePrecomputedMultiscalarMul, -}; +use traits::{MultiscalarMul, VartimeMultiscalarMul, VartimePrecomputedMultiscalarMul}; #[cfg(not(all( feature = "simd_backend", @@ -909,13 +906,6 @@ impl VartimeMultiscalarMul for RistrettoPoint { } } -/// Precomputation for multiscalar multiplication with `RistrettoPoint`s. -// This wraps the inner implementation in a facade type so that we can -// decouple stability of the inner type from the stability of the -// outer type. -#[cfg(feature = "alloc")] -pub struct RistrettoPrecomputation(scalar_mul::precomputed_straus::PrecomputedStraus); - /// Precomputation for variable-time multiscalar multiplication with `RistrettoPoint`s. // This wraps the inner implementation in a facade type so that we can // decouple stability of the inner type from the stability of the @@ -923,42 +913,6 @@ pub struct RistrettoPrecomputation(scalar_mul::precomputed_straus::PrecomputedSt #[cfg(feature = "alloc")] pub struct VartimeRistrettoPrecomputation(scalar_mul::precomputed_straus::VartimePrecomputedStraus); -#[cfg(feature = "alloc")] -impl PrecomputedMultiscalarMul for RistrettoPrecomputation { - type Point = RistrettoPoint; - - fn new(static_points: I) -> Self - where - I: IntoIterator, - I::Item: Borrow, - { - Self(scalar_mul::precomputed_straus::PrecomputedStraus::new( - static_points.into_iter().map(|P| P.borrow().0), - )) - } - - fn mixed_multiscalar_mul( - &self, - static_scalars: I, - dynamic_scalars: J, - dynamic_points: K, - ) -> Self::Point - where - I: IntoIterator, - I::Item: Borrow, - J: IntoIterator, - J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow, - { - RistrettoPoint(self.0.mixed_multiscalar_mul( - static_scalars, - dynamic_scalars, - dynamic_points.into_iter().map(|P| P.borrow().0), - )) - } -} - #[cfg(feature = "alloc")] impl VartimePrecomputedMultiscalarMul for VartimeRistrettoPrecomputation { type Point = RistrettoPoint; @@ -1366,49 +1320,6 @@ mod test { } } - #[test] - fn precomputed_vs_nonprecomputed_multiscalar() { - let mut rng = rand::thread_rng(); - - let B = &::constants::RISTRETTO_BASEPOINT_TABLE; - - let static_scalars = (0..128) - .map(|_| Scalar::random(&mut rng)) - .collect::>(); - - let dynamic_scalars = (0..128) - .map(|_| Scalar::random(&mut rng)) - .collect::>(); - - let check_scalar: Scalar = static_scalars - .iter() - .chain(dynamic_scalars.iter()) - .map(|s| s * s) - .sum(); - - let static_points = static_scalars.iter().map(|s| s * B).collect::>(); - let dynamic_points = dynamic_scalars.iter().map(|s| s * B).collect::>(); - - let precomputation = RistrettoPrecomputation::new(static_points.iter()); - - let P = precomputation.mixed_multiscalar_mul( - &static_scalars, - &dynamic_scalars, - &dynamic_points, - ); - - use traits::MultiscalarMul; - let Q = RistrettoPoint::multiscalar_mul( - static_scalars.iter().chain(dynamic_scalars.iter()), - static_points.iter().chain(dynamic_points.iter()), - ); - - let R = &check_scalar * B; - - assert_eq!(P.compress(), R.compress()); - assert_eq!(Q.compress(), R.compress()); - } - #[test] fn vartime_precomputed_vs_nonprecomputed_multiscalar() { let mut rng = rand::thread_rng(); diff --git a/src/traits.rs b/src/traits.rs index c1166f5..5297aa4 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -225,86 +225,6 @@ pub trait VartimeMultiscalarMul { } } -/// A trait for constant-time multiscalar multiplication with precomputation. -/// -/// A general multiscalar multiplication with precomputation can be written as -/// $$ -/// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, -/// $$ -/// where the \\(B_i\\) are *static* points, for which precomputation -/// is possible, and the \\(A_j\\) are *dynamic* points, for which -/// precomputation is not possible. -pub trait PrecomputedMultiscalarMul: Sized { - /// The type of point to be multiplied, e.g., `RistrettoPoint`. - type Point; - - /// Given the static points \\( B_i \\), perform precomputation - /// and return the precomputation data. - fn new(static_points: I) -> Self - where - I: IntoIterator, - I::Item: Borrow; - - /// Given `static_scalars`, an iterator of (possibly secret) - /// scalars \\(b_i\\), `dynamic_scalars`, an iterator of (possibly - /// secret) scalars \\(a_i\\), and `dynamic_points`, an iterator - /// of points \\(A_i\\), compute - /// $$ - /// Q = a_1 A_1 + \cdots + a_n A_n + b_1 B_1 + \cdots + b_m B_m, - /// $$ - /// where the \\(B_j\\) are the points that were supplied to `new`. - /// - /// It is an error to call this function with iterators of - /// inconsistent lengths. - /// - /// The trait bound aims for maximum flexibility: the inputs must be - /// convertable to iterators (`I: IntoIter`), and the iterator's items - /// must be `Borrow` (or `Borrow`), to allow - /// iterators returning either `Scalar`s or `&Scalar`s. - fn mixed_multiscalar_mul( - &self, - static_scalars: I, - dynamic_scalars: J, - dynamic_points: K, - ) -> Self::Point - where - I: IntoIterator, - I::Item: Borrow, - J: IntoIterator, - J::Item: Borrow, - K: IntoIterator, - K::Item: Borrow; - - /// Given `static_scalars`, an iterator of (possibly secret) - /// scalars \\(b_i\\), compute - /// $$ - /// Q = b_1 B_1 + \cdots + b_m B_m, - /// $$ - /// where the \\(B_j\\) are the points that were supplied to `new`. - /// - /// It is an error to call this function with iterators of - /// inconsistent lengths. - /// - /// The trait bound aims for maximum flexibility: the input must - /// be convertable to iterators (`I: IntoIter`), and the - /// iterator's items must be `Borrow`, to allow iterators - /// returning either `Scalar`s or `&Scalar`s. - fn multiscalar_mul(&self, static_scalars: I) -> Self::Point - where - I: IntoIterator, - I::Item: Borrow, - { - use core::iter; - - Self::mixed_multiscalar_mul( - self, - static_scalars, - iter::empty::(), - iter::empty::(), - ) - } -} - /// A trait for variable-time multiscalar multiplication with precomputation. /// /// A general multiscalar multiplication with precomputation can be written as From 2c3629b30eba4671991cfe5157135df775c3a16c Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 14 Feb 2019 07:16:20 -0800 Subject: [PATCH 49/52] Add a separate benchmark for pure-fixed multiscalar mul. --- benches/dalek_benchmarks.rs | 54 +++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs index e2e3f5b..ad1e0e6 100644 --- a/benches/dalek_benchmarks.rs +++ b/benches/dalek_benchmarks.rs @@ -59,7 +59,7 @@ mod edwards_benches { }); } - criterion_group!{ + criterion_group! { name = edwards_benches; config = Criterion::default(); targets = @@ -110,10 +110,29 @@ mod multiscalar_benches { ); } - fn precomputed_vt_straus_helper(c: &mut Criterion, dynamic_fraction: f64) { + fn vartime_precomputed_pure_static(c: &mut Criterion) { + c.bench_function_over_inputs( + "Variable-time fixed-base multiscalar multiplication", + move |b, &&total_size| { + let static_size = total_size; + + let (static_scalars, static_points) = construct(static_size); + + use curve25519_dalek::edwards::VartimeEdwardsPrecomputation; + use curve25519_dalek::traits::VartimePrecomputedMultiscalarMul; + + let precomp = VartimeEdwardsPrecomputation::new(&static_points); + + b.iter(|| precomp.vartime_multiscalar_mul(&static_scalars)); + }, + &MULTISCALAR_SIZES, + ); + } + + fn vartime_precomputed_helper(c: &mut Criterion, dynamic_fraction: f64) { let label = format!( - "Variable-time mixed-base Straus ({:.2}pct dyn)", - 100.0*dynamic_fraction, + "Variable-time mixed-base multiscalar multiplication ({:.0}pct dyn)", + 100.0 * dynamic_fraction, ); c.bench_function_over_inputs( &label, @@ -141,28 +160,29 @@ mod multiscalar_benches { ); } - fn precomputed_vt_straus_00_pct_dynamic(c: &mut Criterion) { - precomputed_vt_straus_helper(c, 0.0); + fn vartime_precomputed_00_pct_dynamic(c: &mut Criterion) { + vartime_precomputed_helper(c, 0.0); } - fn precomputed_vt_straus_20_pct_dynamic(c: &mut Criterion) { - precomputed_vt_straus_helper(c, 0.2); + fn vartime_precomputed_20_pct_dynamic(c: &mut Criterion) { + vartime_precomputed_helper(c, 0.2); } - fn precomputed_vt_straus_50_pct_dynamic(c: &mut Criterion) { - precomputed_vt_straus_helper(c, 0.5); + fn vartime_precomputed_50_pct_dynamic(c: &mut Criterion) { + vartime_precomputed_helper(c, 0.5); } - criterion_group!{ + criterion_group! { name = multiscalar_benches; // Lower the sample size to run the benchmarks faster config = Criterion::default().sample_size(15); targets = consttime_multiscalar_mul, vartime_multiscalar_mul, - precomputed_vt_straus_00_pct_dynamic, - precomputed_vt_straus_20_pct_dynamic, - precomputed_vt_straus_50_pct_dynamic, + vartime_precomputed_pure_static, + vartime_precomputed_00_pct_dynamic, + vartime_precomputed_20_pct_dynamic, + vartime_precomputed_50_pct_dynamic, } } @@ -198,7 +218,7 @@ mod ristretto_benches { ); } - criterion_group!{ + criterion_group! { name = ristretto_benches; config = Criterion::default(); targets = @@ -219,7 +239,7 @@ mod montgomery_benches { }); } - criterion_group!{ + criterion_group! { name = montgomery_benches; config = Criterion::default(); targets = montgomery_ladder, @@ -251,7 +271,7 @@ mod scalar_benches { ); } - criterion_group!{ + criterion_group! { name = scalar_benches; config = Criterion::default(); targets = From 47967b49f085e108d0cf6bd42846276086e0a617 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 14 Feb 2019 07:59:45 -0800 Subject: [PATCH 50/52] Use rerandomized inputs for variable-time benchmarks. This avoids potentially misleading benchmark results where the memory cost of precomputation becomes "free" as re-running the benchmark loop lifts exactly the required table entries into the highest-level caches. --- benches/dalek_benchmarks.rs | 113 ++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 37 deletions(-) diff --git a/benches/dalek_benchmarks.rs b/benches/dalek_benchmarks.rs index ad1e0e6..bb47634 100644 --- a/benches/dalek_benchmarks.rs +++ b/benches/dalek_benchmarks.rs @@ -2,10 +2,12 @@ extern crate rand; use rand::rngs::OsRng; +use rand::thread_rng; #[macro_use] extern crate criterion; +use criterion::BatchSize; use criterion::Criterion; extern crate curve25519_dalek; @@ -51,11 +53,13 @@ mod edwards_benches { fn vartime_double_base_scalar_mul(c: &mut Criterion) { c.bench_function("Variable-time aA+bB, A variable, B fixed", |bench| { - let B = &constants::ED25519_BASEPOINT_POINT; - let a = Scalar::from(298374928u64).invert(); - let b = Scalar::from(897987897u64).invert(); - let A = B * (b * a); - bench.iter(|| EdwardsPoint::vartime_double_scalar_mul_basepoint(&a, &A, &b)); + let mut rng = thread_rng(); + let A = &Scalar::random(&mut rng) * &constants::ED25519_BASEPOINT_TABLE; + bench.iter_batched( + || (Scalar::random(&mut rng), Scalar::random(&mut rng)), + |(a, b)| EdwardsPoint::vartime_double_scalar_mul_basepoint(&a, &A, &b), + BatchSize::SmallInput, + ); }); } @@ -75,25 +79,39 @@ mod multiscalar_benches { use super::*; use curve25519_dalek::edwards; use curve25519_dalek::edwards::EdwardsPoint; + use curve25519_dalek::edwards::VartimeEdwardsPrecomputation; use curve25519_dalek::traits::MultiscalarMul; use curve25519_dalek::traits::VartimeMultiscalarMul; + use curve25519_dalek::traits::VartimePrecomputedMultiscalarMul; + + fn construct_scalars(n: usize) -> Vec { + let mut rng = thread_rng(); + (0..n).map(|_| Scalar::random(&mut rng)).collect() + } + + fn construct_points(n: usize) -> Vec { + let mut rng = thread_rng(); + (0..n) + .map(|_| &Scalar::random(&mut rng) * &constants::ED25519_BASEPOINT_TABLE) + .collect() + } fn construct(n: usize) -> (Vec, Vec) { - let mut rng = OsRng::new().unwrap(); - let scalars: Vec = (0..n).map(|_| Scalar::random(&mut rng)).collect(); - let points: Vec = scalars - .iter() - .map(|s| s * &constants::ED25519_BASEPOINT_TABLE) - .collect(); - (scalars, points) + (construct_scalars(n), construct_points(n)) } fn consttime_multiscalar_mul(c: &mut Criterion) { c.bench_function_over_inputs( "Constant-time variable-base multiscalar multiplication", |b, &&size| { - let (scalars, points) = construct(size); - b.iter(|| EdwardsPoint::multiscalar_mul(&scalars, &points)); + let points = construct_points(size); + // This is supposed to be constant-time, but we might as well + // rerandomize the scalars for every call just in case. + b.iter_batched( + || construct_scalars(size), + |scalars| EdwardsPoint::multiscalar_mul(&scalars, &points), + BatchSize::SmallInput, + ); }, &MULTISCALAR_SIZES, ); @@ -103,8 +121,16 @@ mod multiscalar_benches { c.bench_function_over_inputs( "Variable-time variable-base multiscalar multiplication", |b, &&size| { - let (scalars, points) = construct(size); - b.iter(|| EdwardsPoint::vartime_multiscalar_mul(&scalars, &points)); + let points = construct_points(size); + // Rerandomize the scalars for every call to prevent + // false timings from better caching (e.g., the CPU + // cache lifts exactly the right table entries for the + // benchmark into the highest cache levels). + b.iter_batched( + || construct_scalars(size), + |scalars| EdwardsPoint::vartime_multiscalar_mul(&scalars, &points), + BatchSize::SmallInput, + ); }, &MULTISCALAR_SIZES, ); @@ -116,14 +142,17 @@ mod multiscalar_benches { move |b, &&total_size| { let static_size = total_size; - let (static_scalars, static_points) = construct(static_size); - - use curve25519_dalek::edwards::VartimeEdwardsPrecomputation; - use curve25519_dalek::traits::VartimePrecomputedMultiscalarMul; - + let static_points = construct_points(static_size); let precomp = VartimeEdwardsPrecomputation::new(&static_points); - - b.iter(|| precomp.vartime_multiscalar_mul(&static_scalars)); + // Rerandomize the scalars for every call to prevent + // false timings from better caching (e.g., the CPU + // cache lifts exactly the right table entries for the + // benchmark into the highest cache levels). + b.iter_batched( + || construct_scalars(static_size), + |scalars| precomp.vartime_multiscalar_mul(&scalars), + BatchSize::SmallInput, + ); }, &MULTISCALAR_SIZES, ); @@ -140,21 +169,31 @@ mod multiscalar_benches { let dynamic_size = ((total_size as f64) * dynamic_fraction) as usize; let static_size = total_size - dynamic_size; - let (static_scalars, static_points) = construct(static_size); - let (dynamic_scalars, dynamic_points) = construct(dynamic_size); - - use curve25519_dalek::edwards::VartimeEdwardsPrecomputation; - use curve25519_dalek::traits::VartimePrecomputedMultiscalarMul; - + let static_points = construct_points(static_size); + let dynamic_points = construct_points(dynamic_size); let precomp = VartimeEdwardsPrecomputation::new(&static_points); - - b.iter(|| { - precomp.vartime_mixed_multiscalar_mul( - &static_scalars, - &dynamic_scalars, - &dynamic_points, - ) - }); + // Rerandomize the scalars for every call to prevent + // false timings from better caching (e.g., the CPU + // cache lifts exactly the right table entries for the + // benchmark into the highest cache levels). Timings + // should be independent of points so we don't + // randomize them. + b.iter_batched( + || { + ( + construct_scalars(static_size), + construct_scalars(dynamic_size), + ) + }, + |(static_scalars, dynamic_scalars)| { + precomp.vartime_mixed_multiscalar_mul( + &static_scalars, + &dynamic_scalars, + &dynamic_points, + ) + }, + BatchSize::SmallInput, + ); }, &MULTISCALAR_SIZES, ); From a38b77b965b69831f802908cd25cb0f40bcf4ff2 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 14 Feb 2019 14:44:49 -0800 Subject: [PATCH 51/52] Add a changelog. --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..298afff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +Entries are listed in reverse chronological order. + +## 1.0.3 + +* Adds `ConstantTimeEq` implementation for compressed points. + +## 1.0.2 + +* Fixes a typo in the naming of variables in Ristretto formulas (no change to functionality). + +## 1.0.1 + +* Depends on the stable `2.0` version of `subtle` instead of `2.0.0-pre.0`. + +## 1.0.0 + +Initial stable release. Yanked due to a dependency mistake (see above). + From a35adbef8424ddb2ddb4659118101c3652ebd236 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 14 Feb 2019 14:49:36 -0800 Subject: [PATCH 52/52] Bump version number to 1.1.0-pre.0 --- CHANGELOG.md | 12 ++++++++++++ Cargo.toml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 298afff..317be17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ Entries are listed in reverse chronological order. +## 1.1.0-pre.0 + +* Restructures the source tree into `serial` and `vector` backends. +* Adds a new IFMA backend which sets speed records. +* Adds support for precomputation for multiscalar multiplication. +* Replaces the `rand` dependency with `rand_core`. +* Generalizes trait bounds on + - `RistrettoPoint::random()` + - `Scalar::random()` + to allow owned and borrowed RNGs and to allow `RngCore` instead of + `Rng`. + ## 1.0.3 * Adds `ConstantTimeEq` implementation for compressed points. diff --git a/Cargo.toml b/Cargo.toml index dea9dfa..ed0cbb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "curve25519-dalek" -version = "1.0.3" +version = "1.1.0-pre.0" authors = ["Isis Lovecruft ", "Henry de Valence "] readme = "README.md"