From 51f04d7ccef91840d06aa507428d92b1293e66b3 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 23 Oct 2017 14:03:23 -0700 Subject: [PATCH 01/54] first avx2 code --- Cargo.toml | 4 + src/avx2/edwards.rs | 104 +++++++++ src/avx2/field.rs | 502 ++++++++++++++++++++++++++++++++++++++++++++ src/avx2/mod.rs | 13 ++ src/lib.rs | 9 +- 5 files changed, 631 insertions(+), 1 deletion(-) create mode 100644 src/avx2/edwards.rs create mode 100644 src/avx2/field.rs create mode 100644 src/avx2/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 470d7e5..e317204 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,10 @@ rustdoc-args = ["--html-in-header", ".cargo/registry/src/github.com-1ecc6299db9e [badges] travis-ci = { repository = "isislovecruft/curve25519-dalek", branch = "master"} +[dependencies] +#stdsimd = { git = "https://github.com/rust-lang-nursery/stdsimd" } +stdsimd = { git = "https://github.com/hdevalence/stdsimd", branch="feature/more-avx2" } + [dependencies.serde] version = "1.0" optional = true diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs new file mode 100644 index 0000000..489894f --- /dev/null +++ b/src/avx2/edwards.rs @@ -0,0 +1,104 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2017 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +//! Extended Twisted Edwards for Curve25519, using AVX2. + +// just going to own it +#![allow(bad_style)] + +use std::convert::From; +use std::ops::Add; + +use stdsimd::simd::u32x8; + +use edwards; + +use avx2::field::FieldElement32x4; + +/// A point on Curve25519, represented in an AVX2-friendly format. +pub(crate) struct ExtendedPoint(FieldElement32x4); + +// XXX need to cfg gate here to handle FieldElement64 +impl From for ExtendedPoint { + fn from(P: edwards::ExtendedPoint) -> ExtendedPoint { + ExtendedPoint(FieldElement32x4::new(&P.X, &P.Y, &P.Z, &P.T)) + } +} + +// XXX need to cfg gate here to handle FieldElement64 +impl From for edwards::ExtendedPoint { + fn from(P: ExtendedPoint) -> edwards::ExtendedPoint { + let tmp = P.0.split(); + edwards::ExtendedPoint{X: tmp[0], Y: tmp[1], Z: tmp[2], T: tmp[3]} + } +} + +impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { + type Output = ExtendedPoint; + + /// Uses a slight tweak of the parallel unified formulas of HWCD'08 + fn add(self, other: &'b ExtendedPoint) -> ExtendedPoint { + unsafe { + use stdsimd::vendor::_mm256_permute2x128_si256; + use stdsimd::vendor::_mm256_permutevar8x32_epi32; + use stdsimd::vendor::_mm256_blend_epi32; + + let mut P: FieldElement32x4 = self.0; + let mut Q: FieldElement32x4 = other.0; + let mut t0: FieldElement32x4 = self.0; + + for i in 0..5 { + t0.0[i] = _mm256_permute2x128_si256(P.0[i].into(), Q.0[i].into(), 32).into(); + } + //println!("t0 = (X1, Y1, X2, Y2)"); + //println!("t0 = {:?}\n", t0.split()); + + let mut t1 = t0.diff_sum(); + //println!("t1 = (S1 S3 S2 S4)"); + //println!("t1 = {:?}\n", t1.split()); + + for i in 0..5 { + Q.0[i] = _mm256_permute2x128_si256(t1.0[i].into(), Q.0[i].into(), 49).into(); + t1.0[i] = _mm256_blend_epi32(t1.0[i].into(), P.0[i].into(), 0b11110000).into(); + } + //println!("Q = (S2 S4 Z2 T2)"); + //println!("Q = {:?}\n", Q.split()); + //println!("t1 = (S1 S3 Z1 T1)"); + //println!("t1 = {:?}\n", t1.split()); + + P = &t1 * &Q; + //println!("P = (S5 S6 S8 S7)"); + //println!("P = {:?}\n", P.split()); + + P.scale_by_curve_constants(); + //println!("P = (S5' S6' S10 S8)"); + //println!("P = {:?}\n", P.split()); + + Q = P.diff_sum(); + //println!("Q = (S11 S14 S12 S13)"); + //println!("Q = {:?}\n", Q.split()); + + let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) + let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBBC) + + for i in 0..5 { + t0.0[i] = _mm256_permutevar8x32_epi32(Q.0[i], c0); + t1.0[i] = _mm256_permutevar8x32_epi32(Q.0[i], c1); + } + //println!("t0 = (S11 S13 S13 S11)"); + //println!("t0 = {:?}\n", t0.split()); + //println!("t1 = (S12 S14 S14 S12)"); + //println!("t1 = {:?}\n", t1.split()); + + ExtendedPoint(&t0 * &t1) + } + } +} + diff --git a/src/avx2/field.rs b/src/avx2/field.rs new file mode 100644 index 0000000..15050c6 --- /dev/null +++ b/src/avx2/field.rs @@ -0,0 +1,502 @@ +// -*- mode: rust; coding: utf-8; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2017 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +//! 4-way vectorized 32bit field arithmetic using AVX2. +//! + +#![allow(bad_style)] + +use std::ops::Mul; + +use stdsimd::simd::{u32x8, i32x8, u64x4}; + +use backend::u32::field::FieldElement32; + +static P_TIMES_2: FieldElement32x4 = FieldElement32x4([ + u32x8::new(134217690, 134217690, 67108862, 67108862, 134217690, 134217690, 67108862, 67108862), + u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862), + u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862), + u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862), + u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862) +]); + +/// A vector of four `FieldElements`, implemented using AVX2. +#[derive(Clone, Copy)] +pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]); + +impl FieldElement32x4 { + pub(crate) fn split(&self) -> [FieldElement32; 4] { + let mut out = [FieldElement32::zero(); 4]; + for i in 0..5 { + out[0].0[2*i ] = self.0[i].extract(0); // + out[1].0[2*i ] = self.0[i].extract(1); // + out[0].0[2*i+1] = self.0[i].extract(2); // `. + out[1].0[2*i+1] = self.0[i].extract(3); // | pre-swapped to avoid + out[2].0[2*i ] = self.0[i].extract(4); // | a cross lane shuffle + out[3].0[2*i ] = self.0[i].extract(5); // .' + out[2].0[2*i+1] = self.0[i].extract(6); // + out[3].0[2*i+1] = self.0[i].extract(7); // + } + + out + } + + pub fn zero() -> FieldElement32x4 { + FieldElement32x4([u32x8::splat(0);5]) + } + + pub fn splat(x: &FieldElement32) -> FieldElement32x4 { + FieldElement32x4::new(x,x,x,x) + } + + pub fn new( + x0: &FieldElement32, + x1: &FieldElement32, + x2: &FieldElement32, + x3: &FieldElement32, + ) -> FieldElement32x4 { + let mut buf = [u32x8::splat(0); 5]; + for i in 0..5 { + buf[i] = u32x8::new(x0.0[2*i ], x1.0[2*i ], x0.0[2*i+1], x1.0[2*i+1], + x2.0[2*i ], x3.0[2*i ], x2.0[2*i+1], x3.0[2*i+1]); + } + + FieldElement32x4(buf) + } + + // Given `self = (A,B,C,D)`, compute `(B - A, B + A, D - C, D + C)`. + pub fn diff_sum(&self) -> FieldElement32x4 { + /// (v0 v1 v2 v3 v4 v5 v6 v7) -> (v1 v0 v3 v2 v5 v4 v7 v6) + #[inline(always)] + fn alternate_32bit_lanes(v: u32x8) -> u32x8 { + unsafe { + use stdsimd::vendor::_mm256_shuffle_epi32; + _mm256_shuffle_epi32(v.as_i32x8(), 0b10_11_00_01).as_u32x8() + } + } + + /// (v0 XX v2 XX v4 XX v6 XX) + /// (XX v1 XX v3 XX v5 XX v7) -> (v0 v1 v2 v3 v4 v5 v6 v7) + #[inline(always)] + fn blend_alternating_32bit_lanes(v1: u32x8, v2: u32x8) -> u32x8 { + unsafe { + use stdsimd::vendor::_mm256_blend_epi32; + _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8() + } + } + + let mut out = [u32x8::splat(0); 5]; + + for i in 0..5 { + let x = self.0[i]; + let p = P_TIMES_2.0[i] ; + let x_shuf = alternate_32bit_lanes(x); + + let diff = (x_shuf + p) - x; + let sum = x + x_shuf; + let diff_sum = blend_alternating_32bit_lanes(diff, sum); + + out[i] = diff_sum; + } + + FieldElement32x4(out) + } + + // Given `self = (A,B,C,D)`, compute `(B + A, B - A, D + C, D - C)`. + pub fn sum_diff(&self) -> FieldElement32x4 { + /// (v0 v1 v2 v3 v4 v5 v6 v7) -> (v1 v0 v3 v2 v5 v4 v7 v6) + #[inline(always)] + #[allow(dead_code)] // XXX + fn alternate_32bit_lanes(v: u32x8) -> u32x8 { + unsafe { + use stdsimd::vendor::_mm256_shuffle_epi32; + _mm256_shuffle_epi32(v.as_i32x8(), 0b10_11_00_01).as_u32x8() + } + } + + /// (v0 XX v2 XX v4 XX v6 XX) + /// (XX v1 XX v3 XX v5 XX v7) -> (v0 v1 v2 v3 v4 v5 v6 v7) + #[inline(always)] + #[allow(dead_code)] // XXX + fn blend_alternating_32bit_lanes(v1: u32x8, v2: u32x8) -> u32x8 { + unsafe { + use stdsimd::vendor::_mm256_blend_epi32; + _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8() + } + } + + let mut out = [u32x8::splat(0); 5]; + + for i in 0..5 { + let x = self.0[i]; + let p = P_TIMES_2.0[i]; + let x_shuf = alternate_32bit_lanes(x); + + let sum = x + x_shuf; + let diff = (x + p) - x_shuf; + let sum_diff = blend_alternating_32bit_lanes(sum, diff); + + out[i] = sum_diff; + } + + FieldElement32x4(out) + } + + pub fn scale_by_curve_constants(&mut self) { + let mut b = [u64x4::splat(0); 10]; + + let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); + let low__p20 = u64x4::splat(0x3ffffed << 20); + let even_p20 = u64x4::splat(0x3ffffff << 20); + let odd__p20 = u64x4::splat(0x1ffffff << 20); + + unsafe { + use stdsimd::vendor::_mm256_mul_epu32; + use stdsimd::vendor::_mm256_blend_epi32; + + let (b0, b1) = unpack_pair(self.0[0]); + let b0 = _mm256_mul_epu32(b0, consts); // need a new binding since now + let b1 = _mm256_mul_epu32(b1, consts); // b0 has type u64x4 + b[0] = _mm256_blend_epi32(b0.into(), (low__p20 - b0).into(), 0b11_00_00_00).into(); + b[1] = _mm256_blend_epi32(b1.into(), (odd__p20 - b1).into(), 0b11_00_00_00).into(); + + let (b2, b3) = unpack_pair(self.0[1]); + let b2 = _mm256_mul_epu32(b2, consts); + let b3 = _mm256_mul_epu32(b3, consts); + b[2] = _mm256_blend_epi32(b2.into(), (even_p20 - b2).into(), 0b11_00_00_00).into(); + b[3] = _mm256_blend_epi32(b3.into(), (odd__p20 - b3).into(), 0b11_00_00_00).into(); + + let (b4, b5) = unpack_pair(self.0[2]); + let b4 = _mm256_mul_epu32(b4, consts); + let b5 = _mm256_mul_epu32(b5, consts); + b[4] = _mm256_blend_epi32(b4.into(), (even_p20 - b4).into(), 0b11_00_00_00).into(); + b[5] = _mm256_blend_epi32(b5.into(), (odd__p20 - b5).into(), 0b11_00_00_00).into(); + + let (b6, b7) = unpack_pair(self.0[3]); + let b6 = _mm256_mul_epu32(b6, consts); + let b7 = _mm256_mul_epu32(b7, consts); + b[6] = _mm256_blend_epi32(b6.into(), (even_p20 - b6).into(), 0b11_00_00_00).into(); + b[7] = _mm256_blend_epi32(b7.into(), (odd__p20 - b7).into(), 0b11_00_00_00).into(); + + let (b8, b9) = unpack_pair(self.0[4]); + let b8 = _mm256_mul_epu32(b8, consts); + let b9 = _mm256_mul_epu32(b9, consts); + b[8] = _mm256_blend_epi32(b8.into(), (even_p20 - b8).into(), 0b11_00_00_00).into(); + b[9] = _mm256_blend_epi32(b9.into(), (odd__p20 - b9).into(), 0b11_00_00_00).into(); + } + + *self = FieldElement32x4::reduce64(b); + } + + pub fn reduce32(&mut self) { + let mut b = [u64x4::splat(0); 10]; + + let (b0, b1) = unpack_pair(self.0[0]); + b[0] = b0.into(); b[1] = b1.into(); + let (b2, b3) = unpack_pair(self.0[1]); + b[2] = b2.into(); b[3] = b3.into(); + let (b4, b5) = unpack_pair(self.0[2]); + b[4] = b4.into(); b[5] = b5.into(); + let (b6, b7) = unpack_pair(self.0[3]); + b[6] = b6.into(); b[7] = b7.into(); + let (b8, b9) = unpack_pair(self.0[4]); + b[8] = b8.into(); b[9] = b9.into(); + + *self = FieldElement32x4::reduce64(b); + } + + pub fn reduce64(mut z: [u64x4; 10]) -> FieldElement32x4 { + // These aren't const because splat isn't a const fn + let LOW_25_BITS: u64x4 = u64x4::splat((1<<25)-1); + let LOW_26_BITS: u64x4 = u64x4::splat((1<<26)-1); + + /// XXX check whether u64x4 >> is this already + #[inline(always)] + fn shift_right(x: u64x4, s: i32) -> u64x4 { + unsafe { + use stdsimd::vendor::_mm256_srli_epi64; + _mm256_srli_epi64(x.into(), s).as_u64x4() + } + } + + // Carry the value from limb i = 0..8 to limb i+1 + let carry = |z: &mut [u64x4; 10], i: usize| { + debug_assert!(i < 9); + if i % 2 == 0 { + // Even limbs have 26 bits + z[i+1] = z[i+1] + shift_right(z[i], 26); + z[i] = z[i] & LOW_26_BITS; + } else { + // Odd limbs have 25 bits + z[i+1] = z[i+1] + shift_right(z[i], 25); + z[i] = z[i] & LOW_25_BITS; + } + }; + + // Perform two halves of the carry chain in parallel. + carry(&mut z, 0); carry(&mut z, 4); + carry(&mut z, 1); carry(&mut z, 5); + carry(&mut z, 2); carry(&mut z, 6); + carry(&mut z, 3); carry(&mut z, 7); + // Since z[3] < 2^64, c < 2^(64-25) = 2^39, + // so z[4] < 2^26 + 2^39 < 2^39.0002 + carry(&mut z, 4); carry(&mut z, 8); + // Now z[4] < 2^26 + // and z[5] < 2^25 + 2^13.0002 < 2^25.0004 (good enough) + + // Last carry has a multiplication by 19. In the serial case we + // do a 64-bit multiplication by 19, but here we want to do a + // 32-bit multiplication. However, if we only know z[9] < 2^64, + // the carry is bounded as c < 2^(64-25) = 2^39, which is too + // big. To ensure c < 2^32, we would need z[9] < 2^57. + // Instead, we split the carry in two, with c = c_0 + c_1*2^26. + + let c = shift_right(z[9], 25); + z[9] = z[9] & LOW_25_BITS; + let mut c0 = c & LOW_26_BITS; // c0 < 2^26; + let mut c1 = shift_right(c, 26); // c1 < 2^(39-26) = 2^13; + + unsafe { + use stdsimd::vendor::_mm256_mul_epu32; + let x19 = u32x8::from(u64x4::splat(19)); + c0 = _mm256_mul_epu32(u32x8::from(c0), x19); // c0 < 2^30.25 + c1 = _mm256_mul_epu32(u32x8::from(c1), x19); // c1 < 2^17.25 + } + + z[0] = z[0] + c0; // z0 < 2^26 + 2^30.25 < 2^30.33 + z[1] = z[1] + c1; // z1 < 2^25 + 2^17.25 < 2^25.0067 + carry(&mut z, 0); // z0 < 2^26, z1 < 2^25.0067 + 2^4.33 = 2^25.007 + + // Now repack the [u64x4; 10] into a FieldElement32x4 + + FieldElement32x4([ + repack_pair(z[0].into(), z[1].into()), + repack_pair(z[2].into(), z[3].into()), + repack_pair(z[4].into(), z[5].into()), + repack_pair(z[6].into(), z[7].into()), + repack_pair(z[8].into(), z[9].into()), + ]) + } +} + +#[inline(always)] +pub fn unpack_pair(src: u32x8) -> (u32x8, u32x8) { + let a: u32x8; + let b: u32x8; + let zero = i32x8::new(0,0,0,0,0,0,0,0); + unsafe { + use stdsimd::vendor::_mm256_unpackhi_epi32; + use stdsimd::vendor::_mm256_unpacklo_epi32; + a = _mm256_unpacklo_epi32(src.as_i32x8(), zero).as_u32x8(); + b = _mm256_unpackhi_epi32(src.as_i32x8(), zero).as_u32x8(); + } + (a,b) +} + +#[inline(always)] +pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { + unsafe { + use stdsimd::vendor::_mm256_shuffle_epi32; + use stdsimd::vendor::_mm256_blend_epi32; + + // Input: x = (a0, 0, b0, 0, c0, 0, d0) + // Input: y = (a1, 0, b1, 0, c1, 0, d1) + + let x_shuffled = _mm256_shuffle_epi32(x.into(), 0b11_01_10_00); + let y_shuffled = _mm256_shuffle_epi32(y.into(), 0b10_00_11_01); + + // x' = (a0, b0, 0, 0, c0, d0, 0, 0) + // y' = ( 0, 0, a1, b1, 0, 0, c1, d1) + + return _mm256_blend_epi32(x_shuffled, y_shuffled, 0b11001100).as_u32x8(); + } +} + +impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { + type Output = FieldElement32x4; + fn mul(self, _rhs: &'b FieldElement32x4) -> FieldElement32x4 { + let mut b = [u32x8::splat(0); 10]; + let mut c = [u64x4::splat(0); 10]; + + let (b0, b1) = unpack_pair(_rhs.0[0]); + b[0] = b0; b[1] = b1; + let (b2, b3) = unpack_pair(_rhs.0[1]); + b[2] = b2; b[3] = b3; + let (b4, b5) = unpack_pair(_rhs.0[2]); + b[4] = b4; b[5] = b5; + let (b6, b7) = unpack_pair(_rhs.0[3]); + b[6] = b6; b[7] = b7; + let (b8, b9) = unpack_pair(_rhs.0[4]); + b[8] = b8; b[9] = b9; + + #[inline(always)] + fn m(x: u32x8, y: u32x8) -> u64x4 { + use stdsimd::vendor::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x,y) } + } + + #[inline(always)] + fn m_lo(x: u32x8, y: u32x8) -> u32x8 { + use stdsimd::vendor::_mm256_mul_epu32; + unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } + } + + let x19 = u32x8::new(19,0,19,0,19,0,19,0); + + macro_rules! loop_body { + ($i:expr) => { + let (ai, ai1) = unpack_pair(self.0[$i/2]); + + c[9] = c[9] + m(ai, b[(100 + 9-$i) % 10]); + b[(100 + 9-$i) % 10] = m_lo(b[(100 + 9-$i) % 10], x19); + c[8] = c[8] + m(ai, b[(100 + 8-$i) % 10]); + c[7] = c[7] + m(ai, b[(100 + 7-$i) % 10]); + c[6] = c[6] + m(ai, b[(100 + 6-$i) % 10]); + c[5] = c[5] + m(ai, b[(100 + 5-$i) % 10]); + c[4] = c[4] + m(ai, b[(100 + 4-$i) % 10]); + c[3] = c[3] + m(ai, b[(100 + 3-$i) % 10]); + c[2] = c[2] + m(ai, b[(100 + 2-$i) % 10]); + c[1] = c[1] + m(ai, b[(100 + 1-$i) % 10]); + c[0] = c[0] + m(ai, b[(100 + 0-$i) % 10]); + + let ai1_2 = ai1 + ai1; + c[9] = c[9] + m(ai1, b[(100 + 9-($i+1)) % 10]); + b[(100 + 9-($i+1)) % 10] = m_lo(b[(100 + 9-($i+1)) % 10], x19); + c[8] = c[8] + m(ai1_2, b[(100 + 8-($i+1)) % 10]); + c[7] = c[7] + m(ai1, b[(100 + 7-($i+1)) % 10]); + c[6] = c[6] + m(ai1_2, b[(100 + 6-($i+1)) % 10]); + c[5] = c[5] + m(ai1, b[(100 + 5-($i+1)) % 10]); + c[4] = c[4] + m(ai1_2, b[(100 + 4-($i+1)) % 10]); + c[3] = c[3] + m(ai1, b[(100 + 3-($i+1)) % 10]); + c[2] = c[2] + m(ai1_2, b[(100 + 2-($i+1)) % 10]); + c[1] = c[1] + m(ai1, b[(100 + 1-($i+1)) % 10]); + c[0] = c[0] + m(ai1_2, b[(100 + 0-($i+1)) % 10]); + }; + } + + loop_body!(0); + loop_body!(2); + loop_body!(4); + loop_body!(6); + loop_body!(8); + + return FieldElement32x4::reduce64(c); + } +} + + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn scale_by_curve_constants() { + let mut x = FieldElement32x4::splat(&FieldElement32::one()); + x.scale_by_curve_constants(); + + let xs = x.split(); + assert_eq!(xs[0], FieldElement32([ 121666,0,0,0,0,0,0,0,0,0])); + assert_eq!(xs[1], FieldElement32([ 121666,0,0,0,0,0,0,0,0,0])); + assert_eq!(xs[2], FieldElement32([2*121666,0,0,0,0,0,0,0,0,0])); + assert_eq!(xs[3], -&FieldElement32([2*121665,0,0,0,0,0,0,0,0,0])); + } + + #[test] + fn diff_sum_vs_serial() { + let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); + let x1 = FieldElement32([10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10109]); + let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); + let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); + + let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + + let result = vec.diff_sum().split(); + + assert_eq!(result[0], &x1 - &x0); + assert_eq!(result[1], &x1 + &x0); + assert_eq!(result[2], &x3 - &x2); + assert_eq!(result[3], &x3 + &x2); + } + + #[test] + fn multiply_vs_serial() { + let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); + let x1 = FieldElement32([10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10109]); + let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); + let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); + + let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let vecprime = vec.clone(); + + let result = (&vec * &vecprime).split(); + + assert_eq!(result[0], &x0 * &x0); + assert_eq!(result[1], &x1 * &x1); + assert_eq!(result[2], &x2 * &x2); + assert_eq!(result[3], &x3 * &x3); + } + + #[test] + fn test_unpack_repack_pair() { + let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); + let x1 = FieldElement32([10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10109]); + let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); + let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); + + let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + + let src = vec.0[0]; + + let (a,b) = unpack_pair(src); + + let expected_a = u32x8::new(10000, 0, 10100, 0, 10200, 0, 10300, 0); + let expected_b = u32x8::new(10001, 0, 10101, 0, 10201, 0, 10301, 0); + + assert_eq!(a, expected_a); + assert_eq!(b, expected_b); + + let expected_src = repack_pair(a,b); + + assert_eq!(src, expected_src); + } + + #[test] + fn new_split_roundtrips() { + let x0 = FieldElement32::from_bytes(&[0x10; 32]); + let x1 = FieldElement32::from_bytes(&[0x11; 32]); + let x2 = FieldElement32::from_bytes(&[0x12; 32]); + let x3 = FieldElement32::from_bytes(&[0x13; 32]); + + let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + + let splits = vec.split(); + + assert_eq!(x0, splits[0]); + assert_eq!(x1, splits[1]); + assert_eq!(x2, splits[2]); + assert_eq!(x3, splits[3]); + } + +} + +#[cfg(all(test, feature = "bench"))] +mod bench { + use test::Bencher; + use super::*; + + #[bench] + fn multiply(b: &mut Bencher) { + let vec = FieldElement32x4::splat(&FieldElement::zero()); + let vecprime = vec.clone(); + + b.iter(|| &vec * &vecprime ); + } +} + diff --git a/src/avx2/mod.rs b/src/avx2/mod.rs new file mode 100644 index 0000000..456c451 --- /dev/null +++ b/src/avx2/mod.rs @@ -0,0 +1,13 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2017 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +pub(crate) mod field; + +pub(crate) mod edwards; diff --git a/src/lib.rs b/src/lib.rs index 2756376..c3a4255 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,7 @@ #![cfg_attr(all(feature = "nightly", feature = "std"), feature(zero_one))] #![allow(unused_features)] -#![deny(missing_docs)] // refuse to compile if documentation is missing +//#![deny(missing_docs)] // refuse to compile if documentation is missing //! # curve25519-dalek //! @@ -50,6 +50,9 @@ extern crate alloc; #[cfg(all(test, feature = "bench"))] extern crate test; +#[cfg(feature = "yolocrypto")] +extern crate stdsimd; + // The `Digest` trait is implemented using `generic_array`, so we need it // too. Hopefully we can eliminate `generic_array` from `Digest` once const // generics land. @@ -91,5 +94,9 @@ pub(crate) mod field; // Arithmetic backends (using u32, u64, etc) live here pub(crate) mod backend; +// XXX this should be in backend +#[cfg(all(feature="yolocrypto", not(feature="radix_51")))] +pub(crate) mod avx2; + // Internal curve models which are not part of the public API. pub(crate) mod curve_models; From 15b88be2d28e4cc024061b01dc7078ff90ac58ce Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 8 Nov 2017 12:11:36 -0800 Subject: [PATCH 02/54] Add serial implementation of the algorithm and a test --- src/avx2/edwards.rs | 85 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs index 489894f..85ff583 100644 --- a/src/avx2/edwards.rs +++ b/src/avx2/edwards.rs @@ -102,3 +102,88 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { } } +#[cfg(test)] +mod test { + use super::*; + + fn serial_add(P: edwards::ExtendedPoint, Q: edwards::ExtendedPoint) -> edwards::ExtendedPoint { + use field_32bit::FieldElement32; + + let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); + let (X2, Y2, Z2, T2) = (Q.X, Q.Y, Q.Z, Q.T); + + let S0 = &Y1 - &X1; // R1 + let S1 = &Y1 + &X1; // R3 + let S2 = &Y2 - &X2; // R2 + let S3 = &Y2 + &X2; // R4 + + let S4 = &S0 * &S2; // R5 = R1 * R2 + let S5 = &S1 * &S3; // R6 = R3 * R4 + let S6 = &T1 * &T2; // R7 + let S7 = &Z1 * &Z2; // R8 + + let S8 = &S6 * &(-&FieldElement32([2*121665,0,0,0,0,0,0,0,0,0])); // R7 + let S9 = &S7 * &FieldElement32([2*121666,0,0,0,0,0,0,0,0,0]); // R8 + let S10 = &S4 * &FieldElement32([ 121666,0,0,0,0,0,0,0,0,0]); // R5 + let S11 = &S5 * &FieldElement32([ 121666,0,0,0,0,0,0,0,0,0]); // R6 + + let S12 = &S11 - &S10; // R1 + let S13 = &S11 + &S10; // R4 + let S14 = &S9 - &S8; // R2 + let S15 = &S9 + &S8; // R3 + + let X3 = &S12 * &S14; // R1 * R2 + let Y3 = &S15 * &S13; // R3 * R4 + let Z3 = &S15 * &S14; // R2 * R3 + let T3 = &S12 * &S13; // R1 * R4 + + edwards::ExtendedPoint{X: X3, Y: Y3, Z: Z3, T: T3} + } + + #[test] + fn serial_add_vs_edwards_extendedpoint() { + use constants; + use scalar::Scalar; + use edwards::Identity; + + println!("Testing id + id"); + let P = edwards::ExtendedPoint::identity(); + let Q = edwards::ExtendedPoint::identity(); + let R: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); + println!("P = {:?}", P); + println!("Q = {:?}", Q); + println!("R = {:?}", R); + println!("P + Q = {:?}", &P + &Q); + assert_eq!(R.compress(), (&P + &Q).compress()); + + println!("Testing id + B"); + let P = edwards::ExtendedPoint::identity(); + let Q = constants::ED25519_BASEPOINT_POINT; + let R: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); + println!("P = {:?}", P); + println!("Q = {:?}", Q); + println!("R = {:?}", R); + println!("P + Q = {:?}", &P + &Q); + assert_eq!(R.compress(), (&P + &Q).compress()); + + println!("Testing B + B"); + let P = constants::ED25519_BASEPOINT_POINT; + let Q = constants::ED25519_BASEPOINT_POINT; + let R: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); + println!("P = {:?}", P); + println!("Q = {:?}", Q); + println!("R = {:?}", R); + println!("P + Q = {:?}", &P + &Q); + assert_eq!(R.compress(), (&P + &Q).compress()); + + println!("Testing B + kB"); + let P = constants::ED25519_BASEPOINT_POINT; + let Q = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from_u64(8475983829); + let R: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); + println!("P = {:?}", P); + println!("Q = {:?}", Q); + println!("R = {:?}", R); + println!("P + Q = {:?}", &P + &Q); + assert_eq!(R.compress(), (&P + &Q).compress()); + } +} From 4f6788c72d927dae82922891f8c483d62c5e451b Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 8 Nov 2017 15:31:02 -0800 Subject: [PATCH 03/54] First working version --- src/avx2/edwards.rs | 196 ++++++++++++++++++++++++++++++-------------- src/avx2/field.rs | 17 ++-- 2 files changed, 141 insertions(+), 72 deletions(-) diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs index 85ff583..3ab6139 100644 --- a/src/avx2/edwards.rs +++ b/src/avx2/edwards.rs @@ -49,53 +49,86 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { use stdsimd::vendor::_mm256_permute2x128_si256; use stdsimd::vendor::_mm256_permutevar8x32_epi32; use stdsimd::vendor::_mm256_blend_epi32; + use stdsimd::vendor::_mm256_shuffle_epi32; - let mut P: FieldElement32x4 = self.0; - let mut Q: FieldElement32x4 = other.0; - let mut t0: FieldElement32x4 = self.0; + let P: &FieldElement32x4 = &self.0; + let Q: &FieldElement32x4 = &other.0; + + let mut t0 = FieldElement32x4::zero(); + let mut t1 = FieldElement32x4::zero(); + + macro_rules! print_vec { + ($x:ident) => { + let splits = $x.split(); + println!("{}[0] = {:?}", stringify!($x), splits[0].to_bytes()); + println!("{}[1] = {:?}", stringify!($x), splits[1].to_bytes()); + println!("{}[2] = {:?}", stringify!($x), splits[2].to_bytes()); + println!("{}[3] = {:?}", stringify!($x), splits[3].to_bytes()); + } + } for i in 0..5 { t0.0[i] = _mm256_permute2x128_si256(P.0[i].into(), Q.0[i].into(), 32).into(); } //println!("t0 = (X1, Y1, X2, Y2)"); - //println!("t0 = {:?}\n", t0.split()); + //print_vec!(t0); + //println!(""); - let mut t1 = t0.diff_sum(); - //println!("t1 = (S1 S3 S2 S4)"); - //println!("t1 = {:?}\n", t1.split()); + t0.diff_sum(); + + //println!("t0 = (S0 S1 S2 S3)"); + //print_vec!(t0); + //println!(""); for i in 0..5 { - Q.0[i] = _mm256_permute2x128_si256(t1.0[i].into(), Q.0[i].into(), 49).into(); - t1.0[i] = _mm256_blend_epi32(t1.0[i].into(), P.0[i].into(), 0b11110000).into(); + t1.0[i] = _mm256_blend_epi32(t0.0[i].into(), P.0[i].into(), 0b11110000).into(); + t0.0[i] = _mm256_permute2x128_si256(t0.0[i].into(), Q.0[i].into(), 49).into(); } - //println!("Q = (S2 S4 Z2 T2)"); - //println!("Q = {:?}\n", Q.split()); - //println!("t1 = (S1 S3 Z1 T1)"); - //println!("t1 = {:?}\n", t1.split()); + //println!("t0 = (S2 S3 Z2 T2)"); + //print_vec!(t0); + //println!(""); - P = &t1 * &Q; - //println!("P = (S5 S6 S8 S7)"); - //println!("P = {:?}\n", P.split()); + //println!("t1 = (S0 S1 Z1 T1)"); + //print_vec!(t1); + //println!(""); + + let mut t2 = &t0 * &t1; + //println!("t2 = (S4 S5 S6 S7)"); + //print_vec!(t2); + //println!(""); - P.scale_by_curve_constants(); - //println!("P = (S5' S6' S10 S8)"); - //println!("P = {:?}\n", P.split()); + t2.scale_by_curve_constants(); + //println!("t2 = (S8 S9 S10 S11)"); + //print_vec!(t2); + //println!(""); + + for i in 0..5 { + let swapped = _mm256_shuffle_epi32(t2.0[i].into(), 0b10_11_00_01); + t2.0[i] = _mm256_blend_epi32(t2.0[i].into(), swapped, 0b11110000).into(); + } + //println!("t2 = (S8 S9 S11 S10)"); + //print_vec!(t2); + //println!(""); - Q = P.diff_sum(); - //println!("Q = (S11 S14 S12 S13)"); - //println!("Q = {:?}\n", Q.split()); + t2.diff_sum(); + //println!("t2 = (S12 S13 S14 S15)"); + //print_vec!(t2); + //println!(""); let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) - let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBBC) + let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) for i in 0..5 { - t0.0[i] = _mm256_permutevar8x32_epi32(Q.0[i], c0); - t1.0[i] = _mm256_permutevar8x32_epi32(Q.0[i], c1); + t0.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c0); + t1.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c1); } //println!("t0 = (S11 S13 S13 S11)"); - //println!("t0 = {:?}\n", t0.split()); + //print_vec!(t0); + //println!(""); + //println!("t1 = (S12 S14 S14 S12)"); - //println!("t1 = {:?}\n", t1.split()); + //print_vec!(t1); + //println!(""); ExtendedPoint(&t0 * &t1) } @@ -112,25 +145,51 @@ mod test { let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); let (X2, Y2, Z2, T2) = (Q.X, Q.Y, Q.Z, Q.T); + macro_rules! print_var { + ($x:ident) => { + println!("{} = {:?}", stringify!($x), $x.to_bytes()); + } + } + let S0 = &Y1 - &X1; // R1 let S1 = &Y1 + &X1; // R3 let S2 = &Y2 - &X2; // R2 let S3 = &Y2 + &X2; // R4 + print_var!(S0); + print_var!(S1); + print_var!(S2); + print_var!(S3); + println!(""); let S4 = &S0 * &S2; // R5 = R1 * R2 let S5 = &S1 * &S3; // R6 = R3 * R4 - let S6 = &T1 * &T2; // R7 - let S7 = &Z1 * &Z2; // R8 + let S6 = &Z1 * &Z2; // R8 + let S7 = &T1 * &T2; // R7 + print_var!(S4); + print_var!(S5); + print_var!(S6); + print_var!(S7); + println!(""); - let S8 = &S6 * &(-&FieldElement32([2*121665,0,0,0,0,0,0,0,0,0])); // R7 - let S9 = &S7 * &FieldElement32([2*121666,0,0,0,0,0,0,0,0,0]); // R8 - let S10 = &S4 * &FieldElement32([ 121666,0,0,0,0,0,0,0,0,0]); // R5 - let S11 = &S5 * &FieldElement32([ 121666,0,0,0,0,0,0,0,0,0]); // R6 + let S8 = &S4 * &FieldElement32([ 121666,0,0,0,0,0,0,0,0,0]); // R5 + let S9 = &S5 * &FieldElement32([ 121666,0,0,0,0,0,0,0,0,0]); // R6 + let S10 = &S6 * &FieldElement32([2*121666,0,0,0,0,0,0,0,0,0]); // R8 + let S11 = &S7 * &(-&FieldElement32([2*121665,0,0,0,0,0,0,0,0,0])); // R7 + print_var!(S8 ); + print_var!(S9 ); + print_var!(S10); + print_var!(S11); + println!(""); - let S12 = &S11 - &S10; // R1 - let S13 = &S11 + &S10; // R4 - let S14 = &S9 - &S8; // R2 - let S15 = &S9 + &S8; // R3 + let S12 = &S9 - &S8; // R1 + let S13 = &S9 + &S8; // R4 + let S14 = &S10 - &S11; // R2 + let S15 = &S10 + &S11; // R3 + print_var!(S12); + print_var!(S13); + print_var!(S14); + print_var!(S15); + println!(""); let X3 = &S12 * &S14; // R1 * R2 let Y3 = &S15 * &S13; // R3 * R4 @@ -140,8 +199,22 @@ mod test { edwards::ExtendedPoint{X: X3, Y: Y3, Z: Z3, T: T3} } + fn addition_test_helper(P: edwards::ExtendedPoint, Q: edwards::ExtendedPoint) { + let R1: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); + let R2: edwards::ExtendedPoint = (&ExtendedPoint::from(P) + &ExtendedPoint::from(Q)).into(); + println!("Testing point addition:"); + println!("P = {:?}", P); + println!("Q = {:?}", Q); + println!("(serial) R1 = {:?}", R1); + println!("(vector) R2 = {:?}", R2); + println!("P + Q = {:?}", &P + &Q); + assert_eq!(R1.compress(), (&P + &Q).compress()); + assert_eq!(R2.compress(), (&P + &Q).compress()); + println!("OK!\n"); + } + #[test] - fn serial_add_vs_edwards_extendedpoint() { + fn addition_vs_serial_add_vs_edwards_extendedpoint() { use constants; use scalar::Scalar; use edwards::Identity; @@ -149,41 +222,40 @@ mod test { println!("Testing id + id"); let P = edwards::ExtendedPoint::identity(); let Q = edwards::ExtendedPoint::identity(); - let R: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); - println!("P = {:?}", P); - println!("Q = {:?}", Q); - println!("R = {:?}", R); - println!("P + Q = {:?}", &P + &Q); - assert_eq!(R.compress(), (&P + &Q).compress()); + addition_test_helper(P, Q); println!("Testing id + B"); let P = edwards::ExtendedPoint::identity(); let Q = constants::ED25519_BASEPOINT_POINT; - let R: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); - println!("P = {:?}", P); - println!("Q = {:?}", Q); - println!("R = {:?}", R); - println!("P + Q = {:?}", &P + &Q); - assert_eq!(R.compress(), (&P + &Q).compress()); + addition_test_helper(P, Q); println!("Testing B + B"); let P = constants::ED25519_BASEPOINT_POINT; let Q = constants::ED25519_BASEPOINT_POINT; - let R: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); - println!("P = {:?}", P); - println!("Q = {:?}", Q); - println!("R = {:?}", R); - println!("P + Q = {:?}", &P + &Q); - assert_eq!(R.compress(), (&P + &Q).compress()); + addition_test_helper(P, Q); println!("Testing B + kB"); let P = constants::ED25519_BASEPOINT_POINT; let Q = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from_u64(8475983829); - let R: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); - println!("P = {:?}", P); - println!("Q = {:?}", Q); - println!("R = {:?}", R); - println!("P + Q = {:?}", &P + &Q); - assert_eq!(R.compress(), (&P + &Q).compress()); + addition_test_helper(P, Q); } } + +#[cfg(all(test, feature = "bench"))] +mod bench { + use test::Bencher; + use super::*; + + use constants; + use scalar::Scalar; + + #[bench] + fn point_addition(b: &mut Bencher) { + let B = &constants::ED25519_BASEPOINT_TABLE; + let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); + let Q = ExtendedPoint::from(B * &Scalar::from_u64(98932328)); + + b.iter(|| &P + &Q ); + } +} + diff --git a/src/avx2/field.rs b/src/avx2/field.rs index 15050c6..ba1b001 100644 --- a/src/avx2/field.rs +++ b/src/avx2/field.rs @@ -71,8 +71,8 @@ impl FieldElement32x4 { FieldElement32x4(buf) } - // Given `self = (A,B,C,D)`, compute `(B - A, B + A, D - C, D + C)`. - pub fn diff_sum(&self) -> FieldElement32x4 { + // Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)`. + pub fn diff_sum(&mut self) { /// (v0 v1 v2 v3 v4 v5 v6 v7) -> (v1 v0 v3 v2 v5 v4 v7 v6) #[inline(always)] fn alternate_32bit_lanes(v: u32x8) -> u32x8 { @@ -92,8 +92,6 @@ impl FieldElement32x4 { } } - let mut out = [u32x8::splat(0); 5]; - for i in 0..5 { let x = self.0[i]; let p = P_TIMES_2.0[i] ; @@ -103,10 +101,8 @@ impl FieldElement32x4 { let sum = x + x_shuf; let diff_sum = blend_alternating_32bit_lanes(diff, sum); - out[i] = diff_sum; + self.0[i] = diff_sum; } - - FieldElement32x4(out) } // Given `self = (A,B,C,D)`, compute `(B + A, B - A, D + C, D - C)`. @@ -415,9 +411,10 @@ mod test { let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); - let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + vec.diff_sum(); - let result = vec.diff_sum().split(); + let result = vec.split(); assert_eq!(result[0], &x1 - &x0); assert_eq!(result[1], &x1 + &x0); @@ -493,7 +490,7 @@ mod bench { #[bench] fn multiply(b: &mut Bencher) { - let vec = FieldElement32x4::splat(&FieldElement::zero()); + let vec = FieldElement32x4::splat(&FieldElement32::zero()); let vecprime = vec.clone(); b.iter(|| &vec * &vecprime ); From e2a2b3a3b54a3cff6fb9f732bbcb62dbceebdb66 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 9 Nov 2017 10:24:41 -0800 Subject: [PATCH 04/54] Add comment to mul draft --- src/avx2/field.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/avx2/field.rs b/src/avx2/field.rs index ba1b001..0fd53dd 100644 --- a/src/avx2/field.rs +++ b/src/avx2/field.rs @@ -344,14 +344,23 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } } - let x19 = u32x8::new(19,0,19,0,19,0,19,0); + let v19 = u32x8::new(19,0,19,0,19,0,19,0); + + // XXX clean up this horrifying abomination + // + // The idea is to take the standard "schoolbook multiplication square" (see + // FieldElement32), and walk up each column from top to bottom, then left to right. + // + // Instead of multiplying by 19 in a precomputation, we overwrite the b[i] value with + // b[i]*19 as soon as we will no longer need it. + // macro_rules! loop_body { ($i:expr) => { let (ai, ai1) = unpack_pair(self.0[$i/2]); c[9] = c[9] + m(ai, b[(100 + 9-$i) % 10]); - b[(100 + 9-$i) % 10] = m_lo(b[(100 + 9-$i) % 10], x19); + b[(100 + 9-$i) % 10] = m_lo(b[(100 + 9-$i) % 10], v19); c[8] = c[8] + m(ai, b[(100 + 8-$i) % 10]); c[7] = c[7] + m(ai, b[(100 + 7-$i) % 10]); c[6] = c[6] + m(ai, b[(100 + 6-$i) % 10]); @@ -364,7 +373,7 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { let ai1_2 = ai1 + ai1; c[9] = c[9] + m(ai1, b[(100 + 9-($i+1)) % 10]); - b[(100 + 9-($i+1)) % 10] = m_lo(b[(100 + 9-($i+1)) % 10], x19); + b[(100 + 9-($i+1)) % 10] = m_lo(b[(100 + 9-($i+1)) % 10], v19); c[8] = c[8] + m(ai1_2, b[(100 + 8-($i+1)) % 10]); c[7] = c[7] + m(ai1, b[(100 + 7-($i+1)) % 10]); c[6] = c[6] + m(ai1_2, b[(100 + 6-($i+1)) % 10]); From 9e383ffccb16d8b1331a5fd5ed3a16870ff7e162 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 9 Nov 2017 10:24:58 -0800 Subject: [PATCH 05/54] Add doubling test harness --- src/avx2/edwards.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs index 3ab6139..bd9ece0 100644 --- a/src/avx2/edwards.rs +++ b/src/avx2/edwards.rs @@ -214,7 +214,7 @@ mod test { } #[test] - fn addition_vs_serial_add_vs_edwards_extendedpoint() { + fn vector_addition_vs_serial_addition_vs_edwards_extendedpoint() { use constants; use scalar::Scalar; use edwards::Identity; @@ -239,6 +239,60 @@ mod test { let Q = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from_u64(8475983829); addition_test_helper(P, Q); } + + fn serial_double(P: edwards::ExtendedPoint) -> edwards::ExtendedPoint { + let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); + + let S0 = &X1 + &Y1; // R1 + + let S1 = X1.square(); + let S2 = Y1.square(); + let S3 = Z1.square(); + let S4 = S0.square(); + + let S5 = &S1 + &S2; + let S6 = &S1 - &S2; + let S7 = &S3 + &S3; + let S8 = &S7 + &S6; + let S9 = &S5 - &S4; + + let X3 = &S8 * &S9; + let Y3 = &S5 * &S6; + let Z3 = &S8 * &S6; + let T3 = &S5 * &S9; + + edwards::ExtendedPoint{X: X3, Y: Y3, Z: Z3, T: T3} + } + + fn doubling_test_helper(P: edwards::ExtendedPoint) { + let R1: edwards::ExtendedPoint = serial_double(P.into()).into(); + println!("Testing point doubling:"); + println!("P = {:?}", P); + println!("(serial) R1 = {:?}", R1); + //println!("(vector) R2 = {:?}", R2); + println!("P + P = {:?}", &P + &P); + assert_eq!(R1.compress(), (&P + &P).compress()); + println!("OK!\n"); + } + + #[test] + fn vector_doubling_vs_serial_doubling_vs_edwards_extendedpoint() { + use constants; + use scalar::Scalar; + use edwards::Identity; + + println!("Testing [2]id"); + let P = edwards::ExtendedPoint::identity(); + doubling_test_helper(P); + + println!("Testing [2]B"); + let P = constants::ED25519_BASEPOINT_POINT; + doubling_test_helper(P); + + println!("Testing [2]([k]B)"); + let P = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from_u64(8475983829); + doubling_test_helper(P); + } } #[cfg(all(test, feature = "bench"))] From 2f32f6355c6e03d9f1d553842b8d67c70b1fae03 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 9 Nov 2017 16:37:50 -0800 Subject: [PATCH 06/54] Add doubling skeleton and scalar mult --- src/avx2/edwards.rs | 284 +++++++++++++++++++++++++++++++++++++++++++- src/avx2/field.rs | 30 ++++- 2 files changed, 306 insertions(+), 8 deletions(-) diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs index bd9ece0..2c7d10a 100644 --- a/src/avx2/edwards.rs +++ b/src/avx2/edwards.rs @@ -14,15 +14,22 @@ #![allow(bad_style)] use std::convert::From; -use std::ops::Add; +use std::ops::{Add, Mul, Neg}; -use stdsimd::simd::u32x8; +use stdsimd::simd::{u32x8, i32x8}; + +use subtle::ConditionallyAssignable; use edwards; +use scalar::Scalar; + +use traits::Identity; use avx2::field::FieldElement32x4; +use avx2::field::P_TIMES_2; /// A point on Curve25519, represented in an AVX2-friendly format. +#[derive(Copy, Clone, Debug)] pub(crate) struct ExtendedPoint(FieldElement32x4); // XXX need to cfg gate here to handle FieldElement64 @@ -40,6 +47,165 @@ impl From for edwards::ExtendedPoint { } } +impl ConditionallyAssignable for ExtendedPoint { + fn conditional_assign(&mut self, other: &ExtendedPoint, choice: u8) { + self.0.conditional_assign(&other.0, choice); + } +} + +impl Identity for ExtendedPoint { + fn identity() -> ExtendedPoint { + ExtendedPoint(FieldElement32x4([ + u32x8::new(0,1,0,0,1,0,0,0), + u32x8::splat(0), + u32x8::splat(0), + u32x8::splat(0), + u32x8::splat(0), + ])) + } +} + +impl<'a> Neg for &'a ExtendedPoint { + type Output = ExtendedPoint; + + fn neg(self) -> ExtendedPoint { + let mut neg = *self; + neg.0.mask_negate(0b10100101); + neg + } +} + +impl ExtendedPoint { + fn double(&self) -> ExtendedPoint { + unsafe { + use stdsimd::vendor::_mm256_permute2x128_si256; + use stdsimd::vendor::_mm256_permutevar8x32_epi32; + use stdsimd::vendor::_mm256_blend_epi32; + use stdsimd::vendor::_mm256_shuffle_epi32; + + macro_rules! print_vec { + ($x:ident) => { + let splits = $x.split(); + println!("{}[0] = {:?}", stringify!($x), splits[0].to_bytes()); + println!("{}[1] = {:?}", stringify!($x), splits[1].to_bytes()); + println!("{}[2] = {:?}", stringify!($x), splits[2].to_bytes()); + println!("{}[3] = {:?}", stringify!($x), splits[3].to_bytes()); + } + } + + let P = &self.0; + + let mut t0 = FieldElement32x4::zero(); + let mut t1 = FieldElement32x4::zero(); + + // Set t0 = (X1 Y1 X1 Y1) + t0.0[0] = _mm256_permute2x128_si256(P.0[0].into(), P.0[0].into(), 0b0000_0000).into(); + t0.0[1] = _mm256_permute2x128_si256(P.0[1].into(), P.0[1].into(), 0b0000_0000).into(); + t0.0[2] = _mm256_permute2x128_si256(P.0[2].into(), P.0[2].into(), 0b0000_0000).into(); + t0.0[3] = _mm256_permute2x128_si256(P.0[3].into(), P.0[3].into(), 0b0000_0000).into(); + t0.0[4] = _mm256_permute2x128_si256(P.0[4].into(), P.0[4].into(), 0b0000_0000).into(); + + // Set t1 = (Y1 X1 Y1 X1) + t1.0[0] = _mm256_shuffle_epi32(t0.0[0].into(), 0b10_11_00_01).into(); + t1.0[1] = _mm256_shuffle_epi32(t0.0[1].into(), 0b10_11_00_01).into(); + t1.0[2] = _mm256_shuffle_epi32(t0.0[2].into(), 0b10_11_00_01).into(); + t1.0[3] = _mm256_shuffle_epi32(t0.0[3].into(), 0b10_11_00_01).into(); + t1.0[4] = _mm256_shuffle_epi32(t0.0[4].into(), 0b10_11_00_01).into(); + + // Set t0 = (X1+Y1 X1+Y1 X1+Y1 X1+Y1) + t0.0[0] = t0.0[0] + t1.0[0]; + t0.0[1] = t0.0[1] + t1.0[1]; + t0.0[2] = t0.0[2] + t1.0[2]; + t0.0[3] = t0.0[3] + t1.0[3]; + t0.0[4] = t0.0[4] + t1.0[4]; + + // Set t0 = (X1 Y1 Z1 X1+Y1) + t0.0[0] = _mm256_blend_epi32(t0.0[0].into(), P.0[0].into(), 0b01011111).into(); + t0.0[1] = _mm256_blend_epi32(t0.0[1].into(), P.0[1].into(), 0b01011111).into(); + t0.0[2] = _mm256_blend_epi32(t0.0[2].into(), P.0[2].into(), 0b01011111).into(); + t0.0[3] = _mm256_blend_epi32(t0.0[3].into(), P.0[3].into(), 0b01011111).into(); + t0.0[4] = _mm256_blend_epi32(t0.0[4].into(), P.0[4].into(), 0b01011111).into(); + + t1 = &t0 * &t0; // replace with .square() + + // Now t1 = (S1 S2 S3 S4) + + let c0 = u32x8::new(0,0,2,2,0,0,2,2); // (ABCD) -> (AAAA) + let c1 = u32x8::new(1,1,3,3,1,1,3,3); // (ABCD) -> (BBBB) + + // Horror block goes here: we want to compute the following table: + // We know that the bit-excess b is bounded by eps, since S1 S2 S3 S4 + // are the outputs of a squaring, so they're freshly reduced. + // + // + | S1 | S1 | S1 | S1 | + // + | S2 | | | S2 | + // + | | | S3 | | + // + | | | S3 | | + // + | | 2p | 2p | 2p | + // - | | S2 | S2 | | + // - | | | | S4 | + // ======================= + // S5 S6 S8 S9 + // + // Bounds for even / odd limbs: + // + // + | 2^26 | 2^26 | 2^26 | 2^26 | + | 2^25 | 2^25 | 2^25 | 2^25 | + // + | 2^26 | | | 2^26 | + | 2^25 | | | 2^25 | + // + | | | 2^26 | | + | | | 2^25 | | + // + | | | 2^26 | | + | | | 2^25 | | + // + | | 2^27 | 2^27 | 2^27 | + | | 2^26 | 2^26 | 2^26 | + // - | | 0 | 0 | | - | | 0 | 0 | | + // - | | | | 0 | - | | | | 0 | + // =================================== =================================== + // < 2^27 2^27.59 2^28.33 2^28 2^26 2^26.59 2^27.33 2^27 + // + // So, the bit-excess for (S5 S6 S8 S9) is (1, 1.59, 2.33, 2). + // + // However the multiplication routine only allows (1.75, 1.75, 1.75, 1.75). + // + // This is because we need to have 19*y[i] < 2^32. Otherwise I think we could get b < 2.5. + // + // Can we tighten these bounds to avoid a reduction? Alternately, can we do better than + // the 64-bit reduction that reduce32() calls internally? + // + // Also, can we do better than the mess below? + for i in 0..5 { + let zero = i32x8::splat(0); + let S1 = _mm256_permutevar8x32_epi32(t1.0[i], c0); + let S2 = _mm256_permutevar8x32_epi32(t1.0[i], c1); + let S3_2 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into(), 0b01010000).into(); + t0.0[i] = (P_TIMES_2.0[i] + S3_2) + S1; + t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into(), 0b10100101).into(); + let S4 = _mm256_blend_epi32(zero, t1.0[i].into(), 0b10100000); + let sub = _mm256_blend_epi32(S2.into(), S4, 0b10100101).into(); + t0.0[i] = t0.0[i] - sub; + } + + // This is really sad, see above + t0.reduce32(); + + let c0 = u32x8::new(4,0,6,2,4,0,6,2); // (ABCD) -> (CACA) + let c1 = u32x8::new(5,1,7,3,1,5,3,7); // (ABCD) -> (DBBD) + + for i in 0..5 { + let tmp = t0.0[i]; + t0.0[i] = _mm256_permutevar8x32_epi32(tmp, c0); + t1.0[i] = _mm256_permutevar8x32_epi32(tmp, c1); + } + + ExtendedPoint(&t0 * &t1) + } + } + + pub fn mult_by_pow_2(&self, k: u32) -> ExtendedPoint { + let mut tmp: ExtendedPoint = *self; + for _ in 0..k { + tmp = tmp.double(); + } + tmp + } +} + impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { type Output = ExtendedPoint; @@ -134,13 +300,56 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { } } } + +impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { + type Output = ExtendedPoint; + /// Scalar multiplication: compute `scalar * self`. + /// + /// Uses a window of size 4. + fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { + use traits::select_precomputed_point; + + // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] + let mut lookup_table: [ExtendedPoint; 8] = [*self; 8]; + for i in 0..7 { + lookup_table[i+1] = self + &lookup_table[i]; + } + + // Setting s = scalar, compute + // + // s = s_0 + s_1*16^1 + ... + s_63*16^63, + // + // with `-8 ≤ s_i < 8` for `0 ≤ i < 63` and `-8 ≤ s_63 ≤ 8`. + let scalar_digits = scalar.to_radix_16(); + + // Compute s*P as + // + // s*P = P*(s_0 + s_1*16^1 + s_2*16^2 + ... + s_63*16^63) + // s*P = P*s_0 + P*s_1*16^1 + P*s_2*16^2 + ... + P*s_63*16^63 + // s*P = P*s_0 + 16*(P*s_1 + 16*(P*s_2 + 16*( ... + P*s_63)...)) + // + // We sum right-to-left. + let mut Q = ExtendedPoint::identity(); + for i in (0..64).rev() { + // Q = 16*Q + Q = Q.mult_by_pow_2(4); + // R = s_i * Q + let R = select_precomputed_point(scalar_digits[i], &lookup_table); + // Q = Q + R + Q = &Q + &R; + } + Q + } +} #[cfg(test)] mod test { use super::*; + use constants; + fn serial_add(P: edwards::ExtendedPoint, Q: edwards::ExtendedPoint) -> edwards::ExtendedPoint { - use field_32bit::FieldElement32; + use backend::u32::field::FieldElement32; let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); let (X2, Y2, Z2, T2) = (Q.X, Q.Y, Q.Z, Q.T); @@ -217,7 +426,6 @@ mod test { fn vector_addition_vs_serial_addition_vs_edwards_extendedpoint() { use constants; use scalar::Scalar; - use edwards::Identity; println!("Testing id + id"); let P = edwards::ExtendedPoint::identity(); @@ -243,18 +451,37 @@ mod test { fn serial_double(P: edwards::ExtendedPoint) -> edwards::ExtendedPoint { let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); + macro_rules! print_var { + ($x:ident) => { + println!("{} = {:?}", stringify!($x), $x.to_bytes()); + } + } + let S0 = &X1 + &Y1; // R1 + print_var!(S0); + println!(""); let S1 = X1.square(); let S2 = Y1.square(); let S3 = Z1.square(); let S4 = S0.square(); + print_var!(S1); + print_var!(S2); + print_var!(S3); + print_var!(S4); + println!(""); let S5 = &S1 + &S2; let S6 = &S1 - &S2; let S7 = &S3 + &S3; let S8 = &S7 + &S6; let S9 = &S5 - &S4; + print_var!(S5); + print_var!(S6); + print_var!(S7); + print_var!(S8); + print_var!(S9); + println!(""); let X3 = &S8 * &S9; let Y3 = &S5 * &S6; @@ -266,12 +493,14 @@ mod test { fn doubling_test_helper(P: edwards::ExtendedPoint) { let R1: edwards::ExtendedPoint = serial_double(P.into()).into(); + let R2: edwards::ExtendedPoint = ExtendedPoint::from(P).double().into(); println!("Testing point doubling:"); println!("P = {:?}", P); println!("(serial) R1 = {:?}", R1); - //println!("(vector) R2 = {:?}", R2); + println!("(vector) R2 = {:?}", R2); println!("P + P = {:?}", &P + &P); assert_eq!(R1.compress(), (&P + &P).compress()); + assert_eq!(R2.compress(), (&P + &P).compress()); println!("OK!\n"); } @@ -279,7 +508,6 @@ mod test { fn vector_doubling_vs_serial_doubling_vs_edwards_extendedpoint() { use constants; use scalar::Scalar; - use edwards::Identity; println!("Testing [2]id"); let P = edwards::ExtendedPoint::identity(); @@ -293,6 +521,33 @@ mod test { let P = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from_u64(8475983829); doubling_test_helper(P); } + + #[test] + fn identity_trait_vs_edwards_identity() { + let id1: edwards::ExtendedPoint = ExtendedPoint::identity().into(); + let id2: edwards::ExtendedPoint = edwards::ExtendedPoint::identity(); + assert_eq!(id1.compress(), id2.compress()); + } + + #[test] + fn neg_vs_edwards_neg() { + let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); + let Bneg = -&B; + assert_eq!(edwards::ExtendedPoint::from(Bneg).compress(), + (-&constants::ED25519_BASEPOINT_POINT).compress()); + } + + #[test] + fn scalar_mult_vs_edwards_scalar_mult() { + let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); + // some random bytes + let s = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + + let R1 = edwards::ExtendedPoint::from(&B * &s); + let R2 = &constants::ED25519_BASEPOINT_TABLE * &s; + + assert_eq!(R1.compress(), R2.compress()); + } } #[cfg(all(test, feature = "bench"))] @@ -311,5 +566,22 @@ mod bench { b.iter(|| &P + &Q ); } + + #[bench] + fn point_doubling(b: &mut Bencher) { + let B = &constants::ED25519_BASEPOINT_TABLE; + let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); + + b.iter(|| P.double() ); + } + + #[bench] + fn scalar_mult(b: &mut Bencher) { + let B = &constants::ED25519_BASEPOINT_TABLE; + let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); + let s = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + + b.iter(|| &P * &s ); + } } diff --git a/src/avx2/field.rs b/src/avx2/field.rs index 0fd53dd..cca421d 100644 --- a/src/avx2/field.rs +++ b/src/avx2/field.rs @@ -19,7 +19,7 @@ use stdsimd::simd::{u32x8, i32x8, u64x4}; use backend::u32::field::FieldElement32; -static P_TIMES_2: FieldElement32x4 = FieldElement32x4([ +pub(crate) static P_TIMES_2: FieldElement32x4 = FieldElement32x4([ u32x8::new(134217690, 134217690, 67108862, 67108862, 134217690, 134217690, 67108862, 67108862), u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862), u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862), @@ -28,9 +28,22 @@ static P_TIMES_2: FieldElement32x4 = FieldElement32x4([ ]); /// A vector of four `FieldElements`, implemented using AVX2. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]); +use subtle::ConditionallyAssignable; + +impl ConditionallyAssignable for FieldElement32x4 { + fn conditional_assign(&mut self, other: &FieldElement32x4, choice: u8) { + let mask = (-(choice as i32)) as u32; + let mask_vec = u32x8::splat(mask); + for i in 0..5 { + self.0[i] = self.0[i] ^ (mask_vec & (self.0[i] ^ other.0[i])); + } + } +} + + impl FieldElement32x4 { pub(crate) fn split(&self) -> [FieldElement32; 4] { let mut out = [FieldElement32::zero(); 4]; @@ -71,6 +84,19 @@ impl FieldElement32x4 { FieldElement32x4(buf) } + // Negate variables in lanes where mask is set + // XXX fix up api + pub fn mask_negate(&mut self, mask: u8) { + unsafe { + use stdsimd::vendor::_mm256_blend_epi32; + for i in 0..5 { + let negated = P_TIMES_2.0[i] - self.0[i]; + self.0[i] = _mm256_blend_epi32(self.0[i].into(), negated.into(), mask as i32).into(); + } + } + self.reduce32(); + } + // Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)`. pub fn diff_sum(&mut self) { /// (v0 v1 v2 v3 v4 v5 v6 v7) -> (v1 v0 v3 v2 v5 v4 v7 v6) From 34ae1b15e00096c22376ee7b54ac5a647a364c23 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sun, 12 Nov 2017 17:30:01 -0800 Subject: [PATCH 07/54] Add a squaring implementation --- src/avx2/edwards.rs | 5 +++- src/avx2/field.rs | 72 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs index 2c7d10a..ca73b87 100644 --- a/src/avx2/edwards.rs +++ b/src/avx2/edwards.rs @@ -126,7 +126,7 @@ impl ExtendedPoint { t0.0[3] = _mm256_blend_epi32(t0.0[3].into(), P.0[3].into(), 0b01011111).into(); t0.0[4] = _mm256_blend_epi32(t0.0[4].into(), P.0[4].into(), 0b01011111).into(); - t1 = &t0 * &t0; // replace with .square() + t1 = t0.square(); // Now t1 = (S1 S2 S3 S4) @@ -167,6 +167,9 @@ impl ExtendedPoint { // // Can we tighten these bounds to avoid a reduction? Alternately, can we do better than // the 64-bit reduction that reduce32() calls internally? + // + // Or, could we do the arithmetic on the intermediate [u64x4;10], then do + // the reduction we'd need to do for the squaring? // // Also, can we do better than the mess below? for i in 0..5 { diff --git a/src/avx2/field.rs b/src/avx2/field.rs index cca421d..53c6405 100644 --- a/src/avx2/field.rs +++ b/src/avx2/field.rs @@ -341,6 +341,60 @@ pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { } } +impl FieldElement32x4 { + pub fn square(&self) -> FieldElement32x4 { + #[inline(always)] + fn m(x: u32x8, y: u32x8) -> u64x4 { + use stdsimd::vendor::_mm256_mul_epu32; + unsafe { _mm256_mul_epu32(x,y) } + } + + #[inline(always)] + fn m_lo(x: u32x8, y: u32x8) -> u32x8 { + use stdsimd::vendor::_mm256_mul_epu32; + unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } + } + + let v19 = u32x8::new(19,0,19,0,19,0,19,0); + + let mut z = [u64x4::splat(0); 10]; + + let (x0, x1) = unpack_pair(self.0[0]); + let (x2, x3) = unpack_pair(self.0[1]); + let (x4, x5) = unpack_pair(self.0[2]); + let (x6, x7) = unpack_pair(self.0[3]); + let (x8, x9) = unpack_pair(self.0[4]); + + let x0_2 = x0 << 1; + let x1_2 = x1 << 1; + let x2_2 = x2 << 1; + let x3_2 = x3 << 1; + let x4_2 = x4 << 1; + let x5_2 = x5 << 1; + let x6_2 = x6 << 1; + let x7_2 = x7 << 1; + + let x5_19 = m_lo(v19, x5); + let x6_19 = m_lo(v19, x6); + let x7_19 = m_lo(v19, x7); + let x8_19 = m_lo(v19, x8); + let x9_19 = m_lo(v19, x9); + + z[0] = m(x0, x0) + m(x2_2,x8_19) + m(x4_2,x6_19) + ((m(x1_2,x9_19) + m(x3_2,x7_19) + m(x5,x5_19)) << 1); + z[1] = m(x0_2,x1) + m(x3_2,x8_19) + m(x5_2,x6_19) + ((m(x2,x9_19) + m(x4,x7_19)) << 1); + z[2] = m(x0_2,x2) + m(x1_2,x1) + m(x4_2,x8_19) + m(x6,x6_19) + ((m(x3_2,x9_19) + m(x5_2,x7_19)) << 1); + z[3] = m(x0_2,x3) + m(x1_2,x2) + m(x5_2,x8_19) + ((m(x4,x9_19) + m(x6,x7_19)) << 1); + z[4] = m(x0_2,x4) + m(x1_2,x3_2) + m(x2, x2) + m(x6_2,x8_19) + ((m(x5_2,x9_19) + m(x7,x7_19)) << 1); + z[5] = m(x0_2,x5) + m(x1_2,x4) + m(x2_2,x3) + m(x7_2,x8_19) + ((m(x6,x9_19)) << 1); + z[6] = m(x0_2,x6) + m(x1_2,x5_2) + m(x2_2,x4) + m(x3_2,x3) + m(x8,x8_19) + ((m(x7_2,x9_19)) << 1); + z[7] = m(x0_2,x7) + m(x1_2,x6) + m(x2_2,x5) + m(x3_2,x4) + ((m(x8,x9_19)) << 1); + z[8] = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1); + z[9] = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5); + + return FieldElement32x4::reduce64(z); + } +} + impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { type Output = FieldElement32x4; fn mul(self, _rhs: &'b FieldElement32x4) -> FieldElement32x4 { @@ -457,6 +511,24 @@ mod test { assert_eq!(result[3], &x3 + &x2); } + #[test] + fn square_vs_serial() { + let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); + let x1 = FieldElement32([10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10109]); + let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); + let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); + + let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + + let result = vec.square().split(); + + assert_eq!(result[0], &x0 * &x0); + assert_eq!(result[1], &x1 * &x1); + assert_eq!(result[2], &x2 * &x2); + assert_eq!(result[3], &x3 * &x3); + } + + #[test] fn multiply_vs_serial() { let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); From e7ec5b3dd19a11189a270a128ac1ee8c22870e56 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 13 Nov 2017 09:52:42 -0800 Subject: [PATCH 08/54] Add basepoint table code --- src/avx2/edwards.rs | 119 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs index ca73b87..ba13ec7 100644 --- a/src/avx2/edwards.rs +++ b/src/avx2/edwards.rs @@ -30,7 +30,7 @@ use avx2::field::P_TIMES_2; /// A point on Curve25519, represented in an AVX2-friendly format. #[derive(Copy, Clone, Debug)] -pub(crate) struct ExtendedPoint(FieldElement32x4); +pub struct ExtendedPoint(FieldElement32x4); // XXX need to cfg gate here to handle FieldElement64 impl From for ExtendedPoint { @@ -344,6 +344,93 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { Q } } + +#[derive(Clone)] +pub struct EdwardsBasepointTable(pub [[ExtendedPoint; 8]; 32]); + +impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsBasepointTable { + type Output = ExtendedPoint; + + /// Construct an `ExtendedPoint` from a `Scalar`, `scalar`, by + /// computing the multiple `aB` of the basepoint `B`. + /// + /// Precondition: the scalar must be reduced. + /// + /// The computation proceeds as follows, as described on page 13 + /// of the Ed25519 paper. Write the scalar `a` in radix 16 with + /// coefficients in [-8,8), i.e., + /// + /// a = a_0 + a_1*16^1 + ... + a_63*16^63, + /// + /// with -8 ≤ a_i < 8. Then + /// + /// a*B = a_0*B + a_1*16^1*B + ... + a_63*16^63*B. + /// + /// Grouping even and odd coefficients gives + /// + /// a*B = a_0*16^0*B + a_2*16^2*B + ... + a_62*16^62*B + /// + a_1*16^1*B + a_3*16^3*B + ... + a_63*16^63*B + /// = (a_0*16^0*B + a_2*16^2*B + ... + a_62*16^62*B) + /// + 16*(a_1*16^0*B + a_3*16^2*B + ... + a_63*16^62*B). + /// + /// We then use the `select_precomputed_point` function, which + /// takes `-8 ≤ x < 8` and `[16^2i * B, ..., 8 * 16^2i * B]`, + /// and returns `x * 16^2i * B` in constant time. + fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { + let e = scalar.to_radix_16(); + let mut h = ExtendedPoint::identity(); + + for i in (0..64).filter(|x| x % 2 == 1) { + h = &h + &edwards::select_precomputed_point(e[i], &self.0[i/2]); + } + + h = h.mult_by_pow_2(4); + + for i in (0..64).filter(|x| x % 2 == 0) { + h = &h + &edwards::select_precomputed_point(e[i], &self.0[i/2]); + } + + h + } +} + +impl<'a, 'b> Mul<&'a EdwardsBasepointTable> for &'b Scalar { + type Output = ExtendedPoint; + + /// Given `self` a table of precomputed multiples of the point `B`, compute `B * s`. + fn mul(self, basepoint_table: &'a EdwardsBasepointTable) -> ExtendedPoint { + basepoint_table * &self + } +} + +impl EdwardsBasepointTable { + /// Create a table of precomputed multiples of `basepoint`. + pub fn create(basepoint: &ExtendedPoint) -> EdwardsBasepointTable { + // Create the table storage + // XXX can we skip the initialization without too much unsafety? + // stick 30K on the stack and call it a day. + let mut table = EdwardsBasepointTable([[ExtendedPoint::identity(); 8]; 32]); + let mut P = *basepoint; + for i in 0..32 { + // P = (16^2)^i * B + let mut jP = P; + for j in 1..9 { + // table[i][j-1] is supposed to be j*(16^2)^i*B + table.0[i][j-1] = jP; + jP = &P + &jP; + } + P = P.mult_by_pow_2(8); + } + table + } + + /// Get the basepoint for this table as an `ExtendedPoint`. + pub fn basepoint(&self) -> ExtendedPoint { + // self.0[0][0] has 1*(16^2)^0*B + self.0[0][0] + } +} + #[cfg(test)] mod test { @@ -551,6 +638,20 @@ mod test { assert_eq!(R1.compress(), R2.compress()); } + + #[test] + fn scalar_mult_vs_basepoint_table_scalar_mult() { + let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); + let B_table = EdwardsBasepointTable::create(&B); + // some random bytes + let s = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + + let P1 = &B * &s; + let P2 = &B_table * &s; + + assert_eq!(edwards::ExtendedPoint::from(P1).compress(), + edwards::ExtendedPoint::from(P2).compress()); + } } #[cfg(all(test, feature = "bench"))] @@ -586,5 +687,21 @@ mod bench { b.iter(|| &P * &s ); } + + #[bench] + fn basepoint_table_creation(b: &mut Bencher) { + let B = ExtendedPoint::from(constants::ED25519_BASEPOINT_POINT); + + b.iter(|| EdwardsBasepointTable::create(&B) ); + } + + #[bench] + fn basepoint_mult(b: &mut Bencher) { + let B = ExtendedPoint::from(constants::ED25519_BASEPOINT_POINT); + let table = EdwardsBasepointTable::create(&B); + let s = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + + b.iter(|| &table * &s ); + } } From 7354b569bbe0e7619e7174b862c7f5a382e62218 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 13 Nov 2017 23:47:38 -0800 Subject: [PATCH 09/54] Add 32bit reduction code --- src/avx2/field.rs | 62 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/src/avx2/field.rs b/src/avx2/field.rs index 53c6405..b98ab91 100644 --- a/src/avx2/field.rs +++ b/src/avx2/field.rs @@ -218,20 +218,58 @@ impl FieldElement32x4 { } pub fn reduce32(&mut self) { - let mut b = [u64x4::splat(0); 10]; - let (b0, b1) = unpack_pair(self.0[0]); - b[0] = b0.into(); b[1] = b1.into(); - let (b2, b3) = unpack_pair(self.0[1]); - b[2] = b2.into(); b[3] = b3.into(); - let (b4, b5) = unpack_pair(self.0[2]); - b[4] = b4.into(); b[5] = b5.into(); - let (b6, b7) = unpack_pair(self.0[3]); - b[6] = b6.into(); b[7] = b7.into(); - let (b8, b9) = unpack_pair(self.0[4]); - b[8] = b8.into(); b[9] = b9.into(); + let shifts = i32x8::new(26,26,25,25,26,26,25,25); + let masks = u32x8::new((1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1, + (1<<26)-1, (1<<26)-1, (1<<25)-1, (1<<25)-1); - *self = FieldElement32x4::reduce64(b); + let carry = |v: u32x8| -> u32x8 { + unsafe { + use stdsimd::vendor::_mm256_srlv_epi32; + _mm256_srlv_epi32(v.into(), shifts).into() + } + }; + + let swap_lanes = |v: u32x8| -> u32x8 { + unsafe { + use stdsimd::vendor::_mm256_shuffle_epi32; + _mm256_shuffle_epi32(v.into(), 0b01_00_11_10).into() + } + }; + + let combine = |v_lo: u32x8, v_hi: u32x8| -> u32x8 { + unsafe { + use stdsimd::vendor::_mm256_blend_epi32; + _mm256_blend_epi32(v_lo.into(), v_hi.into(), 0b11_00_11_00).into() + } + }; + + let v = &mut self.0; + + let c10 = swap_lanes(carry(v[0])); + v[0] = (v[0] & masks) + combine(u32x8::splat(0), c10); + let c32 = swap_lanes(carry(v[1])); + v[1] = (v[1] & masks) + combine(c10, c32); + let c54 = swap_lanes(carry(v[2])); + v[2] = (v[2] & masks) + combine(c32, c54); + let c76 = swap_lanes(carry(v[3])); + v[3] = (v[3] & masks) + combine(c54, c76); + let c98 = swap_lanes(carry(v[4])); + v[4] = (v[4] & masks) + combine(c76, c98); + + // Still need to account for c9 + // c98 = (c9, c9, c8, c8, c9, c9, c8, c8) + // + let c9_19: u32x8; + unsafe { + use stdsimd::vendor::_mm256_mul_epu32; + use stdsimd::vendor::_mm256_shuffle_epi32; + let c9_spread: u32x8 = _mm256_shuffle_epi32(c98.into(), 0b11_01_10_00).into(); + let c9_19_spread: u32x8 = _mm256_mul_epu32(c9_spread, u64x4::splat(19).into()).into(); + c9_19 = _mm256_shuffle_epi32(c9_19_spread.into(), 0b11_01_10_00).into(); + } + + v[0] = v[0] + c9_19; } pub fn reduce64(mut z: [u64x4; 10]) -> FieldElement32x4 { From 1ba7cb1e2c69783f46c18fd2860946c367ffb8d5 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 14 Nov 2017 14:06:13 -0800 Subject: [PATCH 10/54] Add stub implementations of multiscalar mult --- src/avx2/edwards.rs | 204 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 203 insertions(+), 1 deletion(-) diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs index ba13ec7..f10f9c0 100644 --- a/src/avx2/edwards.rs +++ b/src/avx2/edwards.rs @@ -431,7 +431,129 @@ impl EdwardsBasepointTable { } } - +/// Given a vector of (possibly secret) scalars and a vector of +/// (possibly secret) points, compute `c_1 P_1 + ... + c_n P_n`. +/// +/// This function has the same behaviour as +/// `vartime::multiscalar_mult` but is constant-time. +/// +/// # Input +/// +/// A vector of `Scalar`s and a vector of `ExtendedPoints`. It is an +/// error to call this function with two vectors of different lengths. +/// +/// XXX need to clear memory +#[cfg(any(feature = "alloc", feature = "std"))] +pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> ExtendedPoint + where I: IntoIterator, + J: IntoIterator +{ + use edwards::select_precomputed_point; + //assert_eq!(scalars.len(), points.len()); + + let lookup_tables: Vec<_> = points.into_iter() + .map(|P_i| { + // Construct a lookup table of [P_i,2*P_i,3*P_i,4*P_i,5*P_i,6*P_i,7*P_i] + let mut lookup_table: [ExtendedPoint; 8] = [*P_i; 8]; + for i in 0..7 { + lookup_table[i+1] = P_i + &lookup_table[i]; + } + lookup_table + }).collect(); + + // Setting s_i = i-th scalar, compute + // + // s_i = s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63, + // + // with `-8 ≤ s_{i,j} < 8` for `0 ≤ j < 63` and `-8 ≤ s_{i,63} ≤ 8`. + let scalar_digits_list: Vec<_> = scalars.into_iter() + .map(|c| c.to_radix_16()).collect(); + + // Compute s_1*P_1 + ... + s_n*P_n: since + // + // s_i*P_i = P_i*(s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63) + // s_i*P_i = P_i*s_{i,0} + P_i*s_{i,1}*16^1 + ... + P_i*s_{i,63}*16^63 + // s_i*P_i = P_i*s_{i,0} + 16*(P_i*s_{i,1} + 16*( ... + 16*P_i*s_{i,63})...) + // + // we have the two-dimensional sum + // + // s_1*P_1 = P_1*s_{1,0} + 16*(P_1*s_{1,1} + 16*( ... + 16*P_1*s_{1,63})...) + // + s_2*P_2 = + P_2*s_{2,0} + 16*(P_2*s_{2,1} + 16*( ... + 16*P_2*s_{2,63})...) + // ... + // + s_n*P_n = + P_n*s_{n,0} + 16*(P_n*s_{n,1} + 16*( ... + 16*P_n*s_{n,63})...) + // + // We sum column-wise top-to-bottom, then right-to-left, + // multiplying by 16 only once per column. + // + // This provides the speedup over doing n independent scalar + // mults: we perform 63 multiplications by 16 instead of 63*n + // multiplications, saving 252*(n-1) doublings. + let mut Q = ExtendedPoint::identity(); + // XXX this algorithm makes no effort to be cache-aware; maybe it could be improved? + for j in (0..64).rev() { + Q = Q.mult_by_pow_2(4); + let it = scalar_digits_list.iter().zip(lookup_tables.iter()); + for (s_i, lookup_table_i) in it { + // R_i = s_{i,j} * P_i + let R_i = select_precomputed_point(s_i[j], lookup_table_i); + // Q = Q + R_i + Q = &Q + &R_i; + } + } + Q +} + +pub mod vartime { + //! Variable-time operations on curve points, useful for non-secret data. + use super::*; + + /// Given a vector of public scalars and a vector of (possibly secret) + /// points, compute `c_1 P_1 + ... + c_n P_n`. + /// + /// # Input + /// + /// A vector of `Scalar`s and a vector of `ExtendedPoints`. It is an + /// error to call this function with two vectors of different lengths. + /// + /// XXX need to clear memory + #[cfg(any(feature = "alloc", feature = "std"))] + pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> ExtendedPoint + where I: IntoIterator, + J: IntoIterator + { + //assert_eq!(scalars.len(), points.len()); + + let nafs: Vec<_> = scalars.into_iter() + .map(|c| c.non_adjacent_form()).collect(); + let odd_multiples: Vec<_> = points.into_iter() + .map(|P| { + // Construct a lookup table of [P_i,2*P_i,3*P_i,4*P_i,5*P_i,6*P_i,7*P_i] + let P2 = P.double(); + let mut lookup_table: [ExtendedPoint; 8] = [*P; 8]; + for i in 0..7 { + lookup_table[i+1] = &P2 + &lookup_table[i]; + } + lookup_table + }).collect(); + + let mut Q = ExtendedPoint::identity(); + + for i in (0..255).rev() { + Q = Q.double(); + + for (naf, odd_multiple) in nafs.iter().zip(odd_multiples.iter()) { + if naf[i] > 0 { + Q = &Q + &odd_multiple[( naf[i]/2) as usize]; + } else if naf[i] < 0 { + // XXX impl Sub + Q = &Q + &(-&odd_multiple[(-naf[i]/2) as usize]); + } + } + } + Q + } +} + #[cfg(test)] mod test { use super::*; @@ -652,11 +774,50 @@ mod test { assert_eq!(edwards::ExtendedPoint::from(P1).compress(), edwards::ExtendedPoint::from(P2).compress()); } + + #[test] + fn multiscalar_mult_vs_adding_scalar_mults() { + let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); + let s1 = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + let s2 = Scalar([165, 30, 79, 89, 58, 24, 195, 245, 248, 146, 203, 236, 119, 43, 64, 119, 196, 111, 188, 251, 248, 53, 234, 59, 215, 28, 218, 13, 59, 120, 14, 4]); + + let P1 = &B * &s2; + let P2 = &B * &s1; + + let R = &(&P1 * &s1) + &(&P2 * &s2); + + let R_multiscalar = multiscalar_mult(&[s1, s2], &[P1, P2]); + + assert_eq!(edwards::ExtendedPoint::from(R).compress(), + edwards::ExtendedPoint::from(R_multiscalar).compress()); + } + + mod vartime { + use super::*; + + #[test] + fn multiscalar_mult_vs_adding_scalar_mults() { + let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); + let s1 = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + let s2 = Scalar([165, 30, 79, 89, 58, 24, 195, 245, 248, 146, 203, 236, 119, 43, 64, 119, 196, 111, 188, 251, 248, 53, 234, 59, 215, 28, 218, 13, 59, 120, 14, 4]); + + let P1 = &B * &s2; + let P2 = &B * &s1; + + let R = &(&P1 * &s1) + &(&P2 * &s2); + + let R_multiscalar = vartime::multiscalar_mult(&[s1, s2], &[P1, P2]); + + assert_eq!(edwards::ExtendedPoint::from(R).compress(), + edwards::ExtendedPoint::from(R_multiscalar).compress()); + } + } } #[cfg(all(test, feature = "bench"))] mod bench { use test::Bencher; + use rand::OsRng; use super::*; use constants; @@ -703,5 +864,46 @@ mod bench { b.iter(|| &table * &s ); } + + #[bench] + fn ten_fold_scalar_mult(b: &mut Bencher) { + let mut csprng: OsRng = OsRng::new().unwrap(); + // Create 10 random scalars + let scalars: Vec<_> = (0..10).map(|_| Scalar::random(&mut csprng)).collect(); + // Create 10 points (by doing scalar mults) + let B = &constants::ED25519_BASEPOINT_POINT; + let points: Vec<_> = scalars.iter().map(|s| ExtendedPoint::from(B * &s)).collect(); + + b.iter(|| multiscalar_mult(&scalars, &points)); + } + + mod vartime { + use super::super::*; + use super::{constants, Bencher, OsRng}; + + #[bench] + fn double_scalar_mult(b: &mut Bencher) { + let mut csprng: OsRng = OsRng::new().unwrap(); + // Create 2 random scalars + let s1 = Scalar::random(&mut csprng); + let s2 = Scalar::random(&mut csprng); + let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); + let P = &B * &s1; + + b.iter(|| vartime::multiscalar_mult(&[s1, s2], &[B, P])); + } + + #[bench] + fn ten_fold_scalar_mult(b: &mut Bencher) { + let mut csprng: OsRng = OsRng::new().unwrap(); + // Create 10 random scalars + let scalars: Vec<_> = (0..10).map(|_| Scalar::random(&mut csprng)).collect(); + // Create 10 points (by doing scalar mults) + let B = &constants::ED25519_BASEPOINT_POINT; + let points: Vec<_> = scalars.iter().map(|s| ExtendedPoint::from(B * &s)).collect(); + + b.iter(|| vartime::multiscalar_mult(&scalars, &points)); + } + } } From 2b37a65d66795670a2da93bd0d6d57f7707d4315 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 14 Nov 2017 17:16:02 -0800 Subject: [PATCH 11/54] Remove debugging code --- src/avx2/edwards.rs | 53 --------------------------------------------- 1 file changed, 53 deletions(-) diff --git a/src/avx2/edwards.rs b/src/avx2/edwards.rs index f10f9c0..c283e08 100644 --- a/src/avx2/edwards.rs +++ b/src/avx2/edwards.rs @@ -83,16 +83,6 @@ impl ExtendedPoint { use stdsimd::vendor::_mm256_blend_epi32; use stdsimd::vendor::_mm256_shuffle_epi32; - macro_rules! print_vec { - ($x:ident) => { - let splits = $x.split(); - println!("{}[0] = {:?}", stringify!($x), splits[0].to_bytes()); - println!("{}[1] = {:?}", stringify!($x), splits[1].to_bytes()); - println!("{}[2] = {:?}", stringify!($x), splits[2].to_bytes()); - println!("{}[3] = {:?}", stringify!($x), splits[3].to_bytes()); - } - } - let P = &self.0; let mut t0 = FieldElement32x4::zero(); @@ -226,63 +216,27 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { let mut t0 = FieldElement32x4::zero(); let mut t1 = FieldElement32x4::zero(); - macro_rules! print_vec { - ($x:ident) => { - let splits = $x.split(); - println!("{}[0] = {:?}", stringify!($x), splits[0].to_bytes()); - println!("{}[1] = {:?}", stringify!($x), splits[1].to_bytes()); - println!("{}[2] = {:?}", stringify!($x), splits[2].to_bytes()); - println!("{}[3] = {:?}", stringify!($x), splits[3].to_bytes()); - } - } - for i in 0..5 { t0.0[i] = _mm256_permute2x128_si256(P.0[i].into(), Q.0[i].into(), 32).into(); } - //println!("t0 = (X1, Y1, X2, Y2)"); - //print_vec!(t0); - //println!(""); t0.diff_sum(); - //println!("t0 = (S0 S1 S2 S3)"); - //print_vec!(t0); - //println!(""); - for i in 0..5 { t1.0[i] = _mm256_blend_epi32(t0.0[i].into(), P.0[i].into(), 0b11110000).into(); t0.0[i] = _mm256_permute2x128_si256(t0.0[i].into(), Q.0[i].into(), 49).into(); } - //println!("t0 = (S2 S3 Z2 T2)"); - //print_vec!(t0); - //println!(""); - - //println!("t1 = (S0 S1 Z1 T1)"); - //print_vec!(t1); - //println!(""); let mut t2 = &t0 * &t1; - //println!("t2 = (S4 S5 S6 S7)"); - //print_vec!(t2); - //println!(""); t2.scale_by_curve_constants(); - //println!("t2 = (S8 S9 S10 S11)"); - //print_vec!(t2); - //println!(""); for i in 0..5 { let swapped = _mm256_shuffle_epi32(t2.0[i].into(), 0b10_11_00_01); t2.0[i] = _mm256_blend_epi32(t2.0[i].into(), swapped, 0b11110000).into(); } - //println!("t2 = (S8 S9 S11 S10)"); - //print_vec!(t2); - //println!(""); t2.diff_sum(); - //println!("t2 = (S12 S13 S14 S15)"); - //print_vec!(t2); - //println!(""); let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) @@ -291,13 +245,6 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { t0.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c0); t1.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c1); } - //println!("t0 = (S11 S13 S13 S11)"); - //print_vec!(t0); - //println!(""); - - //println!("t1 = (S12 S14 S14 S12)"); - //print_vec!(t1); - //println!(""); ExtendedPoint(&t0 * &t1) } From ed24d1c5fa5701df6c9d5022b59d54553193a061 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 17 Nov 2017 13:25:38 -0800 Subject: [PATCH 12/54] Move AVX2 code into a backend --- Cargo.toml | 1 + build.rs | 2 ++ src/{ => backend}/avx2/edwards.rs | 11 ++++++----- src/{ => backend}/avx2/field.rs | 0 src/{ => backend}/avx2/mod.rs | 0 src/backend/mod.rs | 4 ++++ src/lib.rs | 4 ---- 7 files changed, 13 insertions(+), 9 deletions(-) rename src/{ => backend}/avx2/edwards.rs (99%) rename src/{ => backend}/avx2/field.rs (100%) rename src/{ => backend}/avx2/mod.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index e317204..ac1b8d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,7 @@ rand = "0.3" generic-array = "^0.8" digest = "0.6" arrayref = "0.3.4" +stdsimd = { git = "https://github.com/hdevalence/stdsimd", branch="feature/more-avx2" } [build-dependencies.serde] version = "1.0" diff --git a/build.rs b/build.rs index 13d9715..4fba073 100644 --- a/build.rs +++ b/build.rs @@ -21,6 +21,8 @@ use std::path::Path; // For instance, this shouldn't exist here at all, but it does. #[cfg(feature = "serde")] extern crate serde; +#[cfg(feature = "yolocrypto")] +extern crate stdsimd; // Public modules diff --git a/src/avx2/edwards.rs b/src/backend/avx2/edwards.rs similarity index 99% rename from src/avx2/edwards.rs rename to src/backend/avx2/edwards.rs index c283e08..4db18f9 100644 --- a/src/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -25,8 +25,8 @@ use scalar::Scalar; use traits::Identity; -use avx2::field::FieldElement32x4; -use avx2::field::P_TIMES_2; +use backend::avx2::field::FieldElement32x4; +use backend::avx2::field::P_TIMES_2; /// A point on Curve25519, represented in an AVX2-friendly format. #[derive(Copy, Clone, Debug)] @@ -324,17 +324,18 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsBasepointTable { /// takes `-8 ≤ x < 8` and `[16^2i * B, ..., 8 * 16^2i * B]`, /// and returns `x * 16^2i * B` in constant time. fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { + use traits::select_precomputed_point; let e = scalar.to_radix_16(); let mut h = ExtendedPoint::identity(); for i in (0..64).filter(|x| x % 2 == 1) { - h = &h + &edwards::select_precomputed_point(e[i], &self.0[i/2]); + h = &h + &select_precomputed_point(e[i], &self.0[i/2]); } h = h.mult_by_pow_2(4); for i in (0..64).filter(|x| x % 2 == 0) { - h = &h + &edwards::select_precomputed_point(e[i], &self.0[i/2]); + h = &h + &select_precomputed_point(e[i], &self.0[i/2]); } h @@ -395,7 +396,7 @@ pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> ExtendedPoint where I: IntoIterator, J: IntoIterator { - use edwards::select_precomputed_point; + use traits::select_precomputed_point; //assert_eq!(scalars.len(), points.len()); let lookup_tables: Vec<_> = points.into_iter() diff --git a/src/avx2/field.rs b/src/backend/avx2/field.rs similarity index 100% rename from src/avx2/field.rs rename to src/backend/avx2/field.rs diff --git a/src/avx2/mod.rs b/src/backend/avx2/mod.rs similarity index 100% rename from src/avx2/mod.rs rename to src/backend/avx2/mod.rs diff --git a/src/backend/mod.rs b/src/backend/mod.rs index be3a347..b6845da 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -28,3 +28,7 @@ pub mod u32; #[cfg(feature="radix_51")] pub mod u64; +/// Code using AVX2. +#[cfg(all(feature="yolocrypto", not(feature="radix_51")))] +pub mod avx2; + diff --git a/src/lib.rs b/src/lib.rs index c3a4255..2106e88 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -94,9 +94,5 @@ pub(crate) mod field; // Arithmetic backends (using u32, u64, etc) live here pub(crate) mod backend; -// XXX this should be in backend -#[cfg(all(feature="yolocrypto", not(feature="radix_51")))] -pub(crate) mod avx2; - // Internal curve models which are not part of the public API. pub(crate) mod curve_models; From 81ecef89eec5ab17ef57cf97e95a3b6517c2ad24 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sun, 19 Nov 2017 14:30:32 -0800 Subject: [PATCH 13/54] Connect AVX2 and u64 backends --- src/backend/avx2/edwards.rs | 10 ++-- src/backend/avx2/field.rs | 107 +++++++++++++++++++++--------------- src/backend/mod.rs | 2 +- 3 files changed, 68 insertions(+), 51 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 4db18f9..5a0d01a 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -509,7 +509,7 @@ mod test { use constants; fn serial_add(P: edwards::ExtendedPoint, Q: edwards::ExtendedPoint) -> edwards::ExtendedPoint { - use backend::u32::field::FieldElement32; + use backend::u64::field::FieldElement64; let (X1, Y1, Z1, T1) = (P.X, P.Y, P.Z, P.T); let (X2, Y2, Z2, T2) = (Q.X, Q.Y, Q.Z, Q.T); @@ -540,10 +540,10 @@ mod test { print_var!(S7); println!(""); - let S8 = &S4 * &FieldElement32([ 121666,0,0,0,0,0,0,0,0,0]); // R5 - let S9 = &S5 * &FieldElement32([ 121666,0,0,0,0,0,0,0,0,0]); // R6 - let S10 = &S6 * &FieldElement32([2*121666,0,0,0,0,0,0,0,0,0]); // R8 - let S11 = &S7 * &(-&FieldElement32([2*121665,0,0,0,0,0,0,0,0,0])); // R7 + let S8 = &S4 * &FieldElement64([ 121666,0,0,0,0]); // R5 + let S9 = &S5 * &FieldElement64([ 121666,0,0,0,0]); // R6 + let S10 = &S6 * &FieldElement64([2*121666,0,0,0,0]); // R8 + let S11 = &S7 * &(-&FieldElement64([2*121665,0,0,0,0])); // R7 print_var!(S8 ); print_var!(S9 ); print_var!(S10); diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index b98ab91..e145ac0 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -17,7 +17,7 @@ use std::ops::Mul; use stdsimd::simd::{u32x8, i32x8, u64x4}; -use backend::u32::field::FieldElement32; +use backend::u64::field::FieldElement64; pub(crate) static P_TIMES_2: FieldElement32x4 = FieldElement32x4([ u32x8::new(134217690, 134217690, 67108862, 67108862, 134217690, 134217690, 67108862, 67108862), @@ -45,17 +45,23 @@ impl ConditionallyAssignable for FieldElement32x4 { impl FieldElement32x4 { - pub(crate) fn split(&self) -> [FieldElement32; 4] { - let mut out = [FieldElement32::zero(); 4]; + pub(crate) fn split(&self) -> [FieldElement64; 4] { + let mut out = [FieldElement64::zero(); 4]; for i in 0..5 { - out[0].0[2*i ] = self.0[i].extract(0); // - out[1].0[2*i ] = self.0[i].extract(1); // - out[0].0[2*i+1] = self.0[i].extract(2); // `. - out[1].0[2*i+1] = self.0[i].extract(3); // | pre-swapped to avoid - out[2].0[2*i ] = self.0[i].extract(4); // | a cross lane shuffle - out[3].0[2*i ] = self.0[i].extract(5); // .' - out[2].0[2*i+1] = self.0[i].extract(6); // - out[3].0[2*i+1] = self.0[i].extract(7); // + + let a_2i = self.0[i].extract(0) as u64; // + let b_2i = self.0[i].extract(1) as u64; // + let a_2i_1 = self.0[i].extract(2) as u64; // `. + let b_2i_1 = self.0[i].extract(3) as u64; // | pre-swapped to avoid + let c_2i = self.0[i].extract(4) as u64; // | a cross lane shuffle + let d_2i = self.0[i].extract(5) as u64; // .' + let c_2i_1 = self.0[i].extract(6) as u64; // + let d_2i_1 = self.0[i].extract(7) as u64; // + + out[0].0[i] = a_2i + (a_2i_1 << 26); + out[1].0[i] = b_2i + (b_2i_1 << 26); + out[2].0[i] = c_2i + (c_2i_1 << 26); + out[3].0[i] = d_2i + (d_2i_1 << 26); } out @@ -65,23 +71,34 @@ impl FieldElement32x4 { FieldElement32x4([u32x8::splat(0);5]) } - pub fn splat(x: &FieldElement32) -> FieldElement32x4 { + pub fn splat(x: &FieldElement64) -> FieldElement32x4 { FieldElement32x4::new(x,x,x,x) } pub fn new( - x0: &FieldElement32, - x1: &FieldElement32, - x2: &FieldElement32, - x3: &FieldElement32, + x0: &FieldElement64, + x1: &FieldElement64, + x2: &FieldElement64, + x3: &FieldElement64, ) -> FieldElement32x4 { let mut buf = [u32x8::splat(0); 5]; + let low_26_bits = (1 << 26) - 1; for i in 0..5 { - buf[i] = u32x8::new(x0.0[2*i ], x1.0[2*i ], x0.0[2*i+1], x1.0[2*i+1], - x2.0[2*i ], x3.0[2*i ], x2.0[2*i+1], x3.0[2*i+1]); + let a_2i = (x0.0[i] & low_26_bits) as u32; + let a_2i_1 = (x0.0[i] >> 26) as u32; + let b_2i = (x1.0[i] & low_26_bits) as u32; + let b_2i_1 = (x1.0[i] >> 26) as u32; + let c_2i = (x2.0[i] & low_26_bits) as u32; + let c_2i_1 = (x2.0[i] >> 26) as u32; + let d_2i = (x3.0[i] & low_26_bits) as u32; + let d_2i_1 = (x3.0[i] >> 26) as u32; + + buf[i] = u32x8::new(a_2i, b_2i, a_2i_1, b_2i_1, c_2i, d_2i, c_2i_1, d_2i_1); } - FieldElement32x4(buf) + let mut out = FieldElement32x4(buf); + out.reduce32(); + return out; } // Negate variables in lanes where mask is set @@ -521,22 +538,22 @@ mod test { #[test] fn scale_by_curve_constants() { - let mut x = FieldElement32x4::splat(&FieldElement32::one()); + let mut x = FieldElement32x4::splat(&FieldElement64::one()); x.scale_by_curve_constants(); let xs = x.split(); - assert_eq!(xs[0], FieldElement32([ 121666,0,0,0,0,0,0,0,0,0])); - assert_eq!(xs[1], FieldElement32([ 121666,0,0,0,0,0,0,0,0,0])); - assert_eq!(xs[2], FieldElement32([2*121666,0,0,0,0,0,0,0,0,0])); - assert_eq!(xs[3], -&FieldElement32([2*121665,0,0,0,0,0,0,0,0,0])); + assert_eq!(xs[0], FieldElement64([ 121666,0,0,0,0])); + assert_eq!(xs[1], FieldElement64([ 121666,0,0,0,0])); + assert_eq!(xs[2], FieldElement64([2*121666,0,0,0,0])); + assert_eq!(xs[3], -&FieldElement64([2*121665,0,0,0,0])); } #[test] fn diff_sum_vs_serial() { - let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); - let x1 = FieldElement32([10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10109]); - let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); - let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); + let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); vec.diff_sum(); @@ -551,10 +568,10 @@ mod test { #[test] fn square_vs_serial() { - let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); - let x1 = FieldElement32([10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10109]); - let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); - let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); + let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); @@ -569,10 +586,10 @@ mod test { #[test] fn multiply_vs_serial() { - let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); - let x1 = FieldElement32([10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10109]); - let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); - let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); + let x0 = FieldElement64([10000, 10001, 10002, 10003, 10004]); + let x1 = FieldElement64([10100, 10101, 10102, 10103, 10104]); + let x2 = FieldElement64([10200, 10201, 10202, 10203, 10204]); + let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); let vecprime = vec.clone(); @@ -587,10 +604,10 @@ mod test { #[test] fn test_unpack_repack_pair() { - let x0 = FieldElement32([10000, 10001, 10002, 10003, 10004, 10005, 10006, 10007, 10008, 10009]); - let x1 = FieldElement32([10100, 10101, 10102, 10103, 10104, 10105, 10106, 10107, 10108, 10109]); - let x2 = FieldElement32([10200, 10201, 10202, 10203, 10204, 10205, 10206, 10207, 10208, 10209]); - let x3 = FieldElement32([10300, 10301, 10302, 10303, 10304, 10305, 10306, 10307, 10308, 10309]); + let x0 = FieldElement64([10000 + (10001 << 26), 0, 0, 0, 0]); + let x1 = FieldElement64([10100 + (10101 << 26), 0, 0, 0, 0]); + let x2 = FieldElement64([10200 + (10201 << 26), 0, 0, 0, 0]); + let x3 = FieldElement64([10300 + (10301 << 26), 0, 0, 0, 0]); let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); @@ -611,10 +628,10 @@ mod test { #[test] fn new_split_roundtrips() { - let x0 = FieldElement32::from_bytes(&[0x10; 32]); - let x1 = FieldElement32::from_bytes(&[0x11; 32]); - let x2 = FieldElement32::from_bytes(&[0x12; 32]); - let x3 = FieldElement32::from_bytes(&[0x13; 32]); + let x0 = FieldElement64::from_bytes(&[0x10; 32]); + let x1 = FieldElement64::from_bytes(&[0x11; 32]); + let x2 = FieldElement64::from_bytes(&[0x12; 32]); + let x3 = FieldElement64::from_bytes(&[0x13; 32]); let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); @@ -635,7 +652,7 @@ mod bench { #[bench] fn multiply(b: &mut Bencher) { - let vec = FieldElement32x4::splat(&FieldElement32::zero()); + let vec = FieldElement32x4::splat(&FieldElement64::zero()); let vecprime = vec.clone(); b.iter(|| &vec * &vecprime ); diff --git a/src/backend/mod.rs b/src/backend/mod.rs index b6845da..12957d6 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -29,6 +29,6 @@ pub mod u32; pub mod u64; /// Code using AVX2. -#[cfg(all(feature="yolocrypto", not(feature="radix_51")))] +#[cfg(all(feature="yolocrypto", feature="radix_51"))] pub mod avx2; From 841e0d5b64d67b0798d22440869c8cbeaa8056b4 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sun, 19 Nov 2017 14:53:21 -0800 Subject: [PATCH 14/54] Fix wrong feature for benchmarks --- src/edwards.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/edwards.rs b/src/edwards.rs index f8101df..c831075 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -1054,7 +1054,7 @@ mod test { /// Test precomputed basepoint mult #[test] - #[cfg(feature="basepoint_table_creation")] + #[cfg(feature="precomputed_tables")] fn test_precomputed_basepoint_mult() { let table = EdwardsBasepointTable::create(&constants::ED25519_BASEPOINT_POINT); let aB_1 = &constants::ED25519_BASEPOINT_TABLE * &A_SCALAR; @@ -1325,14 +1325,14 @@ mod bench { } #[bench] - #[cfg(feature="basepoint_table_creation")] + #[cfg(feature="precomputed_tables")] fn create_basepoint_table(b: &mut Bencher) { let aB = &constants::ED25519_BASEPOINT_TABLE * &A_SCALAR; b.iter(|| EdwardsBasepointTable::create(&aB)); } #[bench] - #[cfg(feature="basepoint_table_creation")] + #[cfg(feature="precomputed_tables")] fn ten_fold_scalar_mult(b: &mut Bencher) { let mut csprng: OsRng = OsRng::new().unwrap(); // Create 10 random scalars @@ -1356,7 +1356,7 @@ mod bench { } #[bench] - #[cfg(feature="basepoint_table_creation")] + #[cfg(feature="precomputed_tables")] fn ten_fold_scalar_mult(b: &mut Bencher) { let mut csprng: OsRng = OsRng::new().unwrap(); // Create 10 random scalars From f28635ab4e6580115ff657bb914c96548a7dbfd9 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Sun, 19 Nov 2017 14:53:45 -0800 Subject: [PATCH 15/54] Add a new 'avx2_backend' yolocrypto feature --- Cargo.toml | 6 +++--- src/backend/mod.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ac1b8d6..390a2c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,11 +70,11 @@ nightly = ["radix_51", "subtle/nightly"] default = ["std"] std = ["rand", "subtle/std"] alloc = [] -# This isn't used at the moment, but keep it around for future yolocrypto features. -yolocrypto = [] +yolocrypto = ["avx2_backend"] bench = [] # Radix-51 arithmetic using u128 radix_51 = [] # Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation. precomputed_tables = [] - +# experimental avx2 support +avx2_backend = ["radix_51"] diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 12957d6..e68786e 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -29,6 +29,6 @@ pub mod u32; pub mod u64; /// Code using AVX2. -#[cfg(all(feature="yolocrypto", feature="radix_51"))] +#[cfg(all(feature="yolocrypto", feature="avx2_backend"))] pub mod avx2; From 208180dc7223c6997ea0cb9a25db8e9527a4ca09 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 20 Nov 2017 12:20:22 -0800 Subject: [PATCH 16/54] Simplify mul, square implementations --- src/backend/avx2/field.rs | 118 ++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 70 deletions(-) diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index e145ac0..b911c1c 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -412,8 +412,6 @@ impl FieldElement32x4 { let v19 = u32x8::new(19,0,19,0,19,0,19,0); - let mut z = [u64x4::splat(0); 10]; - let (x0, x1) = unpack_pair(self.0[0]); let (x2, x3) = unpack_pair(self.0[1]); let (x4, x5) = unpack_pair(self.0[2]); @@ -435,37 +433,24 @@ impl FieldElement32x4 { let x8_19 = m_lo(v19, x8); let x9_19 = m_lo(v19, x9); - z[0] = m(x0, x0) + m(x2_2,x8_19) + m(x4_2,x6_19) + ((m(x1_2,x9_19) + m(x3_2,x7_19) + m(x5,x5_19)) << 1); - z[1] = m(x0_2,x1) + m(x3_2,x8_19) + m(x5_2,x6_19) + ((m(x2,x9_19) + m(x4,x7_19)) << 1); - z[2] = m(x0_2,x2) + m(x1_2,x1) + m(x4_2,x8_19) + m(x6,x6_19) + ((m(x3_2,x9_19) + m(x5_2,x7_19)) << 1); - z[3] = m(x0_2,x3) + m(x1_2,x2) + m(x5_2,x8_19) + ((m(x4,x9_19) + m(x6,x7_19)) << 1); - z[4] = m(x0_2,x4) + m(x1_2,x3_2) + m(x2, x2) + m(x6_2,x8_19) + ((m(x5_2,x9_19) + m(x7,x7_19)) << 1); - z[5] = m(x0_2,x5) + m(x1_2,x4) + m(x2_2,x3) + m(x7_2,x8_19) + ((m(x6,x9_19)) << 1); - z[6] = m(x0_2,x6) + m(x1_2,x5_2) + m(x2_2,x4) + m(x3_2,x3) + m(x8,x8_19) + ((m(x7_2,x9_19)) << 1); - z[7] = m(x0_2,x7) + m(x1_2,x6) + m(x2_2,x5) + m(x3_2,x4) + ((m(x8,x9_19)) << 1); - z[8] = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1); - z[9] = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5); + let z0 = m(x0, x0) + m(x2_2,x8_19) + m(x4_2,x6_19) + ((m(x1_2,x9_19) + m(x3_2,x7_19) + m(x5,x5_19)) << 1); + let z1 = m(x0_2,x1) + m(x3_2,x8_19) + m(x5_2,x6_19) + ((m(x2,x9_19) + m(x4,x7_19)) << 1); + let z2 = m(x0_2,x2) + m(x1_2,x1) + m(x4_2,x8_19) + m(x6,x6_19) + ((m(x3_2,x9_19) + m(x5_2,x7_19)) << 1); + let z3 = m(x0_2,x3) + m(x1_2,x2) + m(x5_2,x8_19) + ((m(x4,x9_19) + m(x6,x7_19)) << 1); + let z4 = m(x0_2,x4) + m(x1_2,x3_2) + m(x2, x2) + m(x6_2,x8_19) + ((m(x5_2,x9_19) + m(x7,x7_19)) << 1); + let z5 = m(x0_2,x5) + m(x1_2,x4) + m(x2_2,x3) + m(x7_2,x8_19) + ((m(x6,x9_19)) << 1); + let z6 = m(x0_2,x6) + m(x1_2,x5_2) + m(x2_2,x4) + m(x3_2,x3) + m(x8,x8_19) + ((m(x7_2,x9_19)) << 1); + let z7 = m(x0_2,x7) + m(x1_2,x6) + m(x2_2,x5) + m(x3_2,x4) + ((m(x8,x9_19)) << 1); + let z8 = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1); + let z9 = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5); - return FieldElement32x4::reduce64(z); + FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } } impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { type Output = FieldElement32x4; fn mul(self, _rhs: &'b FieldElement32x4) -> FieldElement32x4 { - let mut b = [u32x8::splat(0); 10]; - let mut c = [u64x4::splat(0); 10]; - - let (b0, b1) = unpack_pair(_rhs.0[0]); - b[0] = b0; b[1] = b1; - let (b2, b3) = unpack_pair(_rhs.0[1]); - b[2] = b2; b[3] = b3; - let (b4, b5) = unpack_pair(_rhs.0[2]); - b[4] = b4; b[5] = b5; - let (b6, b7) = unpack_pair(_rhs.0[3]); - b[6] = b6; b[7] = b7; - let (b8, b9) = unpack_pair(_rhs.0[4]); - b[8] = b8; b[9] = b9; #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { @@ -479,55 +464,48 @@ impl<'a, 'b> Mul<&'b FieldElement32x4> for &'a FieldElement32x4 { unsafe { u32x8::from(_mm256_mul_epu32(x,y)) } } + let (x0, x1) = unpack_pair(self.0[0]); + let (x2, x3) = unpack_pair(self.0[1]); + let (x4, x5) = unpack_pair(self.0[2]); + let (x6, x7) = unpack_pair(self.0[3]); + let (x8, x9) = unpack_pair(self.0[4]); + + let (y0, y1) = unpack_pair(_rhs.0[0]); + let (y2, y3) = unpack_pair(_rhs.0[1]); + let (y4, y5) = unpack_pair(_rhs.0[2]); + let (y6, y7) = unpack_pair(_rhs.0[3]); + let (y8, y9) = unpack_pair(_rhs.0[4]); + let v19 = u32x8::new(19,0,19,0,19,0,19,0); - // XXX clean up this horrifying abomination - // - // The idea is to take the standard "schoolbook multiplication square" (see - // FieldElement32), and walk up each column from top to bottom, then left to right. - // - // Instead of multiplying by 19 in a precomputation, we overwrite the b[i] value with - // b[i]*19 as soon as we will no longer need it. - // + let y1_19 = m_lo(v19, y1); // This fits in a u32 + let y2_19 = m_lo(v19, y2); // iff 26 + b + lg(19) < 32 + let y3_19 = m_lo(v19, y3); // if b < 32 - 26 - 4.248 = 1.752 + let y4_19 = m_lo(v19, y4); + let y5_19 = m_lo(v19, y5); // below, b<2.5: this is a bottleneck, + let y6_19 = m_lo(v19, y6); // could be avoided by promoting to + let y7_19 = m_lo(v19, y7); // u64 here instead of in m() + let y8_19 = m_lo(v19, y8); + let y9_19 = m_lo(v19, y9); - macro_rules! loop_body { - ($i:expr) => { - let (ai, ai1) = unpack_pair(self.0[$i/2]); + let x1_2 = x1 + x1; // This fits in a u32 iff 25 + b + 1 < 32 + let x3_2 = x3 + x3; // iff b < 6 + let x5_2 = x5 + x5; + let x7_2 = x7 + x7; + let x9_2 = x9 + x9; - c[9] = c[9] + m(ai, b[(100 + 9-$i) % 10]); - b[(100 + 9-$i) % 10] = m_lo(b[(100 + 9-$i) % 10], v19); - c[8] = c[8] + m(ai, b[(100 + 8-$i) % 10]); - c[7] = c[7] + m(ai, b[(100 + 7-$i) % 10]); - c[6] = c[6] + m(ai, b[(100 + 6-$i) % 10]); - c[5] = c[5] + m(ai, b[(100 + 5-$i) % 10]); - c[4] = c[4] + m(ai, b[(100 + 4-$i) % 10]); - c[3] = c[3] + m(ai, b[(100 + 3-$i) % 10]); - c[2] = c[2] + m(ai, b[(100 + 2-$i) % 10]); - c[1] = c[1] + m(ai, b[(100 + 1-$i) % 10]); - c[0] = c[0] + m(ai, b[(100 + 0-$i) % 10]); + let z0 = m(x0,y0) + m(x1_2,y9_19) + m(x2,y8_19) + m(x3_2,y7_19) + m(x4,y6_19) + m(x5_2,y5_19) + m(x6,y4_19) + m(x7_2,y3_19) + m(x8,y2_19) + m(x9_2,y1_19); + let z1 = m(x0,y1) + m(x1,y0) + m(x2,y9_19) + m(x3,y8_19) + m(x4,y7_19) + m(x5,y6_19) + m(x6,y5_19) + m(x7,y4_19) + m(x8,y3_19) + m(x9,y2_19); + let z2 = m(x0,y2) + m(x1_2,y1) + m(x2,y0) + m(x3_2,y9_19) + m(x4,y8_19) + m(x5_2,y7_19) + m(x6,y6_19) + m(x7_2,y5_19) + m(x8,y4_19) + m(x9_2,y3_19); + let z3 = m(x0,y3) + m(x1,y2) + m(x2,y1) + m(x3,y0) + m(x4,y9_19) + m(x5,y8_19) + m(x6,y7_19) + m(x7,y6_19) + m(x8,y5_19) + m(x9,y4_19); + let z4 = m(x0,y4) + m(x1_2,y3) + m(x2,y2) + m(x3_2,y1) + m(x4,y0) + m(x5_2,y9_19) + m(x6,y8_19) + m(x7_2,y7_19) + m(x8,y6_19) + m(x9_2,y5_19); + let z5 = m(x0,y5) + m(x1,y4) + m(x2,y3) + m(x3,y2) + m(x4,y1) + m(x5,y0) + m(x6,y9_19) + m(x7,y8_19) + m(x8,y7_19) + m(x9,y6_19); + let z6 = m(x0,y6) + m(x1_2,y5) + m(x2,y4) + m(x3_2,y3) + m(x4,y2) + m(x5_2,y1) + m(x6,y0) + m(x7_2,y9_19) + m(x8,y8_19) + m(x9_2,y7_19); + let z7 = m(x0,y7) + m(x1,y6) + m(x2,y5) + m(x3,y4) + m(x4,y3) + m(x5,y2) + m(x6,y1) + m(x7,y0) + m(x8,y9_19) + m(x9,y8_19); + let z8 = m(x0,y8) + m(x1_2,y7) + m(x2,y6) + m(x3_2,y5) + m(x4,y4) + m(x5_2,y3) + m(x6,y2) + m(x7_2,y1) + m(x8,y0) + m(x9_2,y9_19); + let z9 = m(x0,y9) + m(x1,y8) + m(x2,y7) + m(x3,y6) + m(x4,y5) + m(x5,y4) + m(x6,y3) + m(x7,y2) + m(x8,y1) + m(x9,y0); - let ai1_2 = ai1 + ai1; - c[9] = c[9] + m(ai1, b[(100 + 9-($i+1)) % 10]); - b[(100 + 9-($i+1)) % 10] = m_lo(b[(100 + 9-($i+1)) % 10], v19); - c[8] = c[8] + m(ai1_2, b[(100 + 8-($i+1)) % 10]); - c[7] = c[7] + m(ai1, b[(100 + 7-($i+1)) % 10]); - c[6] = c[6] + m(ai1_2, b[(100 + 6-($i+1)) % 10]); - c[5] = c[5] + m(ai1, b[(100 + 5-($i+1)) % 10]); - c[4] = c[4] + m(ai1_2, b[(100 + 4-($i+1)) % 10]); - c[3] = c[3] + m(ai1, b[(100 + 3-($i+1)) % 10]); - c[2] = c[2] + m(ai1_2, b[(100 + 2-($i+1)) % 10]); - c[1] = c[1] + m(ai1, b[(100 + 1-($i+1)) % 10]); - c[0] = c[0] + m(ai1_2, b[(100 + 0-($i+1)) % 10]); - }; - } - - loop_body!(0); - loop_body!(2); - loop_body!(4); - loop_body!(6); - loop_body!(8); - - return FieldElement32x4::reduce64(c); + FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } } From 77766b3422ad17e08556baf9d08031e75d3fca16 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 20 Nov 2017 12:38:36 -0800 Subject: [PATCH 17/54] Add benchmark for conversion to avx2 format --- src/backend/avx2/edwards.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 5a0d01a..cdb47ba 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -771,6 +771,21 @@ mod bench { use constants; use scalar::Scalar; + #[bench] + fn conversion_into__avx2_format(b: &mut Bencher) { + let B = constants::ED25519_BASEPOINT_POINT; + + b.iter(|| ExtendedPoint::from(B)); + } + + #[bench] + fn conversion_outof_avx2_format(b: &mut Bencher) { + let B = constants::ED25519_BASEPOINT_POINT; + let B_avx2 = ExtendedPoint::from(B); + + b.iter(|| edwards::ExtendedPoint::from(B_avx2)); + } + #[bench] fn point_addition(b: &mut Bencher) { let B = &constants::ED25519_BASEPOINT_TABLE; From 97fe2f0bf410b781b0c036981cbc05fb20d3898b Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 20 Nov 2017 15:27:44 -0800 Subject: [PATCH 18/54] Try to connect the AVX2 backend to the ExtendedPoint frontend --- build.rs | 1 + src/edwards.rs | 65 ++++++++++++++++++++++++++++---------------------- src/lib.rs | 3 ++- 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/build.rs b/build.rs index 4fba073..9604ec7 100644 --- a/build.rs +++ b/build.rs @@ -1,4 +1,5 @@ #![cfg_attr(feature = "nightly", feature(i128_type))] +#![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![allow(unused_variables)] #![allow(non_snake_case)] #![allow(dead_code)] diff --git a/src/edwards.rs b/src/edwards.rs index c831075..45354a0 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -432,38 +432,47 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { /// For scalar multiplication of a basepoint, /// `EdwardsBasepointTable` is approximately 4x faster. fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { - // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] - let P = self.to_projective_niels(); - let mut lookup_table: [ProjectiveNielsPoint; 8] = [P; 8]; - for i in 0..7 { - lookup_table[i+1] = (self + &lookup_table[i]) - .to_extended().to_projective_niels(); + // If we built with AVX2, use the AVX2 backend. + #[cfg(all(target_feature = "avx2", feature = "avx2_backend"))] { + use backend::avx2::edwards as edwards_avx2; + let P_avx2 = edwards_avx2::ExtendedPoint::from(*self); + return ExtendedPoint::from(&P_avx2 * scalar); } + // Otherwise, proceed as normal: + #[cfg(not(all(target_feature = "avx2", feature = "avx2_backend")))] { + // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] + let P = self.to_projective_niels(); + let mut lookup_table: [ProjectiveNielsPoint; 8] = [P; 8]; + for i in 0..7 { + lookup_table[i+1] = (self + &lookup_table[i]) + .to_extended().to_projective_niels(); + } - // Setting s = scalar, compute - // - // s = s_0 + s_1*16^1 + ... + s_63*16^63, - // - // with `-8 ≤ s_i < 8` for `0 ≤ i < 63` and `-8 ≤ s_63 ≤ 8`. - let scalar_digits = scalar.to_radix_16(); + // Setting s = scalar, compute + // + // s = s_0 + s_1*16^1 + ... + s_63*16^63, + // + // with `-8 ≤ s_i < 8` for `0 ≤ i < 63` and `-8 ≤ s_63 ≤ 8`. + let scalar_digits = scalar.to_radix_16(); - // Compute s*P as - // - // s*P = P*(s_0 + s_1*16^1 + s_2*16^2 + ... + s_63*16^63) - // s*P = P*s_0 + P*s_1*16^1 + P*s_2*16^2 + ... + P*s_63*16^63 - // s*P = P*s_0 + 16*(P*s_1 + 16*(P*s_2 + 16*( ... + P*s_63)...)) - // - // We sum right-to-left. - let mut Q = ExtendedPoint::identity(); - for i in (0..64).rev() { - // Q = 16*Q - Q = Q.mult_by_pow_2(4); - // R = s_i * Q - let R = select_precomputed_point(scalar_digits[i], &lookup_table); - // Q = Q + R - Q = (&Q + &R).to_extended(); + // Compute s*P as + // + // s*P = P*(s_0 + s_1*16^1 + s_2*16^2 + ... + s_63*16^63) + // s*P = P*s_0 + P*s_1*16^1 + P*s_2*16^2 + ... + P*s_63*16^63 + // s*P = P*s_0 + 16*(P*s_1 + 16*(P*s_2 + 16*( ... + P*s_63)...)) + // + // We sum right-to-left. + let mut Q = ExtendedPoint::identity(); + for i in (0..64).rev() { + // Q = 16*Q + Q = Q.mult_by_pow_2(4); + // R = s_i * Q + let R = select_precomputed_point(scalar_digits[i], &lookup_table); + // Q = Q + R + Q = (&Q + &R).to_extended(); + } + Q } - Q } } diff --git a/src/lib.rs b/src/lib.rs index 2106e88..3fcb3ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,11 +11,12 @@ #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(feature = "alloc", feature(alloc))] #![cfg_attr(feature = "nightly", feature(i128_type))] +#![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "bench", feature(test))] #![cfg_attr(all(feature = "nightly", feature = "std"), feature(zero_one))] #![allow(unused_features)] -//#![deny(missing_docs)] // refuse to compile if documentation is missing +#![deny(missing_docs)] // refuse to compile if documentation is missing //! # curve25519-dalek //! From 912fc5d412990a90d7db16456d37991cf9cd75f4 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 20 Nov 2017 15:44:34 -0800 Subject: [PATCH 19/54] Never build avx2 without avx2 --- src/backend/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index e68786e..a4c8b55 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -29,6 +29,6 @@ pub mod u32; pub mod u64; /// Code using AVX2. -#[cfg(all(feature="yolocrypto", feature="avx2_backend"))] +#[cfg(all(target_feature="avx2", feature="yolocrypto", feature="avx2_backend"))] pub mod avx2; From b5305b4e3068dd3007f81202f3241ac15f2397f5 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Tue, 21 Nov 2017 12:27:32 -0800 Subject: [PATCH 20/54] Connect multiscalar_mult to the AVX2 backend --- src/backend/avx2/edwards.rs | 46 ++++++----- src/edwards.rs | 152 ++++++++++++++++++++---------------- 2 files changed, 112 insertions(+), 86 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index cdb47ba..93ba438 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -391,20 +391,25 @@ impl EdwardsBasepointTable { /// error to call this function with two vectors of different lengths. /// /// XXX need to clear memory +/// +/// XXX this takes `edwards::ExtendedPoints` because we have to alloc scratch space here anyways, +/// and we need some space to store the converted points, so we may as well do the conversion here. +/// maybe there's a better way to avoid code duplication... #[cfg(any(feature = "alloc", feature = "std"))] -pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> ExtendedPoint +pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> edwards::ExtendedPoint where I: IntoIterator, - J: IntoIterator + J: IntoIterator { use traits::select_precomputed_point; //assert_eq!(scalars.len(), points.len()); let lookup_tables: Vec<_> = points.into_iter() - .map(|P_i| { - // Construct a lookup table of [P_i,2*P_i,3*P_i,4*P_i,5*P_i,6*P_i,7*P_i] - let mut lookup_table: [ExtendedPoint; 8] = [*P_i; 8]; + .map(|P| { + let P = ExtendedPoint::from(*P); + // Construct a lookup table of [P,2*P,3*P,4*P,5*P,6*P,7*P] + let mut lookup_table: [ExtendedPoint; 8] = [P; 8]; for i in 0..7 { - lookup_table[i+1] = P_i + &lookup_table[i]; + lookup_table[i+1] = &P + &lookup_table[i]; } lookup_table }).collect(); @@ -448,7 +453,7 @@ pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> ExtendedPoint Q = &Q + &R_i; } } - Q + Q.into() } pub mod vartime { @@ -464,10 +469,12 @@ pub mod vartime { /// error to call this function with two vectors of different lengths. /// /// XXX need to clear memory + /// + /// XXX see note on consttime multiscalar mul #[cfg(any(feature = "alloc", feature = "std"))] - pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> ExtendedPoint + pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> edwards::ExtendedPoint where I: IntoIterator, - J: IntoIterator + J: IntoIterator { //assert_eq!(scalars.len(), points.len()); @@ -475,9 +482,10 @@ pub mod vartime { .map(|c| c.non_adjacent_form()).collect(); let odd_multiples: Vec<_> = points.into_iter() .map(|P| { - // Construct a lookup table of [P_i,2*P_i,3*P_i,4*P_i,5*P_i,6*P_i,7*P_i] + let P = ExtendedPoint::from(*P); + // Construct a lookup table of [P,2*P,3*P,4*P,5*P,6*P,7*P] let P2 = P.double(); - let mut lookup_table: [ExtendedPoint; 8] = [*P; 8]; + let mut lookup_table: [ExtendedPoint; 8] = [P; 8]; for i in 0..7 { lookup_table[i+1] = &P2 + &lookup_table[i]; } @@ -498,7 +506,7 @@ pub mod vartime { } } } - Q + Q.into() } } @@ -734,10 +742,10 @@ mod test { let R = &(&P1 * &s1) + &(&P2 * &s2); - let R_multiscalar = multiscalar_mult(&[s1, s2], &[P1, P2]); + let R_multiscalar = multiscalar_mult(&[s1, s2], &[P1.into(), P2.into()]); assert_eq!(edwards::ExtendedPoint::from(R).compress(), - edwards::ExtendedPoint::from(R_multiscalar).compress()); + R_multiscalar.compress()); } mod vartime { @@ -754,10 +762,10 @@ mod test { let R = &(&P1 * &s1) + &(&P2 * &s2); - let R_multiscalar = vartime::multiscalar_mult(&[s1, s2], &[P1, P2]); + let R_multiscalar = vartime::multiscalar_mult(&[s1, s2], &[P1.into(), P2.into()]); assert_eq!(edwards::ExtendedPoint::from(R).compress(), - edwards::ExtendedPoint::from(R_multiscalar).compress()); + R_multiscalar.compress()); } } } @@ -835,7 +843,7 @@ mod bench { let scalars: Vec<_> = (0..10).map(|_| Scalar::random(&mut csprng)).collect(); // Create 10 points (by doing scalar mults) let B = &constants::ED25519_BASEPOINT_POINT; - let points: Vec<_> = scalars.iter().map(|s| ExtendedPoint::from(B * &s)).collect(); + let points: Vec<_> = scalars.iter().map(|s| B * &s).collect(); b.iter(|| multiscalar_mult(&scalars, &points)); } @@ -850,7 +858,7 @@ mod bench { // Create 2 random scalars let s1 = Scalar::random(&mut csprng); let s2 = Scalar::random(&mut csprng); - let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); + let B = constants::ED25519_BASEPOINT_POINT; let P = &B * &s1; b.iter(|| vartime::multiscalar_mult(&[s1, s2], &[B, P])); @@ -863,7 +871,7 @@ mod bench { let scalars: Vec<_> = (0..10).map(|_| Scalar::random(&mut csprng)).collect(); // Create 10 points (by doing scalar mults) let B = &constants::ED25519_BASEPOINT_POINT; - let points: Vec<_> = scalars.iter().map(|s| ExtendedPoint::from(B * &s)).collect(); + let points: Vec<_> = scalars.iter().map(|s| B * &s).collect(); b.iter(|| vartime::multiscalar_mult(&scalars, &points)); } diff --git a/src/edwards.rs b/src/edwards.rs index 45354a0..3bf73a8 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -510,59 +510,68 @@ pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> ExtendedPoint where I: IntoIterator, J: IntoIterator { - //assert_eq!(scalars.len(), points.len()); + // If we built with AVX2, use the AVX2 backend. + #[cfg(all(target_feature = "avx2", feature = "avx2_backend"))] { + use backend::avx2::edwards as edwards_avx2; - let lookup_tables: Vec<_> = points.into_iter() - .map(|P_i| { - // Construct a lookup table of [P_i,2*P_i,3*P_i,4*P_i,5*P_i,6*P_i,7*P_i] - let mut lookup_table = [P_i.to_projective_niels(); 8]; - for j in 0..7 { - lookup_table[j+1] = (P_i + &lookup_table[j]) - .to_extended().to_projective_niels(); - } - lookup_table - }).collect(); - - // Setting s_i = i-th scalar, compute - // - // s_i = s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63, - // - // with `-8 ≤ s_{i,j} < 8` for `0 ≤ j < 63` and `-8 ≤ s_{i,63} ≤ 8`. - let scalar_digits_list: Vec<_> = scalars.into_iter() - .map(|c| c.to_radix_16()).collect(); - - // Compute s_1*P_1 + ... + s_n*P_n: since - // - // s_i*P_i = P_i*(s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63) - // s_i*P_i = P_i*s_{i,0} + P_i*s_{i,1}*16^1 + ... + P_i*s_{i,63}*16^63 - // s_i*P_i = P_i*s_{i,0} + 16*(P_i*s_{i,1} + 16*( ... + 16*P_i*s_{i,63})...) - // - // we have the two-dimensional sum - // - // s_1*P_1 = P_1*s_{1,0} + 16*(P_1*s_{1,1} + 16*( ... + 16*P_1*s_{1,63})...) - // + s_2*P_2 = + P_2*s_{2,0} + 16*(P_2*s_{2,1} + 16*( ... + 16*P_2*s_{2,63})...) - // ... - // + s_n*P_n = + P_n*s_{n,0} + 16*(P_n*s_{n,1} + 16*( ... + 16*P_n*s_{n,63})...) - // - // We sum column-wise top-to-bottom, then right-to-left, - // multiplying by 16 only once per column. - // - // This provides the speedup over doing n independent scalar - // mults: we perform 63 multiplications by 16 instead of 63*n - // multiplications, saving 252*(n-1) doublings. - let mut Q = ExtendedPoint::identity(); - // XXX this impl makes no effort to be cache-aware; maybe it could be improved? - for j in (0..64).rev() { - Q = Q.mult_by_pow_2(4); - let it = scalar_digits_list.iter().zip(lookup_tables.iter()); - for (s_i, lookup_table_i) in it { - // R_i = s_{i,j} * P_i - let R_i = select_precomputed_point(s_i[j], lookup_table_i); - // Q = Q + R_i - Q = (&Q + &R_i).to_extended(); - } + edwards_avx2::multiscalar_mult(scalars, points) + } + // Otherwise, proceed as normal: + #[cfg(not(all(target_feature = "avx2", feature = "avx2_backend")))] { + //assert_eq!(scalars.len(), points.len()); + + let lookup_tables: Vec<_> = points.into_iter() + .map(|P_i| { + // Construct a lookup table of [P_i,2*P_i,3*P_i,4*P_i,5*P_i,6*P_i,7*P_i] + let mut lookup_table = [P_i.to_projective_niels(); 8]; + for j in 0..7 { + lookup_table[j+1] = (P_i + &lookup_table[j]) + .to_extended().to_projective_niels(); + } + lookup_table + }).collect(); + + // Setting s_i = i-th scalar, compute + // + // s_i = s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63, + // + // with `-8 ≤ s_{i,j} < 8` for `0 ≤ j < 63` and `-8 ≤ s_{i,63} ≤ 8`. + let scalar_digits_list: Vec<_> = scalars.into_iter() + .map(|c| c.to_radix_16()).collect(); + + // Compute s_1*P_1 + ... + s_n*P_n: since + // + // s_i*P_i = P_i*(s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63) + // s_i*P_i = P_i*s_{i,0} + P_i*s_{i,1}*16^1 + ... + P_i*s_{i,63}*16^63 + // s_i*P_i = P_i*s_{i,0} + 16*(P_i*s_{i,1} + 16*( ... + 16*P_i*s_{i,63})...) + // + // we have the two-dimensional sum + // + // s_1*P_1 = P_1*s_{1,0} + 16*(P_1*s_{1,1} + 16*( ... + 16*P_1*s_{1,63})...) + // + s_2*P_2 = + P_2*s_{2,0} + 16*(P_2*s_{2,1} + 16*( ... + 16*P_2*s_{2,63})...) + // ... + // + s_n*P_n = + P_n*s_{n,0} + 16*(P_n*s_{n,1} + 16*( ... + 16*P_n*s_{n,63})...) + // + // We sum column-wise top-to-bottom, then right-to-left, + // multiplying by 16 only once per column. + // + // This provides the speedup over doing n independent scalar + // mults: we perform 63 multiplications by 16 instead of 63*n + // multiplications, saving 252*(n-1) doublings. + let mut Q = ExtendedPoint::identity(); + // XXX this impl makes no effort to be cache-aware; maybe it could be improved? + for j in (0..64).rev() { + Q = Q.mult_by_pow_2(4); + let it = scalar_digits_list.iter().zip(lookup_tables.iter()); + for (s_i, lookup_table_i) in it { + // R_i = s_{i,j} * P_i + let R_i = select_precomputed_point(s_i[j], lookup_table_i); + // Q = Q + R_i + Q = (&Q + &R_i).to_extended(); + } + } + Q } - Q } /// A precomputed table of multiples of a basepoint, for accelerating @@ -801,30 +810,39 @@ pub mod vartime { where I: IntoIterator, J: IntoIterator { - //assert_eq!(scalars.len(), points.len()); + // If we built with AVX2, use the AVX2 backend. + #[cfg(all(target_feature = "avx2", feature = "avx2_backend"))] { + use backend::avx2::edwards as edwards_avx2; - let nafs: Vec<_> = scalars.into_iter() - .map(|c| c.non_adjacent_form()).collect(); - let odd_multiples: Vec<_> = points.into_iter() - .map(|P| OddMultiples::create(P)).collect(); + edwards_avx2::vartime::multiscalar_mult(scalars, points) + } + // Otherwise, proceed as normal: + #[cfg(not(all(target_feature = "avx2", feature = "avx2_backend")))] { + //assert_eq!(scalars.len(), points.len()); - let mut r = ProjectivePoint::identity(); + let nafs: Vec<_> = scalars.into_iter() + .map(|c| c.non_adjacent_form()).collect(); + let odd_multiples: Vec<_> = points.into_iter() + .map(|P| OddMultiples::create(P)).collect(); - for i in (0..255).rev() { - let mut t = r.double(); + let mut r = ProjectivePoint::identity(); - for (naf, odd_multiple) in nafs.iter().zip(odd_multiples.iter()) { - if naf[i] > 0 { - t = &t.to_extended() + &odd_multiple[( naf[i]/2) as usize]; - } else if naf[i] < 0 { - t = &t.to_extended() - &odd_multiple[(-naf[i]/2) as usize]; + for i in (0..255).rev() { + let mut t = r.double(); + + for (naf, odd_multiple) in nafs.iter().zip(odd_multiples.iter()) { + if naf[i] > 0 { + t = &t.to_extended() + &odd_multiple[( naf[i]/2) as usize]; + } else if naf[i] < 0 { + t = &t.to_extended() - &odd_multiple[(-naf[i]/2) as usize]; + } } + + r = t.to_projective(); } - r = t.to_projective(); + r.to_extended() } - - r.to_extended() } /// Given a point \\(A\\) and scalars \\(a\\) and \\(b\\), compute the point From 3d435a1f4fcc781bd61724485992ad1469c3b82e Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 24 Nov 2017 19:46:06 -0800 Subject: [PATCH 21/54] Fix up tests to use new Scalar API --- src/backend/avx2/edwards.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 93ba438..6523eee 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -709,7 +709,7 @@ mod test { fn scalar_mult_vs_edwards_scalar_mult() { let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); // some random bytes - let s = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + let s = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); let R1 = edwards::ExtendedPoint::from(&B * &s); let R2 = &constants::ED25519_BASEPOINT_TABLE * &s; @@ -722,7 +722,7 @@ mod test { let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); let B_table = EdwardsBasepointTable::create(&B); // some random bytes - let s = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + let s = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); let P1 = &B * &s; let P2 = &B_table * &s; @@ -734,8 +734,8 @@ mod test { #[test] fn multiscalar_mult_vs_adding_scalar_mults() { let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); - let s1 = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); - let s2 = Scalar([165, 30, 79, 89, 58, 24, 195, 245, 248, 146, 203, 236, 119, 43, 64, 119, 196, 111, 188, 251, 248, 53, 234, 59, 215, 28, 218, 13, 59, 120, 14, 4]); + let s1 = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + let s2 = Scalar::from_bits([165, 30, 79, 89, 58, 24, 195, 245, 248, 146, 203, 236, 119, 43, 64, 119, 196, 111, 188, 251, 248, 53, 234, 59, 215, 28, 218, 13, 59, 120, 14, 4]); let P1 = &B * &s2; let P2 = &B * &s1; @@ -754,8 +754,8 @@ mod test { #[test] fn multiscalar_mult_vs_adding_scalar_mults() { let B: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.into(); - let s1 = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); - let s2 = Scalar([165, 30, 79, 89, 58, 24, 195, 245, 248, 146, 203, 236, 119, 43, 64, 119, 196, 111, 188, 251, 248, 53, 234, 59, 215, 28, 218, 13, 59, 120, 14, 4]); + let s1 = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + let s2 = Scalar::from_bits([165, 30, 79, 89, 58, 24, 195, 245, 248, 146, 203, 236, 119, 43, 64, 119, 196, 111, 188, 251, 248, 53, 234, 59, 215, 28, 218, 13, 59, 120, 14, 4]); let P1 = &B * &s2; let P2 = &B * &s1; From ba071f12aace55184142c29e5952e6db9e1e1340 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 27 Nov 2017 14:06:04 -0800 Subject: [PATCH 22/54] Write up notes on the AVX2 backend --- src/backend/avx2/mod.rs | 299 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 299 insertions(+) diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index 456c451..7a06512 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -8,6 +8,305 @@ // - Isis Agora Lovecruft // - Henry de Valence +//! An implementation of group operations on the twisted Edwards form of +//! Curve25519, using AVX2 to implement the 4-way parallel formulas of +//! Hisil, Wong, Carter, and Dawson (HWCD). +//! +//! Their 2008 paper _Twisted Edwards Curves Revisited_, which +//! introduced the extended coordinates used in other parts of `-dalek`, +//! also describes 4-way parallel formulas for point addition and +//! doubling: +//! +//! * a unified addition algorithm taking an effective \\(2\mathbf M + +//! 1\mathbf D\\); +//! +//! * a doubling algorithm taking an effective \\(1\mathbf M + 1\mathbf +//! S\\); +//! +//! * a dedicated (i.e., for distinct points) addition algorithm taking +//! an effective \\(2 \mathbf M \\). +//! +//! Here \\(\mathbf M\\) and \\(\mathbf S\\) represent the cost of +//! multiplication and squaring of generic field elements and \\(\mathbf +//! D\\) represents the cost of multiplication by a curve constant. +//! +//! Currently, this implementation uses only the first two algorithms. +//! +//! # Parallel formulas +//! +//! The doubling formula is presented in the HWCD paper as follows: +//! +//! | Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | +//! |------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +//! | | idle | idle | idle | \\( R\_1 \gets X\_1 + Y\_1 \\) | +//! | \\(1\mathbf S\\) | \\( R\_2 \gets X\_1\^2 \\) | \\( R\_3 \gets Y\_1\^2 \\) | \\( R\_4 \gets Z\_1\^2 \\) | \\( R\_5 \gets R\_1\^2 \\) | +//! | | \\( R\_6 \gets R\_2 + R\_3 \\) | \\( R\_7 \gets R\_2 - R\_3 \\) | \\( R\_4 \gets 2 R\_4 \\) | idle | +//! | | idle | \\( R\_1 \gets R\_4 + R\_7 \\) | idle | \\( R\_2 \gets R\_6 - R\_5 \\) | +//! | \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_6 R\_7 \\) | \\( T\_3 \gets R\_2 R\_6 \\) | \\( Z\_3 \gets R\_1 R\_7 \\) | +//! +//! and the unified addition algorithm is presented as follows: +//! +//! | Cost | Processor 1 | Processor 2 | Processor 3 | Processor 4 | +//! |------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +//! | | \\( R\_1 \gets Y\_1 - X\_1 \\) | \\( R\_2 \gets Y\_2 - X\_2 \\) | \\( R\_3 \gets Y\_1 + X\_1 \\) | \\( R\_4 \gets Y\_2 + X\_2 \\) | +//! | \\(1\mathbf M\\) | \\( R\_5 \gets R\_1 R\_2 \\) | \\( R\_6 \gets R\_3 R\_4 \\) | \\( R\_7 \gets T\_1 T\_2 \\) | \\( R\_8 \gets Z\_1 Z\_2 \\) | +//! | \\(1\mathbf D\\) | idle | idle | \\( R\_7 \gets k R\_7 \\) | \\( R\_8 \gets 2 R\_8 \\) | +//! | | \\( R\_1 \gets R\_6 - R\_5 \\) | \\( R\_2 \gets R\_8 - R\_7 \\) | \\( R\_3 \gets R\_8 + R\_7 \\) | \\( R\_4 \gets R\_6 + R\_5 \\) | +//! | \\(1\mathbf M\\) | \\( X\_3 \gets R\_1 R\_2 \\) | \\( Y\_3 \gets R\_3 R\_4 \\) | \\( T\_3 \gets R\_1 R\_4 \\) | \\( Z\_3 \gets R\_2 R\_3 \\) | +//! +//! Here \\( k = 2d \\) is a curve constant. +//! +//! # Implementation strategy +//! +//! For a software implementation, each "processor"'s operations are too +//! low-latency to parallelize across threads. However, the main cost +//! is in the multiplication and squaring steps, which share a single +//! instruction. +//! +//! Our strategy is to implement 4-wide multiplication and squaring using one +//! 64-bit AVX2 lane for each field element. Field elements are +//! represented in the usual way as 10 `u32` limbs. The addition and +//! subtraction steps are done largely serially, using masking to handle +//! the instruction divergence. +//! +//! The remaining obstacle to parallelism is the multiplication by the curve constant \\(k = 2d\\). In the Curve25519 case, this is +//! +//! $$ k \equiv 2 \frac{-121665}{121666} \\ \equiv 16295367250680780974490674513165176452449235426866156013048779062215315747161 \pmod p. $$ +//! +//! HWCD suggest parallelising this step by breaking \\(k\\) into four +//! parts as \\(k = k_0 + 2\^n k_1 + 2\^{2n} k_2 + 2\^{3n} k_3 \\) and +//! computing \\(k_i R_7 \\) in parallel. However, this would be +//! somewhat awkward in our case, since we would normally represent +//! \\(k\\) as \\( 10 \\) 32-bit limbs, and \\(10 \\) is not divisible +//! by \\(4\\), so we would need a specialized routine to perform a +//! vectorized multiplication by 64-bit constants. +//! +//! Instead, since we are working projectively, we can multiply +//! \\(R_7\\) by \\( -2\cdot 121665 \\) and multiply the other three +//! variables by \\(121666\\). This trick was suggested by Mike +//! Hamburg. Ignoring the sign for the moment, since +//! \\(2 \cdot 121666 < 2\^{18}\\), all these constants fit in 32 bits, +//! so this can be done in parallel as a scaling by \\( (121666, 121666, +//! 2\cdot 121665, 2\cdot 121666) \\). To handle the sign, we use +//! masking to negate one of the field elements. +//! +//! Since we're primarily interested in Ristretto performance, not +//! Curve25519 performance, we could alternately work on the +//! \\(4\\)-isogenous "IsoEd25519" curve, which has \\(d = 121665\\). +//! However, this would only save the negation step, since multiplying +//! one field element by a 32-bit constant is not much easier than +//! multiplying four field elements by 32-bit constants, and it would +//! prevent accelerating Curve25519, so we don't make this choice. +//! +//! The 4-wide formulas of the HWCD paper do not seem to have been +//! implemented using SIMD before. The HWCD paper also describes and +//! analyzes a 2-wide variant of the Montgomery ladder; this strategy was +//! used by Tung Chou's `sandy2x` implementation, which used a 2-wide +//! field implementation in 128-bit registers. Curiously, however, +//! although the `sandy2x` paper cites the HWCD paper for extended +//! twisted Edwards coordinates, it does not mention the 4-wide HWCD +//! Edwards formulas or that the 2-wide Montgomery formulas it uses were +//! previously published there. +//! +//! HWCD also suggest using a mixed representation, passing between \\( +//! \mathbb P\^3 \\) "extended" coordinates and \\( \mathbb P\^2 \\) +//! "projective" coordinates, where doubling is slightly cheaper (saving +//! about \\(\mathbf 1M\\). This approach is used for the +//! non-vectorized `u32` and `u64` backends, and more +//! details on the different coordinate systems can be found in the +//! `curve_models` module documentation. +//! +//! This optimization is not used for the parallel formulas, which are +//! therefore slightly less efficient when counting the total number of +//! multiplications and squarings. In addition, the parallel formulas +//! can only use a \\( 32 \times 32 \rightarrow 64 \\)-bit multiplier +//! instead of a \\( 64 \times 64 \rightarrow 128\\)-bit multiplier. +//! +//! When used for constant-time variable-base scalar multiplication, +//! this strategy (using AVX2) gives a significant speedup over the +//! serial implementation (using the \\(64 \times 64\\) multiplier) of +//! approximately 1.6x for Skylake-X with `target_cpu=skylake` (using AVX2), of +//! approximately 1.8x for Skylake-X with `target_cpu=skylake-avx512` (using the extra +//! `ymm16..ymm31` registers from AVX512VL), and of approximately 1.0x +//! for Ryzen (which implements AVX2 at half rate). +//! +//! # Tweaked formulas +//! +//! After tweaking the formulas as described above, we obtain the +//! following. To avoid confusion with the original HWCD formulas, +//! temporary variables are named \\(S\\) instead of \\(R\\) and are in +//! static single-assignment (SSA) form. +//! +//! ## Addition +//! +//! To add points \\(P_1 = (X_1 : Y_1 : Z_1 : T_1) \\) and \\(P_2 = (X_2 +//! : Y_2 : Z_2 : T_2 ) \\), we compute +//! +//! $$ +//! \begin{aligned} +//! S\_0 &\gets Y\_1 - X\_1 \\\\ +//! S\_1 &\gets Y\_1 + X\_1 \\\\ +//! S\_2 &\gets Y\_2 - X\_2 \\\\ +//! S\_3 &\gets Y\_2 + X\_2 +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! S\_4 &\gets S\_0 S\_2 \\\\ +//! S\_5 &\gets S\_1 S\_3 \\\\ +//! S\_6 &\gets Z\_1 Z\_2 \\\\ +//! S\_7 &\gets T\_1 T\_2 +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! S\_8 &\gets S\_4 \cdot 121666 \\\\ +//! S\_9 &\gets S\_5 \cdot 121666 \\\\ +//! S\_{10} &\gets S\_6 \cdot 2 \cdot 121666 \\\\ +//! S\_{11} &\gets S\_7 \cdot 2 \cdot (-121665) +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! S\_{12} &\gets S\_9 - S\_8 \\\\ +//! S\_{13} &\gets S\_9 + S\_8 \\\\ +//! S\_{14} &\gets S\_{10} - S\_{11} \\\\ +//! S\_{15} &\gets S\_{10} - S\_{11} +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! X\_3 &\gets S\_{12} S\_{14} \\\\ +//! Y\_3 &\gets S\_{15} S\_{13} \\\\ +//! Z\_3 &\gets S\_{15} S\_{14} \\\\ +//! T\_3 &\gets S\_{12} S\_{13} +//! \end{aligned} +//! $$ +//! +//! to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). +//! +//! ## Doubling +//! +//! To double a point \\( P = (X\_1 : Y\_1 : Z\_1 : T\_1) \\), we compute +//! +//! $$ S\_0 \gets X\_1 + Y\_1 $$ +//! +//! $$ +//! \begin{aligned} +//! S\_1 &\gets X\_1\^2 \\\\ +//! S\_2 &\gets Y\_1\^2 \\\\ +//! S\_3 &\gets Z\_1\^2 \\\\ +//! S\_4 &\gets S\_0\^2 +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! S\_5 &\gets S\_1 + S\_2 \\\\ +//! S\_6 &\gets S\_1 - S\_2 \\\\ +//! S\_7 &\gets 2S\_3 \\\\ +//! S\_8 &\gets S\_7 + S\_6 = S\_1 + 2S\_3 - S\_2 \\\\ +//! S\_9 &\gets S\_5 - S\_4 = S\_1 + S\_2 - S\_4 +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! X\_3 &\gets S\_8 S\_9 \\\\ +//! Y\_3 &\gets S\_5 S\_6 \\\\ +//! Z\_3 &\gets S\_8 S\_6 \\\\ +//! T\_3 &\gets S\_5 S\_9 +//! \end{aligned} +//! $$ +//! +//! to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = [2]P\_1 \\). +//! +//! In practice, we compute \\( (S\_5, S\_6, S\_7, S\_9 ) \\) as +//! +//! $$ +//! \begin{matrix} +//! & S\_1 & S\_1 & S\_1 & S\_1 \\\\ +//! +& S\_2 & & & S\_2 \\\\ +//! +& & & S\_3 & \\\\ +//! +& & & S\_3 & \\\\ +//! +& & 2p & 2p & 2p \\\\ +//! -& & S\_2 & S\_2 & \\\\ +//! -& & & & S\_4 \\\\ +//! =& S\_5 & S\_6 & S\_8 & S\_9 +//! \end{matrix} +//! $$ +//! +//! adding multiples of \\(p\\) to prevent underflow. This results in +//! 32-bit limbs which are just too large for multiplication, so we +//! perform a reduction. However, since we just need to reduce the +//! excess in each limb, not a full reduction, it's enough to perform +//! each carry in parallel. +//! +//! With some finesse, it may be possible to rearrange this +//! computation to avoid the extra carry pass, but this is not yet +//! implemented. +//! +//! # Field element representation +//! +//! The field element representation is oriented around the AVX2 +//! `vpmuluqdq` instruction, which multiplies the low 32 bits of each +//! 64-bit lane of each operand to produce a 64-bit result. +//! +//! ```text,no_run +//! (a1 ?? b1 ?? c1 ?? d1 ??) +//! (a2 ?? b2 ?? c2 ?? d2 ??) +//! +//! (a1*a2 b1*b2 c1*c2 d1*d2) +//! ``` +//! +//! To unpack 32-bit values into 64-bit lanes for use in multiplication +//! it would be convenient to use the `vpunpck[lh]dq` instructions, +//! which unpack and interleave the low and high 32-bit lanes of two +//! source vectors. +//! However, the AVX2 versions of these instructions are designed to +//! operate only within 128-bit lanes of the 256-bit vectors, so that +//! interleaving the low lanes of `(a0 b0 c0 d0 a1 b1 c1 d1)` with zero +//! gives `(a0 00 b0 00 a1 00 b1 00)`. Instead, we pre-shuffle the data +//! layout as `(a0 b0 a1 b1 c0 d0 c1 d1)` so that we can unpack the +//! "low" and "high" parts as +//! +//! ```text,no_run +//! (a0 00 b0 00 c0 00 d0 00) +//! (a1 00 b1 00 c1 00 d1 00) +//! ``` +//! +//! The data layout for a vector of four field elements \\( (a,b,c,d) +//! \\) with limbs \\( a_0, a_1, \ldots, a_9 \\) is as `[u32x8; 5]` in +//! the form +//! +//! ```text,no_run +//! (a0 b0 a1 b1 c0 d0 c1 d1) +//! (a2 b2 a3 b3 c2 d2 c3 d3) +//! (a4 b4 a5 b5 c4 d4 c5 d5) +//! (a6 b6 a7 b7 c6 d6 c7 d7) +//! (a8 b8 a9 b9 c8 d8 c9 d9) +//! ``` +//! +//! Since this breaks cleanly into two 128-bit lanes, it may be possible +//! to adapt it to 128-bit vector instructions such as NEON without too +//! much difficulty. +//! +//! We don't attempt to use AVX2 for serial field element computations +//! such as inversion, since wherever we have AVX2 we also have `mulx`. +//! However, it might be useful for batched inverse square-root +//! computations, which can't be batched in the same way inversions can. +//! +//! # Implementation details +//! +//! The implementation uses the unstable `stdsimd` crate to provide AVX2 +//! intrinsics, and the code is not yet cleanly factored between the +//! field element parts and the point parts. + + pub(crate) mod field; pub(crate) mod edwards; From 309486644280d5ff0a1c0bc8347b46e141087c52 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 27 Nov 2017 15:35:37 -0800 Subject: [PATCH 23/54] Fix up after Scalar API changes --- src/backend/avx2/edwards.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 6523eee..a4dd1ac 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -815,7 +815,7 @@ mod bench { fn scalar_mult(b: &mut Bencher) { let B = &constants::ED25519_BASEPOINT_TABLE; let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); - let s = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + let s = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); b.iter(|| &P * &s ); } @@ -831,7 +831,7 @@ mod bench { fn basepoint_mult(b: &mut Bencher) { let B = ExtendedPoint::from(constants::ED25519_BASEPOINT_POINT); let table = EdwardsBasepointTable::create(&B); - let s = Scalar([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); + let s = Scalar::from_bits([233, 1, 233, 147, 113, 78, 244, 120, 40, 45, 103, 51, 224, 199, 189, 218, 96, 140, 211, 112, 39, 194, 73, 216, 173, 33, 102, 93, 76, 200, 84, 12]); b.iter(|| &table * &s ); } From 9213dc0bbb2c8b25669c7ea82a351238a08cc385 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 30 Nov 2017 11:05:06 -0800 Subject: [PATCH 24/54] Implement point subtraction --- src/backend/avx2/edwards.rs | 144 +++++++++++++++++++++++++++++++----- src/backend/avx2/field.rs | 80 +++++++++++++++++--- 2 files changed, 192 insertions(+), 32 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index a4dd1ac..0ebbc52 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -14,7 +14,7 @@ #![allow(bad_style)] use std::convert::From; -use std::ops::{Add, Mul, Neg}; +use std::ops::{Add, Sub, Mul, Neg}; use stdsimd::simd::{u32x8, i32x8}; @@ -70,6 +70,7 @@ impl<'a> Neg for &'a ExtendedPoint { fn neg(self) -> ExtendedPoint { let mut neg = *self; + // (X Y Z T) -> (-X Y Z -T) neg.0.mask_negate(0b10100101); neg } @@ -216,36 +217,114 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { let mut t0 = FieldElement32x4::zero(); let mut t1 = FieldElement32x4::zero(); + // set t0 = (X1 Y1 X2 Y2) for i in 0..5 { t0.0[i] = _mm256_permute2x128_si256(P.0[i].into(), Q.0[i].into(), 32).into(); } + // set t0 = (Y1-X1 Y1+X1 Y2-X2 Y2+X2) = (S0 S1 S2 S3) t0.diff_sum(); + // set t1 = (S0 S1 Z1 T1) + // set t0 = (S2 S3 Z2 T2) for i in 0..5 { t1.0[i] = _mm256_blend_epi32(t0.0[i].into(), P.0[i].into(), 0b11110000).into(); t0.0[i] = _mm256_permute2x128_si256(t0.0[i].into(), Q.0[i].into(), 49).into(); } + // set t2 = (S0*S2 S1*S3 Z1*Z2 T1*T2) = (S4 S5 S6 S7) let mut t2 = &t0 * &t1; - - t2.scale_by_curve_constants(); - - for i in 0..5 { - let swapped = _mm256_shuffle_epi32(t2.0[i].into(), 0b10_11_00_01); - t2.0[i] = _mm256_blend_epi32(t2.0[i].into(), swapped, 0b11110000).into(); - } + // set t2 = (S8 S9 S10 S11) + t2.scale_by_curve_constants(true); + + // set t2 = (S8 S9 S11 S10) + t2.swap_CD(); + + // set t2 = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) t2.diff_sum(); let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) + // set t0 = (S12 S15 S15 S12) + // set t1 = (S14 S13 S14 S13) for i in 0..5 { t0.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c0); t1.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c1); } + // return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) + ExtendedPoint(&t0 * &t1) + } + } +} + +impl<'a, 'b> Sub<&'b ExtendedPoint> for &'a ExtendedPoint { + type Output = ExtendedPoint; + + /// Uses a slight tweak of the parallel unified formulas of HWCD'08 + fn sub(self, other: &'b ExtendedPoint) -> ExtendedPoint { + unsafe { + use stdsimd::vendor::_mm256_permute2x128_si256; + use stdsimd::vendor::_mm256_permutevar8x32_epi32; + use stdsimd::vendor::_mm256_blend_epi32; + use stdsimd::vendor::_mm256_shuffle_epi32; + + let P: &FieldElement32x4 = &self.0; + let Q: &FieldElement32x4 = &other.0; + + let mut t0 = FieldElement32x4::zero(); + let mut t1 = FieldElement32x4::zero(); + + // set t0 = (X1 Y1 X2 Y2) + for i in 0..5 { + t0.0[i] = _mm256_permute2x128_si256(P.0[i].into(), Q.0[i].into(), 32).into(); + } + + // Since we're subtracting instead of adding, we want to add the point (-X2 Y2 Z2 -T2). + // Set (X2' Y2' Z2' T2') = (-X2 Y2 Z2 -T2) + // + // so S2 = Y2 - X2' = Y2 - (-X2) = Y2 + X2 + // and S3 = Y2 + X2' = Y2 + (-X2) = Y2 - X2 + + // set t0 = (Y1-X1 Y1+X1 Y2-X2 Y2+X2) = (S0 S1 S3 S2) + t0.diff_sum(); + + // set t0 = (S0 S1 S2 S3) + t0.swap_CD(); + + // set t1 = (S0 S1 Z1 T1) + // set t0 = (S2 S3 Z2 T2) = (S2 S3 Z2' -T2') + for i in 0..5 { + t1.0[i] = _mm256_blend_epi32(t0.0[i].into(), P.0[i].into(), 0b11110000).into(); + t0.0[i] = _mm256_permute2x128_si256(t0.0[i].into(), Q.0[i].into(), 49).into(); + } + + // set t2 = (S0*S2 S1*S3 Z1*Z2 T1*T2 ) + // = (S0*S2 S1*S3 Z1*Z2' -T1*T2') = (S4 S5 S6 -S7) + let mut t2 = &t0 * &t1; + + // set t2 = (S8 S9 S10 S11) + t2.scale_by_curve_constants(false); + + // set t2 = (S8 S9 S11 S10) + t2.swap_CD(); + + // set t2 = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) + t2.diff_sum(); + + let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) + let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) + + // set t0 = (S12 S15 S15 S12) + // set t1 = (S14 S13 S14 S13) + for i in 0..5 { + t0.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c0); + t1.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c1); + } + + // return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) ExtendedPoint(&t0 * &t1) } } @@ -502,7 +581,7 @@ pub mod vartime { Q = &Q + &odd_multiple[( naf[i]/2) as usize]; } else if naf[i] < 0 { // XXX impl Sub - Q = &Q + &(-&odd_multiple[(-naf[i]/2) as usize]); + Q = &Q - &odd_multiple[(-naf[i]/2) as usize]; } } } @@ -577,40 +656,65 @@ mod test { } fn addition_test_helper(P: edwards::ExtendedPoint, Q: edwards::ExtendedPoint) { - let R1: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); - let R2: edwards::ExtendedPoint = (&ExtendedPoint::from(P) + &ExtendedPoint::from(Q)).into(); + // Test the serial implementation of the parallel addition formulas + let R_serial: edwards::ExtendedPoint = serial_add(P.into(), Q.into()).into(); + // Test the vector implementation of the parallel addition formulas + let R_vector: edwards::ExtendedPoint = (&ExtendedPoint::from(P) + &ExtendedPoint::from(Q)).into(); + // Test the vector implementation of the parallel subtraction formulas + let S_vector: edwards::ExtendedPoint = (&ExtendedPoint::from(P) - &ExtendedPoint::from(Q)).into(); + println!("Testing point addition:"); println!("P = {:?}", P); println!("Q = {:?}", Q); - println!("(serial) R1 = {:?}", R1); - println!("(vector) R2 = {:?}", R2); - println!("P + Q = {:?}", &P + &Q); - assert_eq!(R1.compress(), (&P + &Q).compress()); - assert_eq!(R2.compress(), (&P + &Q).compress()); + println!("R = P + Q = {:?}", &P + &Q); + println!("R_serial = {:?}", R_serial); + println!("R_vector = {:?}", R_vector); + println!("S = P - Q = {:?}", &P - &Q); + println!("S_vector = {:?}", S_vector); + assert_eq!(R_serial.compress(), (&P + &Q).compress()); + assert_eq!(R_vector.compress(), (&P + &Q).compress()); + assert_eq!(S_vector.compress(), (&P - &Q).compress()); println!("OK!\n"); } + #[test] + fn sub_vs_add_minus() { + let P: ExtendedPoint = edwards::ExtendedPoint::identity().into(); + let Q: ExtendedPoint = edwards::ExtendedPoint::identity().into(); + + let mQ = -&Q; + + println!("sub"); + let R1: edwards::ExtendedPoint = (&P - &Q).into(); + println!("add neg"); + let R2: edwards::ExtendedPoint = (&P + &mQ).into(); + + assert_eq!(R2.compress(), edwards::ExtendedPoint::identity().compress()); + assert_eq!(R1.compress(), edwards::ExtendedPoint::identity().compress()); + } + + #[test] fn vector_addition_vs_serial_addition_vs_edwards_extendedpoint() { use constants; use scalar::Scalar; - println!("Testing id + id"); + println!("Testing id +- id"); let P = edwards::ExtendedPoint::identity(); let Q = edwards::ExtendedPoint::identity(); addition_test_helper(P, Q); - println!("Testing id + B"); + println!("Testing id +- B"); let P = edwards::ExtendedPoint::identity(); let Q = constants::ED25519_BASEPOINT_POINT; addition_test_helper(P, Q); - println!("Testing B + B"); + println!("Testing B +- B"); let P = constants::ED25519_BASEPOINT_POINT; let Q = constants::ED25519_BASEPOINT_POINT; addition_test_helper(P, Q); - println!("Testing B + kB"); + println!("Testing B +- kB"); let P = constants::ED25519_BASEPOINT_POINT; let Q = &constants::ED25519_BASEPOINT_TABLE * &Scalar::from_u64(8475983829); addition_test_helper(P, Q); diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index b911c1c..b76d454 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -114,6 +114,18 @@ impl FieldElement32x4 { self.reduce32(); } + // Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` + pub fn swap_CD(&mut self) { + unsafe { + use stdsimd::vendor::_mm256_shuffle_epi32; + use stdsimd::vendor::_mm256_blend_epi32; + for i in 0..5 { + let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01); + self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b11110000).into(); + } + } + } + // Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)`. pub fn diff_sum(&mut self) { /// (v0 v1 v2 v3 v4 v5 v6 v7) -> (v1 v0 v3 v2 v5 v4 v7 v6) @@ -188,7 +200,17 @@ impl FieldElement32x4 { FieldElement32x4(out) } - pub fn scale_by_curve_constants(&mut self) { + /// Let `self` \\(= (A, B, C, D) \\). + /// + /// If `negate_121665 = true`, compute + /// + /// $$( 121666A, 121666B, 2\cdot 121666C, -2\cdot 121665 D).$$ + /// + /// If `negate_121665 = false`, compute + /// + /// $$( 121666A, 121666B, 2\cdot 121666C, 2\cdot 121665 D).$$ + /// + pub fn scale_by_curve_constants(&mut self, negate_121665: bool) { let mut b = [u64x4::splat(0); 10]; let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); @@ -203,32 +225,57 @@ impl FieldElement32x4 { let (b0, b1) = unpack_pair(self.0[0]); let b0 = _mm256_mul_epu32(b0, consts); // need a new binding since now let b1 = _mm256_mul_epu32(b1, consts); // b0 has type u64x4 - b[0] = _mm256_blend_epi32(b0.into(), (low__p20 - b0).into(), 0b11_00_00_00).into(); - b[1] = _mm256_blend_epi32(b1.into(), (odd__p20 - b1).into(), 0b11_00_00_00).into(); + if negate_121665 { + b[0] = _mm256_blend_epi32(b0.into(), (low__p20 - b0).into(), 0b11_00_00_00).into(); + b[1] = _mm256_blend_epi32(b1.into(), (odd__p20 - b1).into(), 0b11_00_00_00).into(); + } else { + b[0] = b0; + b[1] = b1; + } let (b2, b3) = unpack_pair(self.0[1]); let b2 = _mm256_mul_epu32(b2, consts); let b3 = _mm256_mul_epu32(b3, consts); - b[2] = _mm256_blend_epi32(b2.into(), (even_p20 - b2).into(), 0b11_00_00_00).into(); - b[3] = _mm256_blend_epi32(b3.into(), (odd__p20 - b3).into(), 0b11_00_00_00).into(); + if negate_121665 { + b[2] = _mm256_blend_epi32(b2.into(), (even_p20 - b2).into(), 0b11_00_00_00).into(); + b[3] = _mm256_blend_epi32(b3.into(), (odd__p20 - b3).into(), 0b11_00_00_00).into(); + } else { + b[2] = b2; + b[3] = b3; + } let (b4, b5) = unpack_pair(self.0[2]); let b4 = _mm256_mul_epu32(b4, consts); let b5 = _mm256_mul_epu32(b5, consts); - b[4] = _mm256_blend_epi32(b4.into(), (even_p20 - b4).into(), 0b11_00_00_00).into(); - b[5] = _mm256_blend_epi32(b5.into(), (odd__p20 - b5).into(), 0b11_00_00_00).into(); + if negate_121665 { + b[4] = _mm256_blend_epi32(b4.into(), (even_p20 - b4).into(), 0b11_00_00_00).into(); + b[5] = _mm256_blend_epi32(b5.into(), (odd__p20 - b5).into(), 0b11_00_00_00).into(); + } else { + b[4] = b4; + b[5] = b5; + } let (b6, b7) = unpack_pair(self.0[3]); let b6 = _mm256_mul_epu32(b6, consts); let b7 = _mm256_mul_epu32(b7, consts); - b[6] = _mm256_blend_epi32(b6.into(), (even_p20 - b6).into(), 0b11_00_00_00).into(); - b[7] = _mm256_blend_epi32(b7.into(), (odd__p20 - b7).into(), 0b11_00_00_00).into(); + if negate_121665 { + b[6] = _mm256_blend_epi32(b6.into(), (even_p20 - b6).into(), 0b11_00_00_00).into(); + b[7] = _mm256_blend_epi32(b7.into(), (odd__p20 - b7).into(), 0b11_00_00_00).into(); + } else { + b[6] = b6; + b[7] = b7; + } let (b8, b9) = unpack_pair(self.0[4]); let b8 = _mm256_mul_epu32(b8, consts); let b9 = _mm256_mul_epu32(b9, consts); - b[8] = _mm256_blend_epi32(b8.into(), (even_p20 - b8).into(), 0b11_00_00_00).into(); - b[9] = _mm256_blend_epi32(b9.into(), (odd__p20 - b9).into(), 0b11_00_00_00).into(); + if negate_121665 { + b[8] = _mm256_blend_epi32(b8.into(), (even_p20 - b8).into(), 0b11_00_00_00).into(); + b[9] = _mm256_blend_epi32(b9.into(), (odd__p20 - b9).into(), 0b11_00_00_00).into(); + } else { + b[8] = b8; + b[9] = b9; + } } *self = FieldElement32x4::reduce64(b); @@ -517,13 +564,22 @@ mod test { #[test] fn scale_by_curve_constants() { let mut x = FieldElement32x4::splat(&FieldElement64::one()); - x.scale_by_curve_constants(); + x.scale_by_curve_constants(true); let xs = x.split(); assert_eq!(xs[0], FieldElement64([ 121666,0,0,0,0])); assert_eq!(xs[1], FieldElement64([ 121666,0,0,0,0])); assert_eq!(xs[2], FieldElement64([2*121666,0,0,0,0])); assert_eq!(xs[3], -&FieldElement64([2*121665,0,0,0,0])); + + let mut y = FieldElement32x4::splat(&FieldElement64::one()); + y.scale_by_curve_constants(false); + + let ys = y.split(); + assert_eq!(ys[0], FieldElement64([ 121666,0,0,0,0])); + assert_eq!(ys[1], FieldElement64([ 121666,0,0,0,0])); + assert_eq!(ys[2], FieldElement64([2*121666,0,0,0,0])); + assert_eq!(ys[3], FieldElement64([2*121665,0,0,0,0])); } #[test] From e3579995c0ff7736e3b9f07891114eff633bb97f Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 30 Nov 2017 12:39:23 -0800 Subject: [PATCH 25/54] Add double-base scalar vartime for AVX2 --- src/backend/avx2/constants.rs | 76 ++++++++++++++++++++++++++++ src/backend/avx2/edwards.rs | 93 ++++++++++++++++++++++++++++++----- src/backend/avx2/mod.rs | 2 + 3 files changed, 158 insertions(+), 13 deletions(-) create mode 100644 src/backend/avx2/constants.rs diff --git a/src/backend/avx2/constants.rs b/src/backend/avx2/constants.rs new file mode 100644 index 0000000..a84d7e9 --- /dev/null +++ b/src/backend/avx2/constants.rs @@ -0,0 +1,76 @@ +// -*- mode: rust; -*- +// +// This file is part of curve25519-dalek. +// Copyright (c) 2016-2017 Isis Lovecruft, Henry de Valence +// See LICENSE for licensing information. +// +// Authors: +// - Isis Agora Lovecruft +// - Henry de Valence + +//! This module contains constants used by the AVX2 backend. + +use stdsimd::simd::u32x8; + +use backend::avx2::field::FieldElement32x4; +use backend::avx2::edwards::ExtendedPoint; + +/// Odd multiples of the Ed25519 basepoint: +pub static ODD_MULTIPLES_OF_BASEPOINT: [ExtendedPoint; 8] = [ + ExtendedPoint(FieldElement32x4([ + u32x8::new(52811034, 40265304, 25909283, 26843545, 1, 28827043, 0, 27438313), + u32x8::new(16144682, 13421772, 17082669, 20132659, 0, 39759291, 0, 244362), + u32x8::new(27570973, 26843545, 30858332, 6710886, 0, 8635006, 0, 11264893), + u32x8::new(40966398, 53687091, 8378388, 13421772, 0, 19351346, 0, 13413597), + u32x8::new(20764389, 40265318, 8758491, 26843545, 0, 16611511, 0, 27139452), + ])), + ExtendedPoint(FieldElement32x4([ + u32x8::new(63703867, 19156774, 608100, 2486757, 12685460, 3173753, 21649412, 16313381), + u32x8::new(52397038, 65858675, 26775664, 16661035, 14269998, 9080558, 1059463, 28938752), + u32x8::new( 5461635, 28034025, 23358301, 1245198, 1367765, 20288887, 31111942, 18395221), + u32x8::new( 1886934, 32436996, 681756, 18977693, 8129860, 40112764, 25764567, 11876840), + u32x8::new(63042604, 52399761, 22087481, 29829870, 8565820, 33723612, 28645162, 8502864), + ])), + ExtendedPoint(FieldElement32x4([ + u32x8::new(14879397, 3951036, 9454671, 16606238, 23529732, 44147004, 11890541, 17067526), + u32x8::new(58509479, 57216664, 9671992, 32001147, 60966207, 11801823, 10808378, 15115613), + u32x8::new(54854992, 39210911, 8112050, 1353604, 1337416, 35520540, 32967851, 17786030), + u32x8::new(59007462, 40864509, 26240923, 30403852, 28456403, 21546582, 32732450, 21005910), + u32x8::new(40711675, 22446613, 9664668, 12483629, 26142305, 56254715, 15439904, 214849), + ])), + ExtendedPoint(FieldElement32x4([ + u32x8::new(52231579, 51632644, 173613, 7677257, 26374424, 45994428, 5303371, 1425942), + u32x8::new(38126791, 48854506, 23252518, 30611978, 49977504, 66706952, 1076178, 27100873), + u32x8::new(26349427, 63077566, 20258199, 3884787, 33226507, 2371423, 5787271, 18628170), + u32x8::new(15005754, 22729577, 4978944, 2522289, 1404784, 56367795, 22517039, 29271243), + u32x8::new(22748934, 35977548, 25561257, 31734126, 22775284, 32000077, 927866, 2278697), + ])), + ExtendedPoint(FieldElement32x4([ + u32x8::new(66090281, 61980626, 23780289, 6519561, 62542590, 47174086, 28818882, 15661068), + u32x8::new(17433715, 12931425, 12232056, 7885877, 44179512, 35590146, 32787344, 22631048), + u32x8::new(43729883, 6870635, 15782399, 11810556, 2652935, 31800505, 23683367, 13638649), + u32x8::new(64007953, 40242373, 32810277, 20180235, 20399465, 48133835, 32913956, 19094667), + u32x8::new(56562708, 40269142, 18953105, 9027935, 35700921, 12896915, 14757156, 22773619), + ])), + ExtendedPoint(FieldElement32x4([ + u32x8::new(65129016, 34709402, 25132940, 13788431, 3661652, 16914498, 27409409, 18941039), + u32x8::new(42488074, 49427602, 6177212, 20812339, 41644653, 2977316, 12162542, 5293661), + u32x8::new( 7981168, 12223605, 6239200, 20403609, 20710415, 4828170, 11627702, 4431044), + u32x8::new(65817142, 96824, 25021652, 16364722, 50410869, 24651857, 6979034, 33176209), + u32x8::new(33008344, 8687253, 27859668, 28796356, 30192014, 11975680, 11991047, 27710707), + ])), + ExtendedPoint(FieldElement32x4([ + u32x8::new(14676653, 50945941, 13489249, 31456262, 47726639, 21761847, 3324839, 7843947), + u32x8::new(53352326, 8688989, 12944061, 12994004, 50113821, 37990636, 1537898, 20483689), + u32x8::new(46786852, 15572264, 24004728, 7566233, 32596174, 34437796, 23201722, 3431551), + u32x8::new(49025674, 52497128, 13273618, 10266201, 66795206, 2887684, 30966565, 33449990), + u32x8::new(53210238, 65839385, 15458877, 18409918, 24777464, 25586795, 15335748, 12323382), + ])), + ExtendedPoint(FieldElement32x4([ + u32x8::new(57816016, 23106045, 24948505, 27413507, 32551424, 26145165, 22632568, 27527446), + u32x8::new(53022711, 40974949, 14110533, 30646997, 51399118, 53289754, 32528560, 15822835), + u32x8::new(23810949, 51779690, 17532625, 21326637, 60314333, 43761996, 4852905, 3474945), + u32x8::new(13323962, 10752742, 16431634, 26425049, 24258356, 53260846, 19756601, 19546842), + u32x8::new(17403634, 52199608, 32323720, 5313255, 48522162, 33376516, 31903659, 15291466), + ])), +]; diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 0ebbc52..50ca57d 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -14,7 +14,7 @@ #![allow(bad_style)] use std::convert::From; -use std::ops::{Add, Sub, Mul, Neg}; +use std::ops::{Index, Add, Sub, Mul, Neg}; use stdsimd::simd::{u32x8, i32x8}; @@ -28,9 +28,11 @@ use traits::Identity; use backend::avx2::field::FieldElement32x4; use backend::avx2::field::P_TIMES_2; +use backend::avx2; + /// A point on Curve25519, represented in an AVX2-friendly format. #[derive(Copy, Clone, Debug)] -pub struct ExtendedPoint(FieldElement32x4); +pub struct ExtendedPoint(pub(super) FieldElement32x4); // XXX need to cfg gate here to handle FieldElement64 impl From for ExtendedPoint { @@ -539,6 +541,79 @@ pub mod vartime { //! Variable-time operations on curve points, useful for non-secret data. use super::*; + /// Holds odd multiples 1A, 3A, ..., 15A of a point A. + struct OddMultiples([ExtendedPoint; 8]); + + impl OddMultiples { + fn create(A: ExtendedPoint) -> OddMultiples { + // XXX would be great to skip this initialization + let mut Ai = [A; 8]; + let A2 = A.double(); + for i in 0..7 { + Ai[i+1] = &A2 + &Ai[i]; + } + // Now Ai = [A, 3A, 5A, 7A, 9A, 11A, 13A, 15A] + OddMultiples(Ai) + } + } + + impl Index for OddMultiples { + type Output = ExtendedPoint; + + fn index(&self, _index: usize) -> &ExtendedPoint { + &(self.0[_index]) + } + } + + /// Given a point `A` and scalars `a` and `b`, compute the point + /// `aA+bB`, where `B` is the Ed25519 basepoint (i.e., `B = (x,4/5)` + /// with x positive). + /// + /// This is the same as calling the iterator-based function, but slightly faster. + pub fn double_scalar_mult_basepoint(a: &Scalar, + A: &edwards::ExtendedPoint, + b: &Scalar) -> edwards::ExtendedPoint { + let a_naf = a.non_adjacent_form(); + let b_naf = b.non_adjacent_form(); + + // Find starting index + let mut i: usize = 255; + for j in (0..255).rev() { + i = j; + if a_naf[i] != 0 || b_naf[i] != 0 { + break; + } + } + + let odd_multiples_of_A = OddMultiples::create((*A).into()); + let odd_multiples_of_B = &avx2::constants::ODD_MULTIPLES_OF_BASEPOINT; + + let mut Q = ExtendedPoint::identity(); + + loop { + Q = Q.double(); + + if a_naf[i] > 0 { + Q = &Q + &odd_multiples_of_A[( a_naf[i]/2) as usize]; + } else if a_naf[i] < 0 { + Q = &Q - &odd_multiples_of_A[(-a_naf[i]/2) as usize]; + } + + if b_naf[i] > 0 { + Q = &Q + &odd_multiples_of_B[( b_naf[i]/2) as usize]; + } else if b_naf[i] < 0 { + Q = &Q - &odd_multiples_of_B[(-b_naf[i]/2) as usize]; + } + + if i == 0 { + break; + } + i -= 1; + } + + Q.into() + } + /// Given a vector of public scalars and a vector of (possibly secret) /// points, compute `c_1 P_1 + ... + c_n P_n`. /// @@ -559,17 +634,9 @@ pub mod vartime { let nafs: Vec<_> = scalars.into_iter() .map(|c| c.non_adjacent_form()).collect(); + let odd_multiples: Vec<_> = points.into_iter() - .map(|P| { - let P = ExtendedPoint::from(*P); - // Construct a lookup table of [P,2*P,3*P,4*P,5*P,6*P,7*P] - let P2 = P.double(); - let mut lookup_table: [ExtendedPoint; 8] = [P; 8]; - for i in 0..7 { - lookup_table[i+1] = &P2 + &lookup_table[i]; - } - lookup_table - }).collect(); + .map(|P| OddMultiples::create((*P).into()) ).collect(); let mut Q = ExtendedPoint::identity(); @@ -965,7 +1032,7 @@ mod bench { let B = constants::ED25519_BASEPOINT_POINT; let P = &B * &s1; - b.iter(|| vartime::multiscalar_mult(&[s1, s2], &[B, P])); + b.iter(|| vartime::double_scalar_mult_basepoint(&s2, &P, &s1) ); } #[bench] diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index 7a06512..6b48fb4 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -310,3 +310,5 @@ pub(crate) mod field; pub(crate) mod edwards; + +pub(crate) mod constants; From 1103c5c43ac71643b59a84c55cf76fed7da1b457 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 30 Nov 2017 13:15:06 -0800 Subject: [PATCH 26/54] Implement Sub by pre-negating the point --- src/backend/avx2/edwards.rs | 71 ++++--------------------------------- src/backend/avx2/field.rs | 66 +++++++--------------------------- 2 files changed, 19 insertions(+), 118 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 50ca57d..49576b4 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -238,7 +238,7 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { let mut t2 = &t0 * &t1; // set t2 = (S8 S9 S10 S11) - t2.scale_by_curve_constants(true); + t2.scale_by_curve_constants(); // set t2 = (S8 S9 S11 S10) t2.swap_CD(); @@ -265,70 +265,12 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { impl<'a, 'b> Sub<&'b ExtendedPoint> for &'a ExtendedPoint { type Output = ExtendedPoint; - /// Uses a slight tweak of the parallel unified formulas of HWCD'08 + /// Implement subtraction by negating the point and adding. + /// + /// Empirically, this seems about the same cost as a custom subtraction impl (maybe because the + /// benefit is cancelled by increased code size?) fn sub(self, other: &'b ExtendedPoint) -> ExtendedPoint { - unsafe { - use stdsimd::vendor::_mm256_permute2x128_si256; - use stdsimd::vendor::_mm256_permutevar8x32_epi32; - use stdsimd::vendor::_mm256_blend_epi32; - use stdsimd::vendor::_mm256_shuffle_epi32; - - let P: &FieldElement32x4 = &self.0; - let Q: &FieldElement32x4 = &other.0; - - let mut t0 = FieldElement32x4::zero(); - let mut t1 = FieldElement32x4::zero(); - - // set t0 = (X1 Y1 X2 Y2) - for i in 0..5 { - t0.0[i] = _mm256_permute2x128_si256(P.0[i].into(), Q.0[i].into(), 32).into(); - } - - // Since we're subtracting instead of adding, we want to add the point (-X2 Y2 Z2 -T2). - // Set (X2' Y2' Z2' T2') = (-X2 Y2 Z2 -T2) - // - // so S2 = Y2 - X2' = Y2 - (-X2) = Y2 + X2 - // and S3 = Y2 + X2' = Y2 + (-X2) = Y2 - X2 - - // set t0 = (Y1-X1 Y1+X1 Y2-X2 Y2+X2) = (S0 S1 S3 S2) - t0.diff_sum(); - - // set t0 = (S0 S1 S2 S3) - t0.swap_CD(); - - // set t1 = (S0 S1 Z1 T1) - // set t0 = (S2 S3 Z2 T2) = (S2 S3 Z2' -T2') - for i in 0..5 { - t1.0[i] = _mm256_blend_epi32(t0.0[i].into(), P.0[i].into(), 0b11110000).into(); - t0.0[i] = _mm256_permute2x128_si256(t0.0[i].into(), Q.0[i].into(), 49).into(); - } - - // set t2 = (S0*S2 S1*S3 Z1*Z2 T1*T2 ) - // = (S0*S2 S1*S3 Z1*Z2' -T1*T2') = (S4 S5 S6 -S7) - let mut t2 = &t0 * &t1; - - // set t2 = (S8 S9 S10 S11) - t2.scale_by_curve_constants(false); - - // set t2 = (S8 S9 S11 S10) - t2.swap_CD(); - - // set t2 = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) - t2.diff_sum(); - - let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) - let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) - - // set t0 = (S12 S15 S15 S12) - // set t1 = (S14 S13 S14 S13) - for i in 0..5 { - t0.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c0); - t1.0[i] = _mm256_permutevar8x32_epi32(t2.0[i], c1); - } - - // return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) - ExtendedPoint(&t0 * &t1) - } + self + &(-other) } } @@ -647,7 +589,6 @@ pub mod vartime { if naf[i] > 0 { Q = &Q + &odd_multiple[( naf[i]/2) as usize]; } else if naf[i] < 0 { - // XXX impl Sub Q = &Q - &odd_multiple[(-naf[i]/2) as usize]; } } diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index b76d454..b6e6397 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -202,15 +202,9 @@ impl FieldElement32x4 { /// Let `self` \\(= (A, B, C, D) \\). /// - /// If `negate_121665 = true`, compute - /// + /// Compute /// $$( 121666A, 121666B, 2\cdot 121666C, -2\cdot 121665 D).$$ - /// - /// If `negate_121665 = false`, compute - /// - /// $$( 121666A, 121666B, 2\cdot 121666C, 2\cdot 121665 D).$$ - /// - pub fn scale_by_curve_constants(&mut self, negate_121665: bool) { + pub fn scale_by_curve_constants(&mut self) { let mut b = [u64x4::splat(0); 10]; let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); @@ -225,57 +219,32 @@ impl FieldElement32x4 { let (b0, b1) = unpack_pair(self.0[0]); let b0 = _mm256_mul_epu32(b0, consts); // need a new binding since now let b1 = _mm256_mul_epu32(b1, consts); // b0 has type u64x4 - if negate_121665 { - b[0] = _mm256_blend_epi32(b0.into(), (low__p20 - b0).into(), 0b11_00_00_00).into(); - b[1] = _mm256_blend_epi32(b1.into(), (odd__p20 - b1).into(), 0b11_00_00_00).into(); - } else { - b[0] = b0; - b[1] = b1; - } + b[0] = _mm256_blend_epi32(b0.into(), (low__p20 - b0).into(), 0b11_00_00_00).into(); + b[1] = _mm256_blend_epi32(b1.into(), (odd__p20 - b1).into(), 0b11_00_00_00).into(); let (b2, b3) = unpack_pair(self.0[1]); let b2 = _mm256_mul_epu32(b2, consts); let b3 = _mm256_mul_epu32(b3, consts); - if negate_121665 { - b[2] = _mm256_blend_epi32(b2.into(), (even_p20 - b2).into(), 0b11_00_00_00).into(); - b[3] = _mm256_blend_epi32(b3.into(), (odd__p20 - b3).into(), 0b11_00_00_00).into(); - } else { - b[2] = b2; - b[3] = b3; - } + b[2] = _mm256_blend_epi32(b2.into(), (even_p20 - b2).into(), 0b11_00_00_00).into(); + b[3] = _mm256_blend_epi32(b3.into(), (odd__p20 - b3).into(), 0b11_00_00_00).into(); let (b4, b5) = unpack_pair(self.0[2]); let b4 = _mm256_mul_epu32(b4, consts); let b5 = _mm256_mul_epu32(b5, consts); - if negate_121665 { - b[4] = _mm256_blend_epi32(b4.into(), (even_p20 - b4).into(), 0b11_00_00_00).into(); - b[5] = _mm256_blend_epi32(b5.into(), (odd__p20 - b5).into(), 0b11_00_00_00).into(); - } else { - b[4] = b4; - b[5] = b5; - } + b[4] = _mm256_blend_epi32(b4.into(), (even_p20 - b4).into(), 0b11_00_00_00).into(); + b[5] = _mm256_blend_epi32(b5.into(), (odd__p20 - b5).into(), 0b11_00_00_00).into(); let (b6, b7) = unpack_pair(self.0[3]); let b6 = _mm256_mul_epu32(b6, consts); let b7 = _mm256_mul_epu32(b7, consts); - if negate_121665 { - b[6] = _mm256_blend_epi32(b6.into(), (even_p20 - b6).into(), 0b11_00_00_00).into(); - b[7] = _mm256_blend_epi32(b7.into(), (odd__p20 - b7).into(), 0b11_00_00_00).into(); - } else { - b[6] = b6; - b[7] = b7; - } + b[6] = _mm256_blend_epi32(b6.into(), (even_p20 - b6).into(), 0b11_00_00_00).into(); + b[7] = _mm256_blend_epi32(b7.into(), (odd__p20 - b7).into(), 0b11_00_00_00).into(); let (b8, b9) = unpack_pair(self.0[4]); let b8 = _mm256_mul_epu32(b8, consts); let b9 = _mm256_mul_epu32(b9, consts); - if negate_121665 { - b[8] = _mm256_blend_epi32(b8.into(), (even_p20 - b8).into(), 0b11_00_00_00).into(); - b[9] = _mm256_blend_epi32(b9.into(), (odd__p20 - b9).into(), 0b11_00_00_00).into(); - } else { - b[8] = b8; - b[9] = b9; - } + b[8] = _mm256_blend_epi32(b8.into(), (even_p20 - b8).into(), 0b11_00_00_00).into(); + b[9] = _mm256_blend_epi32(b9.into(), (odd__p20 - b9).into(), 0b11_00_00_00).into(); } *self = FieldElement32x4::reduce64(b); @@ -564,22 +533,13 @@ mod test { #[test] fn scale_by_curve_constants() { let mut x = FieldElement32x4::splat(&FieldElement64::one()); - x.scale_by_curve_constants(true); + x.scale_by_curve_constants(); let xs = x.split(); assert_eq!(xs[0], FieldElement64([ 121666,0,0,0,0])); assert_eq!(xs[1], FieldElement64([ 121666,0,0,0,0])); assert_eq!(xs[2], FieldElement64([2*121666,0,0,0,0])); assert_eq!(xs[3], -&FieldElement64([2*121665,0,0,0,0])); - - let mut y = FieldElement32x4::splat(&FieldElement64::one()); - y.scale_by_curve_constants(false); - - let ys = y.split(); - assert_eq!(ys[0], FieldElement64([ 121666,0,0,0,0])); - assert_eq!(ys[1], FieldElement64([ 121666,0,0,0,0])); - assert_eq!(ys[2], FieldElement64([2*121666,0,0,0,0])); - assert_eq!(ys[3], FieldElement64([2*121665,0,0,0,0])); } #[test] From bf5e3581d2802cb0336d8583c8b39b8540117644 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 30 Nov 2017 15:24:26 -0800 Subject: [PATCH 27/54] Update AVX2 docs --- src/backend/avx2/field.rs | 8 ++++---- src/backend/avx2/mod.rs | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index b6e6397..cd71d6a 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -101,8 +101,8 @@ impl FieldElement32x4 { return out; } - // Negate variables in lanes where mask is set - // XXX fix up api + /// Negate variables in lanes where mask is set + /// XXX fix up api pub fn mask_negate(&mut self, mask: u8) { unsafe { use stdsimd::vendor::_mm256_blend_epi32; @@ -114,7 +114,7 @@ impl FieldElement32x4 { self.reduce32(); } - // Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` + /// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` pub fn swap_CD(&mut self) { unsafe { use stdsimd::vendor::_mm256_shuffle_epi32; @@ -126,7 +126,7 @@ impl FieldElement32x4 { } } - // Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)`. + /// Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)`. pub fn diff_sum(&mut self) { /// (v0 v1 v2 v3 v4 v5 v6 v7) -> (v1 v0 v3 v2 v5 v4 v7 v6) #[inline(always)] diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index 6b48fb4..6fcdb1c 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -12,7 +12,7 @@ //! Curve25519, using AVX2 to implement the 4-way parallel formulas of //! Hisil, Wong, Carter, and Dawson (HWCD). //! -//! Their 2008 paper _Twisted Edwards Curves Revisited_, which +//! Their 2008 paper [_Twisted Edwards Curves Revisited_][hwcd08], which //! introduced the extended coordinates used in other parts of `-dalek`, //! also describes 4-way parallel formulas for point addition and //! doubling: @@ -101,12 +101,15 @@ //! The 4-wide formulas of the HWCD paper do not seem to have been //! implemented using SIMD before. The HWCD paper also describes and //! analyzes a 2-wide variant of the Montgomery ladder; this strategy was -//! used by Tung Chou's `sandy2x` implementation, which used a 2-wide -//! field implementation in 128-bit registers. Curiously, however, -//! although the `sandy2x` paper cites the HWCD paper for extended +//! used in 2015 by Tung Chou's `sandy2x` implementation, which used a 2-wide +//! field implementation in 128-bit vector registers. Curiously, however, +//! although the [`sandy2x` paper][sandy2x] cites the HWCD paper for extended //! twisted Edwards coordinates, it does not mention the 4-wide HWCD //! Edwards formulas or that the 2-wide Montgomery formulas it uses were -//! previously published there. +//! previously published there. There is also a 2015 paper by Hernández +//! and López on using AVX2 for the X25519 Montgomery ladder, but +//! neither the paper nor the code are publicly available, and it apparently +//! gives only a [slight speedup][avx2trac]. //! //! HWCD also suggest using a mixed representation, passing between \\( //! \mathbb P\^3 \\) "extended" coordinates and \\( \mathbb P\^2 \\) @@ -118,9 +121,12 @@ //! //! This optimization is not used for the parallel formulas, which are //! therefore slightly less efficient when counting the total number of -//! multiplications and squarings. In addition, the parallel formulas -//! can only use a \\( 32 \times 32 \rightarrow 64 \\)-bit multiplier -//! instead of a \\( 64 \times 64 \rightarrow 128\\)-bit multiplier. +//! field multiplications and squarings. In particular, vectorized doublings +//! are less efficient than serial doublings. +//! In addition, the parallel formulas can only use a \\( 32 \times 32 +//! \rightarrow 64 \\)-bit integer multiplier, so the speedup from +//! vectorization must overcome the disadvantage of losing the \\( 64 +//! \times 64 \rightarrow 128\\)-bit (serial) integer multiplier. //! //! When used for constant-time variable-base scalar multiplication, //! this strategy (using AVX2) gives a significant speedup over the @@ -305,6 +311,10 @@ //! The implementation uses the unstable `stdsimd` crate to provide AVX2 //! intrinsics, and the code is not yet cleanly factored between the //! field element parts and the point parts. +//! +//! [sandy2x]: https://eprint.iacr.org/2015/943.pdf +//! [avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28 +//! [hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf pub(crate) mod field; From fbd0e74357c4f2fc2fad11eae1e2029a2da65eef Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 1 Dec 2017 12:02:16 -0800 Subject: [PATCH 28/54] Fix Cargo.toml to use upstream stdsimd --- Cargo.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 390a2c5..9d791de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,8 +25,7 @@ rustdoc-args = ["--html-in-header", ".cargo/registry/src/github.com-1ecc6299db9e travis-ci = { repository = "isislovecruft/curve25519-dalek", branch = "master"} [dependencies] -#stdsimd = { git = "https://github.com/rust-lang-nursery/stdsimd" } -stdsimd = { git = "https://github.com/hdevalence/stdsimd", branch="feature/more-avx2" } +stdsimd = { git = "https://github.com/rust-lang-nursery/stdsimd" } [dependencies.serde] version = "1.0" @@ -59,7 +58,7 @@ rand = "0.3" generic-array = "^0.8" digest = "0.6" arrayref = "0.3.4" -stdsimd = { git = "https://github.com/hdevalence/stdsimd", branch="feature/more-avx2" } +stdsimd = { git = "https://github.com/rust-lang-nursery/stdsimd" } [build-dependencies.serde] version = "1.0" From 3814beeafeb05b485f5a393fa28e70eab6b820d3 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 1 Dec 2017 13:59:40 -0800 Subject: [PATCH 29/54] Fix typo and add note on AVX512VL --- src/backend/avx2/mod.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index 6fcdb1c..8d48321 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -180,7 +180,7 @@ //! S\_{12} &\gets S\_9 - S\_8 \\\\ //! S\_{13} &\gets S\_9 + S\_8 \\\\ //! S\_{14} &\gets S\_{10} - S\_{11} \\\\ -//! S\_{15} &\gets S\_{10} - S\_{11} +//! S\_{15} &\gets S\_{10} + S\_{11} //! \end{aligned} //! $$ //! @@ -312,6 +312,20 @@ //! intrinsics, and the code is not yet cleanly factored between the //! field element parts and the point parts. //! +//! When compiling with AVX512VL, LLVM is able to use the extra +//! `ymm16..ymm31` registers to reduce register pressure, and avoid +//! spills during field multiplication. This gives a small but +//! noticeable speedup. +//! +//! The addition and subtraction steps involve masking, to apply +//! operations to a single lane of the vector. AVX512VL extends the +//! predication features of AVX512 to AVX2 code and would probably be +//! beneficial. Unfortunately, LLVM is currently unable to lower `op + +//! blend` into an AVX512VL masked operation. However, the explicitly +//! masked versions of the intrinsics seem to produce the same LLVM IR +//! as an `op + blend`, so hopefully this will improve as the AVX512 +//! support in LLVM improves. +//! //! [sandy2x]: https://eprint.iacr.org/2015/943.pdf //! [avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28 //! [hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf From 2c29d9a2ad785c545bfa8a27f02209d38582ef80 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 4 Dec 2017 10:22:31 -0800 Subject: [PATCH 30/54] tweak docs --- src/backend/avx2/mod.rs | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index 8d48321..e9c3846 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -100,16 +100,20 @@ //! //! The 4-wide formulas of the HWCD paper do not seem to have been //! implemented using SIMD before. The HWCD paper also describes and -//! analyzes a 2-wide variant of the Montgomery ladder; this strategy was -//! used in 2015 by Tung Chou's `sandy2x` implementation, which used a 2-wide -//! field implementation in 128-bit vector registers. Curiously, however, -//! although the [`sandy2x` paper][sandy2x] cites the HWCD paper for extended -//! twisted Edwards coordinates, it does not mention the 4-wide HWCD -//! Edwards formulas or that the 2-wide Montgomery formulas it uses were -//! previously published there. There is also a 2015 paper by Hernández -//! and López on using AVX2 for the X25519 Montgomery ladder, but -//! neither the paper nor the code are publicly available, and it apparently -//! gives only a [slight speedup][avx2trac]. +//! analyzes a 2-wide variant of the Montgomery ladder (for comparison +//! with parallel Edwards formulas); this strategy was used in 2015 by +//! Tung Chou's `sandy2x` implementation, which used a 2-wide field +//! implementation in 128-bit vector registers. +//! +//! Curiously, however, although the [`sandy2x` paper][sandy2x] also +//! implements Edwards arithmetic, and cites the HWCD paper, it doesn't +//! mention or discuss the parallel formulas from HWCD, or that the +//! 2-wide Montgomery formulas it uses were previously published there. +//! There is also a 2015 paper by Hernández and López on using AVX2 for +//! the X25519 Montgomery ladder, but neither the paper nor the code are +//! publicly available, and it apparently gives only a [slight +//! speedup][avx2trac], suggesting that it also overlooked the +//! HWCD formulas. //! //! HWCD also suggest using a mixed representation, passing between \\( //! \mathbb P\^3 \\) "extended" coordinates and \\( \mathbb P\^2 \\) @@ -119,7 +123,7 @@ //! details on the different coordinate systems can be found in the //! `curve_models` module documentation. //! -//! This optimization is not used for the parallel formulas, which are +//! This optimization is not compatible with the parallel formulas, which are //! therefore slightly less efficient when counting the total number of //! field multiplications and squarings. In particular, vectorized doublings //! are less efficient than serial doublings. @@ -136,6 +140,17 @@ //! `ymm16..ymm31` registers from AVX512VL), and of approximately 1.0x //! for Ryzen (which implements AVX2 at half rate). //! +//! (Note: since testing this, the experimental `llvm50` Rust branch +//! used to compile the experimental `stdsimd` intrinsics have fallen +//! out of sync and it is no longer possible to compile for +//! `skylake-avx512`. This is why all of this branch is part of the +//! `yolocrypto` feature, pending upstream work.) +//! +//! However, since the relative cost of doubling and addition has +//! changed, the optimal tradeoffs for window size etc. in scalar +//! multiplication have probably also changed and should be +//! re-evaluated. +//! //! # Tweaked formulas //! //! After tweaking the formulas as described above, we obtain the @@ -301,6 +316,10 @@ //! to adapt it to 128-bit vector instructions such as NEON without too //! much difficulty. //! +//! Going the other direction, to extend this to AVX512, we could either +//! run two point operations in parallel in lower and upper halves of +//! the registers, or use 2-way parallelism within a field operation. +//! //! We don't attempt to use AVX2 for serial field element computations //! such as inversion, since wherever we have AVX2 we also have `mulx`. //! However, it might be useful for batched inverse square-root From 7d2d87441b3ab9830514dc3a519fb1cc95774084 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 4 Dec 2017 11:05:24 -0800 Subject: [PATCH 31/54] Connect double_scalar_mult_basepoint to AVX2 backend --- src/edwards.rs | 81 ++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/src/edwards.rs b/src/edwards.rs index 3bf73a8..25bbb69 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -849,49 +849,60 @@ pub mod vartime { /// \\(aA+bB\\), where \\(B\\) is the Ed25519 basepoint (i.e., \\(B = (x,4/5)\\) /// with x positive). #[cfg(feature="precomputed_tables")] - pub fn double_scalar_mult_basepoint(a: &Scalar, - A: &ExtendedPoint, - b: &Scalar) -> ExtendedPoint { - let a_naf = a.non_adjacent_form(); - let b_naf = b.non_adjacent_form(); + pub fn double_scalar_mult_basepoint( + a: &Scalar, + A: &ExtendedPoint, + b: &Scalar, + ) -> ExtendedPoint { + // If we built with AVX2, use the AVX2 backend. + #[cfg(all(target_feature = "avx2", feature = "avx2_backend"))] { + use backend::avx2::edwards as edwards_avx2; - // Find starting index - let mut i: usize = 255; - for j in (0..255).rev() { - i = j; - if a_naf[i] != 0 || b_naf[i] != 0 { - break; - } + edwards_avx2::vartime::double_scalar_mult_basepoint(a, A, b) } + // Otherwise, proceed as normal: + #[cfg(not(all(target_feature = "avx2", feature = "avx2_backend")))] { + let a_naf = a.non_adjacent_form(); + let b_naf = b.non_adjacent_form(); - let odd_multiples_of_A = OddMultiples::create(A); - let odd_multiples_of_B = &constants::AFFINE_ODD_MULTIPLES_OF_BASEPOINT; - - let mut r = ProjectivePoint::identity(); - loop { - let mut t = r.double(); - - if a_naf[i] > 0 { - t = &t.to_extended() + &odd_multiples_of_A[( a_naf[i]/2) as usize]; - } else if a_naf[i] < 0 { - t = &t.to_extended() - &odd_multiples_of_A[(-a_naf[i]/2) as usize]; + // Find starting index + let mut i: usize = 255; + for j in (0..255).rev() { + i = j; + if a_naf[i] != 0 || b_naf[i] != 0 { + break; + } } - if b_naf[i] > 0 { - t = &t.to_extended() + &odd_multiples_of_B[( b_naf[i]/2) as usize]; - } else if b_naf[i] < 0 { - t = &t.to_extended() - &odd_multiples_of_B[(-b_naf[i]/2) as usize]; + let odd_multiples_of_A = OddMultiples::create(A); + let odd_multiples_of_B = &constants::AFFINE_ODD_MULTIPLES_OF_BASEPOINT; + + let mut r = ProjectivePoint::identity(); + loop { + let mut t = r.double(); + + if a_naf[i] > 0 { + t = &t.to_extended() + &odd_multiples_of_A[( a_naf[i]/2) as usize]; + } else if a_naf[i] < 0 { + t = &t.to_extended() - &odd_multiples_of_A[(-a_naf[i]/2) as usize]; + } + + if b_naf[i] > 0 { + t = &t.to_extended() + &odd_multiples_of_B[( b_naf[i]/2) as usize]; + } else if b_naf[i] < 0 { + t = &t.to_extended() - &odd_multiples_of_B[(-b_naf[i]/2) as usize]; + } + + r = t.to_projective(); + + if i == 0 { + break; + } + i -= 1; } - r = t.to_projective(); - - if i == 0 { - break; - } - i -= 1; + r.to_extended() } - - r.to_extended() } } From d62fc7caf1ae2533c6516d3c2c479ee89c3be693 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Thu, 7 Dec 2017 10:21:20 -0800 Subject: [PATCH 32/54] Rearrange signs to avoid a subtraction --- src/backend/avx2/edwards.rs | 12 +++++++----- src/backend/avx2/field.rs | 37 ++++++++++++------------------------- src/backend/avx2/mod.rs | 16 ++++++++++------ 3 files changed, 29 insertions(+), 36 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 49576b4..b720c9d 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -237,17 +237,19 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { // set t2 = (S0*S2 S1*S3 Z1*Z2 T1*T2) = (S4 S5 S6 S7) let mut t2 = &t0 * &t1; - // set t2 = (S8 S9 S10 S11) + //// set t2 = (S8 S9 S10 S11) + // set t2 = (121666*S4 121666*S5 2*121666*S6 2*121665*S7) + // = ( S8 S9 S10 -S11) t2.scale_by_curve_constants(); - // set t2 = (S8 S9 S11 S10) + // set t2 = (S8 S9 -S11 S10) t2.swap_CD(); - // set t2 = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) + // set t2 = (S9-S8 S9+S8 S10+S11 S10-S11) = (S12 S13 S15 S14) t2.diff_sum(); - let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) - let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) + let c0 = u32x8::new(0,4,2,6,4,0,6,2); // (ABCD) -> (ACCA) + let c1 = u32x8::new(5,1,7,3,5,1,7,3); // (ABCD) -> (DBDB) // set t0 = (S12 S15 S15 S12) // set t1 = (S14 S13 S14 S13) diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index cd71d6a..2e44797 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -203,48 +203,35 @@ impl FieldElement32x4 { /// Let `self` \\(= (A, B, C, D) \\). /// /// Compute - /// $$( 121666A, 121666B, 2\cdot 121666C, -2\cdot 121665 D).$$ + /// $$( 121666A, 121666B, 2\cdot 121666C, 2\cdot 121665 D).$$ pub fn scale_by_curve_constants(&mut self) { let mut b = [u64x4::splat(0); 10]; let consts = u32x8::new(121666, 0, 121666, 0, 2*121666, 0, 2*121665, 0); - let low__p20 = u64x4::splat(0x3ffffed << 20); - let even_p20 = u64x4::splat(0x3ffffff << 20); - let odd__p20 = u64x4::splat(0x1ffffff << 20); unsafe { use stdsimd::vendor::_mm256_mul_epu32; use stdsimd::vendor::_mm256_blend_epi32; let (b0, b1) = unpack_pair(self.0[0]); - let b0 = _mm256_mul_epu32(b0, consts); // need a new binding since now - let b1 = _mm256_mul_epu32(b1, consts); // b0 has type u64x4 - b[0] = _mm256_blend_epi32(b0.into(), (low__p20 - b0).into(), 0b11_00_00_00).into(); - b[1] = _mm256_blend_epi32(b1.into(), (odd__p20 - b1).into(), 0b11_00_00_00).into(); + b[0] = _mm256_mul_epu32(b0, consts); + b[1] = _mm256_mul_epu32(b1, consts); let (b2, b3) = unpack_pair(self.0[1]); - let b2 = _mm256_mul_epu32(b2, consts); - let b3 = _mm256_mul_epu32(b3, consts); - b[2] = _mm256_blend_epi32(b2.into(), (even_p20 - b2).into(), 0b11_00_00_00).into(); - b[3] = _mm256_blend_epi32(b3.into(), (odd__p20 - b3).into(), 0b11_00_00_00).into(); + b[2] = _mm256_mul_epu32(b2, consts); + b[3] = _mm256_mul_epu32(b3, consts); let (b4, b5) = unpack_pair(self.0[2]); - let b4 = _mm256_mul_epu32(b4, consts); - let b5 = _mm256_mul_epu32(b5, consts); - b[4] = _mm256_blend_epi32(b4.into(), (even_p20 - b4).into(), 0b11_00_00_00).into(); - b[5] = _mm256_blend_epi32(b5.into(), (odd__p20 - b5).into(), 0b11_00_00_00).into(); + b[4] = _mm256_mul_epu32(b4, consts); + b[5] = _mm256_mul_epu32(b5, consts); let (b6, b7) = unpack_pair(self.0[3]); - let b6 = _mm256_mul_epu32(b6, consts); - let b7 = _mm256_mul_epu32(b7, consts); - b[6] = _mm256_blend_epi32(b6.into(), (even_p20 - b6).into(), 0b11_00_00_00).into(); - b[7] = _mm256_blend_epi32(b7.into(), (odd__p20 - b7).into(), 0b11_00_00_00).into(); + b[6] = _mm256_mul_epu32(b6, consts); + b[7] = _mm256_mul_epu32(b7, consts); let (b8, b9) = unpack_pair(self.0[4]); - let b8 = _mm256_mul_epu32(b8, consts); - let b9 = _mm256_mul_epu32(b9, consts); - b[8] = _mm256_blend_epi32(b8.into(), (even_p20 - b8).into(), 0b11_00_00_00).into(); - b[9] = _mm256_blend_epi32(b9.into(), (odd__p20 - b9).into(), 0b11_00_00_00).into(); + b[8] = _mm256_mul_epu32(b8, consts); + b[9] = _mm256_mul_epu32(b9, consts); } *self = FieldElement32x4::reduce64(b); @@ -539,7 +526,7 @@ mod test { assert_eq!(xs[0], FieldElement64([ 121666,0,0,0,0])); assert_eq!(xs[1], FieldElement64([ 121666,0,0,0,0])); assert_eq!(xs[2], FieldElement64([2*121666,0,0,0,0])); - assert_eq!(xs[3], -&FieldElement64([2*121665,0,0,0,0])); + assert_eq!(xs[3], FieldElement64([2*121665,0,0,0,0])); } #[test] diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index e9c3846..1b041e1 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -87,9 +87,9 @@ //! Hamburg. Ignoring the sign for the moment, since //! \\(2 \cdot 121666 < 2\^{18}\\), all these constants fit in 32 bits, //! so this can be done in parallel as a scaling by \\( (121666, 121666, -//! 2\cdot 121665, 2\cdot 121666) \\). To handle the sign, we use -//! masking to negate one of the field elements. +//! -2\cdot 121665, 2\cdot 121666) \\). //! +//! How do we handle the sign? //! Since we're primarily interested in Ristretto performance, not //! Curve25519 performance, we could alternately work on the //! \\(4\\)-isogenous "IsoEd25519" curve, which has \\(d = 121665\\). @@ -97,6 +97,7 @@ //! one field element by a 32-bit constant is not much easier than //! multiplying four field elements by 32-bit constants, and it would //! prevent accelerating Curve25519, so we don't make this choice. +//! Instead, we flip the sign later by swapping two intermediate variables (see below). //! //! The 4-wide formulas of the HWCD paper do not seem to have been //! implemented using SIMD before. The HWCD paper also describes and @@ -186,7 +187,7 @@ //! S\_8 &\gets S\_4 \cdot 121666 \\\\ //! S\_9 &\gets S\_5 \cdot 121666 \\\\ //! S\_{10} &\gets S\_6 \cdot 2 \cdot 121666 \\\\ -//! S\_{11} &\gets S\_7 \cdot 2 \cdot (-121665) +//! S\_{11} &\gets S\_7 \cdot 2 \cdot 121665 //! \end{aligned} //! $$ //! @@ -194,8 +195,8 @@ //! \begin{aligned} //! S\_{12} &\gets S\_9 - S\_8 \\\\ //! S\_{13} &\gets S\_9 + S\_8 \\\\ -//! S\_{14} &\gets S\_{10} - S\_{11} \\\\ -//! S\_{15} &\gets S\_{10} + S\_{11} +//! S\_{15} &\gets S\_{10} - S\_{11} \\\\ +//! S\_{14} &\gets S\_{10} + S\_{11} //! \end{aligned} //! $$ //! @@ -208,7 +209,10 @@ //! \end{aligned} //! $$ //! -//! to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). +//! to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). Notice that by multiplying +//! \\( S\_{11} \\) by \\(121665\\) instead of by \\(-121665\\), we save a negation; since we use +//! \\( S\_{11} \\) to compute \\( S\_{10} \pm S\_{11} \\), flipping the sign of \\( S\_{11} \\) +//! swaps \\( S\_{14} \\) and \\( S\_{15} \\). //! //! ## Doubling //! From 640888198ef48ca39e85aa746d8dc8bb401f3bca Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 8 Dec 2017 17:49:35 -0800 Subject: [PATCH 33/54] Eliminate a carry pass through tighter bounds checks --- src/backend/avx2/edwards.rs | 33 +++++++-------- src/backend/avx2/field.rs | 81 +++++++++++++++++++++++++++++++------ 2 files changed, 83 insertions(+), 31 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index b720c9d..07316ec 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -26,7 +26,7 @@ use scalar::Scalar; use traits::Identity; use backend::avx2::field::FieldElement32x4; -use backend::avx2::field::P_TIMES_2; +use backend::avx2::field::P_TIMES_2_MASKED; use backend::avx2; @@ -119,9 +119,9 @@ impl ExtendedPoint { t0.0[3] = _mm256_blend_epi32(t0.0[3].into(), P.0[3].into(), 0b01011111).into(); t0.0[4] = _mm256_blend_epi32(t0.0[4].into(), P.0[4].into(), 0b01011111).into(); - t1 = t0.square(); + t1 = t0.square(0b11_00_00_00); - // Now t1 = (S1 S2 S3 S4) + // Now t1 = (S1 S2 S3 -S4) let c0 = u32x8::new(0,0,2,2,0,0,2,2); // (ABCD) -> (AAAA) let c1 = u32x8::new(1,1,3,3,1,1,3,3); // (ABCD) -> (BBBB) @@ -134,9 +134,9 @@ impl ExtendedPoint { // + | S2 | | | S2 | // + | | | S3 | | // + | | | S3 | | - // + | | 2p | 2p | 2p | + // + | | | |-S4 | + // + | | 2p | 2p | | // - | | S2 | S2 | | - // - | | | | S4 | // ======================= // S5 S6 S8 S9 // @@ -146,17 +146,17 @@ impl ExtendedPoint { // + | 2^26 | | | 2^26 | + | 2^25 | | | 2^25 | // + | | | 2^26 | | + | | | 2^25 | | // + | | | 2^26 | | + | | | 2^25 | | - // + | | 2^27 | 2^27 | 2^27 | + | | 2^26 | 2^26 | 2^26 | + // + | | | | 2^26 | + | | | | 2^25 | + // + | | 2^27 | 2^27 | | + | | 2^26 | 2^26 | | // - | | 0 | 0 | | - | | 0 | 0 | | - // - | | | | 0 | - | | | | 0 | // =================================== =================================== - // < 2^27 2^27.59 2^28.33 2^28 2^26 2^26.59 2^27.33 2^27 + // < 2^27 2^27.59 2^28.33 2^27.59 2^26 2^26.59 2^27.33 2^27.59 // - // So, the bit-excess for (S5 S6 S8 S9) is (1, 1.59, 2.33, 2). + // So, the bit-excess for (S5 S6 S8 S9) is (1, 1.59, 2.33, 1.59). // // However the multiplication routine only allows (1.75, 1.75, 1.75, 1.75). // - // This is because we need to have 19*y[i] < 2^32. Otherwise I think we could get b < 2.5. + // This is because we need to have 19*y[i] < 2^32. Otherwise the bound would be b < 2.5. // // Can we tighten these bounds to avoid a reduction? Alternately, can we do better than // the 64-bit reduction that reduce32() calls internally? @@ -169,17 +169,14 @@ impl ExtendedPoint { let zero = i32x8::splat(0); let S1 = _mm256_permutevar8x32_epi32(t1.0[i], c0); let S2 = _mm256_permutevar8x32_epi32(t1.0[i], c1); - let S3_2 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into(), 0b01010000).into(); - t0.0[i] = (P_TIMES_2.0[i] + S3_2) + S1; + let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into(), 0b01010000).into(); + // tmp0 = (0 0 2*S3 -S4) + let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into(), t1.0[i].into(), 0b10100000).into(); + t0.0[i] = (P_TIMES_2_MASKED.0[i] + tmp0) + S1; t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into(), 0b10100101).into(); - let S4 = _mm256_blend_epi32(zero, t1.0[i].into(), 0b10100000); - let sub = _mm256_blend_epi32(S2.into(), S4, 0b10100101).into(); - t0.0[i] = t0.0[i] - sub; + t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into(), zero, 0b10100101).into(); } - // This is really sad, see above - t0.reduce32(); - let c0 = u32x8::new(4,0,6,2,4,0,6,2); // (ABCD) -> (CACA) let c1 = u32x8::new(5,1,7,3,1,5,3,7); // (ABCD) -> (DBBD) diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 2e44797..fc6f00d 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -27,6 +27,14 @@ pub(crate) static P_TIMES_2: FieldElement32x4 = FieldElement32x4([ u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862) ]); +pub(crate) static P_TIMES_2_MASKED: FieldElement32x4 = FieldElement32x4([ + u32x8::new( 0, 134217690, 0, 67108862, 134217690, 0, 67108862, 0), + u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), + u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), + u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), + u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0) +]); + /// A vector of four `FieldElements`, implemented using AVX2. #[derive(Clone, Copy, Debug)] pub(crate) struct FieldElement32x4(pub(crate) [u32x8; 5]); @@ -400,7 +408,13 @@ pub fn repack_pair(x: u32x8, y: u32x8) -> u32x8 { } impl FieldElement32x4 { - pub fn square(&self) -> FieldElement32x4 { + /// Square this field element, then conditionally negate according to `neg_mask`; for instance, + /// `neg_mask = 0b11_00_00_00` negates the \\( D \\) value. + /// + /// # Precondition + /// + /// Limbs must be bounded by bit-excess \\( b < 2.0 \\). + pub fn square(&self, neg_mask: u8) -> FieldElement32x4 { #[inline(always)] fn m(x: u32x8, y: u32x8) -> u64x4 { use stdsimd::vendor::_mm256_mul_epu32; @@ -436,16 +450,55 @@ impl FieldElement32x4 { let x8_19 = m_lo(v19, x8); let x9_19 = m_lo(v19, x9); - let z0 = m(x0, x0) + m(x2_2,x8_19) + m(x4_2,x6_19) + ((m(x1_2,x9_19) + m(x3_2,x7_19) + m(x5,x5_19)) << 1); - let z1 = m(x0_2,x1) + m(x3_2,x8_19) + m(x5_2,x6_19) + ((m(x2,x9_19) + m(x4,x7_19)) << 1); - let z2 = m(x0_2,x2) + m(x1_2,x1) + m(x4_2,x8_19) + m(x6,x6_19) + ((m(x3_2,x9_19) + m(x5_2,x7_19)) << 1); - let z3 = m(x0_2,x3) + m(x1_2,x2) + m(x5_2,x8_19) + ((m(x4,x9_19) + m(x6,x7_19)) << 1); - let z4 = m(x0_2,x4) + m(x1_2,x3_2) + m(x2, x2) + m(x6_2,x8_19) + ((m(x5_2,x9_19) + m(x7,x7_19)) << 1); - let z5 = m(x0_2,x5) + m(x1_2,x4) + m(x2_2,x3) + m(x7_2,x8_19) + ((m(x6,x9_19)) << 1); - let z6 = m(x0_2,x6) + m(x1_2,x5_2) + m(x2_2,x4) + m(x3_2,x3) + m(x8,x8_19) + ((m(x7_2,x9_19)) << 1); - let z7 = m(x0_2,x7) + m(x1_2,x6) + m(x2_2,x5) + m(x3_2,x4) + ((m(x8,x9_19)) << 1); - let z8 = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1); - let z9 = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5); + let mut z0 = m(x0, x0) + m(x2_2,x8_19) + m(x4_2,x6_19) + ((m(x1_2,x9_19) + m(x3_2,x7_19) + m(x5,x5_19)) << 1); + let mut z1 = m(x0_2,x1) + m(x3_2,x8_19) + m(x5_2,x6_19) + ((m(x2,x9_19) + m(x4,x7_19)) << 1); + let mut z2 = m(x0_2,x2) + m(x1_2,x1) + m(x4_2,x8_19) + m(x6,x6_19) + ((m(x3_2,x9_19) + m(x5_2,x7_19)) << 1); + let mut z3 = m(x0_2,x3) + m(x1_2,x2) + m(x5_2,x8_19) + ((m(x4,x9_19) + m(x6,x7_19)) << 1); + let mut z4 = m(x0_2,x4) + m(x1_2,x3_2) + m(x2, x2) + m(x6_2,x8_19) + ((m(x5_2,x9_19) + m(x7,x7_19)) << 1); + let mut z5 = m(x0_2,x5) + m(x1_2,x4) + m(x2_2,x3) + m(x7_2,x8_19) + ((m(x6,x9_19)) << 1); + let mut z6 = m(x0_2,x6) + m(x1_2,x5_2) + m(x2_2,x4) + m(x3_2,x3) + m(x8,x8_19) + ((m(x7_2,x9_19)) << 1); + let mut z7 = m(x0_2,x7) + m(x1_2,x6) + m(x2_2,x5) + m(x3_2,x4) + ((m(x8,x9_19)) << 1); + let mut z8 = m(x0_2,x8) + m(x1_2,x7_2) + m(x2_2,x6) + m(x3_2,x5_2) + m(x4,x4) + ((m(x9,x9_19)) << 1); + let mut z9 = m(x0_2,x9) + m(x1_2,x8) + m(x2_2,x7) + m(x3_2,x6) + m(x4_2,x5); + + #[inline(always)] + fn mask_neg(x: u64x4, p: u64x4, mask: u8) -> u64x4 { + unsafe { + use stdsimd::vendor::_mm256_blend_epi32; + _mm256_blend_epi32(x.into(), (p - x).into(), mask as i32).into() + } + } + + // The biggest z_i is bounded as z_i < 249*2^(51 + 2*b); + // if b < 1.5 we get z_i < 4485585228861014016. + // + // The limbs of the multiples of p are bounded above by + // + // 0x3fffffff << 37 = 9223371899415822336 < 2^63 + // + // and below by + // + // 0x1fffffff << 37 = 4611685880988434432 + // > 4485585228861014016 + // + // So these multiples of p are big enough to avoid underflow + // in subtraction, and small enough to fit within u64 + // with room for a carry. + + let low__p37 = u64x4::splat(0x3ffffed << 37); + let even_p37 = u64x4::splat(0x3ffffff << 37); + let odd__p37 = u64x4::splat(0x1ffffff << 37); + + z0 = mask_neg(z0, low__p37, neg_mask); + z1 = mask_neg(z1, odd__p37, neg_mask); + z2 = mask_neg(z2, even_p37, neg_mask); + z3 = mask_neg(z3, odd__p37, neg_mask); + z4 = mask_neg(z4, even_p37, neg_mask); + z5 = mask_neg(z5, odd__p37, neg_mask); + z6 = mask_neg(z6, even_p37, neg_mask); + z7 = mask_neg(z7, odd__p37, neg_mask); + z8 = mask_neg(z8, even_p37, neg_mask); + z9 = mask_neg(z9, odd__p37, neg_mask); FieldElement32x4::reduce64([z0, z1, z2, z3, z4, z5, z6, z7, z8, z9]) } @@ -556,12 +609,14 @@ mod test { let vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); - let result = vec.square().split(); + let neg_mask = 0b11_00_00_00; + + let result = vec.square(neg_mask).split(); assert_eq!(result[0], &x0 * &x0); assert_eq!(result[1], &x1 * &x1); assert_eq!(result[2], &x2 * &x2); - assert_eq!(result[3], &x3 * &x3); + assert_eq!(result[3], -&(&x3 * &x3)); } From 70f710eafed7f6740ee8fe126846bdad7f092c52 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 15 Dec 2017 13:56:20 -0800 Subject: [PATCH 34/54] Use the LookupTable struct in AVX2 code --- src/backend/avx2/edwards.rs | 107 +++++++++++++----------------------- 1 file changed, 38 insertions(+), 69 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 07316ec..c67c58d 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -22,6 +22,7 @@ use subtle::ConditionallyAssignable; use edwards; use scalar::Scalar; +use curve_models::window::LookupTable; use traits::Identity; @@ -55,6 +56,12 @@ impl ConditionallyAssignable for ExtendedPoint { } } +impl Default for ExtendedPoint { + fn default() -> ExtendedPoint { + ExtendedPoint::identity() + } +} + impl Identity for ExtendedPoint { fn identity() -> ExtendedPoint { ExtendedPoint(FieldElement32x4([ @@ -273,19 +280,24 @@ impl<'a, 'b> Sub<&'b ExtendedPoint> for &'a ExtendedPoint { } } +impl From for LookupTable { + fn from(P: ExtendedPoint) -> Self { + let mut points = [P; 8]; + for i in 0..7 { + points[i+1] = &P + &points[i]; + } + LookupTable(points) + } +} + impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { type Output = ExtendedPoint; /// Scalar multiplication: compute `scalar * self`. /// /// Uses a window of size 4. fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { - use traits::select_precomputed_point; - // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] - let mut lookup_table: [ExtendedPoint; 8] = [*self; 8]; - for i in 0..7 { - lookup_table[i+1] = self + &lookup_table[i]; - } + let lookup_table = LookupTable::from(*self); // Setting s = scalar, compute // @@ -305,62 +317,36 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { for i in (0..64).rev() { // Q = 16*Q Q = Q.mult_by_pow_2(4); - // R = s_i * Q - let R = select_precomputed_point(scalar_digits[i], &lookup_table); - // Q = Q + R - Q = &Q + &R; + // Q += P*s_i + Q = &Q + &lookup_table.select(scalar_digits[i]); } Q } } #[derive(Clone)] -pub struct EdwardsBasepointTable(pub [[ExtendedPoint; 8]; 32]); +pub struct EdwardsBasepointTable(pub [LookupTable; 32]); impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsBasepointTable { type Output = ExtendedPoint; - /// Construct an `ExtendedPoint` from a `Scalar`, `scalar`, by - /// computing the multiple `aB` of the basepoint `B`. - /// - /// Precondition: the scalar must be reduced. - /// - /// The computation proceeds as follows, as described on page 13 - /// of the Ed25519 paper. Write the scalar `a` in radix 16 with - /// coefficients in [-8,8), i.e., - /// - /// a = a_0 + a_1*16^1 + ... + a_63*16^63, - /// - /// with -8 ≤ a_i < 8. Then - /// - /// a*B = a_0*B + a_1*16^1*B + ... + a_63*16^63*B. - /// - /// Grouping even and odd coefficients gives - /// - /// a*B = a_0*16^0*B + a_2*16^2*B + ... + a_62*16^62*B - /// + a_1*16^1*B + a_3*16^3*B + ... + a_63*16^63*B - /// = (a_0*16^0*B + a_2*16^2*B + ... + a_62*16^62*B) - /// + 16*(a_1*16^0*B + a_3*16^2*B + ... + a_63*16^62*B). - /// - /// We then use the `select_precomputed_point` function, which - /// takes `-8 ≤ x < 8` and `[16^2i * B, ..., 8 * 16^2i * B]`, - /// and returns `x * 16^2i * B` in constant time. fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { - use traits::select_precomputed_point; - let e = scalar.to_radix_16(); - let mut h = ExtendedPoint::identity(); + let a = scalar.to_radix_16(); + + let tables = &self.0; + let mut P = ExtendedPoint::identity(); for i in (0..64).filter(|x| x % 2 == 1) { - h = &h + &select_precomputed_point(e[i], &self.0[i/2]); + P = &P + &tables[i/2].select(a[i]); } - h = h.mult_by_pow_2(4); + P = P.mult_by_pow_2(4); for i in (0..64).filter(|x| x % 2 == 0) { - h = &h + &select_precomputed_point(e[i], &self.0[i/2]); + P = &P + &tables[i/2].select(a[i]); } - h + P } } @@ -376,19 +362,12 @@ impl<'a, 'b> Mul<&'a EdwardsBasepointTable> for &'b Scalar { impl EdwardsBasepointTable { /// Create a table of precomputed multiples of `basepoint`. pub fn create(basepoint: &ExtendedPoint) -> EdwardsBasepointTable { - // Create the table storage - // XXX can we skip the initialization without too much unsafety? - // stick 30K on the stack and call it a day. - let mut table = EdwardsBasepointTable([[ExtendedPoint::identity(); 8]; 32]); + // XXX use init_with + let mut table = EdwardsBasepointTable([LookupTable::default(); 32]); let mut P = *basepoint; for i in 0..32 { // P = (16^2)^i * B - let mut jP = P; - for j in 1..9 { - // table[i][j-1] is supposed to be j*(16^2)^i*B - table.0[i][j-1] = jP; - jP = &P + &jP; - } + table.0[i] = LookupTable::from(P); P = P.mult_by_pow_2(8); } table @@ -396,8 +375,8 @@ impl EdwardsBasepointTable { /// Get the basepoint for this table as an `ExtendedPoint`. pub fn basepoint(&self) -> ExtendedPoint { - // self.0[0][0] has 1*(16^2)^0*B - self.0[0][0] + // self.0[0].select(1) = 1*(16^2)^0*B + self.0[0].select(1) } } @@ -422,19 +401,11 @@ pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> edwards::Extende where I: IntoIterator, J: IntoIterator { - use traits::select_precomputed_point; //assert_eq!(scalars.len(), points.len()); let lookup_tables: Vec<_> = points.into_iter() - .map(|P| { - let P = ExtendedPoint::from(*P); - // Construct a lookup table of [P,2*P,3*P,4*P,5*P,6*P,7*P] - let mut lookup_table: [ExtendedPoint; 8] = [P; 8]; - for i in 0..7 { - lookup_table[i+1] = &P + &lookup_table[i]; - } - lookup_table - }).collect(); + .map(|P| LookupTable::from(ExtendedPoint::from(*P)) ) + .collect(); // Setting s_i = i-th scalar, compute // @@ -469,10 +440,8 @@ pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> edwards::Extende Q = Q.mult_by_pow_2(4); let it = scalar_digits_list.iter().zip(lookup_tables.iter()); for (s_i, lookup_table_i) in it { - // R_i = s_{i,j} * P_i - let R_i = select_precomputed_point(s_i[j], lookup_table_i); - // Q = Q + R_i - Q = &Q + &R_i; + // Q = Q + s_{i,j} * P_i + Q = &Q + &lookup_table_i.select(s_i[j]); } } Q.into() From caf296a54625ed061565eab82e2678a41abb4d00 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 15 Dec 2017 11:52:17 -0800 Subject: [PATCH 35/54] Try to make target_feature work on stable --- src/backend/mod.rs | 2 +- src/edwards.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/backend/mod.rs b/src/backend/mod.rs index a4c8b55..0e2ce3e 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -29,6 +29,6 @@ pub mod u32; pub mod u64; /// Code using AVX2. -#[cfg(all(target_feature="avx2", feature="yolocrypto", feature="avx2_backend"))] +#[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] pub mod avx2; diff --git a/src/edwards.rs b/src/edwards.rs index 3488c2b..b8444b8 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -433,13 +433,13 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { /// `EdwardsBasepointTable` is approximately 4x faster. fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { // If we built with AVX2, use the AVX2 backend. - #[cfg(all(target_feature = "avx2", feature = "avx2_backend"))] { + #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] { use backend::avx2::edwards as edwards_avx2; let P_avx2 = edwards_avx2::ExtendedPoint::from(*self); return ExtendedPoint::from(&P_avx2 * scalar); } // Otherwise, proceed as normal: - #[cfg(not(all(target_feature = "avx2", feature = "avx2_backend")))] { + #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] { // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] let lookup_table = LookupTable::::from(self); @@ -505,13 +505,13 @@ pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> ExtendedPoint J: IntoIterator { // If we built with AVX2, use the AVX2 backend. - #[cfg(all(target_feature = "avx2", feature = "avx2_backend"))] { + #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] { use backend::avx2::edwards as edwards_avx2; edwards_avx2::multiscalar_mult(scalars, points) } // Otherwise, proceed as normal: - #[cfg(not(all(target_feature = "avx2", feature = "avx2_backend")))] { + #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] { //assert_eq!(scalars.len(), points.len()); use clear_on_drop::ClearOnDrop; @@ -806,13 +806,13 @@ pub mod vartime { J: IntoIterator { // If we built with AVX2, use the AVX2 backend. - #[cfg(all(target_feature = "avx2", feature = "avx2_backend"))] { + #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] { use backend::avx2::edwards as edwards_avx2; edwards_avx2::vartime::multiscalar_mult(scalars, points) } // Otherwise, proceed as normal: - #[cfg(not(all(target_feature = "avx2", feature = "avx2_backend")))] { + #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] { //assert_eq!(scalars.len(), points.len()); let nafs: Vec<_> = scalars.into_iter() @@ -850,13 +850,13 @@ pub mod vartime { b: &Scalar, ) -> ExtendedPoint { // If we built with AVX2, use the AVX2 backend. - #[cfg(all(target_feature = "avx2", feature = "avx2_backend"))] { + #[cfg(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2")))] { use backend::avx2::edwards as edwards_avx2; edwards_avx2::vartime::double_scalar_mult_basepoint(a, A, b) } // Otherwise, proceed as normal: - #[cfg(not(all(target_feature = "avx2", feature = "avx2_backend")))] { + #[cfg(not(all(feature="nightly", all(feature="avx2_backend", target_feature="avx2"))))] { let a_naf = a.non_adjacent_form(); let b_naf = b.non_adjacent_form(); From 0f6171b7883f07659c673caee6dbe5ed3adcf9a1 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 15 Dec 2017 12:42:37 -0800 Subject: [PATCH 36/54] Try to make stdsimd an optional dependency --- Cargo.toml | 12 ++++++++---- src/lib.rs | 2 -- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 31ba298..4179b8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,8 +24,9 @@ rustdoc-args = ["--html-in-header", ".cargo/registry/src/github.com-1ecc6299db9e [badges] travis-ci = { repository = "isislovecruft/curve25519-dalek", branch = "master"} -[dependencies] -stdsimd = { git = "https://github.com/rust-lang-nursery/stdsimd" } +[dependencies.stdsimd] +git = "https://github.com/rust-lang-nursery/stdsimd" +optional = true [dependencies.serde] version = "1.0" @@ -62,7 +63,10 @@ generic-array = "^0.8" digest = "0.6" arrayref = "0.3.4" clear_on_drop = "=0.2.3" -stdsimd = { git = "https://github.com/rust-lang-nursery/stdsimd" } + +[build-dependencies.stdsimd] +git = "https://github.com/rust-lang-nursery/stdsimd" +optional = true [build-dependencies.serde] version = "1.0" @@ -80,4 +84,4 @@ radix_51 = [] # Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation. precomputed_tables = [] # experimental avx2 support -avx2_backend = ["radix_51"] +avx2_backend = ["nightly"] diff --git a/src/lib.rs b/src/lib.rs index 8eb2c38..6c4ed32 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,9 +13,7 @@ #![cfg_attr(feature = "nightly", feature(i128_type))] #![cfg_attr(feature = "nightly", feature(cfg_target_feature))] #![cfg_attr(feature = "bench", feature(test))] -#![cfg_attr(all(feature = "nightly", feature = "std"), feature(zero_one))] -#![allow(unused_features)] #![deny(missing_docs)] // refuse to compile if documentation is missing //! # curve25519-dalek From 89753e727455a972774d34e07fba4fa6130e8c45 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 15 Dec 2017 13:07:10 -0800 Subject: [PATCH 37/54] for some reason stdsimd dep isn't picked up if it's optional --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4179b8b..0cf7005 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ travis-ci = { repository = "isislovecruft/curve25519-dalek", branch = "master"} [dependencies.stdsimd] git = "https://github.com/rust-lang-nursery/stdsimd" -optional = true +#optional = true [dependencies.serde] version = "1.0" @@ -66,7 +66,7 @@ clear_on_drop = "=0.2.3" [build-dependencies.stdsimd] git = "https://github.com/rust-lang-nursery/stdsimd" -optional = true +#optional = true [build-dependencies.serde] version = "1.0" From 80813e81b1af0c3a70e62a4ed02489b22e4d8ba2 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Fri, 15 Dec 2017 14:58:48 -0800 Subject: [PATCH 38/54] make diff_sum maskable --- src/backend/avx2/edwards.rs | 4 +- src/backend/avx2/field.rs | 149 ++++++++++++++++-------------------- 2 files changed, 68 insertions(+), 85 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index c67c58d..2185f6b 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -229,7 +229,7 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { } // set t0 = (Y1-X1 Y1+X1 Y2-X2 Y2+X2) = (S0 S1 S2 S3) - t0.diff_sum(); + t0.diff_sum(0xff); // set t1 = (S0 S1 Z1 T1) // set t0 = (S2 S3 Z2 T2) @@ -250,7 +250,7 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { t2.swap_CD(); // set t2 = (S9-S8 S9+S8 S10+S11 S10-S11) = (S12 S13 S15 S14) - t2.diff_sum(); + t2.diff_sum(0xff); let c0 = u32x8::new(0,4,2,6,4,0,6,2); // (ABCD) -> (ACCA) let c1 = u32x8::new(5,1,7,3,5,1,7,3); // (ABCD) -> (DBDB) diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index fc6f00d..0c52739 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -19,13 +19,15 @@ use stdsimd::simd::{u32x8, i32x8, u64x4}; use backend::u64::field::FieldElement64; -pub(crate) static P_TIMES_2: FieldElement32x4 = FieldElement32x4([ - u32x8::new(134217690, 134217690, 67108862, 67108862, 134217690, 134217690, 67108862, 67108862), - u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862), - u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862), - u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862), - u32x8::new(134217726, 134217726, 67108862, 67108862, 134217726, 134217726, 67108862, 67108862) -]); +pub(crate) static P_TIMES_2_LO: u32x8 = + u32x8::new(67108845 << 1, 67108845 << 1, 33554431 << 1, 33554431 << 1, 67108845 << 1, 67108845 << 1, 33554431 << 1, 33554431 << 1); +pub(crate) static P_TIMES_2_HI: u32x8 = + u32x8::new(67108863 << 1, 67108863 << 1, 33554431 << 1, 33554431 << 1, 67108863 << 1, 67108863 << 1, 33554431 << 1, 33554431 << 1); + +pub(crate) static P_TIMES_16_LO: u32x8 = + u32x8::new(67108845 << 4, 67108845 << 4, 33554431 << 4, 33554431 << 4, 67108845 << 4, 67108845 << 4, 33554431 << 4, 33554431 << 4); +pub(crate) static P_TIMES_16_HI: u32x8 = + u32x8::new(67108863 << 4, 67108863 << 4, 33554431 << 4, 33554431 << 4, 67108863 << 4, 67108863 << 4, 33554431 << 4, 33554431 << 4); pub(crate) static P_TIMES_2_MASKED: FieldElement32x4 = FieldElement32x4([ u32x8::new( 0, 134217690, 0, 67108862, 134217690, 0, 67108862, 0), @@ -112,12 +114,14 @@ impl FieldElement32x4 { /// Negate variables in lanes where mask is set /// XXX fix up api pub fn mask_negate(&mut self, mask: u8) { + let mask = mask as i32; unsafe { use stdsimd::vendor::_mm256_blend_epi32; - for i in 0..5 { - let negated = P_TIMES_2.0[i] - self.0[i]; - self.0[i] = _mm256_blend_epi32(self.0[i].into(), negated.into(), mask as i32).into(); - } + self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_16_LO - self.0[0]).into(), mask).into(); + self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_16_HI - self.0[1]).into(), mask).into(); + self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_16_HI - self.0[2]).into(), mask).into(); + self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_16_HI - self.0[3]).into(), mask).into(); + self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_16_HI - self.0[4]).into(), mask).into(); } self.reduce32(); } @@ -134,78 +138,47 @@ impl FieldElement32x4 { } } - /// Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)`. - pub fn diff_sum(&mut self) { - /// (v0 v1 v2 v3 v4 v5 v6 v7) -> (v1 v0 v3 v2 v5 v4 v7 v6) - #[inline(always)] - fn alternate_32bit_lanes(v: u32x8) -> u32x8 { - unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - _mm256_shuffle_epi32(v.as_i32x8(), 0b10_11_00_01).as_u32x8() - } + /// Given `self = (A,B,C,D)`, set `self = (B - A, B + A, D - C, D + C)` according to `mask`. + pub fn diff_sum(&mut self, mask: u8) { + let mask = mask as i32; + unsafe { + use stdsimd::vendor::{_mm256_shuffle_epi32, _mm256_blend_epi32}; + + let x01 = self.0[0]; + let x01_shuf = _mm256_shuffle_epi32(x01.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let v1 = (x01_shuf + P_TIMES_2_LO) - x01; + let v2 = x01_shuf + x01; + let diffsum01 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); + self.0[0] = _mm256_blend_epi32(x01.into(), diffsum01.into(), mask).into(); + + let x23 = self.0[1]; + let x23_shuf = _mm256_shuffle_epi32(x23.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let v1 = (x23_shuf + P_TIMES_2_HI) - x23; + let v2 = x23_shuf + x23; + let diffsum23 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); + self.0[1] = _mm256_blend_epi32(x23.into(), diffsum23.into(), mask).into(); + + let x45 = self.0[2]; + let x45_shuf = _mm256_shuffle_epi32(x45.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let v1 = (x45_shuf + P_TIMES_2_HI) - x45; + let v2 = x45_shuf + x45; + let diffsum45 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); + self.0[2] = _mm256_blend_epi32(x45.into(), diffsum45.into(), mask).into(); + + let x67 = self.0[3]; + let x67_shuf = _mm256_shuffle_epi32(x67.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let v1 = (x67_shuf + P_TIMES_2_HI) - x67; + let v2 = x67_shuf + x67; + let diffsum67 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); + self.0[3] = _mm256_blend_epi32(x67.into(), diffsum67.into(), mask).into(); + + let x89 = self.0[4]; + let x89_shuf = _mm256_shuffle_epi32(x89.as_i32x8(), 0b10_11_00_01).as_u32x8(); + let v1 = (x89_shuf + P_TIMES_2_HI) - x89; + let v2 = x89_shuf + x89; + let diffsum89 = _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8(); + self.0[4] = _mm256_blend_epi32(x89.into(), diffsum89.into(), mask).into(); } - - /// (v0 XX v2 XX v4 XX v6 XX) - /// (XX v1 XX v3 XX v5 XX v7) -> (v0 v1 v2 v3 v4 v5 v6 v7) - #[inline(always)] - fn blend_alternating_32bit_lanes(v1: u32x8, v2: u32x8) -> u32x8 { - unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8() - } - } - - for i in 0..5 { - let x = self.0[i]; - let p = P_TIMES_2.0[i] ; - let x_shuf = alternate_32bit_lanes(x); - - let diff = (x_shuf + p) - x; - let sum = x + x_shuf; - let diff_sum = blend_alternating_32bit_lanes(diff, sum); - - self.0[i] = diff_sum; - } - } - - // Given `self = (A,B,C,D)`, compute `(B + A, B - A, D + C, D - C)`. - pub fn sum_diff(&self) -> FieldElement32x4 { - /// (v0 v1 v2 v3 v4 v5 v6 v7) -> (v1 v0 v3 v2 v5 v4 v7 v6) - #[inline(always)] - #[allow(dead_code)] // XXX - fn alternate_32bit_lanes(v: u32x8) -> u32x8 { - unsafe { - use stdsimd::vendor::_mm256_shuffle_epi32; - _mm256_shuffle_epi32(v.as_i32x8(), 0b10_11_00_01).as_u32x8() - } - } - - /// (v0 XX v2 XX v4 XX v6 XX) - /// (XX v1 XX v3 XX v5 XX v7) -> (v0 v1 v2 v3 v4 v5 v6 v7) - #[inline(always)] - #[allow(dead_code)] // XXX - fn blend_alternating_32bit_lanes(v1: u32x8, v2: u32x8) -> u32x8 { - unsafe { - use stdsimd::vendor::_mm256_blend_epi32; - _mm256_blend_epi32(v1.into(), v2.into(), 0b10101010).as_u32x8() - } - } - - let mut out = [u32x8::splat(0); 5]; - - for i in 0..5 { - let x = self.0[i]; - let p = P_TIMES_2.0[i]; - let x_shuf = alternate_32bit_lanes(x); - - let sum = x + x_shuf; - let diff = (x + p) - x_shuf; - let sum_diff = blend_alternating_32bit_lanes(sum, diff); - - out[i] = sum_diff; - } - - FieldElement32x4(out) } /// Let `self` \\(= (A, B, C, D) \\). @@ -590,7 +563,7 @@ mod test { let x3 = FieldElement64([10300, 10301, 10302, 10303, 10304]); let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); - vec.diff_sum(); + vec.diff_sum(0xff); let result = vec.split(); @@ -598,6 +571,16 @@ mod test { assert_eq!(result[1], &x1 + &x0); assert_eq!(result[2], &x3 - &x2); assert_eq!(result[3], &x3 + &x2); + + let mut vec = FieldElement32x4::new(&x0, &x1, &x2, &x3); + vec.diff_sum(0b01011111); // leave D unchanged + + let result = vec.split(); + + assert_eq!(result[0], &x1 - &x0); + assert_eq!(result[1], &x1 + &x0); + assert_eq!(result[2], &x3 - &x2); + assert_eq!(result[3], x3); } #[test] From c0f64009a790acd27e76f55bcf5ff9038e778d7b Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 18 Dec 2017 11:42:53 -0800 Subject: [PATCH 39/54] Implement readdition using a CachedPoint type --- src/backend/avx2/edwards.rs | 136 ++++++++++++++++++++++++++++++++---- src/backend/avx2/field.rs | 34 ++++++++- 2 files changed, 156 insertions(+), 14 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 2185f6b..cbd0c9a 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -29,6 +29,8 @@ use traits::Identity; use backend::avx2::field::FieldElement32x4; use backend::avx2::field::P_TIMES_2_MASKED; +use backend::avx2::field::{A_LANES, B_LANES, C_LANES, D_LANES, ALL_LANES}; + use backend::avx2; /// A point on Curve25519, represented in an AVX2-friendly format. @@ -74,13 +76,69 @@ impl Identity for ExtendedPoint { } } +/// A cached point with some precomputed variables used for readdition. +#[derive(Copy, Clone, Debug)] +pub struct CachedPoint(pub(super) FieldElement32x4); + +impl From for CachedPoint { + fn from(mut P: ExtendedPoint) -> CachedPoint { + let mut x = P.0; + + // x = (S2 S3 Z2 T2) + x.diff_sum(0b00001111); + + // x = (121666*S2 121666*S3 2*121666*Z2 2*121665*T2) + x.scale_by_curve_constants(); + + // x = (121666*S2 121666*S3 2*121666*Z2 -2*121665*T2) + x.negate(D_LANES); + + CachedPoint(x) + } +} + +impl Default for CachedPoint { + fn default() -> CachedPoint { + CachedPoint::identity() + } +} + +impl Identity for CachedPoint { + fn identity() -> CachedPoint { + CachedPoint(FieldElement32x4([ + u32x8::new(121647, 121666, 0, 0, 243332, 67108845, 0, 33554431), + u32x8::new(67108864, 0, 33554431, 0, 0, 67108863, 0, 33554431), + u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), + u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), + u32x8::new(67108863, 0, 33554431, 0, 0, 67108863, 0, 33554431), + ])) + } +} + +impl ConditionallyAssignable for CachedPoint { + fn conditional_assign(&mut self, other: &CachedPoint, choice: u8) { + self.0.conditional_assign(&other.0, choice); + } +} + +impl<'a> Neg for &'a CachedPoint { + type Output = CachedPoint; + + fn neg(self) -> CachedPoint { + let mut neg = *self; + neg.0.swap_AB(); + neg.0.negate_lazy(D_LANES); + neg + } +} + impl<'a> Neg for &'a ExtendedPoint { type Output = ExtendedPoint; fn neg(self) -> ExtendedPoint { let mut neg = *self; // (X Y Z T) -> (-X Y Z -T) - neg.0.mask_negate(0b10100101); + neg.0.negate(A_LANES | D_LANES); neg } } @@ -206,6 +264,49 @@ impl ExtendedPoint { } } +impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { + type Output = ExtendedPoint; + + /// Uses a slight tweak of the parallel unified formulas of HWCD'08 + fn add(self, other: &'b CachedPoint) -> ExtendedPoint { + unsafe { + use stdsimd::vendor::_mm256_permute2x128_si256; + use stdsimd::vendor::_mm256_permutevar8x32_epi32; + use stdsimd::vendor::_mm256_blend_epi32; + use stdsimd::vendor::_mm256_shuffle_epi32; + + let mut tmp = self.0; + + // tmp = (Y1-X1 Y1+X1 Z1 T1) = (S0 S1 Z1 T1) + tmp.diff_sum(A_LANES | B_LANES); + + // tmp = (S0*S2' S1*S3' Z1*Z2' T1*T2') = (S8 S9 S10 S11) + tmp = &tmp * &other.0; + + // tmp = (S8 S9 S11 S10) + tmp.swap_CD(); + + // tmp = (S9-S8 S9+S8 S10-S11 S10+S11) = (S12 S13 S14 S15) + tmp.diff_sum(ALL_LANES); + + let c0 = u32x8::new(0,5,2,7,5,0,7,2); // (ABCD) -> (ADDA) + let c1 = u32x8::new(4,1,6,3,4,1,6,3); // (ABCD) -> (CBCB) + + // set t0 = (S12 S15 S15 S12) + // set t1 = (S14 S13 S14 S13) + let mut t0 = FieldElement32x4::zero(); + let mut t1 = FieldElement32x4::zero(); + for i in 0..5 { + t0.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c0); + t1.0[i] = _mm256_permutevar8x32_epi32(tmp.0[i], c1); + } + + // return (S12*S14 S15*S13 S15*S14 S12*S13) = (X3 Y3 Z3 T3) + ExtendedPoint(&t0 * &t1) + } + } +} + impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { type Output = ExtendedPoint; @@ -280,11 +381,11 @@ impl<'a, 'b> Sub<&'b ExtendedPoint> for &'a ExtendedPoint { } } -impl From for LookupTable { +impl From for LookupTable { fn from(P: ExtendedPoint) -> Self { - let mut points = [P; 8]; + let mut points = [CachedPoint::from(P); 8]; for i in 0..7 { - points[i+1] = &P + &points[i]; + points[i+1] = (&P + &points[i]).into(); } LookupTable(points) } @@ -297,7 +398,7 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { /// Uses a window of size 4. fn mul(self, scalar: &'b Scalar) -> ExtendedPoint { // Construct a lookup table of [P,2P,3P,4P,5P,6P,7P,8P] - let lookup_table = LookupTable::from(*self); + let lookup_table = LookupTable::::from(*self); // Setting s = scalar, compute // @@ -325,7 +426,7 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a ExtendedPoint { } #[derive(Clone)] -pub struct EdwardsBasepointTable(pub [LookupTable; 32]); +pub struct EdwardsBasepointTable(pub [LookupTable; 32]); impl<'a, 'b> Mul<&'b Scalar> for &'a EdwardsBasepointTable { type Output = ExtendedPoint; @@ -372,12 +473,6 @@ impl EdwardsBasepointTable { } table } - - /// Get the basepoint for this table as an `ExtendedPoint`. - pub fn basepoint(&self) -> ExtendedPoint { - // self.0[0].select(1) = 1*(16^2)^0*B - self.0[0].select(1) - } } /// Given a vector of (possibly secret) scalars and a vector of @@ -639,16 +734,23 @@ mod test { // Test the vector implementation of the parallel subtraction formulas let S_vector: edwards::ExtendedPoint = (&ExtendedPoint::from(P) - &ExtendedPoint::from(Q)).into(); + // Test the vector implementation of the parallel readdition formulas + let cached_Q = CachedPoint::from(ExtendedPoint::from(Q)); + let T_vector: edwards::ExtendedPoint = (&ExtendedPoint::from(P) + &cached_Q).into(); + println!("Testing point addition:"); println!("P = {:?}", P); println!("Q = {:?}", Q); + println!("cached Q = {:?}", cached_Q); println!("R = P + Q = {:?}", &P + &Q); println!("R_serial = {:?}", R_serial); println!("R_vector = {:?}", R_vector); + println!("T_vector = {:?}", T_vector); println!("S = P - Q = {:?}", &P - &Q); println!("S_vector = {:?}", S_vector); assert_eq!(R_serial.compress(), (&P + &Q).compress()); assert_eq!(R_vector.compress(), (&P + &Q).compress()); + assert_eq!(T_vector.compress(), (&P + &Q).compress()); assert_eq!(S_vector.compress(), (&P - &Q).compress()); println!("OK!\n"); } @@ -874,6 +976,16 @@ mod bench { b.iter(|| edwards::ExtendedPoint::from(B_avx2)); } + #[bench] + fn point_readdition(b: &mut Bencher) { + let B = &constants::ED25519_BASEPOINT_TABLE; + let P = ExtendedPoint::from(B * &Scalar::from_u64(83973422)); + let Q = ExtendedPoint::from(B * &Scalar::from_u64(98932328)); + let Q_cached = CachedPoint::from(Q); + + b.iter(|| &P + &Q_cached ); + } + #[bench] fn point_addition(b: &mut Bencher) { let B = &constants::ED25519_BASEPOINT_TABLE; diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 0c52739..12e36c1 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -13,6 +13,13 @@ #![allow(bad_style)] +pub const A_LANES: u8 = 0b0000_0101; +pub const B_LANES: u8 = 0b0000_1010; +pub const C_LANES: u8 = 0b0101_0000; +pub const D_LANES: u8 = 0b1010_0000; + +pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES; + use std::ops::Mul; use stdsimd::simd::{u32x8, i32x8, u64x4}; @@ -111,9 +118,21 @@ impl FieldElement32x4 { return out; } + pub fn negate_lazy(&mut self, mask: u8) { + let mask = mask as i32; + unsafe { + use stdsimd::vendor::_mm256_blend_epi32; + self.0[0] = _mm256_blend_epi32(self.0[0].into(), (P_TIMES_2_LO - self.0[0]).into(), mask).into(); + self.0[1] = _mm256_blend_epi32(self.0[1].into(), (P_TIMES_2_HI - self.0[1]).into(), mask).into(); + self.0[2] = _mm256_blend_epi32(self.0[2].into(), (P_TIMES_2_HI - self.0[2]).into(), mask).into(); + self.0[3] = _mm256_blend_epi32(self.0[3].into(), (P_TIMES_2_HI - self.0[3]).into(), mask).into(); + self.0[4] = _mm256_blend_epi32(self.0[4].into(), (P_TIMES_2_HI - self.0[4]).into(), mask).into(); + } + } + /// Negate variables in lanes where mask is set /// XXX fix up api - pub fn mask_negate(&mut self, mask: u8) { + pub fn negate(&mut self, mask: u8) { let mask = mask as i32; unsafe { use stdsimd::vendor::_mm256_blend_epi32; @@ -126,6 +145,18 @@ impl FieldElement32x4 { self.reduce32(); } + /// Given `self = (A,B,C,D)`, set `self = (B,A,C,D)` + pub fn swap_AB(&mut self) { + unsafe { + use stdsimd::vendor::_mm256_shuffle_epi32; + use stdsimd::vendor::_mm256_blend_epi32; + for i in 0..5 { + let swapped = _mm256_shuffle_epi32(self.0[i].into(), 0b10_11_00_01); + self.0[i] = _mm256_blend_epi32(self.0[i].into(), swapped, 0b00001111).into(); + } + } + } + /// Given `self = (A,B,C,D)`, set `self = (A,B,D,C)` pub fn swap_CD(&mut self) { unsafe { @@ -192,7 +223,6 @@ impl FieldElement32x4 { unsafe { use stdsimd::vendor::_mm256_mul_epu32; - use stdsimd::vendor::_mm256_blend_epi32; let (b0, b1) = unpack_pair(self.0[0]); b[0] = _mm256_mul_epu32(b0, consts); From af3c2b28210d6bdb307bb7c0e26ec6807d9bbb64 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 18 Dec 2017 13:58:37 -0800 Subject: [PATCH 40/54] Document readdition --- src/backend/avx2/mod.rs | 122 +++++++++++++++++++++++++++++++--------- 1 file changed, 95 insertions(+), 27 deletions(-) diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index 1b041e1..48dddb6 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -69,7 +69,8 @@ //! subtraction steps are done largely serially, using masking to handle //! the instruction divergence. //! -//! The remaining obstacle to parallelism is the multiplication by the curve constant \\(k = 2d\\). In the Curve25519 case, this is +//! The remaining obstacle to parallelism is the multiplication by the +//! curve constant \\(k = 2d\\). In the Curve25519 case, this is //! //! $$ k \equiv 2 \frac{-121665}{121666} \\ \equiv 16295367250680780974490674513165176452449235426866156013048779062215315747161 \pmod p. $$ //! @@ -133,25 +134,6 @@ //! vectorization must overcome the disadvantage of losing the \\( 64 //! \times 64 \rightarrow 128\\)-bit (serial) integer multiplier. //! -//! When used for constant-time variable-base scalar multiplication, -//! this strategy (using AVX2) gives a significant speedup over the -//! serial implementation (using the \\(64 \times 64\\) multiplier) of -//! approximately 1.6x for Skylake-X with `target_cpu=skylake` (using AVX2), of -//! approximately 1.8x for Skylake-X with `target_cpu=skylake-avx512` (using the extra -//! `ymm16..ymm31` registers from AVX512VL), and of approximately 1.0x -//! for Ryzen (which implements AVX2 at half rate). -//! -//! (Note: since testing this, the experimental `llvm50` Rust branch -//! used to compile the experimental `stdsimd` intrinsics have fallen -//! out of sync and it is no longer possible to compile for -//! `skylake-avx512`. This is why all of this branch is part of the -//! `yolocrypto` feature, pending upstream work.) -//! -//! However, since the relative cost of doubling and addition has -//! changed, the optimal tradeoffs for window size etc. in scalar -//! multiplication have probably also changed and should be -//! re-evaluated. -//! //! # Tweaked formulas //! //! After tweaking the formulas as described above, we obtain the @@ -187,7 +169,7 @@ //! S\_8 &\gets S\_4 \cdot 121666 \\\\ //! S\_9 &\gets S\_5 \cdot 121666 \\\\ //! S\_{10} &\gets S\_6 \cdot 2 \cdot 121666 \\\\ -//! S\_{11} &\gets S\_7 \cdot 2 \cdot 121665 +//! S\_{11} &\gets S\_7 \cdot -2 \cdot 121665 //! \end{aligned} //! $$ //! @@ -195,8 +177,8 @@ //! \begin{aligned} //! S\_{12} &\gets S\_9 - S\_8 \\\\ //! S\_{13} &\gets S\_9 + S\_8 \\\\ -//! S\_{15} &\gets S\_{10} - S\_{11} \\\\ -//! S\_{14} &\gets S\_{10} + S\_{11} +//! S\_{14} &\gets S\_{10} - S\_{11} \\\\ +//! S\_{15} &\gets S\_{10} + S\_{11} //! \end{aligned} //! $$ //! @@ -209,10 +191,76 @@ //! \end{aligned} //! $$ //! -//! to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). Notice that by multiplying -//! \\( S\_{11} \\) by \\(121665\\) instead of by \\(-121665\\), we save a negation; since we use -//! \\( S\_{11} \\) to compute \\( S\_{10} \pm S\_{11} \\), flipping the sign of \\( S\_{11} \\) -//! swaps \\( S\_{14} \\) and \\( S\_{15} \\). +//! to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). +//! +//! ## Readdition +//! +//! If the point \\( P_2 = (X\_2 : Y\_2 : Z\_2 : T\_2) \\) is fixed, we can precompute +//! +//! $$ +//! \begin{aligned} +//! S\_2 &\gets Y\_2 - X\_2 \\\\ +//! S\_3 &\gets Y\_2 + X\_2 +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! S\_2' &\gets S\_2 \cdot 121666 \\\\ +//! S\_3' &\gets S\_3 \cdot 121666 \\\\ +//! Z\_2' &\gets Z\_2 \cdot 2 \cdot 121666 \\\\ +//! T\_2' &\gets T\_2 \cdot -2 \cdot 121665 \\\\ +//! \end{aligned} +//! $$ +//! +//! to obtain the `CachedPoint` \\( (S\_2', S\_3', Z\_2', T\_2') \\). +//! This precomputation is essentially the same as that suggested in +//! §3.1 of HWCD, with the difference that the multiplication by the curve +//! constant \\( -121665 / 121666 \\) is spread over all four +//! coordinates, to allow a vectorized computation of four +//! multiplications of small constants instead of a serial computation +//! of multiplication by a large constant. +//! +//! To perform readdition of \\(P_1 = (X_1 : Y_1 : Z_1 : T_1) \\) and +//! \\(P_2 = (S\_2', S\_3', Z\_2', T\_2') \\), we compute +//! +//! $$ +//! \begin{aligned} +//! S\_0 &\gets Y\_1 - X\_1 \\\\ +//! S\_1 &\gets Y\_1 + X\_1 +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! S\_8 &\gets S\_0 S\_2' \\\\ +//! S\_9 &\gets S\_1 S\_3' \\\\ +//! S\_{10} &\gets Z\_1 Z\_2' \\\\ +//! S\_{11} &\gets T\_1 T\_2' +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! S\_{12} &\gets S\_9 - S\_8 \\\\ +//! S\_{13} &\gets S\_9 + S\_8 \\\\ +//! S\_{14} &\gets S\_{10} - S\_{11} \\\\ +//! S\_{15} &\gets S\_{10} + S\_{11} +//! \end{aligned} +//! $$ +//! +//! $$ +//! \begin{aligned} +//! X\_3 &\gets S\_{12} S\_{14} \\\\ +//! Y\_3 &\gets S\_{15} S\_{13} \\\\ +//! Z\_3 &\gets S\_{15} S\_{14} \\\\ +//! T\_3 &\gets S\_{12} S\_{13} +//! \end{aligned} +//! $$ +//! +//! to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = P\_1 + P\_2 \\). +//! +//! Compared to the addition formulas above, this saves \\( 1\mathbf D \\). //! //! ## Doubling //! @@ -349,6 +397,26 @@ //! as an `op + blend`, so hopefully this will improve as the AVX512 //! support in LLVM improves. //! +//! When used for constant-time variable-base scalar multiplication, +//! this strategy (using AVX2) gives a significant speedup over the +//! serial implementation (using the \\(64 \times 64\\) multiplier) of +//! approximately 1.6x for Skylake-X with `target_cpu=skylake` (using AVX2), of +//! approximately 1.8x for Skylake-X with `target_cpu=skylake-avx512` (using the extra +//! `ymm16..ymm31` registers from AVX512VL), and of approximately 1.0x +//! for Ryzen (which implements AVX2 at half rate). +//! +//! When used for variable-time double-base scalar multiplication \\( aA +//! + bB \\) for fixed \\(B\\) (as in, e.g., signature verification), +//! this strategy provides a 1.4x speedup on Skylake-X over the same +//! operation as implemented in `ed25519-donna`, the fastest +//! production-quality Ed25519 implementation. +//! +//! (Note: since testing this, the experimental `llvm50` Rust branch +//! used to compile the experimental `stdsimd` intrinsics have fallen +//! out of sync and it is no longer possible to compile for +//! `skylake-avx512`. This is why all of this branch is part of the +//! `yolocrypto` feature, pending upstream work.) +//! //! [sandy2x]: https://eprint.iacr.org/2015/943.pdf //! [avx2trac]: https://trac.torproject.org/projects/tor/ticket/8897#comment:28 //! [hwcd08]: https://www.iacr.org/archive/asiacrypt2008/53500329/53500329.pdf From ce68b8d72bf60ea80a0d63b99189d9b6b11b7434 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 18 Dec 2017 14:48:44 -0800 Subject: [PATCH 41/54] Document bounds yoga in doubling --- src/backend/avx2/edwards.rs | 31 ++---------- src/backend/avx2/mod.rs | 96 +++++++++++++++++++++++++++++-------- 2 files changed, 80 insertions(+), 47 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index cbd0c9a..afa7b13 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -191,9 +191,9 @@ impl ExtendedPoint { let c0 = u32x8::new(0,0,2,2,0,0,2,2); // (ABCD) -> (AAAA) let c1 = u32x8::new(1,1,3,3,1,1,3,3); // (ABCD) -> (BBBB) - // Horror block goes here: we want to compute the following table: - // We know that the bit-excess b is bounded by eps, since S1 S2 S3 S4 - // are the outputs of a squaring, so they're freshly reduced. + // See discussion of bounds in the module-level documentation. + // + // We want to compute // // + | S1 | S1 | S1 | S1 | // + | S2 | | | S2 | @@ -205,31 +205,6 @@ impl ExtendedPoint { // ======================= // S5 S6 S8 S9 // - // Bounds for even / odd limbs: - // - // + | 2^26 | 2^26 | 2^26 | 2^26 | + | 2^25 | 2^25 | 2^25 | 2^25 | - // + | 2^26 | | | 2^26 | + | 2^25 | | | 2^25 | - // + | | | 2^26 | | + | | | 2^25 | | - // + | | | 2^26 | | + | | | 2^25 | | - // + | | | | 2^26 | + | | | | 2^25 | - // + | | 2^27 | 2^27 | | + | | 2^26 | 2^26 | | - // - | | 0 | 0 | | - | | 0 | 0 | | - // =================================== =================================== - // < 2^27 2^27.59 2^28.33 2^27.59 2^26 2^26.59 2^27.33 2^27.59 - // - // So, the bit-excess for (S5 S6 S8 S9) is (1, 1.59, 2.33, 1.59). - // - // However the multiplication routine only allows (1.75, 1.75, 1.75, 1.75). - // - // This is because we need to have 19*y[i] < 2^32. Otherwise the bound would be b < 2.5. - // - // Can we tighten these bounds to avoid a reduction? Alternately, can we do better than - // the 64-bit reduction that reduce32() calls internally? - // - // Or, could we do the arithmetic on the intermediate [u64x4;10], then do - // the reduction we'd need to do for the squaring? - // - // Also, can we do better than the mess below? for i in 0..5 { let zero = i32x8::splat(0); let S1 = _mm256_permutevar8x32_epi32(t1.0[i], c0); diff --git a/src/backend/avx2/mod.rs b/src/backend/avx2/mod.rs index 48dddb6..04462b1 100644 --- a/src/backend/avx2/mod.rs +++ b/src/backend/avx2/mod.rs @@ -63,14 +63,18 @@ //! is in the multiplication and squaring steps, which share a single //! instruction. //! -//! Our strategy is to implement 4-wide multiplication and squaring using one -//! 64-bit AVX2 lane for each field element. Field elements are -//! represented in the usual way as 10 `u32` limbs. The addition and -//! subtraction steps are done largely serially, using masking to handle -//! the instruction divergence. +//! Our strategy is to implement 4-wide multiplication and squaring +//! using one 64-bit AVX2 lane for each field element. Field elements +//! are represented in the usual way as 10 `u32` limbs in radix +//! \\(25.5\\) (i.e., alternating between \\(2\^{26}\\) for even limbs +//! and \\(2\^{25}\\) for odd limbs). This has the effect that passing +//! between the parallel 32-bit AVX2 representation and the serial +//! 64-bit representation amounts to regrouping digits. //! -//! The remaining obstacle to parallelism is the multiplication by the -//! curve constant \\(k = 2d\\). In the Curve25519 case, this is +//! The addition and subtraction steps are done largely serially, using +//! masking to handle the instruction divergence. The remaining +//! obstacle to parallelism is the multiplication by the curve constant +//! \\(k = 2d\\). In the Curve25519 case, this is //! //! $$ k \equiv 2 \frac{-121665}{121666} \\ \equiv 16295367250680780974490674513165176452449235426866156013048779062215315747161 \pmod p. $$ //! @@ -87,8 +91,8 @@ //! variables by \\(121666\\). This trick was suggested by Mike //! Hamburg. Ignoring the sign for the moment, since //! \\(2 \cdot 121666 < 2\^{18}\\), all these constants fit in 32 bits, -//! so this can be done in parallel as a scaling by \\( (121666, 121666, -//! -2\cdot 121665, 2\cdot 121666) \\). +//! so (up to sign) this can be done in parallel as four multiplications +//! by small constants \\( (121666, 121666, 2\cdot 121665, 2\cdot 121666) \\). //! //! How do we handle the sign? //! Since we're primarily interested in Ristretto performance, not @@ -98,7 +102,8 @@ //! one field element by a 32-bit constant is not much easier than //! multiplying four field elements by 32-bit constants, and it would //! prevent accelerating Curve25519, so we don't make this choice. -//! Instead, we flip the sign later by swapping two intermediate variables (see below). +//! Instead, we just negate one lane, and move the \\(1 \mathbf D\\) +//! into precomputation (see below). //! //! The 4-wide formulas of the HWCD paper do not seem to have been //! implemented using SIMD before. The HWCD paper also describes and @@ -298,7 +303,24 @@ //! //! to obtain \\( P\_3 = (X\_3 : Y\_3 : Z\_3 : T\_3) = [2]P\_1 \\). //! -//! In practice, we compute \\( (S\_5, S\_6, S\_7, S\_9 ) \\) as +//! Performing too many intermediate additions and subtractions grows +//! the bounds beyond what is allowed as input to multiplication, +//! forcing an extra carry pass. However, it is just possible to avoid +//! this by rearranging signs. +//! +//! Assume that the bounds on the limbs of each field element are +//! parameterized by \\( b \in \mathbb R \\) representing the excess +//! bits, so that each limb is bounded by either \\( 2\^{25} \\) or \\( +//! 2\^{26} \\). +//! +//! The multiplication routine requires that its inputs are bounded by +//! \\( b < 1.75 \\), in order to fit a multiplication by \\( 19 \\) +//! into 32 bits. Since \\( \lg 19 < 4.25 \\), \\( 19x < 2\^{32} \\) +//! when \\( x < 2\^{27.75} = 2\^{26 + 1.75} \\). However, this is only +//! required for one of the inputs; the other can grow up to \\( b < 2.5 +//! \\). +//! +//! Computing \\( (S\_5, S\_6, S\_8, S\_9 ) \\) as //! //! $$ //! \begin{matrix} @@ -313,15 +335,51 @@ //! \end{matrix} //! $$ //! -//! adding multiples of \\(p\\) to prevent underflow. This results in -//! 32-bit limbs which are just too large for multiplication, so we -//! perform a reduction. However, since we just need to reduce the -//! excess in each limb, not a full reduction, it's enough to perform -//! each carry in parallel. +//! results in bit-excesses \\( (1.00, 1.59, 2.33, 2.00)\\) for +//! \\( (S\_5, S\_6, S\_8, S\_9 ) \\). The products we want to compute +//! are then //! -//! With some finesse, it may be possible to rearrange this -//! computation to avoid the extra carry pass, but this is not yet -//! implemented. +//! $$ +//! \begin{aligned} +//! X\_3 &\gets S\_8 S\_9 \leftrightarrow (2.33, 2.00) \\\\ +//! Y\_3 &\gets S\_5 S\_6 \leftrightarrow (1.00, 1.59) \\\\ +//! Z\_3 &\gets S\_8 S\_6 \leftrightarrow (2.33, 1.59) \\\\ +//! T\_3 &\gets S\_5 S\_9 \leftrightarrow (1.00, 2.00) +//! \end{aligned} +//! $$ +//! +//! which are too large. However, if we flip the sign of \\( S\_4 = +//! S\_0\^2 \\) during squaring, so that we output \\(S\_4' = -S\_4 +//! \pmod p\\), then we can compute +//! +//! $$ +//! \begin{matrix} +//! & S\_1 & S\_1 & S\_1 & S\_1 \\\\ +//! +& S\_2 & & & S\_2 \\\\ +//! +& & & S\_3 & \\\\ +//! +& & & S\_3 & \\\\ +//! +& & & & S\_4' \\\\ +//! +& & 2p & 2p & \\\\ +//! -& & S\_2 & S\_2 & \\\\ +//! =& S\_5 & S\_6 & S\_8 & S\_9 +//! \end{matrix} +//! $$ +//! +//! resulting in bit-excesses \\( (1.00, 1.59, 2.33, 1.59)\\) for +//! \\( (S\_5, S\_6, S\_8, S\_9 ) \\). The products we want to compute +//! are then +//! +//! $$ +//! \begin{aligned} +//! X\_3 &\gets S\_8 S\_9 \leftrightarrow (2.33, 1.59) \\\\ +//! Y\_3 &\gets S\_5 S\_6 \leftrightarrow (1.00, 1.59) \\\\ +//! Z\_3 &\gets S\_8 S\_6 \leftrightarrow (2.33, 1.59) \\\\ +//! T\_3 &\gets S\_5 S\_9 \leftrightarrow (1.00, 1.59) +//! \end{aligned} +//! $$ +//! +//! whose right-hand sides are all bounded with \\( b < 1.75 \\) and +//! whose left-hand sides are all bounded with \\( b < 2.5 \\). //! //! # Field element representation //! From 9ad2188bb8ba082c4262d774dcb5077aba1d7ce9 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Mon, 18 Dec 2017 15:08:49 -0800 Subject: [PATCH 42/54] Try to get travis working if yolocrypto=>nightly --- .travis.yml | 15 ++++++++++----- Cargo.toml | 6 +++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a3e92b..cc2bcfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,11 @@ rust: - nightly env: - - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='yolocrypto' - - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='yolocrypto serde' + - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='' + - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='serde' + - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='nightly' - TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='yolocrypto nightly' - - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto bench' + - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly bench' - TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly bench' - TEST_COMMAND=build EXTRA_FLAGS=--no-default-features FEATURES='' @@ -20,14 +21,18 @@ matrix: # run benchmarks, which causes dalek not to build on stable. See # https://github.com/isislovecruft/curve25519-dalek/pull/38#issuecomment-286027562 - rust: stable - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto bench' + env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly bench' - rust: beta - env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto bench' + env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='nightly bench' - rust: stable env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly bench' - rust: beta env: TEST_COMMAND=bench EXTRA_FLAGS='' FEATURES='yolocrypto nightly bench' # Test nightly features, such as radix_51, only on nightly. + - rust: stable + env: TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='nightly' + - rust: beta + env: TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='nightly' - rust: stable env: TEST_COMMAND=test EXTRA_FLAGS='' FEATURES='yolocrypto nightly' - rust: beta diff --git a/Cargo.toml b/Cargo.toml index 0cf7005..8508b38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ travis-ci = { repository = "isislovecruft/curve25519-dalek", branch = "master"} [dependencies.stdsimd] git = "https://github.com/rust-lang-nursery/stdsimd" -#optional = true +optional = true [dependencies.serde] version = "1.0" @@ -66,7 +66,7 @@ clear_on_drop = "=0.2.3" [build-dependencies.stdsimd] git = "https://github.com/rust-lang-nursery/stdsimd" -#optional = true +optional = true [build-dependencies.serde] version = "1.0" @@ -84,4 +84,4 @@ radix_51 = [] # Include precomputed basepoint tables. This is off by default so that build.rs can generate the tables, and then re-enabled by build.rs in the main-stage compilation. precomputed_tables = [] # experimental avx2 support -avx2_backend = ["nightly"] +avx2_backend = ["nightly", "stdsimd"] From 28f114f127357b957a3a1c021bee2c24bd94050d Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Mon, 18 Dec 2017 01:02:55 +0000 Subject: [PATCH 43/54] Combine differential_add and differential_double in Montgomery ladder. --- src/montgomery.rs | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/montgomery.rs b/src/montgomery.rs index b427ff4..614d401 100644 --- a/src/montgomery.rs +++ b/src/montgomery.rs @@ -31,6 +31,7 @@ use core::ops::{Mul, MulAssign}; use constants; +use constants::APLUS2_OVER_FOUR; use field::FieldElement; use edwards::{ExtendedPoint, CompressedEdwardsY}; use scalar::Scalar; @@ -379,6 +380,44 @@ impl MontgomeryPoint { } } +/// DOCDOC +fn differential_add_and_double(P: &mut MontgomeryPoint, Q: &mut MontgomeryPoint, + difference: &MontgomeryPoint) { + let t0 = &P.U + &P.W; + let t1 = &P.U - &P.W; + let t2 = &Q.U + &Q.W; + let t3 = &Q.U - &Q.W; + + let t4 = t0.square(); // (U_P + W_P)^2 = U_P^2 + 2 U_P W_P + W_P^2 + let t5 = t1.square(); // (U_P - W_P)^2 = U_P^2 - 2 U_P W_P + W_P^2 + + let t6 = &t4 - &t5; // 4 U_P W_P + + let t7 = &t0 * &t3; // (U_P + W_P) (U_Q - W_Q) = U_P U_Q + W_P U_Q - U_P W_Q - W_P W_Q + let t8 = &t1 * &t2; // (U_P - W_P) (U_Q + W_Q) = U_P U_Q - W_P U_Q + U_P W_Q - W_P W_Q + + let t9 = &t7 + &t8; // 2 (U_P U_Q - W_P W_Q) + let t10 = &t7 - &t8; // 2 (W_P U_Q - U_P W_Q) + + let t11 = t9.square(); // 4 (U_P U_Q - W_P W_Q)^2 + let t12 = t10.square(); // 4 (W_P U_Q - U_P W_Q)^2 + + let t13 = &APLUS2_OVER_FOUR * &t6; // (A + 2) U_P U_Q + + let t14 = &t4 * &t5; // ((U_P + W_P)(U_P - W_P))^2 = (U_P^2 - W_P^2)^2 // P'_U + let t15 = &t13 + &t5; // (U_P - W_P)^2 + (A + 2) U_P W_P + + let t16 = &t6 * &t15; // 4 (U_P W_P) ((U_P - W_P)^2 + (A + 2) U_P W_P) // P'_W + + let t17 = &difference.U * &t12; // D_U * 4 (W_P U_Q - U_P W_Q)^2 // Q'_W + let t18 = &difference.W * &t11; // D_W * 4 (U_P U_Q - W_P W_Q)^2 // Q'_U + + P.U = t14; // P'_U = (U_P + W_P)^2 (U_P - W_P)^2 + P.W = t16; // P'_W = (4 U_P W_P) ((U_P - W_P)^2 + ((A + 2)/4) 4 U_P W_P) + Q.U = t18; // Q'_U = D_W * 4 (U_P U_Q - W_P W_Q)^2 + Q.W = t17; +} + /// Multiply this `MontgomeryPoint` by a `Scalar`. /// /// The reader is refered to §5.3 of ["Montgomery Curves and Their Arithmetic" @@ -399,8 +438,7 @@ impl<'a, 'b> Mul<&'b Scalar> for &'a MontgomeryPoint { debug_assert!(mask == 0 || mask == 1); x0.conditional_swap(&mut x1, mask); - x1 = x0.differential_add(&x1, &self); - x0 = x0.differential_double(); + differential_add_and_double(&mut x0, &mut x1, &self); } x0.conditional_swap(&mut x1, bits[0] as u8); x0 From dd80421094c67f442448ab7bbe84a12933c27583 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 3 Jan 2018 12:13:52 -0800 Subject: [PATCH 44/54] Move field constants to constants module --- src/backend/avx2/constants.rs | 36 +++++++++++++++++++++++++++++++++++ src/backend/avx2/edwards.rs | 3 +-- src/backend/avx2/field.rs | 18 +----------------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/backend/avx2/constants.rs b/src/backend/avx2/constants.rs index a84d7e9..087d96c 100644 --- a/src/backend/avx2/constants.rs +++ b/src/backend/avx2/constants.rs @@ -15,6 +15,42 @@ use stdsimd::simd::u32x8; use backend::avx2::field::FieldElement32x4; use backend::avx2::edwards::ExtendedPoint; +/// The low limbs of (2p, 2p, 2p, 2p), so that +/// ```no_run +/// (2p, 2p, 2p, 2p) = [P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI] +/// ``` +pub(crate) static P_TIMES_2_LO: u32x8 = + u32x8::new(67108845 << 1, 67108845 << 1, 33554431 << 1, 33554431 << 1, 67108845 << 1, 67108845 << 1, 33554431 << 1, 33554431 << 1); + +/// The high limbs of (2p, 2p, 2p, 2p), so that +/// ```no_run +/// (2p, 2p, 2p, 2p) = [P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI, P_TIMES_2_HI] +/// ``` +pub(crate) static P_TIMES_2_HI: u32x8 = + u32x8::new(67108863 << 1, 67108863 << 1, 33554431 << 1, 33554431 << 1, 67108863 << 1, 67108863 << 1, 33554431 << 1, 33554431 << 1); + +/// The low limbs of (16p, 16p, 16p, 16p), so that +/// ```no_run +/// (16p, 16p, 16p, 16p) = [P_TIMES_16_LO, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI] +/// ``` +pub(crate) static P_TIMES_16_LO: u32x8 = + u32x8::new(67108845 << 4, 67108845 << 4, 33554431 << 4, 33554431 << 4, 67108845 << 4, 67108845 << 4, 33554431 << 4, 33554431 << 4); + +/// The high limbs of (16p, 16p, 16p, 16p), so that +/// ```no_run +/// (16p, 16p, 16p, 16p) = [P_TIMES_16_LO, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI, P_TIMES_16_HI] +/// ``` +pub(crate) static P_TIMES_16_HI: u32x8 = + u32x8::new(67108863 << 4, 67108863 << 4, 33554431 << 4, 33554431 << 4, 67108863 << 4, 67108863 << 4, 33554431 << 4, 33554431 << 4); + +pub(crate) static P_TIMES_2_MASKED: FieldElement32x4 = FieldElement32x4([ + u32x8::new( 0, 134217690, 0, 67108862, 134217690, 0, 67108862, 0), + u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), + u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), + u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), + u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0) +]); + /// Odd multiples of the Ed25519 basepoint: pub static ODD_MULTIPLES_OF_BASEPOINT: [ExtendedPoint; 8] = [ ExtendedPoint(FieldElement32x4([ diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index afa7b13..2d038b4 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -27,7 +27,6 @@ use curve_models::window::LookupTable; use traits::Identity; use backend::avx2::field::FieldElement32x4; -use backend::avx2::field::P_TIMES_2_MASKED; use backend::avx2::field::{A_LANES, B_LANES, C_LANES, D_LANES, ALL_LANES}; @@ -212,7 +211,7 @@ impl ExtendedPoint { let S3_2: u32x8 = _mm256_blend_epi32(zero, (t1.0[i] + t1.0[i]).into(), 0b01010000).into(); // tmp0 = (0 0 2*S3 -S4) let tmp0: u32x8 = _mm256_blend_epi32(S3_2.into(), t1.0[i].into(), 0b10100000).into(); - t0.0[i] = (P_TIMES_2_MASKED.0[i] + tmp0) + S1; + t0.0[i] = (avx2::constants::P_TIMES_2_MASKED.0[i] + tmp0) + S1; t0.0[i] = t0.0[i] + _mm256_blend_epi32(zero, S2.into(), 0b10100101).into(); t0.0[i] = t0.0[i] - _mm256_blend_epi32(S2.into(), zero, 0b10100101).into(); } diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 12e36c1..496283c 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -26,23 +26,7 @@ use stdsimd::simd::{u32x8, i32x8, u64x4}; use backend::u64::field::FieldElement64; -pub(crate) static P_TIMES_2_LO: u32x8 = - u32x8::new(67108845 << 1, 67108845 << 1, 33554431 << 1, 33554431 << 1, 67108845 << 1, 67108845 << 1, 33554431 << 1, 33554431 << 1); -pub(crate) static P_TIMES_2_HI: u32x8 = - u32x8::new(67108863 << 1, 67108863 << 1, 33554431 << 1, 33554431 << 1, 67108863 << 1, 67108863 << 1, 33554431 << 1, 33554431 << 1); - -pub(crate) static P_TIMES_16_LO: u32x8 = - u32x8::new(67108845 << 4, 67108845 << 4, 33554431 << 4, 33554431 << 4, 67108845 << 4, 67108845 << 4, 33554431 << 4, 33554431 << 4); -pub(crate) static P_TIMES_16_HI: u32x8 = - u32x8::new(67108863 << 4, 67108863 << 4, 33554431 << 4, 33554431 << 4, 67108863 << 4, 67108863 << 4, 33554431 << 4, 33554431 << 4); - -pub(crate) static P_TIMES_2_MASKED: FieldElement32x4 = FieldElement32x4([ - u32x8::new( 0, 134217690, 0, 67108862, 134217690, 0, 67108862, 0), - u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), - u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), - u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0), - u32x8::new( 0, 134217726, 0, 67108862, 134217726, 0, 67108862, 0) -]); +use backend::avx2::constants::{P_TIMES_2_LO, P_TIMES_2_HI, P_TIMES_16_LO, P_TIMES_16_HI}; /// A vector of four `FieldElements`, implemented using AVX2. #[derive(Clone, Copy, Debug)] From d41dbb1fe48f753a5b34d06fbd475bd510a7af6b Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 3 Jan 2018 12:34:25 -0800 Subject: [PATCH 45/54] Replace some binary constants with named constants --- src/backend/avx2/edwards.rs | 24 +++++++++++++++++------- src/backend/avx2/field.rs | 5 +++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 2d038b4..d62b31e 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -29,6 +29,7 @@ use traits::Identity; use backend::avx2::field::FieldElement32x4; use backend::avx2::field::{A_LANES, B_LANES, C_LANES, D_LANES, ALL_LANES}; +use backend::avx2::field::D_LANES64; use backend::avx2; @@ -155,6 +156,12 @@ impl ExtendedPoint { let mut t0 = FieldElement32x4::zero(); let mut t1 = FieldElement32x4::zero(); + // Want to compute (X1 Y1 Z1 X1+Y1). + // Not sure how to do this less expensively than computing + // (X1 Y1 Z1 T1) --(256bit shuffle)--> (X1 Y1 X1 Y1) + // (X1 Y1 X1 Y1) --(2x128b shuffle)--> (Y1 X1 Y1 X1) + // and then adding. + // Set t0 = (X1 Y1 X1 Y1) t0.0[0] = _mm256_permute2x128_si256(P.0[0].into(), P.0[0].into(), 0b0000_0000).into(); t0.0[1] = _mm256_permute2x128_si256(P.0[1].into(), P.0[1].into(), 0b0000_0000).into(); @@ -177,13 +184,15 @@ impl ExtendedPoint { t0.0[4] = t0.0[4] + t1.0[4]; // Set t0 = (X1 Y1 Z1 X1+Y1) - t0.0[0] = _mm256_blend_epi32(t0.0[0].into(), P.0[0].into(), 0b01011111).into(); - t0.0[1] = _mm256_blend_epi32(t0.0[1].into(), P.0[1].into(), 0b01011111).into(); - t0.0[2] = _mm256_blend_epi32(t0.0[2].into(), P.0[2].into(), 0b01011111).into(); - t0.0[3] = _mm256_blend_epi32(t0.0[3].into(), P.0[3].into(), 0b01011111).into(); - t0.0[4] = _mm256_blend_epi32(t0.0[4].into(), P.0[4].into(), 0b01011111).into(); + // why does this intrinsic take an i32 for the imm8 ??? + t0.0[0] = _mm256_blend_epi32(P.0[0].into(), t0.0[0].into(), D_LANES as i32).into(); + t0.0[1] = _mm256_blend_epi32(P.0[1].into(), t0.0[1].into(), D_LANES as i32).into(); + t0.0[2] = _mm256_blend_epi32(P.0[2].into(), t0.0[2].into(), D_LANES as i32).into(); + t0.0[3] = _mm256_blend_epi32(P.0[3].into(), t0.0[3].into(), D_LANES as i32).into(); + t0.0[4] = _mm256_blend_epi32(P.0[4].into(), t0.0[4].into(), D_LANES as i32).into(); - t1 = t0.square(0b11_00_00_00); + // Set t1 = t0^2, negating the D values + t1 = t0.square(D_LANES64); // Now t1 = (S1 S2 S3 -S4) @@ -309,7 +318,8 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { // set t1 = (S0 S1 Z1 T1) // set t0 = (S2 S3 Z2 T2) for i in 0..5 { - t1.0[i] = _mm256_blend_epi32(t0.0[i].into(), P.0[i].into(), 0b11110000).into(); + // why does this intrinsic take an i32 for the imm8 ??? + t1.0[i] = _mm256_blend_epi32(t0.0[i].into(), P.0[i].into(), (C_LANES | D_LANES) as i32).into(); t0.0[i] = _mm256_permute2x128_si256(t0.0[i].into(), Q.0[i].into(), 49).into(); } diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 496283c..cf6533e 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -18,6 +18,11 @@ pub const B_LANES: u8 = 0b0000_1010; pub const C_LANES: u8 = 0b0101_0000; pub const D_LANES: u8 = 0b1010_0000; +pub const A_LANES64: u8 = 0b00_00_00_11; +pub const B_LANES64: u8 = 0b00_00_11_00; +pub const C_LANES64: u8 = 0b00_11_00_00; +pub const D_LANES64: u8 = 0b11_00_00_00; + pub const ALL_LANES: u8 = A_LANES | B_LANES | C_LANES | D_LANES; use std::ops::Mul; From 3686562b2f894853b1630d1eccaf69a7df647c12 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 3 Jan 2018 13:37:12 -0800 Subject: [PATCH 46/54] Clear memory from avx2 multiscalar mult --- src/backend/avx2/edwards.rs | 23 ++++++++++++++++------- src/edwards.rs | 1 - 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index d62b31e..1bc999e 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -470,11 +470,11 @@ impl EdwardsBasepointTable { /// A vector of `Scalar`s and a vector of `ExtendedPoints`. It is an /// error to call this function with two vectors of different lengths. /// -/// XXX need to clear memory -/// /// XXX this takes `edwards::ExtendedPoints` because we have to alloc scratch space here anyways, /// and we need some space to store the converted points, so we may as well do the conversion here. -/// maybe there's a better way to avoid code duplication... +/// maybe there's a better way to avoid code duplication... however we can't quite just write a +/// generic `multiscalar_mult` because the non-vectorized code passes between models and this code +/// doesn't. #[cfg(any(feature = "alloc", feature = "std"))] pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> edwards::ExtendedPoint where I: IntoIterator, @@ -482,17 +482,26 @@ pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> edwards::Extende { //assert_eq!(scalars.len(), points.len()); - let lookup_tables: Vec<_> = points.into_iter() + use clear_on_drop::ClearOnDrop; + let lookup_tables_vec: Vec<_> = points.into_iter() .map(|P| LookupTable::from(ExtendedPoint::from(*P)) ) .collect(); + let lookup_tables = ClearOnDrop::new(lookup_tables_vec); + // Setting s_i = i-th scalar, compute // // s_i = s_{i,0} + s_{i,1}*16^1 + ... + s_{i,63}*16^63, // // with `-8 ≤ s_{i,j} < 8` for `0 ≤ j < 63` and `-8 ≤ s_{i,63} ≤ 8`. - let scalar_digits_list: Vec<_> = scalars.into_iter() - .map(|c| c.to_radix_16()).collect(); + let scalar_digits_vec: Vec<_> = scalars.into_iter() + .map(|c| c.to_radix_16()) + .collect(); + + // The above puts the scalar digits into a heap-allocated Vec. + // To ensure that these are erased, pass ownership of the Vec into a + // ClearOnDrop wrapper. + let scalar_digits = ClearOnDrop::new(scalar_digits_vec); // Compute s_1*P_1 + ... + s_n*P_n: since // @@ -517,7 +526,7 @@ pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> edwards::Extende // XXX this algorithm makes no effort to be cache-aware; maybe it could be improved? for j in (0..64).rev() { Q = Q.mult_by_pow_2(4); - let it = scalar_digits_list.iter().zip(lookup_tables.iter()); + let it = scalar_digits.iter().zip(lookup_tables.iter()); for (s_i, lookup_table_i) in it { // Q = Q + s_{i,j} * P_i Q = &Q + &lookup_table_i.select(s_i[j]); diff --git a/src/edwards.rs b/src/edwards.rs index b8444b8..5fca4e9 100644 --- a/src/edwards.rs +++ b/src/edwards.rs @@ -496,7 +496,6 @@ impl<'a, 'b> Mul<&'b ExtendedPoint> for &'a Scalar { /// A iterable of `Scalar`s and a iterable of `ExtendedPoints`. It is an /// error to call this function with two iterators of different lengths. /// -/// XXX need to clear memory // XXX later when we do more fancy multiscalar mults, we can delegate // based on the iter's size hint -- hdevalence #[cfg(any(feature = "alloc", feature = "std"))] From 628af18a1db9ed7163955ad37ebbabd9721f0541 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 3 Jan 2018 13:57:50 -0800 Subject: [PATCH 47/54] Suppress some warnings --- src/backend/avx2/edwards.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 1bc999e..83dc683 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -81,7 +81,7 @@ impl Identity for ExtendedPoint { pub struct CachedPoint(pub(super) FieldElement32x4); impl From for CachedPoint { - fn from(mut P: ExtendedPoint) -> CachedPoint { + fn from(P: ExtendedPoint) -> CachedPoint { let mut x = P.0; // x = (S2 S3 Z2 T2) @@ -253,10 +253,7 @@ impl<'a, 'b> Add<&'b CachedPoint> for &'a ExtendedPoint { /// Uses a slight tweak of the parallel unified formulas of HWCD'08 fn add(self, other: &'b CachedPoint) -> ExtendedPoint { unsafe { - use stdsimd::vendor::_mm256_permute2x128_si256; use stdsimd::vendor::_mm256_permutevar8x32_epi32; - use stdsimd::vendor::_mm256_blend_epi32; - use stdsimd::vendor::_mm256_shuffle_epi32; let mut tmp = self.0; @@ -299,7 +296,6 @@ impl<'a, 'b> Add<&'b ExtendedPoint> for &'a ExtendedPoint { use stdsimd::vendor::_mm256_permute2x128_si256; use stdsimd::vendor::_mm256_permutevar8x32_epi32; use stdsimd::vendor::_mm256_blend_epi32; - use stdsimd::vendor::_mm256_shuffle_epi32; let P: &FieldElement32x4 = &self.0; let Q: &FieldElement32x4 = &other.0; From 4cbff3983db315f09e8bc72ab077c79861c95491 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 3 Jan 2018 14:01:48 -0800 Subject: [PATCH 48/54] Remove fixme notes --- src/backend/avx2/edwards.rs | 12 ++++-------- src/backend/avx2/field.rs | 1 - 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/backend/avx2/edwards.rs b/src/backend/avx2/edwards.rs index 83dc683..d995859 100644 --- a/src/backend/avx2/edwards.rs +++ b/src/backend/avx2/edwards.rs @@ -37,14 +37,12 @@ use backend::avx2; #[derive(Copy, Clone, Debug)] pub struct ExtendedPoint(pub(super) FieldElement32x4); -// XXX need to cfg gate here to handle FieldElement64 impl From for ExtendedPoint { fn from(P: edwards::ExtendedPoint) -> ExtendedPoint { ExtendedPoint(FieldElement32x4::new(&P.X, &P.Y, &P.Z, &P.T)) } } -// XXX need to cfg gate here to handle FieldElement64 impl From for edwards::ExtendedPoint { fn from(P: ExtendedPoint) -> edwards::ExtendedPoint { let tmp = P.0.split(); @@ -608,17 +606,15 @@ pub mod vartime { Q.into() } - /// Given a vector of public scalars and a vector of (possibly secret) - /// points, compute `c_1 P_1 + ... + c_n P_n`. + /// Given a vector of public scalars and a vector of public points, compute + /// $$ + /// Q = c\_1 P\_1 + \cdots + c\_n P\_n. + /// $$ /// /// # Input /// /// A vector of `Scalar`s and a vector of `ExtendedPoints`. It is an /// error to call this function with two vectors of different lengths. - /// - /// XXX need to clear memory - /// - /// XXX see note on consttime multiscalar mul #[cfg(any(feature = "alloc", feature = "std"))] pub fn multiscalar_mult<'a, 'b, I, J>(scalars: I, points: J) -> edwards::ExtendedPoint where I: IntoIterator, diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index cf6533e..29ae3ee 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -120,7 +120,6 @@ impl FieldElement32x4 { } /// Negate variables in lanes where mask is set - /// XXX fix up api pub fn negate(&mut self, mask: u8) { let mask = mask as i32; unsafe { From f816083575c54e5097948f28ef14d78d5fc26ade Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 3 Jan 2018 14:04:27 -0800 Subject: [PATCH 49/54] Use >> instead of srl intrinsic --- src/backend/avx2/field.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/backend/avx2/field.rs b/src/backend/avx2/field.rs index 29ae3ee..7f422e0 100644 --- a/src/backend/avx2/field.rs +++ b/src/backend/avx2/field.rs @@ -296,25 +296,16 @@ impl FieldElement32x4 { let LOW_25_BITS: u64x4 = u64x4::splat((1<<25)-1); let LOW_26_BITS: u64x4 = u64x4::splat((1<<26)-1); - /// XXX check whether u64x4 >> is this already - #[inline(always)] - fn shift_right(x: u64x4, s: i32) -> u64x4 { - unsafe { - use stdsimd::vendor::_mm256_srli_epi64; - _mm256_srli_epi64(x.into(), s).as_u64x4() - } - } - // Carry the value from limb i = 0..8 to limb i+1 let carry = |z: &mut [u64x4; 10], i: usize| { debug_assert!(i < 9); if i % 2 == 0 { // Even limbs have 26 bits - z[i+1] = z[i+1] + shift_right(z[i], 26); + z[i+1] = z[i+1] + (z[i] >> 26); z[i] = z[i] & LOW_26_BITS; } else { // Odd limbs have 25 bits - z[i+1] = z[i+1] + shift_right(z[i], 25); + z[i+1] = z[i+1] + (z[i] >> 25); z[i] = z[i] & LOW_25_BITS; } }; @@ -337,10 +328,10 @@ impl FieldElement32x4 { // big. To ensure c < 2^32, we would need z[9] < 2^57. // Instead, we split the carry in two, with c = c_0 + c_1*2^26. - let c = shift_right(z[9], 25); + let c = z[9] >> 25; z[9] = z[9] & LOW_25_BITS; - let mut c0 = c & LOW_26_BITS; // c0 < 2^26; - let mut c1 = shift_right(c, 26); // c1 < 2^(39-26) = 2^13; + let mut c0 = c & LOW_26_BITS; // c0 < 2^26; + let mut c1 = c >> 26; // c1 < 2^(39-26) = 2^13; unsafe { use stdsimd::vendor::_mm256_mul_epu32; From 841c026c13609edc8c8bfc12dbf98b1d93b6daa3 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 10 Jan 2018 05:49:47 -0800 Subject: [PATCH 50/54] Remove dead code --- src/montgomery.rs | 96 ----------------------------------------------- 1 file changed, 96 deletions(-) diff --git a/src/montgomery.rs b/src/montgomery.rs index 614d401..2d726ed 100644 --- a/src/montgomery.rs +++ b/src/montgomery.rs @@ -310,74 +310,6 @@ impl MontgomeryPoint { CompressedMontgomeryU(u_affine.to_bytes()) } - - /// Differential addition for single-coordinate Montgomery points. - /// - /// Montgomery coordinates in projective 𝗣¹ space are odd in that 𝗣¹ - /// inherits none of the group structure from E_(A,B). Hence, the mapping - /// of the group operation, `⊕`, is undefined for the pair `(x(P), x(Q))`; - /// that is, given `x(P)` and `x(Q)`, we cannot derive `x(P ⊕ Q)`. This is - /// due to the fact that, in Montgomery coordinates, `x(P)` determines `P` - /// only up to a sign, and thus we cannot differentiate `x(P ⊕ Q)` from - /// `x(P ⊖ Q)`. However, via differential addition, any three of the values - /// `{x(P), x(Q), x(P ⊕ Q), x(P ⊖ Q)}` determines the forth, so we can - /// define *pseudo-addition* for a singular coordinate. - /// - /// # Warning - /// - /// If the `difference` is the identity point, or a two torsion point, the - /// results of this method are not correct, but instead result in `(0:0)` - /// (an invalid projective point in the Montgomery model). - /// - /// The doubling case is degenerate, in that `P ⦵ Q ∉ {O,T}`, where `T` is - /// the two torsion point. - fn differential_add(&self, that: &MontgomeryPoint, - difference: &MontgomeryPoint) -> MontgomeryPoint { - // XXX Do we want these debug assertions? We would need to implement - // XXX is_two_torsion_point(). —isis - // debug_assert!(!difference.is_identity()); // P ⦵ Q ∉ {O,T} - // debug_assert!(!difference.is_two_torsion_point()); - - let v1: FieldElement = &(&self.U + &self.W) * &(&that.U - &that.W); - let v2: FieldElement = &(&self.U - &self.W) * &(&that.U + &that.W); - - MontgomeryPoint { - U: &difference.W * &(&v1 + &v2).square(), // does reduction on square() - W: &difference.U * &(&v1 - &v2).square(), // does reduction on square() - } - } - - /// Pseudo-doubling for single-coordinate Montgomery points. - /// - /// Given a Montgomery U-coordinate of a point `P`, compute the - /// U-coordinate given by - /// - ///     differential_double: x(P) ⟼ x([2]P) - /// - /// # Returns - /// - /// A Montgomery point equal to doubling this one. - /// - // XXX It seems possible that combining the differential_add() and - // XXX differential_double() methods would save a non-trivial amount of - // XXX computation in the ladder. —isis - fn differential_double(&self) -> MontgomeryPoint { - let mut v1: FieldElement; - let v2: FieldElement; - let v3: FieldElement; - - v1 = (&self.U + &self.W).square(); - v2 = (&self.U - &self.W).square(); - - let U: FieldElement = &v1 * &v2; - - v1 -= &v2; - v3 = &(&constants::APLUS2_OVER_FOUR * &v1) + &v2; - - let W: FieldElement = &v1 * &v3; - - MontgomeryPoint{ U: U, W: W } - } } /// DOCDOC @@ -514,14 +446,6 @@ mod test { } - #[test] - fn differential_double_matches_double() { - let p: ExtendedPoint = constants::ED25519_BASEPOINT_POINT.double(); - let q: MontgomeryPoint = BASE_COMPRESSED_MONTGOMERY.decompress().differential_double(); - - assert_eq!(p.to_montgomery().compress(), q.compress()); - } - #[test] #[cfg(feature="precomputed_tables")] fn montgomery_ct_eq_ne() { @@ -544,26 +468,6 @@ mod test { assert_eq!(p1.ct_eq(&p1), 1); } - #[test] - #[cfg(feature="precomputed_tables")] - fn differential_add_matches_edwards_model() { - let mut csprng: OsRng = OsRng::new().unwrap(); - - let s1: Scalar = Scalar::random(&mut csprng); - let s2: Scalar = Scalar::random(&mut csprng); - let p1: ExtendedPoint = &constants::ED25519_BASEPOINT_TABLE * &s1; - let p2: ExtendedPoint = &constants::ED25519_BASEPOINT_TABLE * &s2; - let diff: ExtendedPoint = &p1 - &p2; - - let p1m: MontgomeryPoint = p1.to_montgomery(); - let p2m: MontgomeryPoint = p2.to_montgomery(); - let diffm: MontgomeryPoint = diff.to_montgomery(); - - let result = p1m.differential_add(&p2m, &diffm); - - assert_eq!(result.compress(), (&p1 + &p2).to_montgomery().compress()); - } - #[test] #[cfg(feature="precomputed_tables")] fn ladder_matches_scalarmult() { From a0c93cb730785aab5c8a414577f5fc2a4b740d68 Mon Sep 17 00:00:00 2001 From: Henry de Valence Date: Wed, 10 Jan 2018 06:04:03 -0800 Subject: [PATCH 51/54] fix inconsistency in variable naming --- src/montgomery.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/montgomery.rs b/src/montgomery.rs index 2d726ed..e510c87 100644 --- a/src/montgomery.rs +++ b/src/montgomery.rs @@ -336,29 +336,26 @@ fn differential_add_and_double(P: &mut MontgomeryPoint, Q: &mut MontgomeryPoint, let t13 = &APLUS2_OVER_FOUR * &t6; // (A + 2) U_P U_Q - let t14 = &t4 * &t5; // ((U_P + W_P)(U_P - W_P))^2 = (U_P^2 - W_P^2)^2 // P'_U + let t14 = &t4 * &t5; // ((U_P + W_P)(U_P - W_P))^2 = (U_P^2 - W_P^2)^2 let t15 = &t13 + &t5; // (U_P - W_P)^2 + (A + 2) U_P W_P - let t16 = &t6 * &t15; // 4 (U_P W_P) ((U_P - W_P)^2 + (A + 2) U_P W_P) // P'_W + let t16 = &t6 * &t15; // 4 (U_P W_P) ((U_P - W_P)^2 + (A + 2) U_P W_P) - let t17 = &difference.U * &t12; // D_U * 4 (W_P U_Q - U_P W_Q)^2 // Q'_W - let t18 = &difference.W * &t11; // D_W * 4 (U_P U_Q - W_P W_Q)^2 // Q'_U + let t17 = &difference.U * &t12; // U_D * 4 (W_P U_Q - U_P W_Q)^2 + let t18 = &difference.W * &t11; // W_D * 4 (U_P U_Q - W_P W_Q)^2 - P.U = t14; // P'_U = (U_P + W_P)^2 (U_P - W_P)^2 - P.W = t16; // P'_W = (4 U_P W_P) ((U_P - W_P)^2 + ((A + 2)/4) 4 U_P W_P) - Q.U = t18; // Q'_U = D_W * 4 (U_P U_Q - W_P W_Q)^2 - Q.W = t17; + P.U = t14; // U_{P'} = (U_P + W_P)^2 (U_P - W_P)^2 + P.W = t16; // W_{P'} = (4 U_P W_P) ((U_P - W_P)^2 + ((A + 2)/4) 4 U_P W_P) + Q.U = t18; // U_{Q'} = D_W * 4 (U_P U_Q - W_P W_Q)^2 + Q.W = t17; // W_{Q'} = U_D * 4 (W_P U_Q - U_P W_Q)^2 } /// Multiply this `MontgomeryPoint` by a `Scalar`. -/// -/// The reader is refered to §5.3 of ["Montgomery Curves and Their Arithmetic" -/// by Craig Costello and Benjamin Smith](https://eprint.iacr.org/2017/212.pdf) -/// for an overview of side-channel-free Montgomery laddering algorithms. impl<'a, 'b> Mul<&'b Scalar> for &'a MontgomeryPoint { type Output = MontgomeryPoint; fn mul(self, scalar: &'b Scalar) -> MontgomeryPoint { + // Algorithm 8 of Costello-Smith 2017 let mut x0: MontgomeryPoint = MontgomeryPoint::identity(); let mut x1: MontgomeryPoint = *self; From 47d594e61f67562882b8bf39f8b472c554a42c65 Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Fri, 19 Jan 2018 22:51:49 +0000 Subject: [PATCH 52/54] Add dalek logo now that @tarcieri converted it to an SVG and PNG. * THANKS TO @tarcieri for cleanup and vectorisation. * FIXES part of issue #89: https://github.com/dalek-cryptography/curve25519-dalek/issues/89 --- dalek-logo.png | Bin 0 -> 109731 bytes dalek-logo.svg | 1 + 2 files changed, 1 insertion(+) create mode 100644 dalek-logo.png create mode 100644 dalek-logo.svg diff --git a/dalek-logo.png b/dalek-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..83d6a0c52776e03c7c56efcffdbaa8b0584d881f GIT binary patch literal 109731 zcmeFZcRbc_{6Bh2DG`xMcA_Oan+S=FqU^2gmAx|hl$9+RS&_Z7H<_XAy@jldvO+ko z>(=-8JLiw{*E#3E)1%MhllSd?jo0;>&)4(1eILo)KXaPmGzNn?BPA)WfWaJ}LI0h? zgC}1*FI7z`aoO8o8<$C#gEF1Ed!v4_8uPo2$)`cz~; z^5AN}Gy%@J&SK%09KIEU{RH@A56hl}su+J#C+zC+GdJhWkSOqgFxe?a=EoK~e0b9^RLhNDKZ}x^0kT4#Jasq6euNQeHF^7Fkhb;AaqGi*|B)X$y8oVM49A6p$kWHtd^yp; zt$6gX_U1j9qguXO#DPi|2^Ie6#BxYcYzA4Lre?xDdLl{8Nc!@BKE0^SKzd`Ic;+`X z^NIiYctt)$vSHcw{L#-M@2^uarq?dqJo?eYmVNPO#L>SoYy|&npex7tjvm(D@jPz* z?{@=E!8v39P?CH(+AM~fKKS24%*5$V9(_9KBSLiauRN0fH6pF>|N3YA+5h#=|2>by z|2xzC{{te)d;C8-s{e0fc&op!Z`KwcEsykzQ7QpA1jSgJc0TKhG3zpo&?HZSiw{B{ zNK3zqj$YEyN52+>G|J-y1tn$Q&`^4Nd;81ys^6_lp$xeL(pX(fVlU1*hS7PsJW?W{ zc?{n0IDQYk(juA=3~$AXshY1!G|7C6u|23}+>GTnoN^GEv& z#UKU0svC*s_b@t(#{BQm^`!9d3#o1Bjo{MIuHt-oP@v(5jFoa*~{<@6M76S+iG`g)ot$IMowX$n3$T{ zl%v-$Qm?_ZJ$Pezt?Jf>@ z=yVvw&`PLyg^_#kkU#qL>?z*=%cn<9h8(*kDlOMd#eXO947;cL5zCt8e~TkTi(`Ld zSg|->yXkQXBw8$6HUBFIpscVF!?BBe(sx7xdhNyzBD+-LH00T2@Z| zn+vNS?7I_(``Z83^BOzc-Rop;cE+chgQ8WCYI{7xZjtb>Tg^XxQNS*NE{+K6P!|HQ za~2jA{f|hn`y`yldeO5?==}B0c3iR^8ymZNbpAtF*}hWNcIEovvbVe2+tsD3#|3j? zLh-QzaNJuinJzyJ;2CF2Cq>U&ypaEWb?l|bzcuKgHIS28(hUp@7|I=;H8yDAHF9>| z)k{Na8oYw`!VBb*HWSU!LAc{4EP^@FeD=H~zuEHBp z0yXp4z?$A&%jzA2QB$@-qX8l3<}QVgwMd@9N?j@|X!%8JF+Y$O0rxRiEpV_z-XV*Xd>^?_9kz|IWMNC<+>j3O;{Q&gXWp8 zj#o)YV

wBujqx>C=yLF}x(3nWiC2 zLqkJqRnA4$*4ByD*Qkh=vP;IhE#20mRa&uf!zwws#C5XngZ;ga)zzh?Dxou*&S!+? z)~4!*%k44~T+o;dtVoJpRwL#mm&)BX7yCk!oYsF_DqjURhJK5mpPz6HYi4|CYi$0^ z>c!%&=5lg@zt`6pr3fzEV%vy`i=)}x+A`VS-Sk?)P9B5ZO`SvaI<;s0R9bG$Y`8jJ zwSC1{`9`<&8KKobzov&jJ@tn3|2Oy!G8MMhhkt?&*=+aE@Qgkm-eywDVzWK`^t9P_ z<_n3?S}pFs!C#=kd0%y{lu}xh-nA&%D~!J+bU(*^WD%EGP$U9GgbDtR{hj(Cw~Z@e z1H^r2BI4t7V`8YpmMZsFDr{SjiW++$gT8x#SuLr$x*FCvpXlec?8V7HIt61*!GL{=n1MVu8(tHV8e@`L`P%Y2t;GKN3uWlN?7XPn~Nb4D^V%VCRzzy znxAL0WrgXE)Y2n~_V7=$^K<|Y;h4>jXMG8y)@{sQ{eqxPySuw+quJUuU zLR(uqA;}<>00&bm_3?fXx2^pstgUZ+{2*>2!|*iTtMZvpX~I4Zr{B-?4GpD3?SBp? zR8|U#ZKG3=H0rYjsdkT_xTv9X-T!Tpep`4fm;YOapyrQDITh`l!}}vjj0+P7%M*lS z$O`Fxp(E}d9;i96JMrqS*tY*AP6^@NuAs!~@%ciSb^PTc3s70C_Us`UGH`?Thzt*x zSjMlEFR|2bGiz51W~`LDp|cYlO~}kNom|Su$mpp?Dil15PQ}=?KTFkXm3^3cz%t1= zOYG7pW3vZV!-|j+bay+=#iovrYt?U$*^iiel~3B^H_*9o5r;0)!JiLqmLQIK>}7zRJcrx6>%lCI#H{yO93r6MKpa zl7vZLhN6|w#Plz6INGJ9o{%f|@v7a|KA+tDSh_-<-#7a?APda(9PH`{owNveBJ_|R!~b~Z{u8h$l&qg`N?@P#X= zgX@B{Hb*k@YnZJoXg!6F^`$l)LdwKK8^ONk8~YgtdU`&h3~P0JWvtB!ycR|z6uIT3 zteBXX(z;Ulj}i+JHGh+R&&PiM<+uSU!oZp%F-1PJ*LGS<=I()Qj1vxhI zUeMoN#(Yod!#U+h(SA)RVro3%680Acs_)o16R4eSb;&q-YuLNuv4PrCgp0 zL3kh2e@cI2bMyQI6Z>+Ap?c>D@wg1zsj7B%r#+lbD=91SJ>|K+6&k)hZC$mLMq1)n=Bo%*ma+l%kCR^7`Y&!t;lp!3X;>F zx7`n3pbCIJm))fIdaTPz83m_-e}AraN0~KvsF%1W@s5>(D5|f$5U~3G8e(9=+iMRq zQd94{y1J&=X$UY22&ng@J{$;Tl-_7$b(6O!8CeL$U=F>(@dgX_XRD{eu1dYRV_Fv2 zaO*k5%QPN4EA?CI)CfH_R&+GB?+9C8Dx_&7Qjptj*!v&(pG zAt4hds0xbGkD}JMSb>2BYYg2OG73^{P0iSYP3^7Lz5UJ*`X4(N(_Z{i|UqS9q92`mlA&5WcjS2s(ejCm(>jc4t`Qj0a z$9M0(?3u-g@M8s+18JJk&hJ_0xmbr=s)yautbOgV0_U#!FO`7yM|ueoGG4hN7NKb$ z<+e96@jjmh!|K1?2}GHSK<)q~u2d*T;CGzFWZ@q4Q^+7O-cmYodBmWxMh zp|Bd+s_goB-}GxFd+XP)y%5B2gN_E!2&sLP2ne-ZWLsP4NhQ{BZ#=I|_(Dj(l}t8@ z<6dZ3Y-~1!+L1-PV2xd|Qb+?$_qLYhB3_YvjpDqoUTl_-NrCx_B2=E%VgJX2!$NgP znrK{pvd_-VjU1R2MDopFriqG*f;qz&0wB@?WQ3cmudiPz>1F4OvGZH_?tJM82 zvdS&;HBJZP_Q+kdGgA>jpy>xrw8U9n7_0?ef${w4P~gA$iXl!cU)G?b5xKslpDr5NlfgoIABws0fUaXY&a9s z_87kRoQCbS#WkE~h`R9(&C*y-#CJFhZ~I;-y!Ujx(h1q+%rD0nmPQWKft9?a`>$h);y1G-f z?#Fx{Lf-CmhH^0cz1Ia+t*rRNzjJaCIG7ex#FNU)%ZJ#nj%NcJYI{TK^Q^Tos#e%%b6bo?+Q$0(8nuSqXl-ffLQdho|Bgw84lKQ60Hfs4Jxea?epVU0q=Gx~ z=E?0>XU>oB?Ynu3CcIn~O9-BxOXZ)cTyO6I7sU0ni7-_f$Kx3)s(cM#SNv{&OMCPg z0&-oKP0Jy1>gMpFyhToEvOHWgoUiZKlbm!ScObL8oc|bqQq|((V$zxKcn+=Q8Z76@ z$i6Jw6TkjtFi*U+8{`XB6sXY+%0Ybt15`WQaByS7sGzKzv?WWmU}k$SmgUM+vdjHZ z$x}am%B!hmz@UJ_P`x!XY;Y2!Ie+28O<`f}6DLmKf07-WTfYIy3~t7)MmSlNzc}w$ zttvQgk;GGwB~SiG5W{jMVRmL_K2Tt#^S=`Wr@pt-lM^mp6W;MHHX8zKX7V-L-B=i| za^X27qGtwUQdCm9`oa;k=ZS&>J?yBit}f}%ItA80_I7vE$9F`m9SYxxCU*Z*zq^_1 zo(=T{l9V{YzI^q%67QLqnc0^6zn-=(L+75Y@=0|4$2Xhun51*3J$rk5lL9DtIR1V< zP7`!LJ3CwI*AE)D$8={O>FXyKTTZOl-l)IH!lGnl^{X~7{Mki#16(9{8E~eIv))YS zuF3VXy8R-+1}X)>n!t^U(5L0*4uG5Q8yQ)$y@8(~{Cer|chq+DGiRhCAlGl#SHh7inFLlB#O@ z8A@(mO|NN|sttv2(L4_u8yjuwhCbBTSU~P+Vr<-%Am$yzaw9<&A}KR7^J^H|gEtq6wI; zfMyd0ik$p>j(6eVDGl~d3XUJcF(1rBjt+lVOHhKovq=UQNId*j=8 zQrn(?g%qQ=vvbn+5_uiStjK!y0X%);=2rbaIvU?Ugqt**DW|Z|R!91#gp#&)F35!8 ztGk2P0`V-0GbZ2>&@SX|Qq0<(0B{B};FO2)E4G&ccZNFLQXtPwZL6h|m6Zk4l-gs< zNlg3?he(nM>X!Fhrs2~s1DmJ2b~l$$NQY-6P508XvSdI(hKo!iDz$QZW(m_vN|fP2 zIizewmJ_gM9L#;B>c=8sfF4HPH;-7@kH5O>3ajbg9MI0K`8&%h7cor^BGIg{Pftl{ zT2R2KzZSiG{``5q>U(`ln~T2J;yRadYjU_u`_Fncr{(26`u6P`KIK=G1wt-EfYK)f zpWsiTaxsSGvz74PP)B6iVlY(3P#r8mWbT5*Lp`<4>WQM_J2iE6MHQ8K+Z!ItFYd&% zlujON_QEaIV1e^keCWRW7DTz=#VJpJ(gxtN-6ieA3_z%q9t!UlLQdpyg;xxmfVh)W z*|C?k#=`4+`}ppX1w#hk-X~A|{lOy_5yI`ZRS<6s#TeEbCG&!6!L*807lI1E&rEt9 zTs{MP1KotEl3rP<@mZ&HK>^R7R#*$^w$S$IN2rDQG>I^Sz6p>&`oFj{;o`HY^g@zI;cS#m(bCID?v6S6WoO~Q-a zXAa63nv0oZjk}Yioc~ODZ-R>SFDNL;J%0SQdd-s+47y`M!E6q^kc!HeUWp`T2)cl` z4z?Cy2;9_Y;K-y-XWt8y8xK7NXvPF5#kdX8q!lhc;FBvuTgcY z0WQ_r)@Hu%G74s#k@3dn(z&pSRb@D#>9=;AoSa-%-^I%`SVZ|S3;%)j>iAErMs&$cw;@KdGv0@`L3V793}FJ~KJN9OUr zxT%@o$DsT&8d*Q}J=E{*9R3z1lnYm7B0v26X@(|2F5+y32;B!rg8TaW)rPIujk_6K zCvj;XnVU1Kt31tvlvU_g!?{4g3SkDO>(_&ur!TYNQa#er%7ItZ<7Ppk%F500zZk+! zjajRzs!~9S1$LT~8t43gC8F(<8m1P3uO5Hk%)xbh?$)t@Giq&Z?KUqNOBu1K_!erI z^d7X|xF=-+%DM{kjt%Fko40OJ4br!6h}Wvxds-2iN>a-?w8_Ts$^&|HH4Ha+5vAil z@I6H{va$+)e$#fpz8%%^LV}Xp?2)~FF=L0+32;iCb)E#ww4>nGAoL`D|E|Zc=W!eo z^CwT9JaZ~}+cw(`o&wZef5AqFHNGPi^nstlmkSC6sF{=?&kJYK%sAZ5JrE@kNg1 zGx=wxr9DzqJRh5P<0jc@)h<2jm$yp^U8A?aK{&`}XP5I!>+V zy^k?DfJ_a8mj=DwNEP0f(#jnuvF`yko51~~7Ztr@W08f))5^+JX@Z*j=e7>DmKGPgf6um=yK-Yx zP!mDNZ*)|Hi-*S)kc1Rj83F4VAAr0W7KRMt`x5W@^^T8Qv<|lpvHpt~a)TIhc0Hs-dsTr5P1 z$&x_QQv{)Fx$oP-;Wkw%F_6kDD-VcVcbKfj`4%s%U30LRJFO*fY1DB#VE1=KjajSO zaVrt){G0wZWg2QVZdKr{`FKciLym7G0oH-#0)V*SsiKC4*$C&uI6=O(gdS*7r3$%h z4Q`oX@=%MS*50B-9mEE4OL=reyPa?rK@Ct9jJY4~8-%^jv#9(fA@p~x-YOSmNX9)^ z{{um)o^Nuh(kd!0Hu?6RD<#LTchbJ12!i|LJ1TztlUFW$h&ak&~u?~e0Cc*@j+p|z;70sXNYblHmS>reNK-fxd;R^#5Z$3l##rphV%LD3dZ zfVrUCdJ{v(zTwc;xNf}nDAqdNHYetiuExE4o)9+ir#%=9>@f<8iuaVM^bC|!H5f?w z%$F9GYjzcjbB;-xAgA=Y%e{E?+(SWEXj51~d>VA3a__1`h)o=$Zb`}Ym!G!(&H?xbI2rH{4GkHX!R+d4Ex@!gZ+!;fj!iuYcu~(L51ch&&$t4IUn4$SBwyEC8!Z&&*sBN*FEZxRdF=6XD*KnOkil z5fu*ToZ2V7dtbLBzz4x_*+wFv5cE!HsY3Y)6nWeh_eO0NTRS9HkbfEa-8B1TK2jX%W=LrT z1=P#j)v9eRD5$@5&*69sNl2aM)jSTc@!RNV%5}2-DN#Nte9G4SKh47ZkfoaasXicz zgj8bP@0dr0rp@h9>PCj#`}HFwC4+^TzR%~B_1lgPc0V!E_k1=6qjQ&ut^v3Pq3SII z0)lcYf4^buz`>kD-DpV*@Ohx;zn!bRRs@`WCk4_7?Z(AV*bQhJyfaX_b#4JH^VGBl znS!P!wGDFQ_|w68SyaC}$Hs&@cc8^GiZy(4dVX5v1%_p`^!%7!ideVi3DE1=*#@i-U* zY$GDZQLHQ{hXod?DnIOUn~Q4!5DC9sX9(4B_^K)-mki?=%B|%Q!(C2HEg^C(VM@x%*K!9;=i{L0R&fi|0NiAydT%8yciXuNl7{>| za3z`9*@}?uCb*oM4!#M|k+Eig7s>96k2w9C4fog3=s11aQ}pJ|n+ouH!rn^Fp&K9q zb)oM8sLma$aJX-m4cQSC7S=X4%;_gQ_JIUo&>mx)CJ@L*lx_sZ;N>nHFALcn0+Idw z38hl+9B*rBVc)!jIh#t)k0jKwY@d0QcyzilDd+S;`6`>6nx1}Rkxh=jNbwygHM2xX z25Hk}#lGI&$m$n|%C zDm27>t_3L-^i0V%H^B#*SXdZ*AW%>OKxzs^##t)O0tpsWTfLh&baL44lljEwBm^yU)R0sAO~ zR4XmlF!N@xGa%a^SGw)WG=_`KDRrq7v@9%grqbuTbVhD^h&P2RLVNFxor{d%i1I*= zW~RVuC1-}I5KRz7R;MdONoOHlX3DMP$S~E!`Z&c$k94dQMf6S`Keju6U)W@~9NO{& z&|_s}AH@+7)op`FEd)v`3njrClIiW+!3o*`7BmG=9w`yXlwG1YqrbQJY}4gzl|IG+ zFy%EaO=}D(B6-j;mmT@%6qkzt)d=av>C^nm!k?}_W zV|*k03S)Z;6!}xnRUnh{Xh2jaNp(+8&y2J*winulhN+-5me<2Ltq(M0Wdo+(V;FLv z78)KNPRn6tVJT(^!-U=ixj>G(u(ajS0%VV+h17U%S67VB3ACteNLE{*iCNttdEM|? zeg@Qg&_$YZhH;Ee@;M69TxGhl-1z(M(9p)l2GpPrL%{+W2)+1$FA*2=s=s3fGB%qz zIY>N-UA{G5)TXkdD={5lImSolDZaN&AV%eJX!H5RMe(Xe0kwa{RenK%tTrV^ul5mi zN%h9cZ^l+V&Ph-AUE!q*0IW^IjvBr7^j?Gp^i1eOd>~sy79@&du1IQeP7{ZnA_ZR< zCx>d^8)(hMJl58x?JM2_JjII_^GfB{+~p%RA_YQ~s-o@@e(q*X7L5 zI*%eWtfii8I}6fO9qcZ#3sZ{dfg&Pwu-(}{rg>T(%nxi=_8m?Ow9z%roH>({0|v4Rxy%^*iN9lNkDIBUz zDGp#GA{Vu?`m|uWwY8N2XG#j15rd$VS~@rct1m`Hf#tk5{9N5fk{TzC_JB-me4tjtwOoIb(2{5YX^i>#TKpPELEu8;=?N#^OFnhD6}Mfb;HRJXwone#Inw_)r_lgCq9KQP4~kS3%Ow zZc{Zk9WqAHxy?q5RC|VbCemFv30^;lJ|33``-&tSsDk+cXi=>|wG&FV9^VuBU0^a| z!a%A!auW+-Vc3pywukz8owvM=;l1*Hg_pCq0e7JxBXARHG0hH+Hfgw-MK2}B7Vc?4C1`O;?uQK=#c#Bcz6av zvpFjHPAZ7q91`Rr`OYpbMNs#ltTJ>HM~4?p z9pgE&%v~hSA}kzVKlYOBdGSgq;CpL^)NH(5rysD}S#xy(PP<u>w)n@z%ajvTp%& zx+Apc-{j(-m+hl3BtcYRD8b@1Ub@=bj)LIv3UcY_WK}lz3J$5t+S}!%`%ii#AW@!W zuF!O;=+^bu(nz`v~lqkf1YEE*Pp;7N%Q8OI8z~m7lqrz(#xy++mcl_;59Gw!tSqS&ux6M9yvr-ChwlOrN z$`>;-@!Yn4L|ZO`=j=swvp21q3TySBL}*EW*LhdF_VvBvov`P;UyBtn#g~?CfoQ?- z)=^+)F#j~eHHcuv#l^bX+64x(bq=*!FMc)hJFb0TKuOhOXfdKz*8UW+JNfTj)alJs z%I@|S{;Mm{c3k4wY?3vU&Gw0qOT&inXC#7t*s#&52J}*2lS)ob=2q0x0R9IEOXqa!fr3V3;-Q+W7srzVZQ65Shd^6ygc-m z6>1LmSF>-OV{16_AA8%YCC|(;yyvh|Roz(-^l0em>}31C{qu`WFYn3jf0*iI2_`gVQ39+R;{{rWc~IiRoVHhVrj{W)Dgj) zm{;GOuGG$m(v=~&x1hjgaDeINO|z`4{1-KCLx8RHIdVnl#h0rtOiLKbx7x$s91GEn=yD=z5 z%V#EUGrd}aARx)W@mIeuhk(4$7H8~i1toay*F|E z7AJPniW%~sm|7xa&*WsQLtJ3b-`96uT&dOqC;srV*dR0;5nx-hzie47R8Uo=*&IYm z`zMZtkM~=v%?x-ASZEvatq|vojEygCaY5N=OwJbik08<0^7fj!yFJ}m-sKqnV|=_V zi$r{oHsn!YH~8EugntIo^`A^Lv}kub>vKzBM+uObEO$gCK}g)+>2Rm{bC4XN)eZQs z%Y73ZkcYnM_U0ygFOJ77s#!1|oc_I+4Yzl!fJxB{%|@yHj>EqXHI|+k8bTw_7^=wD z_Vz&I0HAyB6=r7=3;%tt!G-e)I>AL|%g{`Qv|a^q3o+4mtG(e0#>lt#4PoQGj4eE1f3_P@~+Emh5x}};g4rvK==5U1p4J-#WEOOnm zM>trgGmyxr=R*aYH?Aj8QR!~-1tL4+LUxAH3C8-U0@L!vaAx(5vJO`{X!KZRuLoGZ z5ObJt-8R-8)v@{UwF@zo>X)j48HTG}0|YYmfs+gw&v|9Qm1FtWkS5jJeHr4?EP@UX zxG1YLmS5W2`yu__grTOh2HGRp>(_Q)5my@K;l_l<$FBnI`z$Bc*eQ$!C_QF)M&p0g z)rq#n3jqfS5utLpQe>hMQ&O%1-|TkAQtHEai$9kjyy1>0X+!^`ejC>y?>AN1__!7p zHh^ea*aLLEU^A$kftH;od7X?ud(Siv70B_i0NQL{D66cj+>W{Ezzt9mV1`g@b8KV* zrDc_0U@C1iiL8}nU}tNUN>-_g@!#C+ve@Q#T>BkX^_SmR?F06ds6;3Tq} z^KjLp+DVFbx6_-->5k53?=&>rRLk#pY#<=oLl&V=;DsjO`h|b(B7P?HolqHRX&Gc1 zJ5*q3+7`~7VtdNY-oEMfkR1?mLX!Xj6I`M25Z?e+Y99zMi4A*mq%%qCtzJ1RC+Axc zB1xTkSrx8Kf%etyC5YKU0{-gx&wQt(GU^w8{_LH*v&~lp3{bDt<@mgEozMT3E&+w& zmOBfEPVLCKbZq6_c$^Xm*8n(Ghp3gIr;DAG`8g$zg}`@34IcBO6% z1G;mgR%~`@%u3Q2YU?~wb)fIFT@Sa32A;}o-~hJ=z8`_`xjf8a(e;k{hl#LIq}EK`0W!$n>t>p-NEKN zDZuV)4tww62$iq&zyxrElb3KMwNcqriw8#0Z+K_@67V?K{1eKj!BhyVhKR8Zu_t0; zVj4x@8khDa4y&PwMKQqt3UnbOFOO{nKNl)4L>dS9!%RURxV+bnB6_lz#m~V{AP@Xo z0DdhEITlL*o&{Ws^CZe<+zcnh#|kx^G-X?-c9 zbU4r-7KryJ0PZpq5-zXUAzBw)FNk#3 zih1TnuB@g;vHk&g>1-V6p!IZ~?I{6BVdsgF=e3;=u79rLh+`g*izXa)F2x3uPh6Fnr z{E{|`l|BOifFOW?Tk&`dFg>2AmslXOv_vHfw1fZ;pa3tx!NGy&b_d9Sz9hGjEahpuuVuQo;-g1v{lW6S|T8S zg#mS9jex%uo=r?=eja-Zj2%Hy&i$smr>O0rDN_FZ4HIG{LhPE&0yh8zGjtJ#&szL` zAeB67=cGgOBm8%Y7?HKcTvn@;BI2F{)e?%mEKoK;`?0#ohS1X#FRN&7b9#TcSBHmG zzF|K8a3{XQ`OHhIpM@4<{F*?Ecb@XRsHb&>)OQ~4X`qe}YEMGbMjBm8G58vp0BycV zMTf?o*kX0d??ANa{h{F#|NC*i7MFU!yikV6aTO*baTgATIWH$tyR44$_b8}f&ijF!KI~W|Pw&`CDLObM`pAuLzcw^9=5n`Mra_hg(AHYK! znvnBA2ES@NW*usa(vX$WGIV=^vBUNO0GkMK9bWT-+@P%v8EGVGlanA#9WR5B+rTkN z>_yjx>+gB;DqB3?!GJ8~^YlIK#FQ=QlFH9P_RWrCrm${kA}@sTsppiJtNC4E<}i1B zb^h&>&Md1PssYu-WVUC-~lF>l;a z33-v-`Zu0v(r0|YO+kGXM4@nDX7c8d{NZf{NgaUYDOpfI_)ignm3)VGJ=f1NNS5;+ zu#rCj?Z46yC{FJTELSeKzmAAz8oDuYxIck5P2)5}0+cvj1?5|C>+J%IhEeO#V>rk1 zxj2{GuDiD9Zo1?GN9{c+#RCQCH=z!KDuLv2z#*Y6VpUEEMF4~_4_>yAv-V39DJF)7 zqmKexXGgq|26X1{Deyw1A|sYo&JoBBzBF985RcHw7%wc{&sa&R>_^&nA!O8{kX zF^HOYdN)?kNdcS3SA)I*T}HN!*d;A1s~53B9*`T(uHpHz!JJRfl{HK-154$aC|E+n z0*72rPmj%Tp0_yBEaX*ikcg}nnIHGAo^p492dE4g8T9^Qt@*!we287=#lo}C&{chc zo-bT{cma1Tx`F)!94)caAbPlrgQzBeu>)9R4BXG*0>dEv*ryUev<0R?RQGuv2)05S zonpW~4a9$UCL5dt;x64VH6JeYUEY*5E-Nbw&)$D{xbYA>5yIBtMk-v-30WJHgxh)Mbofft}FB&ZAC-x&3LFmx}!MY3Pf_2Lzu{`zPI=HgD~{FVk_ zOkyHE5b!`aef5+K3vZZm9zpBkhABxC4kmaITepz*0##V33LL#$gf23e+KoPF;bg0q zwD-^L0gC6#1=q)pxBw7Z9z&ZjCCdSFQZ@d{yP+v^u^B**4NA$nhtBvr;)Y-(MF8TQ zI6T+}R;R(#b4R)`=*jGvMVG5-YGwl_-yN1yiCooqr7B<+90~bTva$S%z-6D1QT1aZ zTDX368I>F*l9*r*hFYg@M=AhFCErH`P8o01g)6m2kNP1jBM zfhP@dFbT5G;zI#~Uuxcf*ymMHq%{EH*tf?ZDBPl(qE;vQ=iHK>1 zk?)XQPvvAvUwEWY{_unRVUv-D>m$2AydnyVci-s@l^4-pS5oW8XqpYQ5r#WTpR9Zy z8#O`uQwI0Hud^{7hAg>r)1QKl&#m8C)%Z5mxEJ%$oz{Mm_W^t;GZ+z*&p zaCrDty8vkS4=&pa{{C{Mj@416$mBfqQd!MJC4Ac`sdQtWI(I4)PW$-y93#f%qlZ7j z*1sArGTCP?b|*`R#K#ALTA(`+!~FIY)KC>kKx?5BYEr>94CI%uJzoHOwH}BpmzqdU zkbmsDo^7wEmsmI~PB#2+Ilh3!H@1s9gTo4en&mIAurTe78#lW93x>>iZGI4rJIzr; z@1rYK?SkJ4=(F6{(u(`A;-ReNbZ?iD0(iXS?0U!RdFekt(^{+*3G{h}Uof(qcXHSF zvpD>@OLylEO{!dMXmxwEWwkG)$#I)bevSX(pS$`2!201~kjOj`_2JAa-;3MCT)bv- zlLcOZLiPgi@2(_ixtu)xHh=^#b*Vxa@>xFP6VFAoi8B!Y8&pI_hLD1SA}NO*+#!G?4u8s_ukTOAr(gm^ zceq~gp|G&9en9os1wpv&r~;Qe^NPIBP+r~tjfYXm)4}P_(PYY^spm}$m$`ZlkC2=$ z+1}RnHU#?Q-m8hh&G^_uE&2x651h#(LPNhR!OHPo;Q$!$ZkQPJ0&r^|U*FFkgYFEk zm;x;?kbtFB#E3k#K{mtV4;yrjBXp_cnXbN-Qd@?E+qlTrhxhV*5PT9xfW#Fufv4() zYRp?JV+65sb6qx&*){?}ox!dTH6h>=LOfi|hw^d?m#v>9)dAZOvztLzJp0Z3=GiQu zd#O@pot)<^{`ouLC3Qe>OYRK}2xtTd2D^BkJEF@(Nb;gXP&cu1`Qv&37ScJ*Mr1Tj z!}okrMAxu5)^E(hyHW41l$38FmzLA)`M+@IH$rLRGAHM`krK;`NLcj;)6CeY;)uK7HlmnZ8i;m{raU+U=To$y{JdtL~`Sc z(d`H6lr%K$%$ed0=Z}o!olG>h?4qs4g9oRfqHxU!iHh=tRTC^qI>GycdAhiW4Ck5+ z*X(p5zp7hg_rJ0jnI*zP|6g>Ak0eXnkqkrA(yl}2TPm{#+GKN4X4Py_GUR;LN%ZQ< z_b9l{$j6I5KeV{o9&}1nvd#!-~tIGwpX7^R^(+5aF6-=5N8tC}>DXz)I1S`b* zh|*XdZLc0mlWSc5(U7nQLu>>(cQ9OzNeWkh^MSwI*GZc+?T7Ti^q52@b;v6y9LrW>vZKCF4h1o5)>W{}?Uu^p zUKjb#hL{mzt%Pg~iog%W#Y$@XFoKI+=3?2rE&tTzc*1|4B02Z*qer!Dfs*?9S7$)L zA*7C8RA8ndbc;DV03RO_GqYQmH9|i6P)|dwOT5wYj6;fG8bb$66E$;; z8z19ET3XsQ-BCcHUc=kxIqG;l&RL5m5#m`Rr*azI4NdN&;j?Dw?&(SHJNdpuakod+ z`siq1fok+T6lP}jKUdWld3_LgPSTm?1w%=FSLv;F=&;6d7F-bUj-)aV;ikD{=CTRs z`kSnt5V&>y6z;Q--+58GQ3SUbJs`9v=#F~e;DR8Wq<);q1R+Y||C8izh|Z_a*lL`i zxGlMB8&m&xXFYjBXE@}~A4f8{=5V1YTeCvR=o=_eQgSkA*zeFR_!A-cpf1aE>HpWWuWD;O8(lmdlrMK%E+a=Wwf#TRttA z@A65J`7^EJ;^w+XTf%(o@&WfOolZ?j>8EZ{cJIOR7F6X-*UTJ6}0FCIN^E z%cqfNyLsb={IgiN%6+f@a$;x0(fabC$dKBrvsjz_>=P6ep-^$fB>u{^VYrm|9PZXl zLt~+P0Pb?UZq)k>mF(J>!-Oudr@KYf6LmiJzK;23@Ysx*{;auFQSA6>)fW$y-g7k7WTTg_X*fqkvctnXzTy@2h-aWR6@>A zir@qQsHlHMd?)QsIb`c~#n~6g$R<}RRwW=&40fi!2X|C%+`x~}cH={rbN#6Wl$OJN zp3Ie!-NSdT(i*;CPk-8Ynh!kJ8{qedDJflmeAvTctULj3QIQL~2|}eF076S2qag`- z4uQMdgqWO7q>qj0)L-*V__-81Pk%YUi9`AYm-SAf7~I|6eU^qM5Yo1%^PLHi+~yR( z1Mag3k(7si!M4U zK|IJ5a#+zYE*b->?eQwD+&^335gx=}o1xOx_+WkkYSntk$6v?0?Y5Lr^9u@6efspN z)1-(O^u`LsS<8v)xhziG`Obo4W9#h##J~xB+bE!h!*LHDK5Pa$Hksx+DBvBT8o23`rC!1^Sy#F> zXvQ5Q=tO$$VO#?kn?KM<_}t|_cG2CsMHtI(C;H?GS)p3?h#kP_XE306&G#+>53m#p zA@#p+C8eN*y8P)X$q8ZXLIQ~zHwTtQ#l^=%%QTA|R>$WjYHA91*B$!kgi}*fokq7o$wi+%V<@*>_|L~D zTwC<#Xj#F%=W}duKU>4L>mE73ZQ6Rgc(uaxKEp&kX$T?mjU<&EaXz9US@>l>KBo{l zS;53bEVbt{BjX9UQuHQ!2T)qazh42Ps3Q6;8Zu_#fM*Ys4U8I|y~z5IovyB~kH0_o z_~Z0fu8^EN_ew67zvxmg%d2e zKg?4lK9T_Fk&*pS{IfR*TuS-TBM*p=>0@{#A&Jk%bD$yx7P_S_cm zZ7t~D;Bs}KZcO=4`-=77gaeZ+@RbyB3#ATB+ZVW#r!g@R58=oYAzgzQit#8d<%Mbi zFFifIea?VFaVo`J~co;p-zAnRmE<;u6(>G)agy^aMpgd%bfBh>7PK ziNwinqdH`E^m2g2D~N)lkgwF4?j>er5nj0;_{8=bENbq=@j&z7oRXIiiI1aOauy{? zlNYI}$Mh)8-~qK~dbx{}BB06g zziJ2zi7wkK^ZnVByIaeCAhlE3mf)+oqhopYxNqO)b=T0=4uY@)E@3WwP9*s4+gEU1 z@ZUnzm3$>L0AS1*J4O=n{f(a=1dg86AZ;%z3|Y-!LRDTiU_Otn7!+Myg#aoLa?9CF zCO7%@LxGH*o>aQM1mu5^y18s3E}kILq@;M#{XoK-bk3DENjrqO2A+y0^^XLZ13BxX%ih_^&vusP%`NANr8v+#+81m}s zZBL0IW4x}zhF*+w+chpyr=wT!%);jjkeH6($>Qg=Xkef}%`F%d>LdOXjHnx#h)Icv zcoJ{aTytY{^WNs`7`fEP6(=AkPxL?qUJ@k1qx|CHms=q5&Ww$Txs_dfYGGjkz15r3 z&>$|jHTd)oTy-D;^Xn>5%T`iQ7AMPaY`aM!^~r3A1G|KKaJeC@9NbJP@G6peuiBxP znD4MKF^M5wQha85x_mx!;s-F{AV}Pw!YxlpX=xj`1Ne@O2B4lV+EXX9W?xR6L^*L1 z>M9R2pCw5b?SBoqO{sP~?ObqZ=znkpq-bPK)xh8?P>_@RXnCJcs=kb=fbgjMNk#2N zPPTLi+=qAWpL28~^*na}lmpz0pM@EKOOAQs3$h0jV9}E(p6uQ0(mArUH zFxO>p&VSw7Zk`>S`*Mj84-&1XaBsJB!L}5bYssG0$kz^Zq$DN9Z1N4X(4KTSRs%>} zfm;XAQsd8OsND+Z7FL#*pOR}6?=Cx6R|Xd?Bk#EKL$L_FeJ+cTtBI3Mu(Q{os)E9S zKBA#t38vWZm?-KNZN+Dp55M~Hi1AL0Ks1I#u>KToZocK(K%+wkt7$fX+u~W(B&DS8>gnlu*H}6x;F7c;4SIcU-;;nu{zLH2gs|2; zUEpV(+I;=C;E<56DN`G3>)?0q7;Rm%mHZ%%|Ant(5RYKdeBDpPy?o*YHGA~8cHy9$ zyRoJo&BlU<5iv1@uRePKIpi`k^BKs3jP}cH=E~21|1Em*C<@|lL(j-*lZv>FF;V22 zX`9K17}|@cXx#qj3T}_uSk$<|u{pwp^Y-%n-irXu-v!5o%DPcN>0$lot5k+4~MHB!_Aws}>ko}&v%`eXK z^7OpS&MyC8`4P#53tlzAm5CH`DZbyaV9S3?B@ZJVev71ET1*TVg&v@3Ux7UTaf%Fy zQncV4zvmo(z(9X8c_5{QW=Eq-gZ=?uJZ zw17aiiZP;!F$?7i68h6b0NRX|+pRASt*t#b8_GWcgwy0djkRko50y5 z_g=^b{`G3R1OWj7s!YAXu z*(7Bb*-Ek-B&&?Fib#PDj;_02uiZbNf)f>g0j{+}92n$9+z`SLd z`*c@ocjDqNAy@})ur#g`^4MA6r2c7%{U`OV_R}JVk#%RdunJQw8)(M zIY5-CE65DYm>u>{<#*>Dru)5~2zauZl!*Fr@7+dPnLz{oRT73uhqr!7MSu%1{udDR zPt*&9GnrmtIDqsQ5^faeX>XrA7E~t1E0^~E^37f5ar&935f*jGbxHP-*%ypYVcc_< z-G!LBJn5|5v2EK*NKR@oK);Z)j>`YTK3^c(=&=PO>bPL20{n||qgWT^AE!kFh3yAV zX==`H-^WLX>3Q`s!~Q|a?Wn9Tcz<*j-h=x<^;%_RyYT1tZGE}pxYGiF4N!uXM|kHo zjRTD%B?7D@#Oyo8QK9OL0GGkEg41=Vu?9nz(UyZ;@^Zl6`T!L$PYs0c^Iov?^pqwC z@%SF0!KfQI?zge%H#avIL;vsJqq+T5nT-+12!0TSSDA^KA;gkkOi`LJ5)Ni)hC7S4 ziAfmlAi^WiUm{&NU?NA;G>XnXu_Owf4(|c_(n3~9tUxD6t z0L&!ee;Gs~CMG7;akIz$j!`c}9knVFbtXPv)l2e9@e2|@iV3hyKL1&sIrU z(Y5Mkt151kZG0cO3sJtOZ#DqELpACrxvbCLRko*33tpW4ykV#gr~8p%HYKJ>>&XW@ z6Zah(9t;GjZbh-bMV;?Lr81pESe!GoQ`D0_makOx;)NInGRe9DHh<-U2OZ2Fbc`l< zj3Rgp&p(FjEKhIbIda?M^S>R2>g*X$n*#Da(Nx0qP^MV(FY1q^$B(H>t`X4GGvP*4 z8hw}F01}4unZoxkH%y$C^&hIEYV+NgT@!5b)LSi6vv6#-f4aOclZW%dR-viuPPL`f?UX(^*j(~OmeHed8B&N~AQ zMrRVgS>@nCnJW1tnZb~dkOLJL7sRaJ@|{DNp^(aS$kv!R!eyI)9JPw_hTjl=rjv<) z!C&BgHF1t)Wz=2XW<4q&TD0Zg5AQ%Pz#c;oRF041K%T(lebzPm@ z-`B-gZNm1k=k&#v;XU`1dj7*s*)S}LYyyh#ugV-j4|}YPvEmWv5%_~?5)$hG*eJFM zi;C8OwA8hA%k2UJzn66>w-{GWmJqYj=jtD&8fl&~i?5gkT3Cn2IzUM-TK9}K*ad!We?JC91AJ4@LJaV0HqP8y)Z7LHO39@~@ z{K@K|u0M|r@vO`X4CQ=Iq6G?2uVJIQ`Qu>Cs<*Dmq|^OIKvX`8uiN|mms%24LUaUH5s$wXVxNI_TSVI(C0zga88iUTK!yv(;6AHY z##rPt{lUtnh@$kGakYKh>a}ZsdohdyxFJrg&UHKByMn>#{A|ZB_B_Kf z(v6k2NBEXq$bQgdX#1o4$+Ys`yF*!w0v-PUlE+U!$5;1bnoXOpKYG`Ezx?i}CcCny!Mhk~-Y_+W@dyRZ zq9Qv?%8x4<46eFro7@KvwyEK%ius=*shY^ND-Ft)Aj6iw^$TJG8e>zGBL1SrlY$>Y`r`^Tn=bGqCZNFg{H2a^d5hZtdMn$%Ul+8BCn)IQ1;6o6C-Oh6 zF6+s0=hA5mcg>E!dsiguX5-MNNfE0g5DiZbYUg7FgC|16WIhMqlWyn1071{4GIx243$lCVC8 z-`!uevt*DB(?PZaS65foM+yYwYH+Luhs=3ls^gFNrjGkZHjpD`W)w!*pUyvhx>umh>KmOJBw^4yzjq2@3KnDDXfIqcZ}Y+e+jSWfXR@ZX^Eq z8fE}qNdG%2flu78tjw@{E$?;y16PwUL0ml|oC^B@3+NXx|4eEsp!3!#)6eb1Mb12U zn~)9u`2y72rZ_GhQCH_M3=dKjL527*GxLVNZ&QN8JDWe-z|R|7PZ8eBGeiIo3~p6V zR-MW3`}MU&`oi3kNmn%SOz0XQIiyes=HkqBfq*|CTvD8tp9_2LP#?jM=X9fUZ{#~X zhzWRWnj?eT6e-gOccXF|EI^TKY_0Gi&>7D?3NSh5t7oSw{w|(BTW~=S-XD1S(Q5`l z-b&SMD5yJ=zk~vG!)U&x4PZZo@}uIfB1y^oCImI4(ca!uSxW~TghVp@@hOPT!+sd^ zO;`x76h?xYd1bnI8KP!(WRP@kU)qTFa0lp3nL|KP#)(gZeL zmuDCu$d9_Bi=vEIt=;$Jk1+&_5ErcdFV{cN+Ps=myk5rq_MJQLEiz7Kw(WTy2k^f#OPiLw;7m*5)V1U3z=;r9)58RmMUENgN{WyTqr z)|2@A0qdjWcWZP!ESP z34aFBiuEx;^@$~#qxC-_$?V5a=GL>YGLwt5rwfdnbiB(lw%0|+y%v47C9ut=23>6D zf|sV^)daN72?9W-ng2y{bzm>?q=1;ciOx5D%44iku;AGP5vj-!UJn#A7E9hJ?`q zBPAP~ZG=qa&IKr&L+Rgd0106GlZ5M;qrpMZ99&%63!c3>a{vUfR*CM; zog&bt&2J_bY9szHC7(Fx%yELa81?Ls%Df;3ODFQrr1vPVvgw&Vj4$R#Kyxqk+BKPy zv7u6}RBfden}T;8??G}K8LG!=PcMD8-8WVhW`oyg6>Ec3{xVaSVr~ALAy-3 z1g!UAH#0S0C5b+`?{ZLkkda($le4;^fO)h;!a=`+_m7b{%QRz_nt5y2jw# zN#B^T$qq3VR`%DhSudz>XgK?ONMq}bz1$La)f->kGW7ju%5W@2DJmiY$%@WFBkx?r z^5Y|$qOQmA0@1|nevYZvZI_AAOtg7fh6O8kLA%!KW<1++4k*V#I2m=?tHRRLvoTTY z@L?CtehhmGr&8lwT&qu`J;U3}P_U(TSo}*21JS z_c))`9;%YV5SkTem-=DAegL9lZyBE_7L_pHVu)ZJ<7`SwWYW9y^ zMkzr-<;krzEI8A|ZUz=LJKcB?{SB<@;D52IM0|fCo>QXj#?adJH|!T*N<5jgoMW+; zd%o0j@#G`h(ci)Pnf8;v%SNZ|JWPIwK{}Xw*|-ngp+-%`f~s-_jCGoq74EB^k>1n4dEKR zfmvgref#$Dr7bya%hO+y;I|@9L8Cr@yn14b^brjWh0&%nD#^H}H<$t7{u;i!|H^W6 z@Hag%k{5J4NN;`NiB})lg_E#y-u=}GPOInRrBuc&XeZ1-V1Cyjb8%jpgnUwZ9A3cm zs{&>=&!y!W9g0q&K2x^Y6nbd6i*-rBdM8J&?99T(GFzSznYit*b2a0@n!f01fZYo9;jRdmj)<1q+xM1<`R(EZEeew?& z^v|40qxr@4IMe|FPF?1LL<_|LknC5V9QECL?DAmD!Y?87Rq&7zfAoW7UdA+xL&KMr zpukZbxf(QV*JmT8ME5&|BIsSmqpq>?_kKYPohbd1+HBUHAm%d=OH)mxwr_uxyVK=( zq21k2n&VFtXFmgp&5bB6XY#9;LNmkr^rKC|Pf${HI1svAx?n1rnp+u*rhFrz&qule zPZ}psVfNkIx54g`WXk@!3c&NffL42gTel}WUdHv%d(J$emf&G2&d}umE40hj&{p!3 zrb%>tNX@`wUq=j*kbqT91A!R{78yJPjpc<6y6)Mw>)I<>(~rIJ$s#?(3M-50SW~>pO{unWf+N{9K zHdP4J!TFJTO1A!}L5uU4y|!DysXCTqTV81=3h3!n!y7@JkO@2>k;Z~V8Y8$-`HB?Y zdUIv~=p$n}p3DB+j_rB+D2{o3=bdvCn_j;Bndv$}`CD3_e2))+uNa5d#keYP${xe3 zvCl2|@=1UsD!EdzZ=%py+HjHg@CtCrI-?N$c*YQv-ZkyssG^^i-gU*vy(krMGa=!- z)CQSi_Sz-S37zWn0w9K7B?QWsC(QT$Q;#IEYRvFdjINdHh+$M&<6( z2Mb3#N!R5t3kA%Gv1VYFnFgou2-)O*A|Ou128((8{hoh(hfdS^Of;%+8@ddU%nFZg z=M^B$r)D4IsMpUm30<;uxu=^guF@@y<1i067hL{Ew=Y|Cib_mj<_BV)> z0P`%WV4bQ1)df&e@IQ2Ae3(lc+-)64n#Y=6IA#U|e)3D^ZvG3pgat(ZWHyO+QU1|; zL~BDQtF|@MpHE8)UWPtn)qWlv8hQ{$fGM`MV3=XNj@j=0lu0*YHOZ5x;A+uli@IqD!dxZ=ECs^a zavo_F85$bs?{@;-^b`iR+uT(a>p@z1^oSK5*>O0Gy*;G|3$OX_XRK%|4F?2Tzh4TP zH-Zsrzd83?c|lCKmgIlP8g}1Bqiup7ibTkocB5h^ejQz1bFqz$bI{+(6I+~XQ~f)i zn*7*cq^Gc;=mrT5?S=&Ca@N#(;sQSxvj>xV(ox}`DZxH;w`F~Vq-h6A@9SJrEYQ(x z2XcMS>GoocwRj5fZ~*<)C=-tw8HJEX@}|Rmw0#YXo*2P*L=S<$5E&Db;Z$b<1e$|` z!weR<;Hp=nfJ)6iz2`2WiP?k#I6J3_H$z549Hs`~f1t!BE=Z|7(HivBRP@ZJa0stu zKWMYjK6b2?7X1nbylfRW_q_R@nwFw!{I|!8osv@>8r!`OJD`=imM!`Y3vT=f#kx@d z4ke&jT$uYs=1l+~sUU5EW0AiUOjjPfapgUC&=_u%c1mh;V`F1;dLz+c2Ko|+Bfnx} zuXmk#e4K1);RnGvatcZ{GFLs=PD_c9A_MYxUGN^kWs|1$)fgVVQ$rM>TUhPQCDFCh(q}y%vglVefX6^YUItjNvgh+$umpp_b(Pcad@_ zGpBSVGG%GXXh=j{+;vcEE3?V7eKWhS!^r`Mwf^PHsJ0YSxG7dcBNEW@6c!y$aLuCa z-P^LO(c#=_an9XtBgb-^{3ows{7P?kDKQ_mUxA%QI&V zpR7BHOoXhgggJG}XsM%d_=vr;@W?J^Qs6BA6! zJs)c|8D;BJp2N)KLdF65pcVp5L%ONBZ_61>z%mleKdWsl0{8FNNZYH~-#0k%6HNW@ zd_SIN$$Th`J9xA2ytb-BX4^G58lA!oYjko%6H@$l@sCmWTh8mz<#xTG)8X|hTUgLA zNndAFBnomhzU{R|DcuIH$HBr+oH5=n*e)sAKFXzE0Lt;V4_Amh;KAMy41ZZM`owwh zE6QfmetY!PXhuatQ&S!6qFWuR#*`f32s=}?d9in*X6+u3S-)X94?qyGJj8HZ(krhX zx>pHN-0*4}AmWc<+yc|Kb?@4bg@9I!Z>wjt59kQKOh&?P@|YdErNNpwG-n9@4q^E` zW1$AfgH)VSh%za+N}AFbTjrw>=-)-Md-`LywnR+tR~*~;`|W|X<_P~6p{Tg@!EU-s zgcp=bf_MCAvMK0CbzAojZve@@7C-GWm|N%o*@{w<*B5zye&QVQ{v770maoR-moCNCuehIuo#(8|^4QD}@6XXl4 zaFq07e@bwM*4NjQN!b1D$cTtbuzovX2q?gboT$@Z-`!W_ISz3iMn~FR`kyeGPaHuD z9g7lUcK%bnQs1uI(#2IsOkg=)^LGXj1tLgAu)YzN{Y`xr)?u!rqODB@m-s4_D!8kE zrgCFDk)z5_lPM|WO$wAGpb;dtY1KK-tSTtVDs=xjs77q{3qd&29-D88w%x!vrm*7pIwI(keU2 z%${rFhS}?}Y+%2QO-5(7biwcG4vWXV@NPeVl@;R>&_qQhBz&}(6MZ{-VB=7RRZ~JS zmL9P3@T}pJ-un69fboefq0!p!FG6`zqzWGA^AetzAXNs^=N$o;Q?bVk@~!Ln&q5OK z-ubmO`vThyh*3!I z1oMNy@2YbYSgIW8M;+d(znmTY%)^3uXb%y8zU;)G23X+T*3VGg1i}_vHC=9B^zLAaiv;Up3k^db*?4h3F6BtXE^q20_M zL5Hb-^QH!S<|g0^!Y3!!mHR?f8-zcIltv89z?fK<>p={sKs=B-GD7a_9+x5vfNFNMGWpYez%gI|P zFJH$<92t|8wG`)cM%F2HYd&xu^zwuTPr7;{5 zHHM$ldiRe=wzF8wk0vc1s*=+ys1=hdJvVX=fVI$)x6FLZZfpD#7!lNB!#0 zxNs8y)tI50sXoFJ#H2L&CWDxqb~Ir&Y>pmk$OpPG*GIUbxgGxzKp7~os@VV%&qV2Z zYGlP8&QR!m{5$PuEdM7D>9l{+FlC>MFOP`~XjSq#_fTQoxY6=mdrc*jrfY$@N0%N# z=1B1DXYR(HD9R-=EFfHn(vOfLlhO4O3gV>`0tNUl`&D!Ef=k~fqv$|Y;y6))5f~{< zF`$IBzZx;vKkFt=4W8V}^v@zEHYgaoGzh*gaeeSMhR*TRGkqylqQc9oKUl~oqpy)` z7qv^D$5a)H-|h#7bo$~pOzV&=Y>N6Md|3tz(L+ydyS{hh>c+`F<=|%PXwF)ht^JG8 z6a_VFp# zvK2l4NU0#ZFmLPUi}MpIDCSpYtNlBZ<5)~FckW)CNp2k|ow7dhI1_%Kc6y*{i8oC= ziQFB6z{Y4PJoLjnZuDF?3=_&@m;g{KLgO5pq$rT1fc{u#1e5i7v|I|?gIuRz_~*UA z+mCZ6T9^YzRdaN`CT%)V^&+_t3nf)`z`;|lssvotwox-!V=X71q6X$gOna0|fJh8P zs_;_v!-JRrsB!!RlF(!-FKSZ$eypREy*oZ+G`bsO8>>VcOj7feR(+mVxbAurueblX zuLqpuPJPHb%_jTP7euB=7l&?41tuWA=v(~rW8csy?p$E5%xwTo44_syUz|r(lU!KX zKI(m#Rp#`U_j#i%fIH;A7cT|c=LskEr-r$p+_$Yd$(gO7r4wL_8XVNl}^CHcD`WG!K@1+Tmp;_`_Z8HuJEo!DmUYF#Y_!6#|@ znKcg*FJ2?$D=_pN6kGws&T>2g9jaz_04N1cm-opd%!iHRoOdEJLrBO3THr{VAnWD} zzo^7sojJZl<(#lpf#Jf>6i(VoV>I>3#}UAqR?Gl9xdHgVOl-zy)0Kfjy-T_-bdGuO z-UWT$9pJ#vgHQ%cFQqa4ZVTjqv4Rfod1K)~XNrF!*FU}?zM8V3Vaz{DSUJp8OmhEV z^jPHiDIX5X6$w!=fzt-|{>JD;5unDts=)K}->~h#Y|q_lsn2Q1PRY92K+)NFHQBX? z(c^NY;2j#UGvt6ODW6-IH0RA!2%7a(m$)c_l%rtT`4N0czHNsNc#+dTK7~UvTVUjD zgug>2E${sWWTI8r$AonP)?&oKV;v?e&$@J-vwLuPp-K7Wy?}{L_rFKygNEFsBT6m% z2(=UUegdf7dG418^k%34qe@+xFeKXzjgtA4=i`mkU_fxfbfRk#Wht7zTJ(t@9E-fB z8@ynjH*Ws-HvKdoR3TMI#KvZhJi#>~sT&)oez-kMxe6{)c#*JXq9PAK+?%c1W~4jw zZ*8_I@I;&8BfFxp^7f>)kygX3EuWCLEOD4n>=>7f?kaWLj!u1Sa_$*`Q$Lg34LYMB zYzYesv+9HdDFbRqVHLM(j9b5g>wHz$S6urL?DtVmY#&0P4#>dnCo`P7Yg+bSr@Wy6 zT+w|D#e?1y_CZY!-^IKootO5|9^Ihw;zhT(4Mb+V-c8R-X(1gCmCZtoxdhO^v<-9on6}uV?hoJs2=^x`6s64 zS8F+(v0wXOFi7~A%(LR|BFD|>_wH-H#)wMpinN5I#;+ReFl+AraXpN*YItiFDt>4T zn}#^U#2U@Sn1&zT#WTgle0mYLvI4ugo=y%Rk!#Y8l*isbJ}|bv^CtL=#F2?tLHFcI z(ZSz0b|LWv*#G$iesP5Ce_GK1)*B7}02Dq9$)a1CI$Or{R{m_?SM$c~f@jV*9YNP> z-6AMxj&LCD+`kOY6wus>Y>7i;C7B#t>qnSYUs}_D>5JMuL{qO{ui+jZ5fig5{oeDV zpzTeHCPL=5!t_%hip09v0njLCJHw6w*u?re{jyNk81b`gkc192??qpy=XRLo(jy}s zpcSI2DDWi#KOn*-?I_V{NdmidT_FB!%paE)RG}x^p-=#Fp+NXzxR&U4 z&5*@-H2n#c^1%ZEA)ys$Q}~I<2b`bv*po@B+dXfNoI_jg^=pn-!MhiRZ8usqdLh)k z*(9N#b)26JHn1Zlai@pVnn5yUNf9(>I4zg@z*iGTHFSfC=YAcj?`m3F8-AYuiR-{n z3XFS0vinkc>UI2g-1{ze56O~sh>x8Ag5L$g6qK>uqnu$PCE(M+jdBf%LJq}!%lQ{D z?9Ko~_mbR@##I=Zgjd~C*?fTm>(OrEHlXPw%1A8<(ifUr)fNbW6N{>NYUD&m2oLDT z^nxwbpa}$YL_f|}0C5LsSv4l?{nL91T7mV785>@p`<;_tP@`{Q06#@p2?LEsliEHP zu6#7afS^gj_`x82WCvYAl~~D=pjjulb*uNhy4cUD`Em?F`f&X>v}*kEVV(!&-N@f% zQCv`HfGG?_oj!>ry$R>PPYh2%p6>~T*8N}oJ1^#64w`4g_2Vl7p}-F7^;ENQL`~n{ zoM3_12&+zu^KlPQ5{h!zFokPzhD%dWXee%eFYfO6+zCWi64V*W&L-wsHP3kuli!QF#{lsplJ=u?y%eQZO5O7%0_yc9n| z_zLR(g&Z|ZrbbcpO%G4Sh1s6LIK#YZd2ZtEi8p1_P;cS=crMJNyJ2(}I0Q2BkPV6e z-lxxyS3#*uPj!$@`{I6rMl(Q;xEx)NW}l1+&pMjXH5#Dbr&wRSSC?Zl96&GLF7D{5g9BzVrso}oRG)0xlNKxxz2;Y!SY7x4nzcca0 zx3H=#NAaUPGJbglE#)ogq~>Rw!xJLi4%zVi#83l3Lbasw3xq#a)2djJ5^VU^P!{5u z{QQn>#r=<5?AlHwq;4TOS^heLP+TV+ffH(89&mlFuuyx-8rj9C4M|NgWk9({sZ?`&~-tRT%K~|;+r{8!! zF!t#t9wd-4*6z^r^eirRc7Y`J-2fyolr@>yx={h1F8-R0AZpX=Z?z$5e~RUYfEEvY|G zK5K_G@R)*@)kJ@da$yo248+OdR@~wRJXNt{3kW`VQAZ67hMM;R5{W8hmPg-ZqR^3V zrBxX({K~X2^JRb22G;rC6@T^YM3Gz26wDY4LHm!0nS6=l6Y_~l$W%M=*oRlg(yxVw zZz!~HHIj|je9yQR1N_-hamSDH2|_AxS16ZRDvBYkvXI|bgpl7OGVz*1#q7Vav$Ji8 zbLsNctZ#$=>h$ZZcs?1czr+kgIX2m|U8tk{TeWS^OHvs(%2JOX-&7#+)&_FJUjUZRZtxrU3n0kB z+q!5AloAw}YQO~da|W0O=4V5qfWYVum$+vikVz21M{vtOHNR=Nvf6Z?sYC2$gth0I z)x&L>x5r~Jpre}TMq^&8mhDScxg!hDX15f4?4L7yw;f3-c(Uw?E?AZE2i!zg0974v4(_ zT;b1jcsTU~e8tAm1~I3v+q5Yd$7U~v>B)$zvyp%xti?IW8py~g_^4e&U>DB2l?#ss z{N1)EXD~~H>nAXrD|kiRNbKtj6M@tjp+xBvomuC&%X$W|KBGr}0(#N8U-JMHwmoEb zRv!Cpi93J4wddIdIRbE-V-wBDXM+uiRBb2n8&`nnLr7Ed3%^=)ny$G5I9mNWai3{q z0sSuQ3C1ap)6f2*c`xa;JJ=g4pg*H;< z7*wu!&JFeTtR}kYpN7E7`8~5~lhm!4H)kv@Et$@1@nAfR8TcOj_CEaf^w(x$&&|Yo zfrVeomchp-7v0`=7SJDX2dZIR!^j9h4%@8ubujw_SI3$39PEiM&wl(^t)F=!_ z)?#D}KJc}i*ju;OKLR-Dlh6%=#rroHa{eFy0Yq0A_T9m{afJ_x0S11B5D{Jk^;8Yq9V|s=|G1$=KcJhL0u} zG1k@A(j+BswK&tapa`M`0HfgEy?gge*Qdp__tVD$0nzr|7I?V-jIb53JIu4N1dC+Y zVz{q*6@nvjX&BS>PSIsg;Nto^a|quefG6Px*#`%r^*%>~^*rlevTO!{d;$}_^Ydt_ z=U<=MOO7=0PRV2zcO1}ERmB|K}7O!;c0yv*6pzO?QE!0bfH4u;z(+J+6|wD31*sp1aUk88pQ!Uf`r~ z#J=!_nbn0HO#QYcvCHh)yEous)dU6_FKi011Hq3h+A~quZsF2It;JTdIpfR%2!o)T z4I?jxP=wWtuN33MAGQU7zDJI_Q~ptZ6JY*>H%6$g zK)!cFpiSOOAKuI6@n6PbnsSNRh!-XbL+|ruh_eD>9nHvQfAu+Z*c;vk8>6s(VFRs0 z-`~X_fBoB*#t01%r%*6eUuF(L0NCF$x2Hg}gDmTiwWwBWz5EvN6edSny8I`aqZeQyoqVAT55I zMiTHS1WnUxsvEf8+tyZpi~%+^$xmMI&4~%V6Sj2zd|Ac;;N=l&ignckWROY(EJNPl z5N6=r{7O&Qkv-YLI{C-h;bxxr*sjkoFWWFEU=W2 z7kpgE2XXQ)UKmJwN6dXtrL^S}lK@PmmQZ$;k23(S-j*lP?ea?WAL5)diN0cab%|M~ zLGB?8y0s)wCqzUnF`TpYjL_Ez*tfdKPl;Hd8ICDC)f~If^^>MP$P=>g_orQM{wvTQ3XpFL{(2zb*36#6WdsQ)#DCm{cLbswq*zNpL9&D*&bbB~YvoIq z=99gn(sYJ3;|^ilz@jYfHmQhr62b;$qRfl zE(LW3eAt51Qe0*F9fF>~yA!)*@ z6LuA-GQl1Y8sW9w=E04%&782?rZpH6_lS-hp6;HWCB{eIfuwN?!NNoRwVH%pfynvA zAKg&AfNS)S=W0#rOp%tR<{?1YcDVySK0acw!Y_v*YhIQXy>%J8%ej2S>2C2i4JMz| zO^cnDkgtw`;bZU;FtS<;Es(z4ju5OWTLok$>so%)UV~TZ>6nICe|o0ibKwG5@5#;9 z(m|JlEcfEcL5I=#Z4^`koVC-}>z@$~8{8bCa=w259tJL@l%_dc2CI?h!MwbnyL}Yq zaJwO$Rr|f-n{856DC##-hL!m|6h6H!0|PgF@g2c3;9Bl5DpYSZ+eo9W+Rd-h!RCZa zBtw{77!!P)Fx1l0n)&nRz`bVi1kuj}1NUrRb)vF##^}A4-j9|$9HBE#@j|CVr`C_H z!op0TBMqJ~VS5D-6tKRpE<;Ro^SL~`7I#B@6e#oJ7sJipg$3<+eEQY;<}1vY^1&%+ zi!OcI3S^c%`Er|&<-|Xl*~7!5xU8(It!*7xBQ^M0giiwIzQ4b}>a$#dEhn?%OEcEn zZq57%Mn?(oR@%OOpR#V*&}iFt^ACMnT=?O83Yfe!!1w|7Dx(XscUXcw+}!MamR{JN zCe^&P=tygih@VI1N`+qEU@409`RN@{&8)uH>{(Yl#?z&Hvr&)Tz5({pqV4Cwz zAAxx)6=1*;3Q6dP5XSkK80Hri+E(8Uvhz<&OdJl}$vr;4@35e+x(Y8Eb#BxH#okjH zJ90-6|&bC9YF*e-jf~?%I{HgWLmh#YtXgz#XQ73SU%y<=mckV1+V zdXJuyUAAIH=EhCbf0ql2#gy;VWlK=@FflPjh-~`Mtcu?+V`R`hHR=(CG0L|5}?1JTQ$QrF_ahM+JF#QXv7o zs^{25ZFWjwaNPe*?)u^Y4?mUIg-x;CMlNn$meLTdEI5%3ebk9lIz;tTt~ZmYA5BsKN#b zyf=DcMR%Ep4*&PZC2~3Vb{B@b4{vmOvOvVRx3a2DC^1lCaCjb1$eA z+`^;ceM*h~<#F0UXE^7R*|!tOgUkYX7cf)V5BQeC$H(_P?S|qPwq=!lX|3nIygt~q znZk$iV1tv$?A+Wn-Yw>`w{JlXX?Mbn4FmCN$UQUNH~QPIlSM2*avy3vCiet1b|8I+ zZl$|!*|ndQV1R+=4;-R28MZ>Xcnw;P);O*N4@^cy4i1Nz0@4&>PjRgV<#Du{wUt-+GplJ*Ne?;#KSpS?8 zt7WW++!Fw>er>uc>oUUe0NaT_VNZ($F=hjs>?$_ICP-PW0Q;OU;0vnwXzNH+!oSwf z9l%*63^gQJCT8Zxql3nHFp>`4L7-8v5oIvUA29e!tBb`#wVvpQ`BGuxZ~NTADw=@H zTn6VAQNaOO`As-5?EAWnk_ep18#A}!tV1U*& zvrpx&>X0yYuC01%(ljDX6FOsmGY&=zt6(G?j@Wwuzr)|3c=+Kv?Z? z34R2I7M+79+%4E#kYVYd?7fCxp@fBUV{=Jy@gqIGt(Waf^sB`aXrb{pAU!oFcLQU|XGJRE+^ zy_RDH8>so<3T&{!O5NV8x*m8f91N^3YtoQI_TdHEO@}g2MP30IX=>8Cb?tLnc9sUa z>+^z_B@hp_HO8%CW_BJ)-2T*i>IJ&&B=?OOCm=Y%c@flj4RaC9;y%nTJ-Zd`^Q!^k z267BhYi~HeD{=X}l40pm>V^%0xYv6&sYH=@g)4Ctqb=#F*sUVWmoK5Z$G@RtK}5M2 zl=UatB{@S$yFY=+LO-#pN>4vKpAWPO)Obd$7}`RTV@*aY5o}^OB)!md6F_JmQj|?# zats0z@p%Bb1ZOngCL5m%^BOK(KFP%E?8GYECQym4d3Bu&3$|J4!Kkjh(_A_lV0|+F z6vF1bx_VQJyYwLCm$w4pTf??c#(q_#F&-T5Z2XsUKH8qdW9$ZR`o4Y*#>&CUmX^uV z@FF~95TGj2-feNTY8#y*E~+^4L}>Q4a~c*vs~~iQe9W&i`Ta6b`EhxPv@JDt86sv; zHm5{<2sbzP>Ud2nkF_P$$Lb5tw$z@z8pj%`VfHHH9_E`jrplSDzV!EBMvCDBspK-> zLmrk7$*{dEM+zd7xXxthRM@V>DmGmWY87g$)^kO_iLb|wom+viu|@-Ra?o%RqzKIFFrZ{~D|T@y zfJ6fxa_e#Ut0C}cWeNO-ou)g<5?x6DA{wMOFruxaW+9QQVTP!&CoRn~7;CDnd@M>K+GgxHpYNafTGGK(GC%0*A<(iNG5s{0eC} zh5zr({72L?96K6R&mt%!Bp@SmVoXf+40;uU@W`#!VMc>(S@k||B#VK8fdq9-Yq`Ny zv9m8w1{{0w?}CRs?7hMD70+DDjJHcjvqa&#dy^19rjw`{;D5QvcYb08aHqT11!y56 zJ%xuydYv}29Z>%n7F2JTW3xhOWO8ciARr#_mbr|Kjv#v9xZzrteq?J}@R76)@@ds3 zE#Y_J(UOE`^1OiA8o`2?7htdCc9sHQR{fyd)BsO?k;N7n?+$kG4*Y;qhT~z+?&Fb`VjH(;L?P*_H+N)O8dVo&jhvGO z8TCl*|59z9BYqyJ8dylN69hfGZTp9C&v1JOa9C?2_z&QSTmHnHh6`Rv`$1^H9o_+P zASQ_@^Gt91pXv_{YK5Nh3RKA%JXfnYrJeYoN*c#uGAwA!jyjjN^IDoin|5X3=|&AZ z;+;mY=RJq$^wuD>RXY#@LT^~wCo|qB=LqYE$P5>N$c;7^mQ)bYh>Q=rrP^3(&fMu# z*RVIkGm|e146-ws{35S&VkfcRCT_X#vphC-v|cS(qkc*60ezazN_LmQV&Lr{L#&6m zE8Rq92$?33*P98XhJ)}CITjj_^}Cu~OL~mmBU^tJEX?)GsF793&wcy9@A=wUu)P#G zj4udk`UVZu6EC9tjZtcD7G{fNyAvTWbkx|GkP1L;I#|%hiqz%g;ZX+(>~1t;dKP@6 z=wbT7_u3B^sm|p3ky?6J=oO=&AnxG32?86aR6pbvZ=Awx#V!XZBYaM>@UH^8W{YVVbx&2N0sbL-$I?BtS_ZTxL);%%?6Q zvRtjG*mueHZzF=-6K@UL?j36N_=+Strjb(`gtL=zVHb#mGiYFxw+`0t#P!JJVW7$|<8RI~u2>-;EnWOLUN2q$8a~wc zsrR~yg=sEg^8kCcrGTG~S~{`~A(hra_hi27V09Xn0Mx{$k42-WXliD6xy;A6efuiI z?DlWx@k_{c2Ji?k>o*w-yG+WN(0gJ$#MPwuCri(kD$-Z`^(me1f{WYy7jhdvJoAH1 z(%4o=8pdEq=Uz z9w*)0|0h=EblLxL0Wh*8CIc9cgtzFcIp2Dc#Sr)8F#THZCKtMtPU+3Ib2)LhZ*MR< zZhr|xEgW9EjDu~*#?BlrpNNy9(ay@C0{ir7GYE@O@>vs8Q#Gh{Et-+n&hNrNl7&*= z)Knz99{*dz`yWYf zT0sHEtzV;*y1X#^zY9}JvIzOFFr&^Zii&BxNzx)Atr3&cVyFG>3SYfRE9TG4&arQo z=6{9%e}*quv0I0r8xbAt54};|nwvpEbTBw7Q8#8S&Z5K($8fO^UsXT5<=msCkxwx! z)s|aEb79~fQ98E)^zb0quYm%x?a!*OQ`-?kG7vGl0%iHS-$PNUd7Gi8V?S_e|zj{V0v;GZFZpZDPkC`|fetU>k6kCeme zSO3@UsoCwo+h^65t}7I$a9+P~igCXZ{G0&fpn-IT4Np}^he6>2tj^1^ z$s~j4^1)^Mm7v{$b15+u%SJ!4-=SG=$3)T#>O0p zk3{vMLBrC7U0(<7(^p7!zd?s}e7A!NYGgPtdG*wh%er>$+MBdJTK^vy#LikJBpwWJ zb~Je7V(+XbLlSH20rLsk353y4g5eZ)B zOsBh(1@e;d*TEckj+!m0*?NnSD=-y8o3IBVK;@SAWX1a~Do;a8 zJ^uT59THFT<5MaD0_9K9NaAQ6diV7qRAIyt1jIgp58(4?JjQdeKwVggcF|Jq#~UDM z-@kvSTtc6<87%%=r#Gw6-1gvR?UL7&!eLuxbzQsYBkU;5&mjJQ^uN=E2E~e zisjVQ)d!n2xC-!vkP$S0C1rvF$`8Xbz%+Xfx)V1iltjc{>pD1$Y9E&XBwqro&+gN+ zcsgcuPmakYMklnd>P6Mu5iVMGV33nfR zw^%VZ^v6c6*qpsj+s}D;sDLm0w$R?_iEYRCTej0Mkj{NN9b>W172loAIS4D5%-^v5 zJod~& z#&{1ktzZzUgH}TJvU~q%|CktsqNu2NN5XbJGOVEKA1FMki2!xB)B>#mm;?vL#Nu@D z3lc6&pCAP&_F88Yba!__$@$t@WP)vvRgmfFo5(ex7O=#{ro)(e^5%-E{6a;9K*0WEM*!Q^e<4q zq07$3w#BpyhKQI?U1M=5jzz<1{&TRwCa_PO49H0BCAG#X;xH^E-y(#8v+=~HZRPWV zyAaM>{xt|LkJqOkhTl_qpRj%7XHSzmj^#s~h&9orx!E<~ew!yd;o#G-L|3!#3 zm-$qmg+GPFbn!C4|Jo5Z6s(?K)=(p_Fte~6GBHWW*j){!F9Z_r8q&8S0#`yDuw|d_ zk(FRvRiP+;3rQ?711I))MGif0lMk#YIN-VwVh2x%A)F9@Yo>oqPagu<(VTNq4pEQ~ zrxuL0C*D1%s>}f15gF+Hr{(y?X!TYgJP-rJ58Elh_32{#DybaklOrNnPc+^qoCoaPdlU(p^h;1j9 zK84NAc{2-0m?8rAfsnnQbsXbTr{{2b48ZBUkF242%k}e}=&~~?RZ&J>*0aS~XTka5 zF**iXL4CQ`OTR!IjjsS!?f^)S6x#!Ty%wm<$T%k>uLyYn{F@z1xzw@0 z%?~f7q3~HZOvQts2uhmZ&PyY}4Vb8VcYCbBFCgafHVt38&bDQ&!y4oLgj5E)zJEnU z#fjf-9UWEH;uaX%-@*E;6sy`EH9Ski+lV`N>f2#O$beSY2H0067Cdb6jqH1^CWDxd z1(e5qDp2GRiwF#hj=gw@v@D8G120evJji1oOdDPMDP7#%X6Xd8A%@y*nH?IwUjUh- zo4`cXFreNpHhb$p%m@eo$R%OKdjegdM|r!oxMaeGVicdKxK=>leUzr2?>78)iHwYl zkLMA3y9HvmhRd4sPPM6VP=UfINc33+W?-H$ghPM1)f|~G^_JJ<7juOB4DMySe#bZw1}*5aytyU+)8BtL$r& zhMu$3#iu5K&qFrsss%gHE=`Ap0su86_u1S^%(ch?bYB)_&9}k$= zhOLiR`(jNXS;X~Bo~s|otsny&F|qMd^R*h8{SGQnX@dz$R6=aYF{1FmMA4h4SIHK6 zVAa_i_ht0PMzn0CXa)kDI@;dRaSuv(!U#d|M}=putF`|APc$Az@z*D(r)$AdxdSjF z&lN8c(KNfiL3%$chzOSu4eDdAZxk+mE(4Yt66%Ana|1djSBJ-s#j7T8}n1OR8b>qf|N>9G19XWC( zF>&)h3AZpR5cI5hOHi5uZ=oMM7X9z5OC_X#d8kHU&X6Axf&*9=-RP5ykwpH4IEk=V2ju!-QS=G`*XY2=-l#X{E}{Wgf)-xthc0L$@bl66Zdf28!loC3TGC0^RZDX2ed@K?t| z)HHmt|5ZN%;B&cUoOeapfuiIjjP*n>E)daX}QFSUA&~ zH8;g%U)|0448=j6jSQ`HJNZUO&z*a+L#+gljq)7Y{j(ivB?zk_>}Sz)4+GqQW#AQX z8Ya7?X*9X}he9%R#?qe`RId4dM7;-C&i&gzei7oPG)W~&MX3;NAzf*g#uaTewY09# zu%cbs(Q+5kPNh+SwN|Nn73$MYQD?{nXLuFp8%=lMEc17w;j z=>~>YR#tk)kDu%Da0Zyf89MAmY?iSPmHKl5kc>`y#ZT-Iwd3h9<%bsp>uO?LffP@5 zQ!ON6e;gb8hjI?u(A~%}Y3MN0z#7lG0%%*l&dm6u>JY>cb6*1sh`tTlpGC4Y zv~^eJD|-I3Zk^$5E*Y+{8W{DgP_XF4!Z9?b^=?NtZ70?+6bdQlKt|)X=^w!oCSO6f znHzuro-Z7xs~meuu<}4jx(@}sx@7`viZ_!yh2)2#*TC61f$|{jMhy)NWIV9+%Y$q? zH~+)h#iwH|RAlq)*)u456Yd0dbhNeYOMNu|jB(2eoVn3?c~`-Xj4Q`J=L29i z-2T0Q|E&Yt?Dc5bikwI7T$TUR{eMXegOn<#rr{xu^@g1Z`DW#BWhXvD6y%mDDwh@Q zmf_G0R4j?UDjg>V3hq%OqX4kW8!VE>=DZtvBCJlGa{A<34nY)-5{@mtUje?eL@}Yt z*VnFGu|h2|mJCv$nxegccU~=~VL@Y47ZfRMD$7PK@C`jIFF%PjcA0YxTJD~^o}msW zPMolsR&s%G8BL6#r>j3WQNKfr3U`|HEyo+X3F}E9 z!vaNnoN#B^F>e8yi^}~~g+qt{h3Mi}0j43c9WuaMJD2yu(C0fCDk}Y*=oXRoq==HR zju?h+5`Yh;oJd!?=<4N|zpg{>qcYzEmM%$Z2)!3z3SV2jHp2zXHsC*+&mI7lhF?*q zy`>pRK}1Ur>*yB8_ZleBUx`c^8@lLRyPW^!bQ$~N|$N?q)IA8;=*{q?17_1)^SN4EZdA~f={Z`lW`I7 zU2*wghp&q~1YD8eX`A1Ga!CueL!P3{gtVOp-`^uKrXj|S!UAG+Zmw94<*h&CmP^sX z|Msl@=byvKGs2^}Gxi}6X_QZp3-r+QbF{-AmZ-@S*Lb5+qAK*8{%^9w*?9_#=jhy> zLTFe4$Any{#!~hQgvKAyS=lCuvPPxc#GC_WM;J+5dz}R21 zaH86!0U1Y@s@@$HT0>(} z@f@m`*30>J(s^Q&Xy1{$iE;w-jr5+OGz{PAOBb$eCC=73BH~0u5xbSBFP@zt^AQC# zo_;GMmamsslb|5yu9Jc^Um4AQ z2|}pK!Dem3$i?UrT-A&>4`%_6md(~dV^id-0@Da^#nD1VoEroK`h{1SB!+SHBm_Av zH#G{LxEUMgjt(tcnUC`BMPwjmXk!k9zX`_>Sj3tPpARwv0s_*3KeY&6i~f5v@yZcd zrMe_>Wo>(p@u%2Z_rpcTuZxR|rk)qg%{OCDB^X>|6r^4IFy8pGk2EjXXCae9y60GMG7E+PF9}Rw08yiJVb}u_H^YbSp>|9e znnn3S=vx2+Qu{Yzfed`_GXwlq8!*eHWW8ASi#%sz*X3sA68c1NkiJ~03aNz(j zznu*gr-qw2H#7owewRCrRiDj|g}LeJB2kfq)Knt%*Wittevzucg|7BpWOPsA>`dC@$GiZi z*i^TrqI#&SKZD)9^S6;(T)4dtT|X{I8yf-0p`mIEYu75YLcCP)WkH1y!Xbmt%UKzy zs0#cz1;-OL(V*gWO8*iw*ti?@QyL{dA1hXJnWfu7-f^dELprEH@d2ZMN{dJHM*QE3vNBuffkDu%b) z?;C)#=U;0x9Q&^-HnMgxlWSAMG7?Q+uEo8yv}I_Y!k=t`ol*KoLpX9zXXOq9lftAB zD}u)#{TxP4Yr#$<$w5U^;1&Brr>&ARK*CDFcp$o(qAz#b9c*$!_U>8u9tcSA)%#UW z+WPtE$;m*>FFH)fOtzLdZL>#5eovk|f8IDY7P5tEeCn2)1d1X~b32|m@voE5FW5H% z>=MNn_BvljNMNxRIEp{CwFyJNZv6cDvvy{PbcBF3CJ=GZEv6~<>b~<=8yYrXeWuET;UQ zJhGV@dh2ZbvbhFNjUj%i0nVAAZB&41Z z*U1YSo0*UEK!|? z;^J7|UX)9>4^NkMHEV##KsXD=ohI_yt@<5;!ytT27ST(v&uyD4o%ODNTTs(~VX-mS z>vRbcTZnGGC|UgMRIeQQ(!rz&DXC(Q^2QM3e(1lkE7scDr*&os^R?XOe%pUMtsD^=pX@dZ~tHp(p^VKJ35Az1>m+GkqoR+ zOo`YR!u{IG$h}VA!LSAtyYD}L4#2iK)^8VJZ(J=%2MtvlUGwB6Adt^>jLVO36h(iH zvXcG<3{4dmm;80C4cp-sIpIhOH!#vow^^It7Bu2T)#MGH#eoKqjMz`J$1%9gdEYu; zCG1M@bQUNQ2PPnQA2m_kk36 z>)5HA*#nTtaXKsU!ga}5Ij$`mm&4EhEPX5oV(JE#aNh_%MThwk$P8cHf{z)wlxUA3 zyu8j8HgiY7#f}ypMa(b0GR6-kalFmZ+uY5_bv(orp&qcmzOTXo{a11;)LLqf0V=M_ z&(D{okHtFq065)Y!O2KL`2%1P>*q=;U^>lB_*C{F=I5Or`_7JlO^9AOxJno02r!x? zRq#5>(oH{S96L5)FC^oAQ!FYx8}ixQpO$1d;!JGlP$2Bmg2vw!f&z=x_%zugPu2S; z<6aonp#W1=RW&y-u(BF&$32IfFhTsJjl+7=D&O+-^;Tf_Jg%%ncgt+N0BR-%kEFqR06>Bf+o-_9YJ3b#yj%MD=!!j9rh2OW0~DiKLlt1A4R;PD1k z)LCQ?y;!I0MyJQ^VH-cbc=TN=p%NE;&H;|Tf`Vn3dJ!K+G8EWk>Y+oI1Vl{ zfo1_44E;96+z3E2On9%rwXwep0Nyx$4u}-qm=L2H5vhkt*qeYlg|j)E@7aCm$%S{oqENrExaSs6&|Zp%2*cp8GU+@780{tluqZ$>L1HWEpAPc`YE| zC2#OBk$(RqxoIz1f;~Z9#4=%M?F)^jwju`*uC3_f4seMWEND)avvO9WTKoUK82yy( zb`Iv1=e`L5em&X39lu{O?E3Yxn=-pWTln-}T}-N3e`=~0g8*Qu5fFG=HtA9d+GVIT;Zg z6hgWJb0dnWX!1)hRWKC4S>tVmiK*$@r$x)wgZKk6Jv31KPHhj;}b~FMSu4gRdP{HIiLvl)}DE5s{H- z3fUB0hmL5&6R-Xoc^r4{*qi4?6sm9kKQ2ICo(eHa`xdyZ86@c03uJ9jE+dIM@vrW! z1H?Q+%yAJ&Ck~wGb|zy{;-1 zVfH|hp%E6?;QhK`D{b6yJvbN>=H&r9t-J8hfDl*q>{POo4=B43MZNxX(F+GcaXt?` zHD5=mDLcjWf^u7kU~i%W!bKEaJ?)TNSzN;pTvCC&0H?@r{JEd&HnJG2tX<-IG27=W zWOndkp@+kolZOwb%pU7H45Q(7>}u*PD;Sg$a;BFiTJ_-sZlP(d?$(PEQwGgG_C)4;bwLK{OP3+1u~HqMRd-SI*5ViuZ3GA56+ux4 zn2Aw*tfsKvl2jWZ~S|xsI;`S zP{RfMw6e-6qS3A_tu*9jVeKl$${XJUdLY_#$?xC4qvGPOp%2xpnxFKA%>$8-fnPxA zd|mTvV3~@x#N5a|DJjyiH6#4-Wi>6Q!z5~rav!gY-1EZv6h4>t!MC-B_boUn`lXv#6dGp2FGkWO%JQIC`-NlErf0jf^ahMExJ?Pp z>xFJ8ynEi>PWz;GXM3sk8$%(xf>tBv-yq1_HJ)Ik5OqB@iX}=M?`#k^C|bsq0aU`E zA0}C~*PXHGo|Pwgd}=1R?@(Mt-c_M}e#S&35fE^0kT46v{U`L7ct2n<)rK`vbGS!I zetDItmNx21A5fsJgQ)v8*`7crtyM3_f|;8_WLva|t3g9pXtSDchpb{ux&QONoRItZjy#W4(5+hAFP2Xd@-{6<$-rIA8e993c6V!n&zq zpDWHr$HxbNvrRlW;-4`QZ3FpLe@4RJT{WZ8;W?9txPB08z0)}ZiFnBD^xv3=tV3Q%-(Yh`yn~@!^its`$20hx*p8e^LDzf3 z&l$K#Sk#U$&=zj=(}h7s*Ad+zL42v{3ezBQ&$^o9PP5*$!(y%ILFPYKE@7l2r2G(} zd`P26zC2v-2eC_tF6akg1he!Yyp|(yo-_4#Ek{5prG(4CEZrRE~El3FO9(iL7cH>+Y` z{i_kK@&ZhcbB@DV4_TYXzP;R85S04dJ|BDv3hJi{3VI86G{*Dyj~%(K*EZ*#JYuJr z2$IuZ8hNeu6@P#6ss-|C`4+G!PZ)b?c-hoX;?~z)Tf33k*IMQ0^B547qzaWd^p`s?&Bg|n~so8ImmS1HS>sHhMS7uN>qyM99<^4eYtoK+A( z?dLUcbK64*>Mng12!o!CXn1JaGCp?>DO+T#Nn8p}?5^j?a*i%mkwSn{+=h0fw~G>l zyu=1FU2rPE3)r?V?_lhtB6lC04;?%_k#28b)oa6R4*S8$B`1U0Qm?5CfgVer3 zHFl#B&gT!MA^ot{IEyI^4WGivr-OXt;83IkF znq7B)TK6Vv?2eom9$mf>Py2AuIS0I-xNW?EZDZl3{W8~a%lRSJqf1rX@~m<74qZJMP8eK{e{}yFtm}l8k?YiAScLAsn*hRIq$H?ur^H!iYuyK(s38%2%#jah)5ZsVwAyF^=!wb>PJs_=IhO zdWZ}?>3h^vnl`lmU%B-wP=vsW4|fGhEu#*R>`xU~3gisszS(!P9H1BHwmtn|EWWNF z1clMod)HsL9N&deVN42J_A5~H7{|1Ntx`5RatRZ3JMEIgBmDdLtvh}mfKCMKHL{HP zd$GX+s&TydA|{fs3CCAX>*)v1k9h>dV|a+{0+cn8Z=!>X`~Pb(VNl|4U5~Fe<6Y_} zlpdS7R5w^_%OQXKp4E6%=&>UG;m_Cz;FdCs?^a1VFXi+@VGi(=Vg3F~#5g8o!Y&KzAtL%B`)BBm zQb@?ix6UPK;GIe}V7N-iUbdFIMxKgh;Y(NV=qQJ=kNVCeyT-(8040n)3$9;ZkGd%o zs&aZ)kXR_~|DJ>oy(w)f+ zkeXu!Mas^8v4ES+ao;6$Pk9ka;?Reo?=KyF54ECKFpHcVN`%_JEOvX|(L>gjgpH7- zuq8LkRA?9r$b0UFow6pOV|W&Upp4aCbi9kv8f~=z?Du9XNjw5dU5dPj-Dx(Ws)DH# z1H~7XHqtU~?f>PB+gct()4b_+lq$O6JL`{t`4@!LJd=`I%Rkc_bEb|>FS}w$xV8W- zKVTcZq3_G5PqoEp@Bvt|>Aozbp{eOgAjDi8?U+@L$q1863q-fA-+qHPONCa@IS2^x z*suo#2y5^h(3syJ{bCCIr2pblUsjBJyzmfOtb&{{bL>;JOY$X*i&l&@-;!DM_Hj5S z6;+|Z{BZuu22M_!mP&9l3&(s+?CmqnwU6OV*+tlue^oh*te}Hf(|K9p6hR$OH}-qB zaT|4B8GktnrGdh_VmN++L042Lz7G`@tB|8Y8_G=#{d|2n&ghv#y${xjKLB6tLdDBV zv>HrY<_m6`d=oH`;HD7Sn z)(4p{*O($Hk4$Zpr=WCSgXf0Id|ZwS=R+e}bW~K@yQB)}aEY5<-=kwuzqMSBadyVu zT_3DPbElBS6`tDh2N9J;85)>fNq9lHdO7-#?8Mk4aAYz)#8)caf2h)Ggc{c1K2e7j zc@J1%zxttIbAB>nXKj4~@e4+it?*RIY0i&%g-(Lloq)?rLT{r}<1$-6Psnf5qP_*U zDM(C!rm_lSl~X^y<@>NPD5<{x5`QL7jJt+mLqk`3#l@uyD)-os_{Rw?_wIa)%ItTk zPg5`j6wUt1%AmPpai^xEQ)<82$Hw_E=m&dC3ecCa<3Kp}ORK!ZvGWvglk};=1?%Jn z5H4&KE>0FVA0RC7uP|>Psklxq8P*L(7tbINbMFzva<+5p!%6A|VXVRdi4Y=fb;miC zBVHZ>!L<=fwl^ov{N7HzQvpVZ`t+(w7)wuQY$K6#j1gDWl1V{ zLuWH$T~KtbvqIK_?)E?w(tl!4R1K_+v#S}~8`OxtiCawI_re4yQ^g08*)({;hCk6& zS9i-ksSNJAzePcrcM616&zqaq0i)E-l*QsqNS-> z4K2QPUp0i);96b+0}h&lJe^wn*(Zs(P|6{e%9=O=R0GkH{q+7;LiNFCJctvg=*t#- z;`F3lta7m|q%jg8@_$2i__}GLS!%FpNmnU)(q&+RNfb=;Kw@Q2$RtjE=ymy}q-gOo z?I60L4sQKXQC5t6223s{QYvrQ00yM*RLfNdiYi+21-6TnL$G<6sNyyZgG7iJ+Tw(Q zi^}Xo7tw<=Qiw7Ny=HW--%fHv(GpVoWxOVi2Wp0p4-dZvjMaElxxwJ=sZa5LZF;Og z>Ix731YYuUh=b9z6Lbl3Scd{d)QQc)tXkA)Y)^y4wzVgD;@3Hug7eLx`?5X{U)@*m zfRUOOh;K*Tq>6-mzxun^g+ctk|84~g$O-Zk;#XGzBQUZ)$X*C;w8Q(}X@lb~xzXg+ zfgsQ3oA>-3{JprF37*>jqm`nS*g`1G5Vo%&VhJJBet0{=H(MEw`6>$v`t;;rtOxLN z8YT&d*AksOj%MzQKU;5E_YA`3p}73wTCAU1#IJruBwtYwhjRubK`~p|1fn%jgwq=c znO0?fkG2_E&Y%y=^&S)+;h2j$khA7n>1uthx}u+)aaQkB&d04WF{+o?yT3JO(PtcU!k zW_~RbQw>@?!rw$*pJ|k$X^F!%5Y#t+PaUXP_1Qn1pPhxhB zHx&u)Qko6s7drNN9Jjt-ap=5%kbpB2F(sUi3Kb)%K5r`MjsS|5LoW7bb3vR(NcMkJ zMI26omWxJhT)kUuXl4fbf>l2!&_I~Avpf)S#!`Fj`Siw{hW>xtnfv!MhM(v-Wf_VQ; zi)Y=c*{MDr0$%-UOwCfE6HqDHIW8gL-uDly;C%qPg}SNX8Nqg?-{=Wle$AmPBrF#7 zjuJ-5<(Xwx>a(nP#O1X$5aovVfG-!|pO7zoI8vd{V8EOE97+_99vP(96jr|GWM{t( zfZw{BY*#|N9O4*Qji*4eNud7*ZI`_$gcNiRk&Q&5sxr?In-dZv%6|-Xmt2>SjM2f^h)0)FB55X_@$Y5%(|Sds*EKN-W<^ek>K9?f+&NP>IKyFl&FW&w^oP=2whU+ zP*2N0tVZEk&VF>oN?U!kTR(n3YjRpG;?ZmhF@gS;Zm%<1cikiI&{tHWnyHEN5X;5J%jHiAV=1KHbR%nX|^qX zcB*jHJ2acmz^$Zf((7aihQLohE*Keg$Ze5fgi29AnQ3*DKpT^to){Pj*xs(rYxQ*oK7&xzEy-0bG5(#6cPU2dz3+@RQHmurq~OIPD!h6 zwf1>V&;-I$@7dcn%7rm-A5eCN&gwYKZ@`-1_ie-sLqs*;ld9r~x!So9FQ}d&ucOjj zd^EUT2Ii6=t!dvI1iORt%)xdaeAv9qj2H#3b;b#W#OMXS_YTR|Hij@?*=9X3~(WoF_5^^da@YYGb=~Usq^OuzkW>ceEAs$&y z=4|s)!}47>!06v3!%C{}{fXYaKa?Laj%b5S;0@PmLa{cvdE%VaMF+Vr|D7-eX>sV7@ z?4vOSBNFF$1NWz0YmnjCd2qkq8IE|aT;)*w8L#bfj5<;j zk}6Px-J6tgT_-#nSb@q*^D+nWk8c+e0`SR%#xo%I%n^L%F~z%nU#|X0@&fl;wq^aU zNdfQn<2aNSghqtq)SR!~)S+YE#D=?uM1&3v4~twtf@st?lAgQ&>r;lqjwWIod|{9N zygC#7B1ZJT>9}I7`33Rn zX69NbI@a||Yf>{1cn}o(h?E`Jy6yPyYN6=6j2?Pz9)ORm1|Hi-eJ(%!i#CcKzXWSW z$0{8!FKN)0Jr#E5<3)$hw0~B{KriXs`dIEou#+ZV2)28(o}tAM0y=Y<1s zf5fj^t{@;#4$SRpfV_zlEpHN^tTGl{wsUc4#;U<4^ek7I!^-XjR9^cm>n4HIs)G6E zGTv}^{`-nEpp*m~7+YWur3?uKu8Y{v;dHATw-Zu742}&epedf+cgAZna zR@MJexQA{8?O99aHi7AF_^#FDO6=xg94(nzUtESS5{stX&pe|r3;zJ5q%*gQ4YadE z@Q3(xGWq>R8A1&}TP~jp5Xrp5VC>3TiA#Mhou1*i8Mr3(R;gcI&|4=ZCtH2UT8F=O z4j8%>bkaqsc%NzXJ8M~`cofeg#F15RFOr%i9=wN##=cv&pF0b-2&zj??EX=K7UwH| zxjkR)XFs=U>^34W-KC=ut;oF>5PuR)bh4dEXl!Ui1TqyKv^akk3UK>Woi|Ax5{Kup zH5c;;+a^c!)XyE@-Mb&};qx)64D`eJB!eX!rH0AO0ney?IRy(fOkhz`u|EHXAL}C_ zxw{z1?3z48dVvc6d@_27efFPG^;D>v$+5MCWYBYTJw9JN#*GijZyOb<^Ugh2+r6|llOJyBI z)74AwgpS{h&_RLKT*Xph(U6NVb8#TD-Ju3sbyuj55n>p6lJ^IP3#~n{_sJRfr$_Pr z3R9v6$oqc3-y+T;r74}H{fIk_WN_|ErdW7$fDq zb?eGEo!@2!#}Z;jgQd)xJO!V5C-ip838seFhlPWVH-KjRA-t3KS?cbDh_T25AXXse z>yuD2anq@H^02Mq`=LY~+D?9m05%6l5Qi=br!Nz=&6N`FnLCAqY9i^&{w3m%{;JCR zliB|}wl#NE-g$NNC>gDAm;~VXv6v}&FpJc|25&z^CF0S&=sD0}S8gVe%46r=i)2*y zuNc5Z^bt{ihuL3Iz={ys507!}+02-%xP-JGt|Bk|8t8O)-7!K%fxa~Jm3TH>__CqF5B%i#`7We9}TDa|8?Rmy5hs)5ZX3*`+3~ zs+pokb}ctMQl@0l4M`vQrXfM~E7c$0(EG3f1v9q6yVj+Q%>E?&ZQt0~MWm0y-S4+_5Chp* zpZ{$V;5L+#th1egj~zKz7PnF+m~}TZGnPGK(~FVC*d2m+(zekJL2CT07cYc&OPC}e z5DBmuLMtqu9Kdn6Ayg${cT@kpfnL(y&N-^2{7}8KJ>iwmko_`!1pL0dV2dx9+w(B> zPXKA9_4axo1D4jT8EyUDizajX$}=1j=jY^6N$|%e&JQ-*)wHWN#ExJFmm0mQ#(H<{m#xE7z)N7am!-@3XcG z49^D+fod2SMJ)zC-0n3Elf&DFxt$>KRvmI`N@leeXR3MZ%1+j0*(emcCvL=l&E zUiAf2j_p4~vB{I$UgBU5LW)@1Oktl(9h5O)z-`oLCT$ztM4KQhthVUC|4%Yc{kLBR zl6&80I_S7BrNVUy2TJ|U|GLE})R3(FlS76keqp_b7cpq310N(O>KmA|;HXkLeyeFPI~qR-;MjUpK}D>{@=(4ePIgi8(OTVXrjJB+G_u1Q@d_mAMPLON4+C!f_<^-hYq_Ch0*Wb#5Kk?n5yLGS7c3cl(ovmlA)RGF*ZXNZm5Y5~WT;!Z+dTimF0N z1+5c1nGYKdhO0%tC?4QOV8br^sw1#{AqpM{4WK5RecW&t;0zyHEdeNPY7R2o;jWfl z`c(DqGMLJldq`Kq%4+unmJv#awo`}lh~-uL%(G{oHs`KiyLNBIN*eL3f(FP%FR#w@ zOyxDS`h6W9xAtPExd&w@Wn(fS9FQDI&+))ty2x8<4xizNi&xRDk8&WFGR^#Xl^CR@ z{bKeA%myy$1gY(UlOp93UinSE4L@G#^Q#KG9w^hCa&b;UK1y(k^f;8jJ3+e|J zW;uQokbTNi;Im9qSA$-^mRXw9pdQ!-p2(LbKLN={*hccxUwdUSP#!pSy^Zyg5*89# zi2=DcwVU#&FM@p8qtcY*6fe2+(}LKl6W1x|TqpXU^cO6bOKJ?jKfwW^n5%rscZg$~ zd>R40H~o;s+6*~)EF^cZvU~P?Z}yl^?4Le;kazK25I-l<_J7F7iALU|8q{ki>7;Z! zh`&&d?-XM-k>h#dbSY4k%6%!v!-IvwM>79ltay1Me~m;!XjyjRjsIqx z$XmD8CpCWhQT6r^S%RW_!uye_L|&3Y_-xf&?sYEfTcsYo;w+Wwz1wQ*$WB?#kra$U zYq$8#MzuKa%S0g})=I#hBB2`hR0!D&SPI{?Dkut-2Q5GW#RB&u773I~P{5N885gDm zE&vpQ%aZ%v|0Kx}XoFQ@lO<6(tKawM;CD6`n$&edsD;T3k#;a|mcNRdQ|(f=*8Dc9 z+huuddex-8IfcfD=KE$I#cbtSWvccMcne%>3l0n8&{q{K)~#1La)gMPh76N80`7Bq zSMK({0&Jr~*pl8-^d^40Bv+ASL>o6F{ftmC;AX2D!K47;9NTXRO5oCeBFv zVaUS;HvEykMPZ$~O}D2W^kC!yLp<|UH;Qzz^Bf+^@U1lh^17E^4POcE{KXQrocKNB zHVQAwA(aanp3pNeU!*@JW@qy+&cB#0wu zP@ihnFVW1+8xQHIqTkAp9W2Bu7z|U=?33{7+dcdO9-lYadoogo9zh|;sm=In^3287 ze|ZGnDnnVRy|MOY7gAF%gB2U#W>GO5Z>q&pTY~3_;h7|Rw&d+NRn|iwF5+Kvj*F3k z?o`%&2txFKT)%%knQKL{6q9YQ$t$D8cLsN);!z_vIpG zPBJxUjL18MnedY%l66UV56n-VVs0k-a3Y;dEO4|%U+4z6FP^^TW)d4g+*n4z{oMPI zM?jLt#aKw7ot$V`Q+ogQ{!<-W&xNl86kDTTzGx3!fyx@VDD7>HZzuN{V98FS+}*>? zahpUGmn&)F<$w6$%Gx*SDr@)8zq2~vcb<%D`QO=uS@Doudz{PoVVg}%s7M?oUM}vA z>+fL=bfTlobSg>3S=!Y9i7G}KR#YCm1u2gVwu4+=cYTXY1=EUEtGp$qyO3m#Gve81 z)+qH^YcxKG%{*IFQcd4+$Hc~7C6vR!h@i~;r-$hN-(Xr&ixuZe`ZgnXcS-2m9Xxp` zm#{h|yFMUYF)exDSD^5{l6bTZOr9fU%RfxqVpoqQLT|KcRxk=01^J}KS)>CFlHix~ z^sS z_oExQIe48SLI)K*N1yK2fj6(9!Bsa1+Mvv%q87Pg=ClG);2j!u-9R^BWf^M*DTG=< z_6X#41j~#VFQ3CdxCbBxI8o0==l7iqFkglgCPZkiAfYs#NCL%ZO}sHUQiN_GK^^T7 z7d~N9?_6gZWr)L2fX)eNYP29sLp#Z-p=iZQ!KSgU7~P&f+Q>EJ8KO){{ipylXLAL= zfH*4hroJm#CE#cQeOWhXO{`P#nS{Aidz`E$kT{`0 zNOniM4>94@?syyev9$|-jF`!yqi_CLhF>Z&)q4R)g=T6OUkigv2Uz92PAhq7E@Vc`KB zAZ%^n$Ite<1%5oU0ufil#+$3#e^+7897aaq}S9wM)CgWdY*ott`DSvQ5*;UG8%BNHmX9t@7;v<5+~`D0_1 z?+)px0t24t!(242K4GHAw7^za1yb!&10^zk8atvA`ip|$|iiy1djDI@h>l#^}*L3*HVB%3h zxpYJN@kcb;?UiJ`K~M-+S>B2Q^MpmQa$*Mzg!D+6GPM?AdKe(hGMR=b9-DP6l3FhJ zVuiEYu|ur7V?nCeuDKTf(bKZ?R3R}r90d*pcg1VJZtmWu9=*Ru$q)9XU(d%NI^!fU z2h4vwt-+vRpjyy;ljL!rB-*^}F~f=sWT{8n#KhLYxrU$YB8HOt*jF_}R5NdTG5E`?sAxz`T0(A?P-90SU2Veoi-4&;^n}TDQhd zy=~}qK=e=DD~%mO5Fu4!;w4-(NbGK6Ks6RS^)Z29Hbp{BbS1-u9fA2 zV;Z&aw5b+|^FE&S$aw@r`M*cQX<<-%8AGZT{# z;@&gf)7$PYpMnAMFl1yH))o65mSpz!CXd0(RG~G{?P*u>EK#|TEjwmca*h$FLZl{) zb@*eO1N~b#R$F}?xw_}i%VD5ELq?qcm3{aHtXA^$k!tpXnE>MdUEXceubYUWCY}S` zcqa-VbYNH89olUa_mYnv(-1Pzz|SNf?W3uRoM)Y=jLp&qlNvK#{jf&iS4EefUpa3A zm!jJki?Kp|dS&I>eU+nWS;5i^ZON^lXSI%VmrTL~j0Mbc;#SQ+zLO4uC4h`o@EcGd z?F{Y<2b8R2bw?jR3HL%`@sBP@&(QErPlXGPVeqL)J}8uRB>Cr7s)LZgFewZ{nKxO! z)w+QtNW1|rxxCQO(W%8hMYoiZkkBc+)UXBH17LF6oxmDMjv7qz-i?m`a(gvKPUnM5 zzgovTFXFV-{d{(08TW`$m@6Ep2H^+5Na;nnst9nWe1$0TW7fUY=iueg%{#^Ms-rKU zPyottk*1s9k0t8l-=7imI3X_1#&hkQB$X@(PH-`YU{mVdEHb5p!`UT_t{hqYW*L2;4H;8iZpWKj<5 zZ^9!m@1!e0&u_Ol13Dkm+Eu~zvY0^y1RR8^TJ`GDSBdNAA(>Ns*zQ#HrvjCA!_a2l zIctMHC2_Os7OY!9u@#VzXd1Wsb^37Au=~eE$NXogcN24a5$Q~<;DmqUFnJ7m5`ljs z8_FwU+;Q|sVsJz;HpM&C?DTsAklLH9Ug+cZPuFk5&Pn!*`(+e05(N*UHTo)+Qd@Jd zgNylN+)FO@fhre*{D9FS7iY9Vk%DWUc7;ly`=0o5Bj^_&BGQ9qr;+CR5j(?bbQg;K zuueVWX-X}^8*ef3)e^2`m_l#uLA%}fTz0`QIeB-J<-;K3jwGgct^W|AS5HJZ}>-tLwm9w6EaV8Ga)Yc@-S!`8>^AkAW4qroW1W zuHyHLFb|-VGbuz`GA-O)UUKmOI~&`5j}e^$$H#CVJ~tGl`oO+GiRcL-w8V~>U5>`h$5@+kTT|WUd z%d~c#oud5ihMqNFZK`^@$>{bsDmSe za9nIID=L!ouMp68`Fq~Frs8ZEudsk0N&fhcB0waE?JEg`33ACs2^Nsjv(7pWFf<#a;Uoj^!RNTn@w{+k5)jBoSK~NEwfIV5oSVB%^ys z^(kp?$R9i^#D{Eo52HR{mG`~p98Q+-=*vWU@a#EtHQ)5Oxx_{&JQULq1}}=VYKK+- zT{utseSZQ;2<8wrRZI$-@#tLMn774509Ewmc-Wgl+7&oWD*i zW&>~GR4|~#1z~<@b&C;Xa+0)wvC*`ZuQyre-iYuR;0yB2kQOFan_5e7);oo@rFVZm zKlUkjtQhR9+orOCsTbqn`gpg;q)Gyb1!|k|mto}%GQ~exBbtAF-g{IYoX3JNg;yF!&m0L+Baq0+lGk1l z$rfjF^NeKITCOJ>BV^H`g8U)#qJ(|$zytZPd~%9BVkXW9=V#D z)ESAluX&`2xMr6hJGAZpxB$@qb^2-xNUkDO1}0~uMFE80k3RES`^)BLnf6jzrSOpK z=8;M$opbV8QO94uerX3napMH%I={5S0dz`R}7l>j5Wt1eqmkIzC47!iXd@dqhaxAzmh{qy1g~dJr(T@8qQ&M zs=UxK!R}7JUUTJw&$8$(t|l)U(P(s?9Yn-}wek_Vxy-%Rtg(Je%mF08duQ&gpK!YK zuCiZ5m2w>yjoLyQV51hQcaIN`8BPH@`5br*Sz&M*4F=4RG84h z@ubY}4fcY3vj^1Hb~>8!k3TLTvFpBwT*9lnKSwsmD)?p2mAjK?*Z;p~m$!FHhLvhU zY{kDO60GiPqWfpJcyqXmLp&;l$A|^ID(%f45cO@(CpnmhuXIP^Y81^8zJ+*{;n-Qi zJ^s0-)0C%?|qa^gcN z=6%}SrQ#=30uo7Xq1Uh1;jkp0^CzVoJ<82e8=j(^yvMcL{k_(plRQ5j!MU!lG)$U$ zeXjAspZJ?vu=M*@48XXC5R2|71>6)0%hI(v*^7n~Rx`pmsm8lJ?A6e~Oo*3W{PW}9 z$Q4G)@hV?AfH-QM2{*=w`*@kvOU{e+y3YGsCzl_d9U!gb->TC^Qdxz%I1>x z)JTS=NB`PQ_i;f%){gslJ}CmJ0{`er`Q*z=FoKP_wG!U_8%DtXOq-=Wwb= z-@owNyYX;&_rdiJ{yGAZ>#;4Sa>HkDW{i$cg!}AaPb2Krh3N@m#-~POlRGZ+R?B|L z>e87*KRN0h>~Fk%DDfXA9HbIC7NB;uJ+ML@{fc8TAD)~|QnNn04<59n zJC`R|*{e+6Wur5clh5?9(HXc|L!G9PaPZ62Q61;i#@NMw?xGihkfez!JQEDYTP-%7z2<>|Oe#yeXpmp??rDDdw*fw=GyaRz4 z=CEuh<#ov}RY-Mc%iqf&)_GWV-{HBPG4voc0%Whud;`;}DYU$?zr*lGAMmSK)wi!v z%XDMXKU|a#{Y!t!auWgMe}~gxc7}L$LtnZn7+J8&FnMsN9RHeG`^<*Wn#dj+Pr9u} z{?vRM{7ueP^}jjqIu*(jwyw7yp7kGlj+sj&*zKIbNze6?^~bWm&qxkbd#s3420^&& zd{BkWs`m=_p%fzhZQ=OWRXzgsF6<7TlWKqvaK*MlfYM{~mpn!%<`7)7iV5w4(({YxeEx2|s9vaiaYt zMzEef^hJbU;LNqaKu3})yXDK{nTQO>ho^sxXaUOu1-tm?pUf%eAni49e}uhLrk>0+ zkmOPKjAWmrl;no}SObv>tV16By8}7Almxsy9ACxy2)E@Ev>Mxrh1OL4c}^M0{#CzFW?}%D^{j^Lqw}TBL#!GJFix* zH_@q^`_g|aL9YHjM$oQivvVI|krwpkU!LJ!)Ax`2j)6-!tj-ydh4{i7dj4Ny^4?uuj*6SWB-MIP# zb93)Upb%GEKgV+3LH&|VY4RA`!Sn;4>-OQ(!9 zYg)#vx;jae0UHFXt@Fzrp{&xnt9&cUZL^P3RVjIVC6WgeoYoIbvAiqUc!Jnz*C zy7@&#Z)bkkpqo1E@V{eWjTc~KFrOTbB9_&~k=TV7d%OqZ`?k6{$?MLvE%_bSt6W1Q z3)o=Uw}l@tlQ>q{Pc=&vHxOu~ z*6OsgalX9y+t!@*_m8o7S{V)}l4Ww(0X%h|s%GHYzU%u}vazlaJ}a5JgV!FhNh<{&fanEFZeLag&fpmb|G*$`?-j$1|qOjqzZ;9xagKtJ|=i;lz>^Z%n z&BgOqlaq{<l@GBce)dX{|n@9R?`1c;Vm;3*S zdhd9y_y2wTRf?vllpQZw6=~Rsc&U(`NJgR*Wv>WLlp;||Mksr&V`LC=k}z-6nw&px>lXvP_AW{9adKhcdEWY3KgPJz zS?3pukI|W|MUYFY{G!_MP}bXKXO_7~4L0}Zj$kAguW0vRDAgvh`x5_MhOqJE567P# z4BfgCBmo1{vL2_4%CdGR1job(9c9@)uLtYjYVaKP9oBij!bT;RxZ<6ZUO=uT&63&$ zU&C3Q-wDKT{l~OpJH|TETpJ77*xCDsH&U`N+ECXK z{z$wq?uyDzzA;<;W#j~JCAKi;IvsDs6hznd47%l?X)z$4?BBYd>fF=}Mr7bCi|8}?igm+mMob1K@+pSV>ftx!y#cKNgrSz=%sKT)M^XO}_u zq&;1%kCqOB>WP*rq8?~IfY<9zPH*@tqZPQA{Z&GJJRV)!?c_5Da`zI4{FH1+KSgFA zfk8z$l$;_6GIX6#O+e-2m4Mdgh@9&zu-^9jw=`AvPn_(;a@q>>4G`ub@;lINF- zHosJVnhhJO_!ATdsj&R6Ao+q-uK(Rv*Uz8TS+<6nHnaJd!)KZgFn4>zUUMfUb#oQ5 zoA{bZyR zqU6Moi%YhtUm0RgXuk%ArVUol<9l4VcRE*N%@GS_Z{XX**Av94Nx`$d-#SEBx}Y-t zjs8n|T24TPRB2Pk&1DstbvE<5VRr~1@z}R^Ukjb$Z3(-6c<#?bUVe zagcGqAnz0gALH{m59%B#66e23RNBnS%bH(zh_T0U)O@Vq^(L-2;&XbN`c5~pFO;z@ zU;`^_>{K%H%#$L#fHb}D%I>NVfidJ8e2JZI24iez z+M;El3~j{}sdstHV!aR*)ODKqQ-!R`SMT_}C~2{$GQLl?{d?Q<@BZzN8_>(AQ`5CS1}~)2tbG2#8R0MsdbOgGFH4Ssi)dHy`(V{;6U>aj zAdhSI#I!l0!H$`9SL3#A*5$5HjHkN&`-q{Y@GQrqiwVkST6WiFoG1Mm>(Wy%lJ_34+1^A@3uHYA95}+Uln3$sB0K5W7IXh zioBVt4xhJ$7y-%`*|DQOZGWlEI2j>$M#=B^xi+=kQ9D;>E=+5B_}dE=2jNqw14KQ) zZbpEL>dVL0AxZjslX|U+T!vh9Vys{peO!!mFQJo1J{vQ}3XB>tb~_MC4`x@N&&f_5 z`zHS*8vbY!p^>xo)U4x>C}?WZiytZoHRg4# z46@N%4xRZBxH}@@jIj+&&F{?TM5swhnuv#|Jl6BRp#HVffxYdj=uM)C>R7~yF5_h4 zrthitcXrMby85kkH-u{=n(dbg{-#~O3~l3|%&Q4K+ ztaiC$e#$&X_h2Ykk5;<;(U-rA&1^a*)|l>7zwb&%dMEVg^+aqGLLtU;XbUQ85f=of)B9 zCql~)SMGWUNzk6W8)rTzy~@2*0aeFsyV&Ns*~#Sc@k^&$Y#^10>n&QC_j(t_!pady zEU zx0~OjZz~LompxK`p`)k`q4n?~awSLc%@Mp(HR6N}=<0SZt1BZu_mrT6!z8=D{s=`f z{8ev#>vOu8!qQy7MhN!|44TN#TH8qYckiMeYU^iHKG$#uKjRydT>ObYZ!mYK;(@G6 zZNM|h!%6`gw-9DzTSoLt8m&T)^m%&Q{AT6IMXG6d&ml)N78xAO-Bc1x^fxf z@lRZ>H)Wg8zd5s~C&4dbC?(xEYiuUVc)8P^8;)@gy<6yzMzARkjD(-P89@yqBqRU- zYqXxBFi!QH2c~jWj{lzh;>mfE3r3-noR)x-(2h2X4T;_K#EC%p;XO0|yib^;`?0gV zxxcHj-$jb!V{+TeRS$hzY#be5{Is5%`{N(5;EUM_PQn=L;$zYKbKi_Y6fubSmvFl9 zTz2QJt;EO;$}f&Kzm8i$K@&4sXpS%}dk5P6Yuj>v(ZckWN*>;6XTDws866#+Cj$}f z*UxG?=9-xKd!g%1X-JjErd7dqY`$A&A@L6!N5Av~L2V8H91g_!1mfW1&V3qI9SRlU)%<>--LzwGs$ zK&rwG9n9{WBvjvlao{SjQ~ddXnG?=?%YMW#b;xrw8|a(E3$6EGX7P3`QNqEhxe#fI zCwHr=2V3P?TklL`PGaIpR-=;Y8Z*$Pa#U(M>3;PrpwL&Q6T-=2`@kw15*c}$oV*qm zz(n9h-8*5RS&q^^FE2vhXFbunZrYJD(~CX#m&(d@k^I`E!LgiK6IL6E?|)!U|^VES!Z^&o6m9}HHA=?P(0 zJFy1=0AN`zW80JztN&>J&Xr?jD|4Y@PV(Fj#T4S!hWa~iXx_X8HNH{LdZYJ{)Oo0S z@I0BAEPuScqHMz&GC@*?D3Af+L1jOXQb3)X8n^gh<%17ve{rQVCSuqF_Yg6KOFe<* z^QTYGkzGJIKYA@F0>J7`{E_0>d@M&fG(MDqlbZ51xGprwxf^rz>!ur@ko!xZGwS+w zy!6Cab5ZHB(}i!0pL{PH*DKTcgb6gygnZ9I}<^hVW7yytjfX31{RRINBhG>;Qu-_HSenMonbD zc)n8*c zfel(JA65cYuXr?!g_TO^gf9pE#Xgy7lM+ama05e^QPuP&47({XTW+@Q&%s+BlmcmvQlJD z5;pg15aaxaer-4at-+=Jcb#RpOTNB35AWI0W}wU`m5*yQYV z8O@eprV#xnzpGvTkQDkpew1s93eJ3k%5Ce!lwA%Sj7}S(x zjUtBJ@gBPa-(z;C{}%~I8Qc#ukIv=qc&2%4TE5^)0E;M2{grh>KmQTtNH^+dS~}+a z@FMuL{!-})2HB`!sDN6}YTA?j#&zxF4Q7M~ebdiJ&};4N-pxdbloA@#UA&|jbk%!P zy?=e-pu7d*nbgclQ^$$1L}BCC{x$-}izp!-15+xJ9_j566Z-(QtK?h(eRY^$#9G8I z>}HfKnxC>tA{Wt^uo) z8w$PLwaQqhi8hJ0yYk_#(t##SLI+Y?K@h>SBp)|%ofjNpJtqXJ@vAeHB}k2~@+Y+J zZN1GYxe68!H1K&7e;yIR^z!#Bm)$;#Wk12E zRYLFoV3V~<;SjToR~H=up&O0ozW_15QICb5j1JeJY#JvGi&X#e&*LSLECAByjmCyrVpu~AQ;iCd;k&RvYU43& zdrVi@%wEBwCVVw7@nKFo2FkNt@tm8*`0dv&{dxtsU1b|W@CBJP0Vmb!_M+5cj146u zoK+wZ0*wjD6Mx;?dU1}eAF7S2Nffv$L{|9HOR)9?Scst9x@nz_3J+(>Z)@*!#ydXU z_lv21o^unn)Z!fwWgJnylWko()|Dr)EpPu!p8%nj{jF0aR7*IZR7Ze099He=`3l_= zAst*uKr~T>>YE_quQOUe-MFFIiJ`)Qds~aYWx&%;k03fmg%IZ4taGI84}!S`&CC;& z%}y7&LZ3Ozl?JAZS}Xhs6#3(g!Po^{8H!-1_zw1u(@c^lj$L`xYmHsOODUvp1NGmW zc4EKk-|O5I%!hseeR&HjoEuG>&S5L35z3j-Jd71%Mk74AFq9Q?-QqR4OViF2Oe|4| ze#O(w5I!zZb?k2SNrePl5HB+2d3_i=ltSe->nfHNo6>;5By4B&@8O20TFfpsU zKQ`biM#NZMIAsZ{PD$c=*e1re2r3)1@_byPb-&>aoW+OZ$JI2nGl-KxefG^ zaj%-G035h%^s>P$#Ut55tk4>)8-s}CT&yHmVd5dqVr7)Pp$B_7){zc6i?ecUf7|2q zPLTffG?l8zwtl^n*5O7nrUFwXN_v+QMhOrPmkvmlmX?zBA{6W5Jn|;JIwV*_zAk0> za4AX}gfDO)DVI?;Y=YI{WWRK`6Im%5$|V6G4zj!VHn;5hs(pWMkIp#3RewdXW|kBf zWwNYE2eFa%ddv&M6NJ`{L1!n%yEIe;g+Ai2ez5#_8Yj5<;xY;mO{H1i4EXD& znC5fue_R02-Bu*)gO#|bSV8ARb|{~5PKazR7A`P2etad~F)Fi@AZkaXb*7VTdO(S4 zcid+9@#h5hz#k}cZH*UznPQy)t6g5d5TW}M+9;eVxGs5XX)#B8(nU$Q!ME`?*ZTL& zJ%AL1Hdj(AliEPn&lcLCTSwG@^((*tCvt~&pNR8k<-VE@fZ!dH<%7ru9}i|^M4FYI zp{Xkyfv@8*n9O{TPU`>l z=0LDYTr?z2RaWaK9C-(!1b;yWxR{t`<`~>BFDC4zq+QZ&E}?k()uS%l4sHXg5Eq|L zCv1bNNm;=|w8|T3P(k@o*s__8t&^~kffTPDRQCcQ9lXwn21N`^K<7*VyBYI5y3@g! z$}^>4ND^U#>6;Kb&tAHyq$C^0#d&FXKR40dN6NXb0^i^xq;xOfTR8YVOC$WPGop1w zCchrG0}%*R#z7G8_U5W) z?I(H_tQv^_Y<%!IOa%bD+_6*9#{{Nir*gGAE{IsZf+$Jq%ZM}!$hjXpNKm^#-A!bF zag0R1(KrCg>AKX0?|rry-ZKS$pr#H(T(L5-ZYB8;>I+c#$87#2W?sY%1CQ{I=#_z(fPw|Eoqah2BeAuZ{xDEH zK+Yi0sQ~A#&j-{^-P>AC5f!MAR`=ZUc|Qn{Cl>x6vK?1pR2G)gX`rRR^o5@Z*IgFB8*Nv z)fe!D)Ms?U)3L&-#?ZQe1ZOJt-0aq4FHgF+}19z|WHl@}sC!fK0=n>!mDm=g;Tjyv;1QCEq zskTwDRW)&IKD}bi8saSK4@)R5^0^w<%H3t)WbUE-AZH3u7>gF=&VIkv2xSdG-ceMt zr_fmDrE|e833F}?6Y9CXLUIL4O&l@>Du0l%@45DeJRk|HP)N3g!|X6& zRR{eQHuj<|p#q-G5*tz&zWr7+efjdR0y3@ypVQ!B;I z?CdK7KP*M|Ck9&}EbBHfv4$Aq3n%mg?>6-7(BJO$5WUg@am*;*+7qN5IoDr2h)s!g z&3(3HT{?1Hl-$@_ytWSra(C=XT`(Hn1wRN_LdEq8GNV$2jE@hyZR0+BQumJMUJ1Cw z2~mOWpM<@)^_@agyu+denf{I3^Ctxbe!+M}=B*7O3q-C5&~xnxMFbI0UC*LMkI&49 z7>;B-BIQl|%K1x^ohdSko(ojUTga;BJLqr3LMXmPLJZ#&$#-C@OXl4_*zX-r$Kr3| z?cdh8;R-~)<^*o546j9-iq)t=C<#~gBV0}~;VsAa9d{RTM=?bFGL>iesd`k2YHs0J ztdoP)^csMbHlsh8-cC9HqPN6-XZLfm-gWzr8BPqGl!8D+h(95F2%i<%>;TL9fuad` z*4+aAlU|gehj#NHC(e;gGz2soiigoox{Tx3L zw`cIoc71qs4ig{}zE8d|T64S@0Q+4&9Z_UPpF81lEV_T7Z}};K#;2k8V*?PmXH#H! z5j$g)QGVD}$Pb3;NoyLd{Wz>MYQ*Z{an6ZA41&d5^mz_wnGsl`|3rYrL@d>@ub}wX zoJ^F~2U3@fJOROz2&$ImkFAAJ#*#fz2Si-rZHT3OU!Ql0A-eB6{b^&`iR`b>@h$*{ zO5h($8-=MUchE`Hp9cx@haZ<6Jj}9wasrzD>hVZ`UpzU$O7tfgO~c5knF)d>hW_7X01e9PJOI8gsM$*H<{SVKbv z1~4fSc7!{5!;$hFLRW(YqjWqtB!*FJ`d*73x$x~*C{TL?V|C7+?UUVNn;WI=0=*`* z2j7ak^JDe3jRPSIL9JOKp#|$#PQc7a=BF?gN2eo7jB@Dt$nOOPSy109;7CNGa35(F^N znndj_@87={xnN*O_Lq=72Y5Cti_d1KD<5@1(ymnotAY&1k6uO12*(BJ!~~OGeQfhx zfE`}Bj_IL$TghT0B+|#1GXetW){U!qIQ8B0>F^~E1O8SjWLjEYew28M|xJj-^&Da~MCK?^{T#wmI5T`T|cg)BU|{I$R`yT!3iQ+nmaJ#64@08Q#5oaJf`)|zP; zF%9U4uYn@A6CMD6FL85l+$5So6ca3O{8%kaOpz_70N=g7~ZW+w>`?<`-YHfdpW@W z`2kesBg|9<(}Y-c;cuF<+PM{nbf5X&Zy2Hp+55p4raD2BtEw!saDtO`@D}5hn?5cI zkI>XXUroI3fGi9vpm$;d7D5^YP4{ob#V6SlgYfMEdm_dBVgw*WkZDQ8rtW;Qgox~L z0WWI3nHNskLvS2N^PI#WG(to8v^oL;zz^7MuydXvIN&HGUyG(*Q`3Onu!%ap{ba&~ zl_9*W=$FNzenSdb0bt9)&1s5|5mO>u&pAF$;XN0*Uv*XF>e6>9mpR|r@GDVn zZdz)guk-TqlNDdA_*-@qE7tDOSN?Ci;Efbsk>5Ku!WwSlfzCf1*DH(OH9w45yHYR z$(EP^7eb;dC`Hw_3q~I(+HJpkj{=dkgxO)lP$r?L6Dup*m)R9oC1^v*7Hfh#g@VXx zZ-br75LEvhhhq{ucq?g=xGB*2_U{LFuN@B2|I8JXa&H95 zBN$B}L5M<` z2p9qP?pjH;0g}&3mgLw31J#AxQO{s4XyY8v$Nte6jtYd~9VsL%O0H|jWj(b_jq@0Q z-9HZ|5b<~%L?=Y_iUa=nNU>~p?VaZDjh}}d}f1y(6A#RdGK^X(Alx*%p zI+jAl-|7{EXGzQ`udTXSP9afbKcsFbx?Z3e0Uv5J^glV`=G6}?&tSpl8kBp*WijdX zK=CD@CCVNitx<>M2$Prpqo`qWw{4y(4P_vXNQU|2BSH|4FHM@gsz9a2y=w_5XD2Qj z#BSkohLkGrYr2FUfimq>uXr355qF@=U-Fp|1yoPcu&{S-~-2h6dlUEE7oAZ!( zxm*PD-=abe?;x<>IGL~4z-?kiOoQO#IxUFbp*KEQIY#t*}ayT0vr9G(%Q`k0b%Jab{ zj7=Eulp-5gpm1U>kUWESjwooFcnh&=vy51~qXM##8ayD{znbD+%#t(Z^A>8;93Sx; zo_f;l_>;OYa0Kxz$7}-(T0zetArSPJNcy4fB?!7Hibaz{ErvRhm7)8CSf&~Jy`!R> zXU@?(E`slt=3T$0lAqeu6=FcOIfmy((As~1rO8kRX?<}P7#_|gw%E){YRC*G?epT4-6ZB{vA8MjC;|JPFC_}b$lLwuf&w@C&O}ad;gRaE(CGj znNP-pKmvXM=da7(n2Al|LB!hBlre-K zmd0k|-HKvNa%UCZcb;;~kLq9%$Sl>-+TmU*qvXR^X1`+n^bX$Rf@4B(wkac+wmXMk zUADbB(ZBAOlr?ybU=WAy7c_wqj95sS?^c;Bx_^JP1$T`vZa$g03z3|f6rpiZ?sjN& z1>T(|mW~Y=GwEgI%6DH278;2X7&~NNIR1dHK=k2D?(hAoUv_k$ zOTo1}{qNN5PYu8?4xJ^-o1^@|VBj=B&*aDBL^FFIBWxW}c?Px>n@RJHbpb3K9Ler# zW;@}r0&rou&SYB?pv0f#TDY_yZwda=S{Zd=pLU=>0rDy8Ajc zFY|vGC3L3~LjZR6^7z!U#@2(h<{AFO5yRUu0;7XuN!n4< zAnict&e2=a>3hSL=Ak6Bfb}HS55$*goGI4=q=g|1(A%HRw{}#SELr2-&;rRO5Q-OS$9}tjs?*PdG4|WR#RtGR>Y44d5#E=Ik zjj%u>v{d(A#VJF1$&B>dl)sUx0vpxvqa&>A*Iz|$CEKkhp1%8a*jz-Tv*ZK+@i}om z-8Ln?fHI|Poxkj2|(uH5g&#HkmB^K~S5NNt;`xotN=fo1Oir%Bm{bmden)o^OJO z+K&vqus4S+4;$xF-8pF8*GPKrME~%2peeY6|LQ21v+3Qun#>>qYVLcf-mQw5 zKrB-LdIC}SGHy!=xOK~Eu9XKuoN>iT02xT2x$O^PV(fEP|GOZ?H8U|3NEBs5&qCmP z%>sl%qSe={2C}2uxJOKmR7Ugy3P>graw*0(AF^!7WAH`U^+wnSoH)3a_=e6@AUh%{B!t$LB>% zZy+$ii&{Z^LX;CZkU(6{%H%F59?#8+uYBC2GnZXN@GVg^+$JWbiW=c``bEr3 zN#>7rWeFJ`Cc0&1R0ROX+v7XARb;2}XodBBG;^(Lo`^>R$a;CUn3rN}ha3%w zV2URU%8r81x?3*6CvcUe($t+UAVI9+`o(7qP9d<8YcP?`De;51tpy-__}Rn^?sO}x z0L6pgW1XQ}1i>SI57%SZha^?Z*UN~N$lQ#MjeOuMo_@~J#2~KQvVO#O!OMkSzmhnH zJ%Jbe64HX+MqogL%>Z$=e}fHMkuPdKLNQs{(bUog+Q_$D=fp$Nr==e6vMOh3+6(VX zk>Gvd0q-C0vSfr+M65eUrOl3q4D^y*RB^3|a=m~{egn<-N^23pDrbUwqE6g5w$Jz1 z&1&^39j(++BMl&qFnXvyzkuHA6rRq@#>0~rwZQid71FGY+d&=MK|q7!IOljwjK8X4 zcJ6za4;y#lbFiYV?gYNp&UL<14Cqq!dTRm5!kF|bf|m2!r6Ix9mUpj_nZW-oE~)!5 zgwKlbipT&}|1{kj-!}10v~%}`|KSfj6?w+BoKHXh@s1J}esnA1?Y3}f-V+UDIoqD> zxSK!m-L?iU+in%C(?Rn47FZtz{4>S|TC}K+Q24>uW>sP8FPuPW;y^=2+ux$w&mw;; zS`Dx6LKOQAN3|{_U*fS6^5{{8L{PauAsu*z^F6NXr-2HE#=GeWS0e0bi2Ec+B~PZM zMP5r_$tc_&2jl`0(WnyJdMzzGiLCqeutWt^#v65z?jW;o{)qkQ_m*#JeY0_kKQXS7 zGgP5b2^G%Ng!{`)Cm_0Hf{y{R562g>pA)qp+kDj7SIxJDqdk^-O$;s~kr&6Tc+~{Q z4#3-93H0Fh(;_&{SjZ`hfu;16j0d=cU?+GQh>+E%L~AV(rAjOY$p{eGuU2TP@V^@_ z`wXhymeswBj8+j08>Ka!LSR3gnXFjQ^Vq(K^*A$Y|Mf%1ZkDs4Wh5xEMd=n4_kk#8 z))PZ1G8Xe1&OKCKbP)KM^r9v5SKt8ug9`7)_@ea0(o8ESFMnIiqOZJ{xP( z{7U6vSY`xljv(D+ht0T|o(K~KzFP0`S-8hU6}W@v(1nv)GNGZZeF%3W9KU{gB~lP@ zLbC7#g?1-7;wToWpalqps+BL8zkZEygF>N7BFZt`8F>FGq+jLCm{HaCO^4NDP3A6Z zY3lT@^_|D^vM%tkaxg4Hp%IAr9I=2!JYA}@IWKCzw{c@MGL-^Ya={=@!89)8-3Ww1 z#NPyMC!m1>H*i#6Hf{*%BwvkXgre_$aBfv!^znw;Qd{#|qsS(3CwqB9tXaA5KZl!2Y3Rx;HBF=a;6P7#(RoXYuL?U$XtQ z2aT{);DTD}=28gDY7cC2L3$4%%QZ~%j*jeJQ_7WBLIY+43zw6josllq(Yh$Xn1Gi7 zosi!+!ap}Xu<%M0mHk1GT;LMoqekgtKDSf~Q8b2>7Psw5-a&zM)b%rAA7_l5U+UWC zL|o;-l%a(st!6Dd+SS#y3vL{|nu25cDCl#B^9I&*6jJq&yfLz{DW9#v{6iPMnC=|6 zOfgy~u&f1xw)StoAPF%XXf>Xz2YX!P07zd(ZBIA7)k5M9_#OzDMV-uAm{XRLO$R|+ z6oU{Dx3`mh%b@i-%;iDJM?5z04uzj6SLt(o1eAvNNX8Rux&rSmZ^+u|1o3dqRs&`$kzQJn9Ex)rg2V;)RRb7;vytVdVhDL4Y zr?Orh7rz`(&I4W%=?Tb0fv{9{vs>ZIvE<2t#8zCABQ&4|U}}`uG zlTMAC43ii}#Uum&S;`aa#(lSJ0`v=h zYXQqFP&$|i3J5SDPLfwhgrdy@T&sJ?tcY}++lTKH!$9JP^kIOwzF{DD{G=+a`wrQG zNYKNP-7c2zmn;KTR_o#$T5%l<3k0wt0|=I8Q3e7y^tOa+1o`7e#SwIYA5i`50_gC; z%VR@fX%1<5a5XA$K8DS>ghI`sw&$Gz7#k3&3q8+J{_`t!Z%gTK}NF}G>7L;-|kStH$a%uF+&+&F840CWh zNOfBg^ssp>wg1%-p$(+;B%@qznQGhoKCn{P_{gERvja2bqPd-byWp3yjIku zZ)2Hh-9z{R|h%e(0;lvWj-qIAh9`n zl8|t4st80P?BU6x3*_O!dSKlzoBNmX{<<=2`Q76W6C&U~MHY<#%^B}=9^jHVNg>l* zRBQBO*VcpOnK9~u$V=S23~x9Q|I1sq;7ZcXB~yPTpx^}DKFI+>i`{ICC=&?WiqT#k z_EojDwMRQ*1-_F~O4Y&y5IRy-nM6i;(a}^~>i}J^ZqIE7y7G6#BNN}bal)>Rv%l^6{X>}t z?%@u&#y_BREJf8@0}O;G)j7Ddb97+%4As08P4gJX|A<|-z)SeZMm($uWVDyoOKin) zcmY~x^8oR=!5krAXFxy!VZv;@G28yhYIiIE8B%dyx>OJwl&ns$kj{=}5qqqe!OIeM z(N+WTiR=_gI%7T%_NQLg0K*Qr_mDAUgrY_=^37Hup;G|La!{cC6;IS<_K$BK^}_5C z>l}8wG_|6byn~{n-;|LxxeVA-_X!dbnnUTgxjzBSScgm*mh}-}8@^AJIE@J+)G>9m z0XXQ@`D--ERx%J~(UPIV%H;Q#uAhWQ$4S(z%9AEIP`Ea6m#VBm4Lppfn3g^h;F zvas6-agfKn6W6)Nq}NBEB)ps)88Uc(hk<~2E+@@?bIk3t>S}5$$rJ_SQO@BGZw$o@ zUAC^*eMOq}kqU@Ji86a2^nQYJE-Z^r;b6sSxCDMUbpW(6f)8ak0hEpiV4w>Aru}rqn7`t`1AVMN3BwEXjZJUNo$-wQu@`_d)vsAB%yk>qK!qE?)NR&6{J#?%HG-lNXEhRWI?* z#KGaYP+9BE@-A}S_AiZgO~ zYqAA}gi6j!<;%5xIrC~Dqk6`E@(fv3h;gC#`C^MwKMSlrE}3v{0(lqKH!jTs|2KF# zs}o{<);qWf_>)baK*}(ta@YsHk+giIyu>N>3}5!2Y^8Ys{x8H>vND1H^Ehx1G6VJu zKR~0Cx})PX7Df^30BXx2NzEHhKpfsbRSTtse|t*6AFvMo@2XGZk72~0(SCZbsFYNr zYt>_Ew_H(45dAE8zTp=FJ@soozv*R+$^UI0Je>*lZYclAxNT5lsDU^C?bH$X;#nT3 zmTr-_hRQl`izT|sTMP|oqJF^t`7J8)<%WV)3X`o}8x9rU{KrjNC2vB)y{j{P-_YzG z=5Qi;1sWo`(sr;8DcA#KfoZ`1=euT? zB{Fm`)gIV3aa?s(_&5WEk$>AZ_3YjaAvPFYl`EF>OaMU)K)phhkf#lv)F~GHlSl$nEU=Az3|ydaX!8$@2VQAogS*dorm~`RJ0#t!K57yfrQ)u7A>Jvx}>UV zhDW})$s+RSJWVV3o6BG%$Y2_+NmFiR{r1lmamXBFZ4#HV%}21I4fU{;%ssth$6!YK z@!U*XHSt+l)rDRQ$eV6FfRi<>AA#=^{Q(pv8^~2pT5#)ZF29Lzck-bGbJFS+Sm?Ob zwi4+y7~>o>DJxhipj*mN`f94@0X^Tz>=x5mkaL++t23_!Tv9=|OBS3AO#DNrCxm|=XW__voDA%E z5wXg|7#UWT%H?KM(f+U#2yeK@8a|Plms~@qDqU+T)-v>b0?Pn6*rGNHW6d__0!RXMa%Cp?T`CwC+gY@N9ZAB;j^i({TbKyYT+0-& z2EuPm*X2B|Tm-aF#($vwiUX1*^*H^2;p4=^RcRfXh710UbE`pIs&Vm+m%TJ4p%t_L z4(1DejiSPF8d|3SE5MENA6#t6)}`m(kM;5LiA_G}iAHqI}_)xqxPJbAu zPKmJ2>tA7X0Kv?s^d_gcZLE^L#Gn(Fv-BG4kMG}~M9q>R{~Q~iuKT@tCO%pF@+=G4c%bqG? zk8+wdH3E%{Ezs-GV*_pD^L{!l-Z?K|y@Qkz7e7U~41aLu(b5 z25A7a!_VnWSf9j>!cu>HA9d_P^k5L7D1k1{0t3Ikk!<*WWXeTF{0a8`$3R&ADGJb# z65quGZ09mj<^hFwoaHh?SqOp-j)KqcB7GB+*6xli3YfGC{5PUqv2Goc&gA4ja9=2w zu?!m|uan6CARe-geL;ahX!BT^qyQi%7(?$B%}qSigmfuu{5OQ&`>V}~==_1rQ7DjM zFw9S67gyvxei*QCn3>=}lPD9dctX!&Hk?s3V*~8VYgqO}sFv8K)r$vg=>dQls9FJg zAE|j2P(BHGh-`J_y+{brFzJl^?8f=$n~h3#o7>}*_ypVxDlor7`3zCij^KVd&yK_2 z;{)Exnzxg|89~4^CNE3EI3Bfw3T(o*Uw1x)DxuwTfLM?xGd~8&#s7p|?sC#tp8+`w zEpW>BDpLeW)aMb3W&Z^|B7#e(O5XAbr3JIbgB=zkhpbEh)ziX&Yr`I)X{gB%I5N7S z4D+2v&nf=vwjBFAh?tkx7A1`)(a(|Qa$Lo4&F^yO#g&!DualZB7yd<$ERa!n(c}d< z;+bv!xP+kQR%uwYAdcoz~V3KzKQii(EU+R zO7daDHtyl*(H!Mm2xj(SL(t2Z{nxd{u24!;RLyZSFX+P~J?xwzK9+dN=2cdqOx)k1u-e%uS0l_dM4UWvWLa*$%bj~Itj|@qCw~)vZopXfiXM+KBJ3Z+BPM{&U%=V?gG7mT+z0tO=u1?nAen3PPh+UNvK{(N9CfQ8)kF2{AS=yP? zeaa)hZJ1uv4EAP;B#2+6z`qtQ|@67$%NpRjEY66qhX3M5;e_9lIGF6EAhcg@BWf@(<+ zf0q!@|3Qc|YUC6Rg9CLbHptZu4zwwJh30oS;(;oyK8)qX(Kg0UH6uui-BT zA_zGpIJvI!^faL`*x6sNJX2>?ey#G%9vZu0%!4+v5NvJUE;N~eL_#>_9B_C)f*I}3 zLuvcyLG6=R0)b{&*?LwAugbuW+M ziO%GPi9*KRjQMr_t3BO^W61s&1J{ewdhU@{kkZ+0iAw7bG}P6NJ$;-veq6#Iq7j0= zp%D%+ai$;4~0*cw`Ob7{yjhu6HU_dj^AsY1@+c#90>WA5x*yHn)(U(kcqVrnSPhYsZ zK_aC;NJCBScQO)dcR@#nSs^DO`Vdn%3)SfO9tnRq!IK3N0`4Djw?iC+L-L@I7+FE| z%0&6UEa+|<_1FE_s;{XjDdJoox$e}9O=9xAx^0rg&F#k!zjnFOkH*e!2)VRmLE+%Z zf5As+G&wSNLcM5}ISU$i0H6nxu?a+>SWs#0hJK@P#z;Nxj4R4j0KS{@da=XrI*QJf zZF{zErSO7b?&^B4%-!XzdZ2bvVSAn7ut)P2PH zCy>47gqEhJfn?`V4b0xi&WFJR1i8vVwN0t_e&qu1eKMK2ys-S&_(H#}d81ux4l4>c zy??u&?jmAJz#tR~oFMf!K3E^Z;!etqzTMr-9(T#|iYdsV-6~oPGfes$XD|gsj_BT0 zV_||UH7dbeovopjDIpGb_}`mrk-Yx&OE5bWYh{BT>rk#%lqz{X0odLk|tSSNzCD)`rh!S7rM$j@ipHev@==%wh;E)cIleh+J=9RP{(%01*g6_S9kE$Z2I4sQ^%`j2kNiy zgq~;V_FV}}7#MTBI~RbOEqx+>xjn}*{=w;U2{}1Np-Nx-EFz)VT@HTzgTjn?-3DBk zM`ew^?w2!ruyXqfvg0<+yN)jaJ2rVcQ8?!qce7q3qLH{WNgQbsHuWNgB9llEd6Hk= zotGj*?B9ECmiwZsRPKW+V7%@;HQwyqL$j(Z<8-4}XYRyh6-gb>5=GhranF4q3;8=^ z_I(6bwAE3v6lYWc4buplMPxUA=7YinpKxvpx17siD>XAq^yCEJ1YT<}VzAKBk1~zT zH;qblMF-9*DS&|}K#W^W8nLa5L3_X-QogC4pd&8`nwgT;v=QRP+8F|j+)q)u>RfPp=N z6rH{2KR2P&TummLDGG4#8nF9_A~t(OR*34jcqRM*R>9OtnQ)lQ*+#}CJk;PJQ^yZ% zP(#rW5^fANv-{>wR@h|yL~zh~?FLrK)zn@4WhZ+etzo9XOFq>=HiD&%XA)5_pT~+* z6W@EBt6q4dqI-D*DG_`ZW;;@b@6QtZ2*{8q-`-q!oKzc7u1PX!YxhmBHmnW}0XX|z z&?<(wmVvf~kB;eguV9E_l9hvXYBRiKh;^%(*=lkx3rQ2MO`MRDlxb;BR&UibIk zp2q|@#0MeX(%(dZ;E%}9DLTU_6AofoC&Z7XTGVwUC3OpFAKBmIXh3xePnz{1LO4qG z-<*{9byp&@c?`Wl)Lm})lc$^e1f0_koIWi`!NM-{&o8fw0@_wnyZL+$HKo}3orNO( z_h4O@q10&3B)|9I1rhg&!Q%u;r9glt@O(Gbh8Z6|RT}zcew@QPYY-)D^AuU98FkCX z$v_DW45ixT$+h=e1^wno3r*Y4#Qt%8HU7r}m^1BJ+Y15{J=6PDTJgH0hXX{6jMrnM# z>%!d1+MGU%nO!_kEaDWDzNi8R7Li~es|^1m({E;|vf18*N8=mY=;{JnL~PNyRd$RE zC`Y_UwF(+$8(UjPOf3DPXuR)Hr5$Dz8}YDo=Vr~_4j;yGUTAfkT+3WB!zOA~x`Nt8 zK4~2JclSNQ0X}3iAeVDO9!Y9Kc!_WtCTKZ2zIfiBf_x2>M|ooRQb|)MsA4yo+_5LS zzr82h63^n60``%*CWJ_#BX~|U%*gD(v=^9Va(g@?az;!CnG z4~Ca=i#GzG&o$7cu%4GtUci3?Z38in5MbY{Lo=TdNpQS-ik9$KzK zPGH5;51TV^0t*y_0$Qm*1FH#}z{dx3$=Pm410-xvLA4AG4r%#E?!C~^cO|lR@@C)- zfB$@in~f*%>0{G~NzHczCFffDY0QDCI299e{R=w_4h5C(S^N!XNp66c73=j<9={_C z9b|)7{km@A_s}q#yB#|OEvsM^!t1r^$#4#x2vw8HXTeSWQs{UKO$*jdXP_`eK=>!KpJgXH)-vv%#!4w$w;~i1$&Volb|n z9Lk^`(3bxHUm{a|C7^X2U`?uV2PgndRp`kx-*9Iy$0D&b6_ipJyJg2k*LGuuCyq~0 zTGo^leKJ^n?AMS47j8HH*{Pq71~+S!n3HCW%JaX9YpXH+L|o9+tM6O_0;L zC{o8vaUJ?uW}Lr%R1D`3jt&!(SNC!7m%2*1ad?N&_A;_v+?i{r9Pm&kZ8=bQfR!tn zKN|+8E_G#C5!UL;Sl=d)jlmT@xd*7Mnr-nFT+TZg^nYGkSI{Dro|jtPLcaA}7*8Gf z{T&}2$kQt*5?Gg<-L#yENdUWpYeMP}IbNTwQJ#6LRL{K?TOvlw6z6M(Tx(gYfe=8}O87S!4%(~e^aZTGb1FQxijvj+A9+rLok?U} zH|h1;x45(BULl^%{39V7Tz$#R=xFYRzuW%D1>hO@%c#C^378C)5xzs9UuM{S0=jLn zf$`d1Cvo(?lK}nnZqhqu9Cqaie(J)LUp}u%M!{Kdp_qN!2jbs3Mdp_ zTiqwPVg`g0A8Y|(Ue`9be>L~}cVhrml_g{xBNvg<^usSfL^fkp{c@H9#SBL$`D&QZ zTi@BHd$3!H>Dht%*}9(Rvp}B0JwsIelU3s#4*I#p*t{w(D1AL^u8QX7!+E3Gw&~z5pXS$^qVX@r&7Tx|7=0*^>Zj2_-BcMnexPMMii5<&big7uJ7jTCjm*veEYFu`xtBXW zVPFk$*Za=1nT6#Ey=aF`D~8zP;!7wnmfCtg6C&^fesgM7Jn8PB-@#Y%J zfxFK6lUG*v&@uQKULWkb5Z3-!c&(d^OW)mcrvfveb}%N5^_YYEo#1m$Y92eOnAqxh zWqY8*fO~g~U0wdNrNrdV;{r_KJSPT4pL5-lsx$(JZqXt!w}=!N$Zw4aieuPRdvLx`iPZ zd_-j7&t`_jDnXOwb5(%m*fNJ8xO}S)=MHkdkLo$Ei~KH{PNNO4cg<#70^nKyH6}He z@s%dQ(j0NJ_Cn6Ja;2{^qC-?PKa|UeMgcp@Y_V+=>yM94$R*2XU>+E!L5&L}hoY7> z@%4Jvi<@C~lBZ-Qu4UwE4tu7bS5U`rpz04qWWV?4;4xqb3Xci zj(Rt*aRij>*K>y$?g_X&f2gv!3^A4p+v}5(MH*^+!pnC>5 z_%I%JtuwgKYj|JF2iH)`OWxM~dTxAF_f#s&xcX%_C&vr*^?!R2|5m*Z5`?z#KV*ft zqp(1~Mt_JwKX2~LWW{tU#5@MSjr+^Djr{M2@a*~`K-F%3Vd>v4ICULUT2Zrs;PAf# z&tMCVy-lNidsJP^Atm}jY!C5e63A;%a^FkiMEI!wb~Jc0>|B?ck$20zob@7Q@v0DW ze`f6)R;NY3w@jsbLtlQY=?(0gPYTv^Yil3a+;D?lHCd(>#(mI`O_3SzoRR2o4+*-XUXohaZ>A zqI(8#t8wCDzP#P=JTXARKgoWm+v2~kPVOOTJsbpS-Rl6Buw{ygigvy*U<1R?Al>j! zM^JajaPXg_d`SF1u}t!l8QkOdCcT%((0U^VdAFLd=?`{8n;00^uo$tinl9$W`-_~G zd*3ZnF~HrV9}sg9IyeJ=aXdiQqjyI1z5eO1>V}p!!09lP`fvy&>7~F`QySgj$V@uw z0bW_~`pHY8Qg_C^-fnm=DoQMA{elzj`JL77UIbrl?&H}}ruCN7m_^!*8{ z*}p6m@$K7JLkq8yMHK&-ge-&XW0C&+bc0(XYsh*VMK_?4qvEp5p(G9jh1|dk1;DNk zQ%fYTw4(Ry_|b%9ec}JL_uh|G_kZ~Kscxy1(lC;SQg=j=vLhK~lRa`Ll_au~O;pOJ zk_eFsAyOQY5+T`SQ}#Tvvp?5U@6Y#d`2O(u;hyn2uk##_d0p2-h7SUa z5u+~qHGYbgg{b>ylr!y@oz~wxai2%X;q=-Y)J?(eo0$fFHzc~CgMcDq;*?rWTQnl- zgXefSbRNEOE^|KE9|FhrBW81_D2Q4&Xe|lmffz)K%=(nlNtv$W)9mjfE?ZC( zK`PRdMv6A~p8P1m&>2ie2+G}$1|{V>hJPNrnS-H+iE06>I2R9K2*lJA6tT?J?pY_~ ziZN24Mp2T}>0?I_ZKn^gxYI`?T{pigdYbw0xkIMA+;`s=f#pk_B6*?AgH9GuYq*`2 z$<${+Inj&k327gHkyH2~K|A?ie~_31Epqik*082dl7nGN@D|MtOeM>yOFR}SWMwft0;=WxCK#FH^htomAJv*V6h1`Lfc_%Vn% z7YXWIU*BTdPMtHEOTChN4<{GNVb;d%3+l|Zm3h83r%N2O8`f;N4$|gsi^Ik*CQ@<{ z5)(m(FjwVk*OGO-ZOn6`T<*0vcM%2Z!~$I~)dxBgA&5G^nm;&zJdTOUkxkvx5U+F2 zZRX-0=NZnbR%^igrKiWW5NDwj{R_xot10`o>+%M8W*Zmlot6tpu-rATHeX<>;s}e1 z;)~hLsqt3<`O;gxAA=%r__dp?O|`~gr>ti$?|rhav$k!od|7FEx%P)C8;9aEU}pUO zq-a-f1hLi1g%5fb7O#hnEmqDX9S;IgNCt%U$0itMzkFbdCEUJsu8?_t=M|#}!IIym z^)h6v>3X2VI4Q62vg$^Eb>VP5=nmkltzoD}g#(4C+e36@LUsPs*bNe8F13MBrxs`K z_ZdwuY;fWf$TC~8%LkVybI`*~oZf)xtZ<#0_FOaDlUIu%E>)hIJca;8BO`@6*SY;H zKx9s{iGufZEHlNasP`@Rbuk5_T{oK7a0=A8d7$+p7* zj?_*Y&&ZblVm_i8Rd{vb_LH?qHiwtO5O;zl!_8>&E-N&^q4?Oo=@BCoSL7bDep$AH z(c+HjF$#FOJRR*~8F4LFS3K!xR&R%+Rv^n1c65<|Oret=s#4GNR6~t``yN`L~UH zGkfS9CP!kceh_bb+|gS515!|T6qV;V^W4-qG6u%X2v-)jb^GVKTKhzVp^9!d^C6DW72JE1LF9Z^ zDx*Exx9YC$Eznw}(>f(%PHMi@EAoz>k&%P=_AT^q2956UwXD^zRnnvLK(#}=&dJyT zuuj=Qnf8QN-o=a}9<2{pwk(X)v**qZBqjc^eY`3-jf57PWmGcP;pPgY6GPQnt0iKj zih{59d?ZQ?aU?NyGJZK>`5HZp&ku_aIDH$yr{Qe+-$ek=%&u^EX3?pF3kJ4T242(y6bPL;-E(vzu0Wq%4|mQ%wesJ zjP!6r+A7q*)7vhZsnA%4&2!QS8&hl0^&yTh;=)@=$cz5F2Zg&ljyIgQ1cSawu*V~h3V6)0Vf|lidH$%?75kn zIt!G>*5oa%;Va>=^UDlQx z&DU=6sME^DMz&_Rla4Artg6A8uSi3h<@)l(?$8ZQg?>sQ@pI?O=={aXp(c1*N#nhW<9O^kPG8xdwf1 zi?L~EzGB4F@a|!oUF_m)-^oxo1i8iM#Gjos760tSA|=zM2*81%q}7mEFd_2a&X(Ed zU1&1eH_o_2*aKw03y>(<>6-GW)`dTLKL+cd!VVe{X&@y|jc!v2l!Z!P`H|_O6&){g ziQqHL`f-K4dsi}>N8uVfV4tu3z5(PLN;ypN)LT?i(gAC|S!WE~$#o)$+zCq}+)Qk% z$iejl#0k_8@MUFWo}ewU0Rf;Fa<4(7`{p*;7q>$gm;@9EJ^s}pMOdDMp@6T_Ex4fX7=W^CP27YO@@D6DyUmhIXcI-YiNT@OS8TAP|VU@dN9 zF1Pm(-R~^!VD-^rZj#BcG-2X9U&Vs^(#X=Ym>8+B%f8QhvJz#6)9Qe$$*zjCUp(Dk z^&JG>bty6uRlgcx&%J5Ahd*L0g|%~r;C`_wvc5+xEBh==)rMCMa5jv2jtpO zz|TJtu5`b?zP6UkN0v4py`(S_2u^3>$XJauZgS;Sb zKn7IggSH(!N(>(-=a@j$&c4dW_#XdZbDYAS`4k^6KC7XzH8nXgF(2YoqX_>Pl8V?5 zZs774{Qm3&uXH|{XZj~<@ijGYMeJ(9EN8z-5TjV!lpg-=qhr~_8o|?T#MQ!~*t+rGBa{;I!1M|2yYrD0~#)v;?w!lR4;zGojy8`bKXEklNTW5)ibJbfB zT|}fg$oLf^Gufp#IWt(q!Wl=(Yceuj7yY&(=QUm&V@VDFkXX2&j{5ukdXxfIs<~Oz zG()!NbX7J8QKY_pD~U;dBCQT^fH)*ueNi6|L6(a^NFs(bCy;&nV-HUPYK`Rl9xCLH zLsSG6eqfg_AKZh74#}vi2LlbNhMR%bBGE!|3adWE`*D+wtz9*WcBm*3ok;)#INxx0 z^6>F#?+_q8tn0^25~v1m4t|hVi2@Vt+|+c$;G=DITsrV?gc1wnX5qEW);svnXYfkJ zZM0bwF2rZWIc<=#EuS}&47AO_7h3#%SCVy>K{7cT%?*=H9306vT^9gsaDSF^AimWa z4QRnGySiC`(6E2ND?jAXS=U+-2K2>7fGNUU{h2)tQLG?RBxoM66A}|P!mHUbX9IzD zV=kBJ$N>cyK@$2mkKcP`y==s=)U;;Y+%_)31bjBWoHq#;*@drrEg zBR1PB1%d`Kz9sP&Bfwddf$T20BhJJ_U-UoR{a53+c}cFQAcn8$D6p(#?r@ ze`J9_|J%grMT&Mf&s=_O96H8G=Z0>GK^E~YIu(O%nUM!xaKjL4*UbX{1eLTMCtP%` z|Lfj>Txn=v&4WR5-3|S3;5B|z)J5M99DXpb_wb*KKsPJX*&rbz>dZsEef~x^GG9q= z18bG0uII z*0GdNgtL=Ssd1d>0BQ85p{LH*FRTD56%a+p#L{PZe~t3*_xN3CWb zwnMGHOoI%j26{{=%CU!yDZW1+?Q+oMG+Z>bhFe{nYC0MQ08(b3cOBDG5bb)tx&IyYpBcg_1LgeLv!5fI`X6BnKA3LA z=wLs97AV2(a-hQvzd2AxG zKn&o%4=A7I(@UW3M;0a@TE1RnEFBOK>C-{0wJ=GfgG`PX+{=0K@2O{tWM0M5Wf&9< zqcdxCWDwdiaCWw6|oUluL73(?sw%5K!~ zs-j?<0+ToYjRHA2R{=;?fEdf0#s&}a?+&yl?dfc3J1(L>{}uI9zUE(D_f~9lvi}9{ z-g(auNGwB<(TKyA2<3vcw>J~1r30c{vV5pl<5*QS$a?nYL} zCgD(z=iB2@^H_bRU(cnpB7ew%S5=d5~U`y>z`c!Bq3uNc$6>&R9Z^PW8lg5 z&g;TS`Nc$roL9oH$-;Wy zrX{^3D}s_xdc7-J?b&U6de0!F&=QP6u)zo^hc8gd7JPNz$27|eyl7be^-x(_v#}Gv z++X>^pN`7wOadWKWO9DZkuD03TpphzytCIcL%gO8`F6S7o8reTV!Q zHf}6}x}NvYML`5z`K#h^g8g<$p6iQpI$N#Y1o~Ls+D=OMbT`JblBDR5(3=0j{A6TS zKLoPGMRjj`Nq3Ux4mBaxQw6nkU!l|SYkp>2*nLr)h?u(oz``nMwuzRvQ0gtE4-*@1194U(Wh_r%#}R3WIH`rekO=>veuDCv?@n z|8A9!r5EpWy=&+Sj30uo9~Zx!hfc#SufdPI+A*;krI~q}L!MWOsriJ4edd2U$(b${prH9)$)&39ohJz=lxPb#4#G|@jHw86d2x7TJ9lBg@Ew9&^#Wm2 zc^L@1*DaM0)eHmzPAy;_okFcILm>z zo(4fvSuX~@UhE^`9zq6L<=LFmHz-DIO&-teObkIqN!E)4oLZn0Oer7!Msyw-eOm>_ z7Oa7S_~$O_6msjjAM6ytgx%1R-Vy(+VCSy`%KY{SrZB6o=MdHW^HDzndRZO2tzmR; zC5ql^NH}D zn_~aJSb)8&Z78QiD8w-8#j88VvMkOU;R(hv3_m7=1pgEm<6w}sUY?7q6fQhT|!RCyz?bRt3IryY!>FFXZZ! z6ko^9WyR#8%aU`Av5FTbw+&;iIebp3g0ywRet3FS$Siif?fs zbusbRxM;2zUsbUQeT?o(5sJtDkV1jG(|Jn^yg8rKwUg>NzJas)Xk8a)0R2lsriB;< zwNFFCfs{@<%Q*+t_&WZqyuI(SGl<^;t|=smYF}t=FGR(~5aLqSpaB+u9~Jej(Rg7% z=d8%fx??thJJeoR2g;SmLU}V~f4=E_tlm29&iexlT@Iw`Va-nKg?K=6 z)J0)a;>C*>Hm+^(%R=SIL~r(rwX+eoicw?}8CDVVZQy*xY9tiE?DtGg;)p@-ExhDX zleYb5I+g-BHs*yaL_A-SKR-Gt`)TExE(+yIAtDX-==rV#XC=ltMsCWxAW2l6{LX1_ zZ!fD;pI~Kl=Bq*TWa|6&`5vLptHyKsigw?V&*h4K1Zy{QsP%T431G=-Af~mPqF$P6 z9dXN~@t570EKzk6_&EM_qrRHS z+84mWV?ctKarel7-!ftsD=qC)d<_M7^_hM*M)0^|BT7V7Y7ZBj|Q~Zp%933Ej zY?E4OKpy_#u*$Qx=`&PA73${y`0dCY@_hYO%VD@_n^4=$3j@8?J9gqEk%==Gprv!$ zKV!c`5P?Nr?X$`43JWbGt|QL8Ip-f1g4f{=rIEwXYup8rh0BY$WOPaccv7-BNlE|d zbrGac#w}oPatP&f`-#0p9FPU&qCux2p^u~fz{VP6)2(2zzNlKYjY0`3%NjtUdf0Dp zH&}6w26vm_tvYd|ZI>eEr%fg%Ckx36NQ^kDu3x`?&ChjqAmJCF%)m};7cS?AJ{y}o zequQ|wS6NaM1_@iHVr}-Y?}WZ@|I$x#H=O7w1jBKvSann-S#JQt?lzR8;(m&J&Xuq zk;pA{Fc|-lR9oBr@XCw)kaL{Ic(H+U@ErM=X6#bti0_7 zFytp~J$tI^>sg}bT6#^;>LnW=TY5&;-Q8V$qVkkA+N9fz6cihA5LyH4xabQvVV}1i z^0lr*#pb2k{>sSJ7?FrnH{f!#>-%^wDtYhGu29t~YOD#(;CO6V~d3o^*3$udbb0p-6 zmmG_j1vnA8CLV9O>&)N1!e6lrD=2+}VA-6`J!J{$o6%~;!|_i0EAiX z`-Y0|D9~(?Q^U^A&b|3DsAEL_zB)GPxzT0!ImoccN?JneaulO{E~+tI->x%27E4OI zGm%}$QfWac<}D6&!KY7B2o?8Sx^(G2-(A_|yRxq9Yf;hBSF44$0KG%mx;x3bu6@3& z{W~^h{Cjyhv(ZXlI}GxvTWCPYJF#v>xdObLn#ZB{Nf_5&D^0(9+uOHqe_e|Ep#h5n z=7GJW8^(GSntQxu!t=Tg;dy^el4tqZWbHWpWM-q)_RF8__F6e9=rB3557!V^~vuDx5xSh@Z0E{Dz!w=Z7X-wVafcVw4S^^-3}n6sZb5x^`a zbrt6sAa$Uk{!|6JJ(#5;&bD>m@#fz;^J7?JZ|iIOW^|9IH=Mk?qe~vk78Hsy@I@OM zSD0Z)302DZ$jun@n+z?$t&6{@43nM?0pq<1oAt90JA39|{ zR+NFj31p8A+N+i=&Te{iOdv7yOR4;L%DPDZl&q;6V|~{6lGsyrUV2@MA_o>Es4iwQk-=JKZYcWv{Sn0B0wOGC*WT&#F26iSM^|v29J3ga z4Mq~S{o;ztyU8o+KI%*aF8wT)EjdXep&_Q}yo&wd7(Nj*OR`aZuA{!j-@9746dx79 zpbChvn@IrDrejDi+8PodM+;_0&Oev-DL?BJBW3r6jpDzh_V*sIUZ7-zH8$!spB1a^ zKSA@i(68G=-?#-w$kYtPeQ;8fmC6?Gb8_>A#b(XM%TY`Pa(9!Qp{I{4E~y zmj0DgkXt{{j?&o#N$gFcP}shi8y@5A*tqn2<54koGdCR7{4CkWQBeU<@}y2T!?~NF zyAnUrgn7c3hx}e=hR!0x1k>=IXwph%X|iX};Pr??Lf_zEDOQil#jff9im#==%^24= z{uK4CE2HiZrHllBLBWTLwAk5XSL*AeXFbze^*lE6N|kJ~_SHr;ngxUScB;H3l znEyZ)viK$B0>77k9Dd&{AQtgMEvc^OT;F<{Z&HQy`||zI40%MFv&>~KEW3u*K)Ujj ziB8Fag@Yi9q!R;$0awAxN6weHQzs{<*rd|D)EoEB-)*lmpUl`}h=s9pa9~kmb9|-A z&9Ls#&78uPclC93+dvvF%Zj!8{{8!*KHHAm5mzgt-f8&1rG@sTx=G!iuTv(3gee~D zl~hzNZT>1N<1!Jq%ipJxt!_m~z$*m1kDoq$%Fv}E`(J4E%&xFd<4cz>$7v)ycgl+h zymomP()}jp)U@Ci`>16#15A(TQy}I8(aWggeS`n0ip|bRgQ5C@@86Z=9zMU!A6E)X zeB}MQMq*-2|AW1G`&P)JVH1;RSG8tRoeAzTPO!?&Gtz0#2M-QTJEo7O1ykg1hCX$A zba3BOrIOxilbz8n=R#6%x0Go=FSIL?C0m&IvTr4Y(o}|qPjq9}0Q}Fy)KvLGwtjN9 ze9FnoIsdvZZtN-9_}R$FsH(MY=ypMyK=|fX@RC-mfP&^*^`}<$KaYG>GN#%yR$8x6vI#t4Un4Ti%I2Il36R7w76_ba8#VP}!hRz3%8> zZX7p{bkoF5;1d7qSeb=F)e+u~1o1m!wKDqVi!I}R!{*WknVl|s9x!Kn<;$d#tB}*pwc8UKIzLWPkGDIemi~4{9*nM*Clyw@W}8HzSuM= zTE)_Q#o44)!YMqQ+FOnOp-3e;cicN5u%ST30pH@Gf~-#M%6N{iIfh8;^j-0`N_w!tSnW!yM>^DkDi>f zlCpBVE|cAHvcl`q!N}sXyfVt7txk7TNP&n&Ot_4u1dj~)$o9Wm&8NV)91h_7FB<2Qcq z`>F}}5!7IH=f6nKpCU~!^7ztd#AO?1Ya|zHlqyF0`|lw`GF1;u(Bzh;hCauV>ZO~i zH*5JCrvJ3{yb*ShHB7g7hG9lDRg)W16KPw@lQg*#0-K@~_IetQE!cvwcje(Kj=wJr zkMO-*<*U{%$F5*}rQ+C~5%QU`NoIGBq!jbL^?5!{i&8w2EQndfc9Nx9aNY4;4L`uQh*{ zY5(Nnu+nEe9`8zjz8lLc&nzzGMFO4;#k2GwGY$_(+&ZedG@6A ziCci!(M0EcT0p77=6tTt2JspqoxM8M@xn)s);t>7ICe}-j2(+ox?p>-@v&2O`_&sT z!;hDaxpfsY8QE0sCMRYU=1C?fRk))N4B5lB#|-?7U8;+w>I^%3B6fY}owzlo{pb>F zeND~VSJyJf-!(T}ueR44=~W-IzH<9F=lQf#&GllFzALd9EIm-^qJOH2*{HF$_K8|~lI zyB%yCegY1JFiP|h$?nKtxB#$RgfSRAe8!e*PYtAtx1%KIi=xTLUo>xow(z^x;_hBi z%ny5>!(I9x-|&%?wj(_jdm|mq_o*dnawC*`dd4h5M@n%}LGEk>myejdT=e$K2g3}(BnjsWF^B6(vn>idf5O~n!blNh{;SozoXcEuyGr-n*yqlC zYyYdhp14ah6d(Bea56q~u4D6=;>JS*YrCo!#f59e1;zFz%_w}7Bm>sXB0S0EnVB0z z^`e9DtmC*&%5$T3^vkEgz-|wPG8}{S=)j55d#t;5RV+~52VS~ypBM`cECBRl7i@b% zq#pOf*q1L~K4VO}?QM$ibi>`9CAt>dvvYD(bzh~iV!8XJUliKSaye}1VY?+*eedyI z!I8aPwIS~7`@H9PBR)H~&$VUmnmZ~ve)NEg`w-G;sLbs5Pmc*T?2mSWOTa|-?3*-d zuL%wuRdSkWI*~}x1#pQ#P>R1o1}Oa7>8m^AO#Cf9m1!>(6(#KVoz#AN+d5w3%IEVs z0@;xcyQ?imj*f-ene6x7?rlAD`mG`jib^Dl64c*px121mMmLoSP=Af}8#Zhkn+JN% ze5i$b^hfIQ7ab)FPG+9ex}Mv)M|rau@5IOSna+1t+Uu|#%=1eQiy?lOs7Eb7-uAXW z^IKo@$PaU9mj`IKWHs|uH)TBeQ||Dw=Cnvjp!eqd+L3?#ANMR{|Bo$N*p8+mA3++ zMvl|AAG~2mJV#M;BhZU2;1%9SBxzlBK~A#D#G1jPL#g>1duyx#_rn23 zdw9V?w{y(<*Up9RrYW@>hg8{PoX$)uileOT>`8-Zq_h}mTP||o135oe9?(6_e*|y) z;g~^Yi%a`VxkCp!Z-EePl;q}jSNLm86(AFq?U3tZAJoD3WXZfmx6E8z{^A7i(p*>a17V6Ej((iZukm~4zEDsI~X#m7#IV{`<%ZQo)`MdHyoE~w92ocvRH_rz9+)H)JhO$@E9 zz5d#f`?iU5QBN96qIvXgT!{L>>wql7k{_?n^tI~8cHrI}{m5io|3|AhNPb)S5Biv! zkt`V!DJ0o28#&P1?vwH;ro?vQu>a>WeAQTNP+-lo)xN?1oGsAd?##)~CXPR$_Q@U7 z&+0f*$5kxMoeF(C30A9+)Yx)!a}R3LbI6iCShxScfleSr@29#=Ch@X; zhcTvJ2U@Ot%lFTm-#1o{wOwyn|I9F6C zd0Enr7?pRQkhex3JXX;wA3-0<+KGG>D)GP08_INT_TyU7g^UnBO>nthvB3!dgide* zeGa?!D~5YpGX;O^!eLQQorWPX3goNUMFcC)H(K~o4uk*ECHaV=gusSu*nrQT;tOdh>GvoiCx zrENSvxneoAY@+}6t0l@7MZh#zUWhgrSyrsQCP$9MS{#Yqd@7E7M%RlU1rX!BfV2lR zovrAp;hWndL{_$yZ;{4X>7&t3bdQv6{8e0pSk+{j)=FJ%&BR2G9VS=v)c1NH--mP2 zCxh-9Sx_i9t17mOIiN;J%A_nPb@~8N#bU&G=yduA%0Te6nK}8cgduOBwTFY0h*Tiz z!9*(l^1ECG<>ckbvLVm3y^xIP0`K#3pI_WLSvqT`qA(#KoyDMt5>>OI|RhOYg$Qv-yZstWTYVZNpkX}%`3Ojhm%izb7WYc$%%YN z^l<&&ha~m)Uul*TNrLS0&EKEtZH;S>v}4rI1^goIzL&*XY^Lsq8{4lb=TiY7WV^<- z1*Io`KvM142wEY5U}eY;k3tEAS-%txn;@y{w*4)6Rc4uH*rQzkzZ1n`WLIS#e(i#~yWbp=9Zfw3xt07X@|Ou^`I0+E>;)%-VrEag1z0d@3;_PO;;Ox6&v`uh9# zn^A3*!G^R8?-Osl+UGq7oB;b0ET*oV$bkmP3?SCt3x}j-cd)XAL`PdM*w~DD=k)4R zoh@-wMn6vH9$pO!EzmDpD$fU>qJOe0A61QQZiib+$oIzL-4Zrq#BriLIVEOaF*z|& zaCkodZ~=;|n=x)ui?Env!(__sitO)PqIoM4TZwgs)5z-6@Q5aTfJ&F91ZP}D^e>4Y z`AWxPxC!+bKTzRs!a5n->6ywn7&?O=C`Gyer#4EE$pt1=-I0%WGd_7jzIac)N~3V7 z@`xYcrA6U1`;+_WGL5;^37YpIar3oeaj4=e{fXb6e}2+SvW6Mm(EBjgI>MJN$b1HG zh1et_han|A$3JT&J%z$RJtK2c^JfoWvR5(iMQam%@=4r%sjCr>9}A!VFo`O;Y}kY4 z=K_OBhHa1XH_r+!UCLxUlG%kl;v(ls_VhQ%uUp=l;T*0@QLY8b`P z+jRcJv0y*y$EAglK(vTR!~H`6S3 z2E!j)x`J7iyw;q;qf>rBd!g%v`Z9}=Mk35EGmlP+pCTW1AKCb(tEs%%&s#LPANu(8 z4Gi4ZNYK1~tO&YGDE{(hukQ(m3JOCzK--%z`bv7f-hDx*>SEnBKuO4bTfWICI^Y?YYn%TdGiph>xh1W&eti zAQoh|I$xM-U(D5d4?fQ2zU2IT#gYpdua2pScSsKsir%rMqDgP*z?LvdGc5@*%;_dI z0XefX_1H&%l3yizWzPIaF=8jd21KU%?s!ds<^ki^|J;L*^rlElb#ZYKbPVO!=s}|x z=F4}r9JHZ|+2h0;9@5*4hlkhi^}9;W1_>94V*tcq5#?eDC>__Kz{Q)LVF)_800{ue zM`3RKHY3KMjR-s`hcpkYw!4Q*Mdiyz+e*gD%8E8!!#frA9_vl=@=KFzI+-t7SRCTy z1n4syR^m_KO^{(b&{*&B{E1^+;)x3#s^J}_OI`ymSH*YZ9PVP29J$`q*H zdJk*fkTd-!&_DnE?dA`Twy-nQEsVn9xZ0F-0e*gKX(=HH+}NE_=2|<%X<|8kb#qgh zyHN*BV}gDBG9*Ap2Mr7=-KH~YDE#z9z0^-G&&^0AZUT$YyEq4k?tu@BWX>=N*HJ%Q z+NyWS;Iy22=qARWg2Z|Ty`0&b(rrWCTV%+Rl$R$tJ2w}HXaW81U2v=iKAh;dnMGXm zHoZ%hxH3l^mh{s-1v?6UUNbR~{`ANT2sW7P>%>OFa`OrQ_l!JM?fJ|FJXMdyNTG+` z;U0P`^Vh6|Bx?=6>y0n;x-VWP(yUBOqu^SV=PpuvqkMNQ^vPQ}H_xb?(|~AqH8!_o ztwT2wS-Xjq-j_H%R%s9?<#9VWf?ov6YUN`zxcQq*Tz?*Vp16 zR@w~}p6gZdX;KFQLl`$~=(kyAMj{(Zas7NiHlQ@@uDlKNmBU@P*HKJKGm9b^XhYqB z0E%B!RFtJfSw*F3wkamm>-bYk%54(oP?CRjdGO8H%VfQ!6{*w>_oxVYydbdwrAdBs zL)k7*<#Q}mO`na@W%hI&V870*K`}eyFtODLzbU29eDs!x5oE5HGY;Y1z@?-zb%5$Qt4V?hs4HEC}Ycpf#M-ff)#`-^2AQU_yqs`V;f>mao2bJ?~f%f z{_log*7)BQAGq&xFV>g*j)zJWQvCh*`Otu+P(%x41kz`1o?RsWA$9pNL)W+4k6>ab z6pk28ZvTCrEm6`u%+@b6?WE|Jh75*r@xOx$>RZ+m+QNF(345xG|}2I5*1-(B=cJL zQ)t*;6Pp!Q^irbxJO50dqr^zO_UV*#UlA02yXV~A0}I(Zm_p+<`n&3*e=e=qsQvp) U`|&OIdalek \ No newline at end of file From 385004cadb1b0c6f23847a13e588c43a8487e6b6 Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Fri, 19 Jan 2018 23:01:47 +0000 Subject: [PATCH 53/54] Add dalek logo to README. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 91f533b..e968134 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # curve25519-dalek [![](https://img.shields.io/crates/v/curve25519-dalek.svg)](https://crates.io/crates/curve25519-dalek) [![](https://docs.rs/curve25519-dalek/badge.svg)](https://docs.rs/curve25519-dalek) [![](https://travis-ci.org/isislovecruft/curve25519-dalek.svg?branch=master)](https://travis-ci.org/isislovecruft/curve25519-dalek) + + **A low-level cryptographic library for point, group, field, and scalar operations on a curve isomorphic to the twisted Edwards curve defined by -x²+y² = 1 - 121665/121666 x²y² over GF(2²⁵⁵ - 19).** From 489fad186f77785b90699f834a822d8b821e2d41 Mon Sep 17 00:00:00 2001 From: Isis Lovecruft Date: Sat, 20 Jan 2018 00:00:04 +0000 Subject: [PATCH 54/54] Bump curve25519-dalek version to 0.14.2. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index dcb4747..a05c2ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "curve25519-dalek" -version = "0.14.1" +version = "0.14.2" authors = ["Isis Lovecruft ", "Henry de Valence "] readme = "README.md"